def log_metric_data_common(self, metric_namespace: str, metric_name: str, metrics: List[typing.Union[Metric, dict]], job_level=False):
LOGGER.info("Request save job {} task {} {} on {} {} metric {} {} data".format(self.job_id,
self.task_id,
self.task_version,
self.role,
self.party_id,
metric_namespace,
metric_name))
request_body = {}
request_body['metric_namespace'] = metric_namespace
request_body['metric_name'] = metric_name
request_body['metrics'] = [serialize_b64(metric if isinstance(metric, Metric) else Metric.from_dict(metric), to_str=True) for metric in metrics]
request_body['job_level'] = job_level
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/metric_data/save'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=request_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"log metric(namespace: {metric_namespace}, name: {metric_name}) data error, response code: {response['retcode']}, msg: {response['retmsg']}")
def get_job_conf(cls, job_id, role):
response = api_utils.local_api(
job_id=job_id,
method='POST',
endpoint='/operation/job_config/get',
json_body={"job_id": job_id, "role": role})
return response.get("data")
def load_json_conf(cls, job_id, config_path):
response = api_utils.local_api(
job_id=job_id,
method='POST',
endpoint='/operation/json_conf/load'.format(
),
json_body={"config_path": config_path})
return response.get("data")
def run_do(self):
try:
running_tasks = job_utils.query_task(status='running',
run_ip=get_lan_ip())
stop_job_ids = set()
detect_logger.info('start to detect running job..')
for task in running_tasks:
try:
process_exist = job_utils.check_job_process(
int(task.f_run_pid))
if not process_exist:
detect_logger.info(
'job {} component {} on {} {} task {} {} process does not exist'
.format(task.f_job_id, task.f_component_name,
task.f_role, task.f_party_id,
task.f_task_id, task.f_run_pid))
stop_job_ids.add(task.f_job_id)
except Exception as e:
detect_logger.exception(e)
if stop_job_ids:
schedule_logger.info(
'start to stop jobs: {}'.format(stop_job_ids))
for job_id in stop_job_ids:
jobs = job_utils.query_job(job_id=job_id)
if jobs:
initiator_party_id = jobs[0].f_initiator_party_id
job_work_mode = jobs[0].f_work_mode
if len(jobs) > 1:
# i am initiator
my_party_id = initiator_party_id
else:
my_party_id = jobs[0].f_party_id
initiator_party_id = jobs[0].f_initiator_party_id
api_utils.local_api(
method='POST',
endpoint='/{}/job/stop'.format(API_VERSION),
json_body={'job_id': job_id})
schedule_logger.info(
'send stop job {} command'.format(job_id))
except Exception as e:
detect_logger.exception(e)
finally:
detect_logger.info('finish detect running job')
def update_job(cls, job_info):
LOGGER.info("request update job {} on {} {}".format(
job_info["job_id"], job_info["role"], job_info["party_id"]))
response = api_utils.local_api(
job_id=job_info["job_id"],
method='POST',
endpoint='/party/{}/{}/{}/update'.format(job_info["job_id"],
job_info["role"],
job_info["party_id"]),
json_body=job_info)
return response
def report_task(cls, task_info):
LOGGER.info("request update job {} task {} {} on {} {}".format(
task_info["job_id"], task_info["task_id"],
task_info["task_version"], task_info["role"],
task_info["party_id"]))
response = api_utils.local_api(
job_id=task_info["job_id"],
method='POST',
endpoint='/party/{}/{}/{}/{}/{}/{}/report'.format(
task_info["job_id"], task_info["component_name"],
task_info["task_id"], task_info["task_version"],
task_info["role"], task_info["party_id"]),
json_body=task_info)
return response
def save_component_output_model(self, component_model):
json_body = {"model_id": self.model_id, "model_version": self.model_version, "component_model": component_model}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/component_model/save'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=json_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"create table meta failed:{response['retmsg']}")
def log_component_summary(self, summary_data: dict):
LOGGER.info(
"Request save job {} task {} {} on {} {} component summary".format(
self.job_id, self.task_id, self.task_version, self.role,
self.party_id))
request_body = dict()
request_body["summary"] = summary_data
response = api_utils.local_api(
job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/summary/save'.format(
self.job_id, self.component_name, self.task_id,
self.task_version, self.role, self.party_id),
json_body=request_body)
return response['retcode'] == RetCode.SUCCESS
def get_model_run_parameters(self):
json_body = {"model_id": self.model_id, "model_version": self.model_version}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/model/run_parameters/get'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=json_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"create table meta failed:{response['retmsg']}")
else:
return response["data"]
def get_output_data_info(self, data_name=None):
LOGGER.info(
"Request read job {} task {} {} on {} {} data {} info".format(
self.job_id, self.task_id, self.task_version, self.role,
self.party_id, data_name))
request_body = dict()
request_body["data_name"] = data_name
response = api_utils.local_api(
job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/output_data_info/read'.format(
self.job_id, self.component_name, self.task_id,
self.task_version, self.role, self.party_id),
json_body=request_body)
if response["retcode"] == RetCode.SUCCESS and "data" in response:
return response["data"]
else:
return None
def log_output_data_info(self, data_name: str, table_namespace: str,
table_name: str):
LOGGER.info(
"Request save job {} task {} {} on {} {} data {} info".format(
self.job_id, self.task_id, self.task_version, self.role,
self.party_id, data_name))
request_body = dict()
request_body["data_name"] = data_name
request_body["table_namespace"] = table_namespace
request_body["table_name"] = table_name
response = api_utils.local_api(
job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/output_data_info/save'.format(
self.job_id, self.component_name, self.task_id,
self.task_version, self.role, self.party_id),
json_body=request_body)
return response['retcode'] == RetCode.SUCCESS
def read_component_output_model(self, search_model_alias, tracker):
json_body = {"search_model_alias": search_model_alias, "model_id": self.model_id, "model_version": self.model_version}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/component_model/get'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=json_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"create table meta failed:{response['retmsg']}")
else:
model_buffers = {}
for model_name, v in response['data'].items():
model_buffers[model_name] = tracker.pipelined_model.parse_proto_object(buffer_name=v[0], buffer_object_serialized_string=base64.b64decode(v[1].encode()))
return model_buffers
def read_component_output_model(self, search_model_alias):
json_body = {"search_model_alias": search_model_alias, "model_id": self.model_id, "model_version": self.model_version}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/model/get'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=json_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"get output model failed:{response['retmsg']}")
else:
model_buffers = {}
for model_name, v in response['data'].items():
model_buffers[model_name] = (v[0], base64.b64decode(v[1].encode()))
return model_buffers
def get_job_conf(cls, job_id, role, party_id, component_name=None, task_id=None, task_version=None):
json_body = {
"job_id": job_id,
"role": role,
"party_id": party_id,
}
if component_name is not None and task_id is not None and task_version is not None:
json_body.update({
"component_name": component_name,
"task_id": task_id,
"task_version": task_version,
})
response = api_utils.local_api(
job_id=job_id,
method='POST',
endpoint='/operation/job_config/get',
json_body=json_body,
)
return response.get("data")
def save_component_output_model(self, model_buffers: dict, model_alias: str, user_specified_run_parameters: dict = None):
if not model_buffers:
return
component_model = self.job_tracker.pipelined_model.create_component_model(component_name=self.component_name,
component_module_name=self.module_name,
model_alias=model_alias,
model_buffers=model_buffers,
user_specified_run_parameters=user_specified_run_parameters)
json_body = {"model_id": self.model_id, "model_version": self.model_version, "component_model": component_model}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/model/save'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=json_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"save component output model failed:{response['retmsg']}")
def save_metric_meta_remote(self,
metric_namespace: str,
metric_name: str,
metric_meta: MetricMeta,
job_level: bool = False):
# TODO: In the next version will be moved to tracking api module on arch/api package
schedule_logger(self.job_id).info(
'request save job {} component {} on {} {} {} {} metric meta'.
format(self.job_id, self.component_name, self.role, self.party_id,
metric_namespace, metric_name))
request_body = dict()
request_body['metric_namespace'] = metric_namespace
request_body['metric_name'] = metric_name
request_body['metric_meta'] = serialize_b64(metric_meta, to_str=True)
request_body['job_level'] = job_level
response = api_utils.local_api(
method='POST',
endpoint='/{}/tracking/{}/{}/{}/{}/{}/metric_meta/save'.format(
API_VERSION, self.job_id, self.component_name, self.task_id,
self.role, self.party_id),
json_body=request_body)
return response['retcode'] == 0
def set_metric_meta_common(self,
metric_namespace: str,
metric_name: str,
metric_meta: MetricMeta,
job_level=False):
LOGGER.info(
"Request save job {} task {} {} on {} {} metric {} {} meta".format(
self.job_id, self.task_id, self.task_version, self.role,
self.party_id, metric_namespace, metric_name))
request_body = dict()
request_body['metric_namespace'] = metric_namespace
request_body['metric_name'] = metric_name
request_body['metric_meta'] = serialize_b64(metric_meta, to_str=True)
request_body['job_level'] = job_level
response = api_utils.local_api(
job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/metric_meta/save'.format(
self.job_id, self.component_name, self.task_id,
self.task_version, self.role, self.party_id),
json_body=request_body)
return response['retcode'] == RetCode.SUCCESS
def get_table_meta(self, table_name, table_namespace):
request_body = {"table_name": table_name, "namespace": table_namespace}
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/table_meta/get'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=request_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"create table meta failed:{response['retmsg']}")
else:
data_table_meta = storage.StorageTableMeta(name=table_name,
namespace=table_namespace, new=True)
data_table_meta.set_metas(**response["data"])
data_table_meta.address = storage.StorageTableMeta.create_address(storage_engine=response["data"].get("engine"),
address_dict=response["data"].get("address"))
data_table_meta.part_of_data = deserialize_b64(data_table_meta.part_of_data)
data_table_meta.schema = deserialize_b64(data_table_meta.schema)
return data_table_meta
def create_table_meta(self, table_meta):
request_body = dict()
for k, v in table_meta.to_dict().items():
if k == "part_of_data":
request_body[k] = serialize_b64(v, to_str=True)
elif k == "schema":
request_body[k] = serialize_b64(v, to_str=True)
elif issubclass(type(v), AddressABC):
request_body[k] = v.__dict__
else:
request_body[k] = v
response = api_utils.local_api(job_id=self.job_id,
method='POST',
endpoint='/tracker/{}/{}/{}/{}/{}/{}/table_meta/create'.format(
self.job_id,
self.component_name,
self.task_id,
self.task_version,
self.role,
self.party_id),
json_body=request_body)
if response['retcode'] != RetCode.SUCCESS:
raise Exception(f"create table meta failed:{response['retmsg']}")
