def pipeline_dag_dependency(job_info):
try:
detect_utils.check_config(job_info,
required_arguments=["party_id", "role"])
if job_info.get('job_id'):
jobs = job_utils.query_job(job_id=job_info["job_id"],
party_id=job_info["party_id"],
role=job_info["role"])
if not jobs:
raise Exception('query job {} failed'.format(
job_info.get('job_id', '')))
job = jobs[0]
job_dsl_parser = job_utils.get_job_dsl_parser(
dsl=json_loads(job.f_dsl),
runtime_conf=json_loads(job.f_runtime_conf),
train_runtime_conf=json_loads(job.f_train_runtime_conf))
else:
job_dsl_parser = job_utils.get_job_dsl_parser(
dsl=job_info.get('job_dsl', {}),
runtime_conf=job_info.get('job_runtime_conf', {}),
train_runtime_conf=job_info.get('job_train_runtime_conf', {}))
return job_dsl_parser.get_dependency(role=job_info["role"],
party_id=int(
job_info["party_id"]))
except Exception as e:
stat_logger.exception(e)
raise e
def component_output_model():
request_data = request.json
check_request_parameters(request_data)
job_dsl, job_runtime_conf, train_runtime_conf = job_utils.get_job_configuration(job_id=request_data['job_id'],
role=request_data['role'],
party_id=request_data['party_id'])
model_id = job_runtime_conf['job_parameters']['model_id']
model_version = job_runtime_conf['job_parameters']['model_version']
tracker = Tracking(job_id=request_data['job_id'], component_name=request_data['component_name'],
role=request_data['role'], party_id=request_data['party_id'], model_id=model_id,
model_version=model_version)
dag = job_utils.get_job_dsl_parser(dsl=job_dsl, runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
component = dag.get_component_info(request_data['component_name'])
output_model_json = {}
# There is only one model output at the current dsl version.
output_model = tracker.get_output_model(component.get_output()['model'][0] if component.get_output().get('model') else 'default')
for buffer_name, buffer_object in output_model.items():
if buffer_name.endswith('Param'):
output_model_json = json_format.MessageToDict(buffer_object, including_default_value_fields=True)
if output_model_json:
component_define = tracker.get_component_define()
this_component_model_meta = {}
for buffer_name, buffer_object in output_model.items():
if buffer_name.endswith('Meta'):
this_component_model_meta['meta_data'] = json_format.MessageToDict(buffer_object,
including_default_value_fields=True)
this_component_model_meta.update(component_define)
return get_json_result(retcode=0, retmsg='success', data=output_model_json, meta=this_component_model_meta)
else:
return get_json_result(retcode=0, retmsg='no data', data={})
def component_output_model():
request_data = request.json
check_request_parameters(request_data)
job_dsl, job_runtime_conf, train_runtime_conf = job_utils.get_job_configuration(job_id=request_data['job_id'],
role=request_data['role'],
party_id=request_data['party_id'])
model_id = job_runtime_conf['job_parameters']['model_id']
model_version = job_runtime_conf['job_parameters']['model_version']
tracker = Tracking(job_id=request_data['job_id'], component_name=request_data['component_name'],
role=request_data['role'], party_id=request_data['party_id'], model_id=model_id,
model_version=model_version)
dag = job_utils.get_job_dsl_parser(dsl=job_dsl, runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
component = dag.get_component_info(request_data['component_name'])
output_model_json = {}
if component.get_output().get('model', []):
# There is only one model output at the current dsl version.
output_model = tracker.get_output_model(component.get_output()['model'][0])
for buffer_name, buffer_object in output_model.items():
if buffer_name.endswith('Param'):
output_model_json = json_format.MessageToDict(buffer_object, including_default_value_fields=True)
if output_model_json:
pipeline_output_model = tracker.get_output_model_meta()
this_component_model_meta = {}
for k, v in pipeline_output_model.items():
if k.endswith('_module_name'):
if k == '{}_module_name'.format(request_data['component_name']):
this_component_model_meta['module_name'] = v
else:
k_i = k.split('.')
if '.'.join(k_i[:-1]) == request_data['component_name']:
this_component_model_meta[k] = v
return get_json_result(retcode=0, retmsg='success', data=output_model_json, meta=this_component_model_meta)
else:
return get_json_result(retcode=0, retmsg='no data', data={})
def save_pipeline(job_id, role, party_id, model_id, model_version):
schedule_logger(job_id).info(
'job {} on {} {} start to save pipeline'.format(
job_id, role, party_id))
job_dsl, job_runtime_conf, train_runtime_conf = job_utils.get_job_configuration(
job_id=job_id, role=role, party_id=party_id)
job_parameters = job_runtime_conf.get('job_parameters', {})
job_type = job_parameters.get('job_type', '')
if job_type == 'predict':
return
dag = job_utils.get_job_dsl_parser(
dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
predict_dsl = dag.get_predict_dsl(role=role)
pipeline = pipeline_pb2.Pipeline()
pipeline.inference_dsl = json_dumps(predict_dsl, byte=True)
pipeline.train_dsl = json_dumps(job_dsl, byte=True)
pipeline.train_runtime_conf = json_dumps(job_runtime_conf, byte=True)
pipeline.fate_version = RuntimeConfig.get_env("FATE")
pipeline.model_id = model_id
pipeline.model_version = model_version
job_tracker = Tracking(job_id=job_id,
role=role,
party_id=party_id,
model_id=model_id,
model_version=model_version)
job_tracker.save_pipeline(pipelined_buffer_object=pipeline)
schedule_logger(job_id).info(
'job {} on {} {} save pipeline successfully'.format(
job_id, role, party_id))
def update_job_status(job_id, role, party_id, job_info, create=False):
job_info['f_run_ip'] = RuntimeConfig.JOB_SERVER_HOST
if create:
dsl = json_loads(job_info['f_dsl'])
runtime_conf = json_loads(job_info['f_runtime_conf'])
train_runtime_conf = json_loads(job_info['f_train_runtime_conf'])
if USE_AUTHENTICATION:
authentication_check(src_role=job_info.get('src_role', None), src_party_id=job_info.get('src_party_id', None),
dsl=dsl, runtime_conf=runtime_conf, role=role, party_id=party_id)
save_job_conf(job_id=job_id,
job_dsl=dsl,
job_runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job_parameters = runtime_conf['job_parameters']
job_tracker = Tracking(job_id=job_id, role=role, party_id=party_id,
model_id=job_parameters["model_id"],
model_version=job_parameters["model_version"])
if job_parameters.get("job_type", "") != "predict":
job_tracker.init_pipelined_model()
roles = json_loads(job_info['f_roles'])
partner = {}
show_role = {}
is_initiator = job_info.get('f_is_initiator', 0)
for _role, _role_party in roles.items():
if is_initiator or _role == role:
show_role[_role] = show_role.get(_role, [])
for _party_id in _role_party:
if is_initiator or _party_id == party_id:
show_role[_role].append(_party_id)
if _role != role:
partner[_role] = partner.get(_role, [])
partner[_role].extend(_role_party)
else:
for _party_id in _role_party:
if _party_id != party_id:
partner[_role] = partner.get(_role, [])
partner[_role].append(_party_id)
dag = get_job_dsl_parser(dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
job_args = dag.get_args_input()
dataset = {}
for _role, _role_party_args in job_args.items():
if is_initiator or _role == role:
for _party_index in range(len(_role_party_args)):
_party_id = roles[_role][_party_index]
if is_initiator or _party_id == party_id:
dataset[_role] = dataset.get(_role, {})
dataset[_role][_party_id] = dataset[_role].get(_party_id, {})
for _data_type, _data_location in _role_party_args[_party_index]['args']['data'].items():
dataset[_role][_party_id][_data_type] = '{}.{}'.format(_data_location['namespace'],
_data_location['name'])
job_tracker.log_job_view({'partner': partner, 'dataset': dataset, 'roles': show_role})
else:
job_tracker = Tracking(job_id=job_id, role=role, party_id=party_id)
job_tracker.save_job_info(role=role, party_id=party_id, job_info=job_info, create=create)
def get_parameters(job_id, component_name, role, party_id):
job_conf_dict = job_utils.get_job_conf(job_id)
job_dsl_parser = job_utils.get_job_dsl_parser(dsl=job_conf_dict['job_dsl_path'],
runtime_conf=job_conf_dict['job_runtime_conf_path'],
train_runtime_conf=job_conf_dict['train_runtime_conf_path'])
if job_dsl_parser:
component = job_dsl_parser.get_component_info(component_name)
parameters = component.get_role_parameters()
role_index = parameters[role][0]['role'][role].index(party_id)
return parameters[role][role_index]
def pipeline_dag_dependency(job_id):
try:
jobs = job_utils.query_job(job_id=job_id)
if not jobs:
raise Exception('query job {} failed'.format(job_id))
job = jobs[0]
job_dsl_parser = job_utils.get_job_dsl_parser(dsl=json_loads(job.f_dsl),
runtime_conf=json_loads(job.f_runtime_conf),
train_runtime_conf=json_loads(job.f_train_runtime_conf))
return job_dsl_parser.get_dependency()
except Exception as e:
stat_logger.exception(e)
raise e
def pipeline_dag_dependency(job_info):
try:
if job_info.get('job_id'):
jobs = job_utils.query_job(job_id=job_info.get('job_id', ''))
if not jobs:
raise Exception('query job {} failed'.format(
job_info.get('job_id', '')))
job = jobs[0]
job_dsl_parser = job_utils.get_job_dsl_parser(
dsl=json_loads(job.f_dsl),
runtime_conf=json_loads(job.f_runtime_conf),
train_runtime_conf=json_loads(job.f_train_runtime_conf))
else:
job_dsl_parser = job_utils.get_job_dsl_parser(
dsl=job_info.get('job_dsl', {}),
runtime_conf=job_info.get('job_runtime_conf', {}),
train_runtime_conf=job_info.get('job_train_runtime_conf', {}))
return job_dsl_parser.get_dependency(role=job_info.get('role', ''),
party_id=job_info.get(
'party_id', ''))
except Exception as e:
stat_logger.exception(e)
raise e
def update_job_status(job_id, role, party_id, job_info, create=False):
job_tracker = Tracking(job_id=job_id, role=role, party_id=party_id)
job_info['f_run_ip'] = RuntimeConfig.JOB_SERVER_HOST
if create:
dsl = json_loads(job_info['f_dsl'])
runtime_conf = json_loads(job_info['f_runtime_conf'])
train_runtime_conf = json_loads(job_info['f_train_runtime_conf'])
save_job_conf(job_id=job_id,
job_dsl=dsl,
job_runtime_conf=runtime_conf)
roles = json_loads(job_info['f_roles'])
partner = {}
show_role = {}
is_initiator = job_info.get('f_is_initiator', 0)
for _role, _role_party in roles.items():
if is_initiator or _role == role:
show_role[_role] = show_role.get(_role, [])
for _party_id in _role_party:
if is_initiator or _party_id == party_id:
show_role[_role].append(_party_id)
if _role != role:
partner[_role] = partner.get(_role, [])
partner[_role].extend(_role_party)
else:
for _party_id in _role_party:
if _party_id != party_id:
partner[_role] = partner.get(_role, [])
partner[_role].append(_party_id)
dag = get_job_dsl_parser(dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
job_args = dag.get_args_input()
dataset = {}
for _role, _role_party_args in job_args.items():
if is_initiator or _role == role:
for _party_index in range(len(_role_party_args)):
_party_id = roles[_role][_party_index]
if is_initiator or _party_id == party_id:
dataset[_role] = dataset.get(_role, {})
dataset[_role][_party_id] = dataset[_role].get(_party_id, {})
for _data_type, _data_location in _role_party_args[_party_index]['args']['data'].items():
dataset[_role][_party_id][_data_type] = '{}.{}'.format(_data_location['namespace'],
_data_location['name'])
job_tracker.log_job_view({'partner': partner, 'dataset': dataset, 'roles': show_role})
job_tracker.save_job_info(role=role, party_id=party_id, job_info=job_info, create=create)
def save_pipeline(job_id, role, party_id, model_id, model_version):
job_dsl, job_runtime_conf, train_runtime_conf = job_utils.get_job_configuration(job_id=job_id, role=role,
party_id=party_id)
job_parameters = job_runtime_conf.get('job_parameters', {})
job_type = job_parameters.get('job_type', '')
if job_type == 'predict':
return
dag = job_utils.get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
predict_dsl = dag.get_predict_dsl(role=role)
pipeline = pipeline_pb2.Pipeline()
pipeline.inference_dsl = json_dumps(predict_dsl, byte=True)
pipeline.train_dsl = json_dumps(job_dsl, byte=True)
pipeline.train_runtime_conf = json_dumps(job_runtime_conf, byte=True)
job_tracker = Tracking(job_id=job_id, role=role, party_id=party_id, model_id=model_id,
model_version=model_version)
job_tracker.save_output_model({'Pipeline': pipeline}, 'pipeline')
def run_job(job_id, initiator_role, initiator_party_id):
job_dsl, job_runtime_conf, train_runtime_conf = job_utils.get_job_configuration(
job_id=job_id, role=initiator_role, party_id=initiator_party_id)
job_parameters = job_runtime_conf.get('job_parameters', {})
job_initiator = job_runtime_conf.get('initiator', {})
dag = get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
job_args = dag.get_args_input()
if not job_initiator:
return False
storage.init_storage(job_id=job_id, work_mode=RuntimeConfig.WORK_MODE)
job = Job()
job.f_job_id = job_id
job.f_start_time = current_timestamp()
job.f_status = JobStatus.RUNNING
job.f_update_time = current_timestamp()
TaskScheduler.sync_job_status(
job_id=job_id,
roles=job_runtime_conf['role'],
work_mode=job_parameters['work_mode'],
initiator_party_id=job_initiator['party_id'],
job_info=job.to_json())
top_level_task_status = set()
components = dag.get_next_components(None)
schedule_logger.info('job {} root components is {}'.format(
job.f_job_id, [component.get_name() for component in components],
None))
for component in components:
try:
# run a component as task
run_status = TaskScheduler.run_component(
job_id, job_runtime_conf, job_parameters, job_initiator,
job_args, dag, component)
except Exception as e:
schedule_logger.info(e)
run_status = False
top_level_task_status.add(run_status)
if not run_status:
break
if len(top_level_task_status) == 2:
job.f_status = JobStatus.PARTIAL
elif True in top_level_task_status:
job.f_status = JobStatus.SUCCESS
else:
job.f_status = JobStatus.FAILED
job.f_end_time = current_timestamp()
job.f_elapsed = job.f_end_time - job.f_start_time
if job.f_status == JobStatus.SUCCESS:
job.f_progress = 100
job.f_update_time = current_timestamp()
TaskScheduler.sync_job_status(
job_id=job_id,
roles=job_runtime_conf['role'],
work_mode=job_parameters['work_mode'],
initiator_party_id=job_initiator['party_id'],
job_info=job.to_json())
TaskScheduler.finish_job(job_id=job_id,
job_runtime_conf=job_runtime_conf)
schedule_logger.info('job {} finished, status is {}'.format(
job.f_job_id, job.f_status))
def run_job(job_id, initiator_role, initiator_party_id):
job_dsl, job_runtime_conf, train_runtime_conf = job_utils.get_job_configuration(job_id=job_id,
role=initiator_role,
party_id=initiator_party_id)
job_parameters = job_runtime_conf.get('job_parameters', {})
job_initiator = job_runtime_conf.get('initiator', {})
dag = get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
job_args = dag.get_args_input()
if not job_initiator:
return False
timeout = job_utils.get_timeout(job_id, job_parameters.get("timeout", None), job_runtime_conf, job_dsl)
t = Timer(timeout, TaskScheduler.job_handler, [job_id])
t.start()
job = Job()
job.f_job_id = job_id
job.f_start_time = current_timestamp()
job.f_status = JobStatus.RUNNING
job.f_update_time = current_timestamp()
TaskScheduler.sync_job_status(job_id=job_id, roles=job_runtime_conf['role'],
work_mode=job_parameters['work_mode'],
initiator_party_id=job_initiator['party_id'],
initiator_role=job_initiator['role'],
job_info=job.to_json())
top_level_task_status = set()
components = dag.get_next_components(None)
schedule_logger(job_id).info(
'job {} root components is {}'.format(job.f_job_id, [component.get_name() for component in components],
None))
for component in components:
try:
# run a component as task
run_status = TaskScheduler.run_component(job_id, job_runtime_conf, job_parameters, job_initiator,
job_args, dag,
component)
except Exception as e:
schedule_logger(job_id).exception(e)
run_status = False
top_level_task_status.add(run_status)
if not run_status:
break
if len(top_level_task_status) == 2:
job.f_status = JobStatus.FAILED
elif True in top_level_task_status:
job.f_status = JobStatus.COMPLETE
else:
job.f_status = JobStatus.FAILED
job.f_end_time = current_timestamp()
job.f_elapsed = job.f_end_time - job.f_start_time
if job.f_status == JobStatus.COMPLETE:
job.f_progress = 100
job.f_update_time = current_timestamp()
try:
TaskScheduler.finish_job(job_id=job_id, job_runtime_conf=job_runtime_conf)
except Exception as e:
schedule_logger(job_id).exception(e)
job.f_status = JobStatus.FAILED
if job.f_status == JobStatus.FAILED:
TaskScheduler.stop(job_id=job_id, end_status=JobStatus.FAILED)
try:
TaskScheduler.sync_job_status(job_id=job_id, roles=job_runtime_conf['role'],
work_mode=job_parameters['work_mode'],
initiator_party_id=job_initiator['party_id'],
initiator_role=job_initiator['role'],
job_info=job.to_json())
except Exception as e:
schedule_logger(job_id).exception(e)
schedule_logger(job_id).warning('job {} sync status failed'.format(job.f_job_id))
schedule_logger(job_id).info('job {} finished, status is {}'.format(job.f_job_id, job.f_status))
t.cancel()
def submit_job(job_data):
job_id = generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_utils.check_pipeline_job_runtime_conf(job_runtime_conf)
job_parameters = job_runtime_conf['job_parameters']
job_initiator = job_runtime_conf['initiator']
job_type = job_parameters.get('job_type', '')
if job_type != 'predict':
# generate job model info
job_parameters['model_id'] = '#'.join([dtable_utils.all_party_key(job_runtime_conf['role']), 'model'])
job_parameters['model_version'] = job_id
train_runtime_conf = {}
else:
detect_utils.check_config(job_parameters, ['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
job_tracker = Tracking(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'],
model_id=job_parameters['model_id'], model_version=job_parameters['model_version'])
pipeline_model = job_tracker.get_output_model('pipeline')
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf)
path_dict = save_job_conf(job_id=job_id,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job = Job()
job.f_job_id = job_id
job.f_roles = json_dumps(job_runtime_conf['role'])
job.f_work_mode = job_parameters['work_mode']
job.f_initiator_party_id = job_initiator['party_id']
job.f_dsl = json_dumps(job_dsl)
job.f_runtime_conf = json_dumps(job_runtime_conf)
job.f_train_runtime_conf = json_dumps(train_runtime_conf)
job.f_run_ip = ''
job.f_status = JobStatus.WAITING
job.f_progress = 0
job.f_create_time = current_timestamp()
initiator_role = job_initiator['role']
initiator_party_id = job_initiator['party_id']
if initiator_party_id not in job_runtime_conf['role'][initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(initiator_party_id))
raise Exception("initiator party id error {}".format(initiator_party_id))
get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
TaskScheduler.distribute_job(job=job, roles=job_runtime_conf['role'], job_initiator=job_initiator)
# push into queue
job_event = job_utils.job_event(job_id, initiator_role, initiator_party_id)
try:
RuntimeConfig.JOB_QUEUE.put_event(job_event)
except Exception as e:
raise Exception('push job into queue failed')
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters['model_id']))
board_url = BOARD_DASHBOARD_URL.format(job_id, job_initiator['role'], job_initiator['party_id'])
logs_directory = get_job_log_directory(job_id)
return job_id, path_dict['job_dsl_path'], path_dict['job_runtime_conf_path'], logs_directory, \
{'model_id': job_parameters['model_id'],'model_version': job_parameters['model_version']}, board_url
