def update_job_status(job_id, role, party_id, job_info, create=False):
job_info['f_run_ip'] = RuntimeConfig.JOB_SERVER_HOST
if create:
dsl = json_loads(job_info['f_dsl'])
runtime_conf = json_loads(job_info['f_runtime_conf'])
train_runtime_conf = json_loads(job_info['f_train_runtime_conf'])
if USE_AUTHENTICATION:
authentication_check(src_role=job_info.get('src_role', None), src_party_id=job_info.get('src_party_id', None),
dsl=dsl, runtime_conf=runtime_conf, role=role, party_id=party_id)
save_job_conf(job_id=job_id,
job_dsl=dsl,
job_runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job_parameters = runtime_conf['job_parameters']
job_tracker = Tracking(job_id=job_id, role=role, party_id=party_id,
model_id=job_parameters["model_id"],
model_version=job_parameters["model_version"])
if job_parameters.get("job_type", "") != "predict":
job_tracker.init_pipelined_model()
roles = json_loads(job_info['f_roles'])
partner = {}
show_role = {}
is_initiator = job_info.get('f_is_initiator', 0)
for _role, _role_party in roles.items():
if is_initiator or _role == role:
show_role[_role] = show_role.get(_role, [])
for _party_id in _role_party:
if is_initiator or _party_id == party_id:
show_role[_role].append(_party_id)
if _role != role:
partner[_role] = partner.get(_role, [])
partner[_role].extend(_role_party)
else:
for _party_id in _role_party:
if _party_id != party_id:
partner[_role] = partner.get(_role, [])
partner[_role].append(_party_id)
dag = get_job_dsl_parser(dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
job_args = dag.get_args_input()
dataset = {}
for _role, _role_party_args in job_args.items():
if is_initiator or _role == role:
for _party_index in range(len(_role_party_args)):
_party_id = roles[_role][_party_index]
if is_initiator or _party_id == party_id:
dataset[_role] = dataset.get(_role, {})
dataset[_role][_party_id] = dataset[_role].get(_party_id, {})
for _data_type, _data_location in _role_party_args[_party_index]['args']['data'].items():
dataset[_role][_party_id][_data_type] = '{}.{}'.format(_data_location['namespace'],
_data_location['name'])
job_tracker.log_job_view({'partner': partner, 'dataset': dataset, 'roles': show_role})
else:
job_tracker = Tracking(job_id=job_id, role=role, party_id=party_id)
job_tracker.save_job_info(role=role, party_id=party_id, job_info=job_info, create=create)
def create_job(cls, job_id, role, party_id, job_info):
# parse job configuration
dsl = job_info['dsl']
runtime_conf = job_info['runtime_conf']
train_runtime_conf = job_info['train_runtime_conf']
if USE_AUTHENTICATION:
authentication_check(src_role=job_info.get('src_role', None),
src_party_id=job_info.get(
'src_party_id', None),
dsl=dsl,
runtime_conf=runtime_conf,
role=role,
party_id=party_id)
job_parameters = RunParameters(**runtime_conf['job_parameters'])
job_initiator = runtime_conf['initiator']
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
# save new job into db
if role == job_initiator['role'] and party_id == job_initiator[
'party_id']:
is_initiator = True
else:
is_initiator = False
job_info["status"] = JobStatus.WAITING
roles = job_info['roles']
# this party configuration
job_info["role"] = role
job_info["party_id"] = party_id
job_info["is_initiator"] = is_initiator
job_info["progress"] = 0
engines_info = cls.get_job_engines_address(
job_parameters=job_parameters)
cls.special_role_parameters(role=role, job_parameters=job_parameters)
cls.check_parameters(job_parameters=job_parameters,
engines_info=engines_info)
runtime_conf["job_parameters"] = job_parameters.to_dict()
JobSaver.create_job(job_info=job_info)
job_utils.save_job_conf(job_id=job_id,
job_dsl=dsl,
job_runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
cls.initialize_tasks(job_id, role, party_id, True, job_initiator,
job_parameters, dsl_parser)
cls.initialize_job_tracker(job_id=job_id,
role=role,
party_id=party_id,
job_info=job_info,
is_initiator=is_initiator,
dsl_parser=dsl_parser)
def create_job(cls, job_id, role, party_id, job_info):
# parse job configuration
dsl = job_info['dsl']
runtime_conf = job_info['runtime_conf']
train_runtime_conf = job_info['train_runtime_conf']
if USE_AUTHENTICATION:
authentication_check(src_role=job_info.get('src_role', None), src_party_id=job_info.get('src_party_id', None),
dsl=dsl, runtime_conf=runtime_conf, role=role, party_id=party_id)
dsl_parser = schedule_utils.get_job_dsl_parser(dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
job_parameters = dsl_parser.get_job_parameters().get(role, {}).get(party_id, {})
schedule_logger(job_id).info('job parameters:{}'.format(job_parameters))
job_parameters = RunParameters(**job_parameters)
# save new job into db
if role == job_info["initiator_role"] and party_id == job_info["initiator_party_id"]:
is_initiator = True
else:
is_initiator = False
job_info["status"] = JobStatus.WAITING
# this party configuration
job_info["role"] = role
job_info["party_id"] = party_id
job_info["is_initiator"] = is_initiator
job_info["progress"] = 0
cls.adapt_job_parameters(role=role, job_parameters=job_parameters)
engines_info = cls.get_job_engines_address(job_parameters=job_parameters)
cls.check_parameters(job_parameters=job_parameters, role=role, party_id=party_id, engines_info=engines_info)
job_info["runtime_conf_on_party"]["job_parameters"] = job_parameters.to_dict()
job_utils.save_job_conf(job_id=job_id,
role=role,
job_dsl=dsl,
job_runtime_conf=runtime_conf,
job_runtime_conf_on_party=job_info["runtime_conf_on_party"],
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
cls.initialize_tasks(job_id=job_id, role=role, party_id=party_id, run_on_this_party=True, initiator_role=job_info["initiator_role"], initiator_party_id=job_info["initiator_party_id"], job_parameters=job_parameters, dsl_parser=dsl_parser)
job_parameters = job_info['runtime_conf_on_party']['job_parameters']
roles = job_info['roles']
cls.initialize_job_tracker(job_id=job_id, role=role, party_id=party_id, job_parameters=job_parameters, roles=roles, is_initiator=is_initiator, dsl_parser=dsl_parser)
JobSaver.create_job(job_info=job_info)
def create_job(cls, job_id, role, party_id, job_info):
# parse job configuration
dsl = job_info['dsl']
runtime_conf = job_info['runtime_conf']
train_runtime_conf = job_info['train_runtime_conf']
if USE_AUTHENTICATION:
authentication_check(src_role=job_info.get('src_role', None),
src_party_id=job_info.get(
'src_party_id', None),
dsl=dsl,
runtime_conf=runtime_conf,
role=role,
party_id=party_id)
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
job_parameters = dsl_parser.get_job_parameters(runtime_conf)
schedule_logger(job_id).info(
'job parameters:{}'.format(job_parameters))
dest_user = job_parameters.get(role, {}).get(party_id,
{}).get('user', '')
user = {}
src_party_id = int(
job_info['src_party_id']) if job_info.get('src_party_id') else 0
src_role = job_info.get('src_role', '')
src_user = job_parameters.get(src_role, {}).get(src_party_id, {}).get(
'user', '') if src_role else ''
for _role, party_id_item in job_parameters.items():
user[_role] = {}
for _party_id, _parameters in party_id_item.items():
user[_role][_party_id] = _parameters.get("user", "")
schedule_logger(job_id).info('job user:{}'.format(user))
if USE_DATA_AUTHENTICATION:
job_args = dsl_parser.get_args_input()
schedule_logger(job_id).info('job args:{}'.format(job_args))
dataset_dict = cls.get_dataset(False, role, party_id,
runtime_conf.get("role"), job_args)
dataset_list = []
if dataset_dict.get(role, {}).get(party_id):
for k, v in dataset_dict[role][party_id].items():
dataset_list.append({
"namespace": v.split('.')[0],
"table_name": v.split('.')[1]
})
data_authentication_check(
src_role=job_info.get('src_role'),
src_party_id=job_info.get('src_party_id'),
src_user=src_user,
dest_user=dest_user,
dataset_list=dataset_list)
job_parameters = RunParameters(
**job_parameters.get(role, {}).get(party_id, {}))
# save new job into db
if role == job_info["initiator_role"] and party_id == job_info[
"initiator_party_id"]:
is_initiator = True
else:
is_initiator = False
job_info["status"] = JobStatus.READY
job_info["user_id"] = dest_user
job_info["src_user"] = src_user
job_info["user"] = user
# this party configuration
job_info["role"] = role
job_info["party_id"] = party_id
job_info["is_initiator"] = is_initiator
job_info["progress"] = 0
cls.create_job_parameters_on_party(role=role,
party_id=party_id,
job_parameters=job_parameters)
# update job parameters on party
job_info["runtime_conf_on_party"][
"job_parameters"] = job_parameters.to_dict()
JobSaver.create_job(job_info=job_info)
schedule_logger(job_id).info("start initialize tasks")
initialized_result, provider_group = cls.initialize_tasks(
job_id=job_id,
role=role,
party_id=party_id,
run_on_this_party=True,
initiator_role=job_info["initiator_role"],
initiator_party_id=job_info["initiator_party_id"],
job_parameters=job_parameters,
dsl_parser=dsl_parser)
schedule_logger(job_id).info("initialize tasks success")
for provider_key, group_info in provider_group.items():
for cpn in group_info["components"]:
dsl["components"][cpn]["provider"] = provider_key
roles = job_info['roles']
cls.initialize_job_tracker(job_id=job_id,
role=role,
party_id=party_id,
job_parameters=job_parameters,
roles=roles,
is_initiator=is_initiator,
dsl_parser=dsl_parser)
job_utils.save_job_conf(
job_id=job_id,
role=role,
party_id=party_id,
dsl=dsl,
runtime_conf=runtime_conf,
runtime_conf_on_party=job_info["runtime_conf_on_party"],
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
return {"components": initialized_result}
def create_job(cls, job_id, role, party_id, job_info):
# parse job configuration
dsl = job_info['dsl']
runtime_conf = job_info['runtime_conf']
train_runtime_conf = job_info['train_runtime_conf']
if USE_AUTHENTICATION:
authentication_check(src_role=job_info.get('src_role', None),
src_party_id=job_info.get(
'src_party_id', None),
dsl=dsl,
runtime_conf=runtime_conf,
role=role,
party_id=party_id)
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=dsl,
runtime_conf=runtime_conf,
train_runtime_conf=train_runtime_conf)
job_parameters = dsl_parser.get_job_parameters().get(role, {}).get(
party_id, {})
schedule_logger(job_id).info(
'job parameters:{}'.format(job_parameters))
dest_user = dsl_parser.get_job_parameters().get(role, {}).get(
party_id, {}).get("user", '')
user = {}
src_party_id = int(job_info.get('src_party_id')) if job_info.get(
'src_party_id') else 0
src_user = dsl_parser.get_job_parameters().get(
job_info.get('src_role'), {}).get(src_party_id,
{}).get("user", '')
for _role, party_id_item in dsl_parser.get_job_parameters().items():
user[_role] = {}
for _party_id, _parameters in party_id_item.items():
user[_role][_party_id] = _parameters.get("user", "")
schedule_logger(job_id).info('job user:{}'.format(user))
if USE_DATA_AUTHENTICATION:
job_args = dsl_parser.get_args_input()
schedule_logger(job_id).info('job args:{}'.format(job_args))
dataset_dict = cls.get_dataset(False, role, party_id,
runtime_conf.get("role"), job_args)
dataset_list = []
if dataset_dict.get(role, {}).get(party_id):
for k, v in dataset_dict[role][party_id].items():
dataset_list.append({
"namespace": v.split('.')[0],
"table_name": v.split('.')[1]
})
data_authentication_check(
src_role=job_info.get('src_role'),
src_party_id=job_info.get('src_party_id'),
src_user=src_user,
dest_user=dest_user,
dataset_list=dataset_list)
job_parameters = RunParameters(**job_parameters)
# save new job into db
if role == job_info["initiator_role"] and party_id == job_info[
"initiator_party_id"]:
is_initiator = True
else:
is_initiator = False
job_info["status"] = JobStatus.WAITING
job_info["user_id"] = dest_user
job_info["src_user"] = src_user
job_info["user"] = user
# this party configuration
job_info["role"] = role
job_info["party_id"] = party_id
job_info["is_initiator"] = is_initiator
job_info["progress"] = 0
cls.adapt_job_parameters(role=role, job_parameters=job_parameters)
engines_info = cls.get_job_engines_address(
job_parameters=job_parameters)
cls.check_parameters(job_parameters=job_parameters,
role=role,
party_id=party_id,
engines_info=engines_info)
job_info["runtime_conf_on_party"][
"job_parameters"] = job_parameters.to_dict()
job_utils.save_job_conf(
job_id=job_id,
role=role,
job_dsl=dsl,
job_runtime_conf=runtime_conf,
job_runtime_conf_on_party=job_info["runtime_conf_on_party"],
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
cls.initialize_tasks(job_id=job_id,
role=role,
party_id=party_id,
run_on_this_party=True,
initiator_role=job_info["initiator_role"],
initiator_party_id=job_info["initiator_party_id"],
job_parameters=job_parameters,
dsl_parser=dsl_parser)
job_parameters = job_info['runtime_conf_on_party']['job_parameters']
roles = job_info['roles']
cls.initialize_job_tracker(job_id=job_id,
role=role,
party_id=party_id,
job_parameters=job_parameters,
roles=roles,
is_initiator=is_initiator,
dsl_parser=dsl_parser)
JobSaver.create_job(job_info=job_info)