def get_job_parameters_dict(self, job_parameters: RunParameters = None): if job_parameters: if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: self.job_runtime_conf['job_parameters']['common'] = job_parameters.to_dict() else: self.job_runtime_conf['job_parameters'] = job_parameters.to_dict() return self.job_runtime_conf['job_parameters']
def update_common_parameters(self, common_parameters: RunParameters): if int(self.job_runtime_conf.get("dsl_version", 1)) == 2: if "common" not in self.job_runtime_conf["job_parameters"]: raise RuntimeError("the configuration format for v2 version must be job_parameters:common") self.job_runtime_conf["job_parameters"]["common"] = common_parameters.to_dict() else: self.job_runtime_conf["job_parameters"] = common_parameters.to_dict() return self.job_runtime_conf
def get_common_parameters(self): if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: if "common" not in self.job_runtime_conf["job_parameters"]: raise RuntimeError("the configuration format for v2 version must be job_parameters:common") job_parameters = RunParameters(**self.job_runtime_conf['job_parameters']['common']) self.job_runtime_conf['job_parameters']['common'] = job_parameters.to_dict() else: if "processors_per_node" in self.job_runtime_conf['job_parameters']: self.job_runtime_conf['job_parameters']["eggroll_run"] = \ {"eggroll.session.processors.per.node": self.job_runtime_conf['job_parameters']["processors_per_node"]} job_parameters = RunParameters(**self.job_runtime_conf['job_parameters']) self.job_runtime_conf['job_parameters'] = job_parameters.to_dict() return job_parameters
def create_job(cls, job_id, role, party_id, job_info): # parse job configuration dsl = job_info['dsl'] runtime_conf = job_info['runtime_conf'] train_runtime_conf = job_info['train_runtime_conf'] if USE_AUTHENTICATION: authentication_check(src_role=job_info.get('src_role', None), src_party_id=job_info.get( 'src_party_id', None), dsl=dsl, runtime_conf=runtime_conf, role=role, party_id=party_id) job_parameters = RunParameters(**runtime_conf['job_parameters']) job_initiator = runtime_conf['initiator'] dsl_parser = schedule_utils.get_job_dsl_parser( dsl=dsl, runtime_conf=runtime_conf, train_runtime_conf=train_runtime_conf) # save new job into db if role == job_initiator['role'] and party_id == job_initiator[ 'party_id']: is_initiator = True else: is_initiator = False job_info["status"] = JobStatus.WAITING roles = job_info['roles'] # this party configuration job_info["role"] = role job_info["party_id"] = party_id job_info["is_initiator"] = is_initiator job_info["progress"] = 0 engines_info = cls.get_job_engines_address( job_parameters=job_parameters) cls.special_role_parameters(role=role, job_parameters=job_parameters) cls.check_parameters(job_parameters=job_parameters, engines_info=engines_info) runtime_conf["job_parameters"] = job_parameters.to_dict() JobSaver.create_job(job_info=job_info) job_utils.save_job_conf(job_id=job_id, job_dsl=dsl, job_runtime_conf=runtime_conf, train_runtime_conf=train_runtime_conf, pipeline_dsl=None) cls.initialize_tasks(job_id, role, party_id, True, job_initiator, job_parameters, dsl_parser) cls.initialize_job_tracker(job_id=job_id, role=role, party_id=party_id, job_info=job_info, is_initiator=is_initiator, dsl_parser=dsl_parser)
def create_job(cls, job_id, role, party_id, job_info): # parse job configuration dsl = job_info['dsl'] runtime_conf = job_info['runtime_conf'] train_runtime_conf = job_info['train_runtime_conf'] if USE_AUTHENTICATION: authentication_check(src_role=job_info.get('src_role', None), src_party_id=job_info.get('src_party_id', None), dsl=dsl, runtime_conf=runtime_conf, role=role, party_id=party_id) dsl_parser = schedule_utils.get_job_dsl_parser(dsl=dsl, runtime_conf=runtime_conf, train_runtime_conf=train_runtime_conf) job_parameters = dsl_parser.get_job_parameters().get(role, {}).get(party_id, {}) schedule_logger(job_id).info('job parameters:{}'.format(job_parameters)) job_parameters = RunParameters(**job_parameters) # save new job into db if role == job_info["initiator_role"] and party_id == job_info["initiator_party_id"]: is_initiator = True else: is_initiator = False job_info["status"] = JobStatus.WAITING # this party configuration job_info["role"] = role job_info["party_id"] = party_id job_info["is_initiator"] = is_initiator job_info["progress"] = 0 cls.adapt_job_parameters(role=role, job_parameters=job_parameters) engines_info = cls.get_job_engines_address(job_parameters=job_parameters) cls.check_parameters(job_parameters=job_parameters, role=role, party_id=party_id, engines_info=engines_info) job_info["runtime_conf_on_party"]["job_parameters"] = job_parameters.to_dict() job_utils.save_job_conf(job_id=job_id, role=role, job_dsl=dsl, job_runtime_conf=runtime_conf, job_runtime_conf_on_party=job_info["runtime_conf_on_party"], train_runtime_conf=train_runtime_conf, pipeline_dsl=None) cls.initialize_tasks(job_id=job_id, role=role, party_id=party_id, run_on_this_party=True, initiator_role=job_info["initiator_role"], initiator_party_id=job_info["initiator_party_id"], job_parameters=job_parameters, dsl_parser=dsl_parser) job_parameters = job_info['runtime_conf_on_party']['job_parameters'] roles = job_info['roles'] cls.initialize_job_tracker(job_id=job_id, role=role, party_id=party_id, job_parameters=job_parameters, roles=roles, is_initiator=is_initiator, dsl_parser=dsl_parser) JobSaver.create_job(job_info=job_info)
def create_job(cls, job_id, role, party_id, job_info): # parse job configuration dsl = job_info['dsl'] runtime_conf = job_info['runtime_conf'] train_runtime_conf = job_info['train_runtime_conf'] if USE_AUTHENTICATION: authentication_check(src_role=job_info.get('src_role', None), src_party_id=job_info.get( 'src_party_id', None), dsl=dsl, runtime_conf=runtime_conf, role=role, party_id=party_id) dsl_parser = schedule_utils.get_job_dsl_parser( dsl=dsl, runtime_conf=runtime_conf, train_runtime_conf=train_runtime_conf) job_parameters = dsl_parser.get_job_parameters().get(role, {}).get( party_id, {}) schedule_logger(job_id).info( 'job parameters:{}'.format(job_parameters)) dest_user = dsl_parser.get_job_parameters().get(role, {}).get( party_id, {}).get("user", '') user = {} src_party_id = int(job_info.get('src_party_id')) if job_info.get( 'src_party_id') else 0 src_user = dsl_parser.get_job_parameters().get( job_info.get('src_role'), {}).get(src_party_id, {}).get("user", '') for _role, party_id_item in dsl_parser.get_job_parameters().items(): user[_role] = {} for _party_id, _parameters in party_id_item.items(): user[_role][_party_id] = _parameters.get("user", "") schedule_logger(job_id).info('job user:{}'.format(user)) if USE_DATA_AUTHENTICATION: job_args = dsl_parser.get_args_input() schedule_logger(job_id).info('job args:{}'.format(job_args)) dataset_dict = cls.get_dataset(False, role, party_id, runtime_conf.get("role"), job_args) dataset_list = [] if dataset_dict.get(role, {}).get(party_id): for k, v in dataset_dict[role][party_id].items(): dataset_list.append({ "namespace": v.split('.')[0], "table_name": v.split('.')[1] }) data_authentication_check( src_role=job_info.get('src_role'), src_party_id=job_info.get('src_party_id'), src_user=src_user, dest_user=dest_user, dataset_list=dataset_list) job_parameters = RunParameters(**job_parameters) # save new job into db if role == job_info["initiator_role"] and party_id == job_info[ "initiator_party_id"]: is_initiator = True else: is_initiator = False job_info["status"] = JobStatus.WAITING job_info["user_id"] = dest_user job_info["src_user"] = src_user job_info["user"] = user # this party configuration job_info["role"] = role job_info["party_id"] = party_id job_info["is_initiator"] = is_initiator job_info["progress"] = 0 cls.adapt_job_parameters(role=role, job_parameters=job_parameters) engines_info = cls.get_job_engines_address( job_parameters=job_parameters) cls.check_parameters(job_parameters=job_parameters, role=role, party_id=party_id, engines_info=engines_info) job_info["runtime_conf_on_party"][ "job_parameters"] = job_parameters.to_dict() job_utils.save_job_conf( job_id=job_id, role=role, job_dsl=dsl, job_runtime_conf=runtime_conf, job_runtime_conf_on_party=job_info["runtime_conf_on_party"], train_runtime_conf=train_runtime_conf, pipeline_dsl=None) cls.initialize_tasks(job_id=job_id, role=role, party_id=party_id, run_on_this_party=True, initiator_role=job_info["initiator_role"], initiator_party_id=job_info["initiator_party_id"], job_parameters=job_parameters, dsl_parser=dsl_parser) job_parameters = job_info['runtime_conf_on_party']['job_parameters'] roles = job_info['roles'] cls.initialize_job_tracker(job_id=job_id, role=role, party_id=party_id, job_parameters=job_parameters, roles=roles, is_initiator=is_initiator, dsl_parser=dsl_parser) JobSaver.create_job(job_info=job_info)
def submit(cls, job_data, job_id=None): if not job_id: job_id = job_utils.generate_job_id() schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(job_id, job_data)) job_dsl = job_data.get('job_dsl', {}) job_runtime_conf = job_data.get('job_runtime_conf', {}) job_initiator = job_runtime_conf['initiator'] job_parameters = RunParameters(**job_runtime_conf['job_parameters']) cls.backend_compatibility(job_parameters=job_parameters) job_utils.check_job_runtime_conf(job_runtime_conf) if job_parameters.job_type != 'predict': # generate job model info job_parameters.model_id = model_utils.gen_model_id(job_runtime_conf['role']) job_parameters.model_version = job_id train_runtime_conf = {} else: detect_utils.check_config(job_parameters.to_dict(), ['model_id', 'model_version']) # get inference dsl from pipeline model as job dsl tracker = Tracker(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'], model_id=job_parameters.model_id, model_version=job_parameters.model_version) pipeline_model = tracker.get_output_model('pipeline') if not job_dsl: job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl) train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf) path_dict = job_utils.save_job_conf(job_id=job_id, job_dsl=job_dsl, job_runtime_conf=job_runtime_conf, train_runtime_conf=train_runtime_conf, pipeline_dsl=None) job = Job() job.f_job_id = job_id job.f_dsl = job_dsl job_runtime_conf["job_parameters"] = job_parameters.to_dict() job.f_runtime_conf = job_runtime_conf job.f_train_runtime_conf = train_runtime_conf job.f_roles = job_runtime_conf['role'] job.f_work_mode = job_parameters.work_mode job.f_initiator_role = job_initiator['role'] job.f_initiator_party_id = job_initiator['party_id'] initiator_role = job_initiator['role'] initiator_party_id = job_initiator['party_id'] if initiator_party_id not in job_runtime_conf['role'][initiator_role]: schedule_logger(job_id).info("initiator party id error:{}".format(initiator_party_id)) raise Exception("initiator party id error {}".format(initiator_party_id)) dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job_dsl, runtime_conf=job_runtime_conf, train_runtime_conf=train_runtime_conf) cls.adapt_job_parameters(job_parameters=job_parameters) # update runtime conf job_runtime_conf["job_parameters"] = job_parameters.to_dict() job.f_runtime_conf = job_runtime_conf status_code, response = FederatedScheduler.create_job(job=job) if status_code != FederatedSchedulingStatusCode.SUCCESS: raise Exception("create job failed: {}".format(response)) if job_parameters.work_mode == WorkMode.CLUSTER: # Save the status information of all participants in the initiator for scheduling for role, party_ids in job_runtime_conf["role"].items(): for party_id in party_ids: if role == job_initiator['role'] and party_id == job_initiator['party_id']: continue JobController.initialize_tasks(job_id, role, party_id, False, job_initiator, job_parameters, dsl_parser) # push into queue try: JobQueue.create_event(job_id=job_id, initiator_role=initiator_role, initiator_party_id=initiator_party_id) except Exception as e: raise Exception(f'push job into queue failed:\n{e}') schedule_logger(job_id).info( 'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters.model_id)) board_url = "http://{}:{}{}".format( ServiceUtils.get_item("fateboard", "host"), ServiceUtils.get_item("fateboard", "port"), FATE_BOARD_DASHBOARD_ENDPOINT).format(job_id, job_initiator['role'], job_initiator['party_id']) logs_directory = job_utils.get_job_log_directory(job_id) return job_id, path_dict['job_dsl_path'], path_dict['job_runtime_conf_path'], logs_directory, \ {'model_id': job_parameters.model_id, 'model_version': job_parameters.model_version}, board_url