def pipeline_dag_dependency(job_info): try: detect_utils.check_config(job_info, required_arguments=["party_id", "role"]) component_need_run = {} if job_info.get('job_id'): jobs = JobSaver.query_job(job_id=job_info["job_id"], party_id=job_info["party_id"], role=job_info["role"]) if not jobs: raise Exception('query job {} failed'.format(job_info.get('job_id', ''))) job = jobs[0] dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, runtime_conf=job.f_runtime_conf_on_party, train_runtime_conf=job.f_train_runtime_conf) tasks = JobSaver.query_task(job_id=job_info["job_id"], party_id=job_info["party_id"], role=job_info["role"], only_latest=True) for task in tasks: need_run = task.f_component_parameters.get("ComponentParam", {}).get("need_run", True) component_need_run[task.f_component_name] = need_run else: dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job_info.get('job_dsl', {}), runtime_conf=job_info.get('job_runtime_conf', {}), train_runtime_conf=job_info.get('job_train_runtime_conf', {})) dependency = dsl_parser.get_dependency() dependency["component_need_run"] = component_need_run return dependency except Exception as e: stat_logger.exception(e) raise e
def submit_job(): work_mode = request.json.get('job_runtime_conf', {}).get('job_parameters', {}).get('work_mode', None) detect_utils.check_config({'work_mode': work_mode}, required_arguments=[('work_mode', (WorkMode.CLUSTER, WorkMode.STANDALONE))]) if work_mode == RuntimeConfig.WORK_MODE: job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job( request.json) return get_json_result(retcode=0, retmsg='success', job_id=job_id, data={ 'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'model_info': model_info, 'board_url': board_url, 'logs_directory': logs_directory }) else: if RuntimeConfig.WORK_MODE == WorkMode.CLUSTER and work_mode == WorkMode.STANDALONE: # use cluster standalone job server to execute standalone job return request_execute_server( request=request, execute_host='{}:{}'.format( request.remote_addr, CLUSTER_STANDALONE_JOB_SERVER_PORT)) else: raise Exception( 'server run on standalone can not support cluster mode job')
def download_upload(access_module): request_config = request.json required_arguments = ['work_mode', 'namespace', 'table_name'] if access_module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif access_module == 'download': required_arguments.extend(['output_path']) else: raise Exception( 'can not support this operating: {}'.format(access_module)) detect_utils.check_config(request_config, required_arguments=required_arguments) data = {} if access_module == "upload": data['table_name'] = request_config["table_name"] data['namespace'] = request_config["namespace"] job_dsl, job_runtime_conf = gen_data_access_job_config( request_config, access_module) job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job( { 'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf }) data.update({ 'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'board_url': board_url, 'logs_directory': logs_directory }) return get_json_result(job_id=job_id, data=data)
def get_component_summary(): request_data = request.json try: required_params = ["job_id", "component_name", "role", "party_id"] detect_utils.check_config(request_data, required_params) tracker = Tracker(job_id=request_data["job_id"], component_name=request_data["component_name"], role=request_data["role"], party_id=request_data["party_id"], task_id=request_data.get("task_id", None), task_version=request_data.get("task_version", None)) summary = tracker.read_summary_from_db() if summary: if request_data.get("filename"): temp_filepath = os.path.join(TEMP_DIRECTORY, request_data.get("filename")) with open(temp_filepath, "w") as fout: fout.write(json.dumps(summary, indent=4)) return send_file( open(temp_filepath, "rb"), as_attachment=True, attachment_filename=request_data.get("filename")) else: return get_json_result(data=summary) return error_response( 210, "No component summary found, please check if arguments are specified correctly." ) except Exception as e: stat_logger.exception(e) return error_response(210, str(e))
def bind_model_service(): request_config = request.json if request_config.get('job_id', None): with DB.connection_context(): model = MLModel.get_or_none( MLModel.f_job_id == request_config.get("job_id"), MLModel.f_role == 'guest' ) if model: model_info = model.to_json() request_config['initiator'] = {} request_config['initiator']['party_id'] = str(model_info.get('f_initiator_party_id')) request_config['initiator']['role'] = model_info.get('f_initiator_role') request_config['job_parameters'] = model_info.get('f_runtime_conf').get('job_parameters') request_config['role'] = model_info.get('f_runtime_conf').get('role') for key, value in request_config['role'].items(): for i, v in enumerate(value): value[i] = str(v) request_config.pop('job_id') else: return get_json_result(retcode=101, retmsg="model {} can not be found in database. " "Please check if the model version is valid.".format(request_config.get('job_id'))) if not request_config.get('servings'): # get my party all servings adapter_servings_config(request_config) service_id = request_config.get('service_id') if not service_id: return get_json_result(retcode=101, retmsg='no service id') check_config(request_config, ['initiator', 'role', 'job_parameters']) bind_status, retmsg = publish_model.bind_model_service(config_data=request_config) operation_record(request_config, "bind", "success" if not bind_status else "failed") return get_json_result(retcode=bind_status, retmsg='service id is {}'.format(service_id) if not retmsg else retmsg)
def pipeline_dag_dependency(job_info): try: detect_utils.check_config(job_info, required_arguments=["party_id", "role"]) if job_info.get('job_id'): jobs = JobSaver.query_job(job_id=job_info["job_id"], party_id=job_info["party_id"], role=job_info["role"]) if not jobs: raise Exception('query job {} failed'.format( job_info.get('job_id', ''))) job = jobs[0] job_dsl_parser = schedule_utils.get_job_dsl_parser( dsl=job.f_dsl, runtime_conf=job.f_runtime_conf, train_runtime_conf=job.f_train_runtime_conf) else: job_dsl_parser = schedule_utils.get_job_dsl_parser( dsl=job_info.get('job_dsl', {}), runtime_conf=job_info.get('job_runtime_conf', {}), train_runtime_conf=job_info.get('job_train_runtime_conf', {})) return job_dsl_parser.get_dependency(role=job_info["role"], party_id=int( job_info["party_id"])) except Exception as e: stat_logger.exception(e) raise e
def operate_model(model_operation): request_config = request.json or request.form.to_dict() job_id = generate_job_id() if model_operation not in [ModelOperation.STORE, ModelOperation.RESTORE, ModelOperation.EXPORT, ModelOperation.IMPORT]: raise Exception('Can not support this operating now: {}'.format(model_operation)) required_arguments = ["model_id", "model_version", "role", "party_id"] check_config(request_config, required_arguments=required_arguments) request_config["model_id"] = gen_party_model_id(model_id=request_config["model_id"], role=request_config["role"], party_id=request_config["party_id"]) if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]: if model_operation == ModelOperation.IMPORT: file = request.files.get('file') file_path = os.path.join(TEMP_DIRECTORY, file.filename) try: os.makedirs(os.path.dirname(file_path), exist_ok=True) file.save(file_path) except Exception as e: shutil.rmtree(file_path) raise e request_config['file'] = file_path model = pipelined_model.PipelinedModel(model_id=request_config["model_id"], model_version=request_config["model_version"]) model.unpack_model(file_path) return get_json_result() else: model = pipelined_model.PipelinedModel(model_id=request_config["model_id"], model_version=request_config["model_version"]) archive_file_path = model.packaging_model() return send_file(archive_file_path, attachment_filename=os.path.basename(archive_file_path), as_attachment=True) else: data = {} job_dsl, job_runtime_conf = gen_model_operation_job_config(request_config, model_operation) job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job( {'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf}, job_id=job_id) data.update({'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'board_url': board_url, 'logs_directory': logs_directory}) return get_json_result(job_id=job_id, data=data)
def validate_component_param(): if not request.json or not isinstance(request.json, dict): return error_response(400, 'bad request') required_keys = [ 'component_name', 'component_module_name', ] config_keys = ['role'] dsl_version = int(request.json.get('dsl_version', 0)) if dsl_version == 1: config_keys += ['role_parameters', 'algorithm_parameters'] parser_class = DSLParser elif dsl_version == 2: config_keys += ['component_parameters'] parser_class = DSLParserV2 else: return error_response(400, 'unsupported dsl_version') try: check_config(request.json, required_keys + config_keys) except Exception as e: return error_response(400, str(e)) try: parser_class.validate_component_param( get_federatedml_setting_conf_directory(), {i: request.json[i] for i in config_keys}, *[request.json[i] for i in required_keys]) except Exception as e: return error_response(400, str(e)) return get_json_result()
def check_job_runtime_conf(runtime_conf: typing.Dict): detect_utils.check_config(runtime_conf, ['initiator', 'job_parameters', 'role']) detect_utils.check_config(runtime_conf['initiator'], ['role', 'party_id']) # deal party id runtime_conf['initiator']['party_id'] = int(runtime_conf['initiator']['party_id']) for r in runtime_conf['role'].keys(): for i in range(len(runtime_conf['role'][r])): runtime_conf['role'][r][i] = int(runtime_conf['role'][r][i])
def component_output_data_table(): request_data = request.json detect_utils.check_config(config=request_data, required_arguments=['job_id', 'role', 'party_id', 'component_name']) jobs = JobSaver.query_job(job_id=request_data.get('job_id')) if jobs: job = jobs[0] return jsonify(FederatedScheduler.tracker_command(job, request_data, 'output/table')) else: return get_json_result(retcode=100, retmsg='No found job')
def get_job_table_list(): detect_utils.check_config(config=request.json, required_arguments=['job_id', 'role', 'party_id']) jobs = JobSaver.query_job(**request.json) if jobs: job = jobs[0] tables = get_job_all_table(job) return get_json_result(data=tables) else: return get_json_result(retcode=101, retmsg='no find job')
def submit_job(job_data): job_id = generate_job_id() schedule_logger.info('submit job, job_id {}, body {}'.format(job_id, job_data)) job_dsl = job_data.get('job_dsl', {}) job_runtime_conf = job_data.get('job_runtime_conf', {}) job_utils.check_pipeline_job_runtime_conf(job_runtime_conf) job_parameters = job_runtime_conf['job_parameters'] job_initiator = job_runtime_conf['initiator'] job_type = job_parameters.get('job_type', '') if job_type != 'predict': # generate job model info job_parameters['model_id'] = '#'.join([dtable_utils.all_party_key(job_runtime_conf['role']), 'model']) job_parameters['model_version'] = job_id train_runtime_conf = {} else: detect_utils.check_config(job_parameters, ['model_id', 'model_version']) # get inference dsl from pipeline model as job dsl job_tracker = Tracking(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'], model_id=job_parameters['model_id'], model_version=job_parameters['model_version']) pipeline_model = job_tracker.get_output_model('pipeline') job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl) train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf) job_dsl_path, job_runtime_conf_path = save_job_conf(job_id=job_id, job_dsl=job_dsl, job_runtime_conf=job_runtime_conf) job = Job() job.f_job_id = job_id job.f_roles = json_dumps(job_runtime_conf['role']) job.f_work_mode = job_parameters['work_mode'] job.f_initiator_party_id = job_initiator['party_id'] job.f_dsl = json_dumps(job_dsl) job.f_runtime_conf = json_dumps(job_runtime_conf) job.f_train_runtime_conf = json_dumps(train_runtime_conf) job.f_run_ip = '' job.f_status = JobStatus.WAITING job.f_progress = 0 job.f_create_time = current_timestamp() # save job info TaskScheduler.distribute_job(job=job, roles=job_runtime_conf['role'], job_initiator=job_initiator) # push into queue RuntimeConfig.JOB_QUEUE.put_event({ 'job_id': job_id, "initiator_role": job_initiator['role'], "initiator_party_id": job_initiator['party_id'] } ) schedule_logger.info( 'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters['model_id'])) board_url = BOARD_DASHBOARD_URL.format(job_id, job_initiator['role'], job_initiator['party_id']) return job_id, job_dsl_path, job_runtime_conf_path, {'model_id': job_parameters['model_id'], 'model_version': job_parameters[ 'model_version']}, board_url
def download_upload(access_module): job_id = job_utils.generate_job_id() if access_module == "upload" and UPLOAD_DATA_FROM_CLIENT and not (request.json and request.json.get("use_local_data") == 0): file = request.files['file'] filename = os.path.join(job_utils.get_job_directory(job_id), 'fate_upload_tmp', file.filename) os.makedirs(os.path.dirname(filename), exist_ok=True) try: file.save(filename) except Exception as e: shutil.rmtree(os.path.join(job_utils.get_job_directory(job_id), 'fate_upload_tmp')) raise e job_config = request.args.to_dict() if "namespace" in job_config and "table_name" in job_config: pass else: # higher than version 1.5.1, support eggroll run parameters job_config = json_loads(list(job_config.keys())[0]) job_config['file'] = filename else: job_config = request.json required_arguments = ['work_mode', 'namespace', 'table_name'] if access_module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif access_module == 'download': required_arguments.extend(['output_path']) else: raise Exception('can not support this operating: {}'.format(access_module)) detect_utils.check_config(job_config, required_arguments=required_arguments) data = {} # compatibility if "table_name" in job_config: job_config["name"] = job_config["table_name"] if "backend" not in job_config: job_config["backend"] = 0 for _ in ["work_mode", "backend", "head", "partition", "drop"]: if _ in job_config: job_config[_] = int(job_config[_]) if access_module == "upload": if job_config.get('drop', 0) == 1: job_config["destroy"] = True else: job_config["destroy"] = False data['table_name'] = job_config["table_name"] data['namespace'] = job_config["namespace"] data_table_meta = storage.StorageTableMeta(name=job_config["table_name"], namespace=job_config["namespace"]) if data_table_meta and not job_config["destroy"]: return get_json_result(retcode=100, retmsg='The data table already exists.' 'If you still want to continue uploading, please add the parameter -drop.' ' 0 means not to delete and continue uploading, ' '1 means to upload again after deleting the table') job_dsl, job_runtime_conf = gen_data_access_job_config(job_config, access_module) submit_result = DAGScheduler.submit({'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf}, job_id=job_id) data.update(submit_result) return get_json_result(job_id=job_id, data=data)
def submit_job(): work_mode = JobRuntimeConfigAdapter( request.json.get('job_runtime_conf', {})).get_job_work_mode() detect_utils.check_config({'work_mode': work_mode}, required_arguments=[('work_mode', (WorkMode.CLUSTER, WorkMode.STANDALONE))]) submit_result = DAGScheduler.submit(request.json) return get_json_result(retcode=0, retmsg='success', job_id=submit_result.get("job_id"), data=submit_result)
def bind_model_service(): request_config = request.json if request_config.get('job_id', None): retcode, retmsg, res_data = model_utils.query_model_info( request_config['job_id'], 'guest') if res_data: model_info = res_data[0] request_config['initiator'] = {} request_config['initiator']['party_id'] = str( model_info.get('f_initiator_party_id')) request_config['initiator']['role'] = model_info.get( 'f_initiator_role') runtime_conf = model_info.get( 'f_runtime_conf', {}) if model_info.get( 'f_runtime_conf', {}) else model_info.get( 'f_train_runtime_conf', {}) adapter = JobRuntimeConfigAdapter(runtime_conf) job_parameters = adapter.get_common_parameters().to_dict() request_config[ 'job_parameters'] = job_parameters if job_parameters else model_info.get( 'f_train_runtime_conf', {}).get('job_parameters') roles = runtime_conf.get('role') request_config['role'] = roles if roles else model_info.get( 'f_train_runtime_conf', {}).get('role') for key, value in request_config['role'].items(): for i, v in enumerate(value): value[i] = str(v) request_config.pop('job_id') else: return get_json_result( retcode=101, retmsg="model {} can not be found in database. " "Please check if the model version is valid.".format( request_config.get('job_id'))) if not request_config.get('servings'): # get my party all servings request_config['servings'] = RuntimeConfig.SERVICE_DB.get_urls( 'servings') service_id = request_config.get('service_id') if not service_id: return get_json_result(retcode=101, retmsg='no service id') detect_utils.check_config(request_config, ['initiator', 'role', 'job_parameters']) bind_status, retmsg = publish_model.bind_model_service(request_config) operation_record(request_config, "bind", "success" if not bind_status else "failed") return get_json_result( retcode=bind_status, retmsg='service id is {}'.format(service_id) if not retmsg else retmsg)
def load_checkpoints(): required_args = [ 'role', 'party_id', 'model_id', 'model_version', 'component_name' ] try: check_config(request.json, required_args) except Exception as e: abort(error_response(400, str(e))) checkpoint_manager = CheckpointManager(**{ i: request.json[i] for i in required_args }, mkdir=False) checkpoint_manager.load_checkpoints_from_disk() return checkpoint_manager
def get_url(): request_data = request.json detect_utils.check_config( config=request_data, required_arguments=['job_id', 'role', 'party_id']) jobs = JobSaver.query_job(job_id=request_data.get('job_id'), role=request_data.get('role'), party_id=request_data.get('party_id')) if jobs: board_urls = [] for job in jobs: board_url = job_utils.get_board_url(job.f_job_id, job.f_role, job.f_party_id) board_urls.append(board_url) return get_json_result(data={'board_url': board_urls}) else: return get_json_result(retcode=101, retmsg='no found job')
def table_add(): request_data = request.json detect_utils.check_config(request_data, required_arguments=[ "engine", "address", "namespace", "name", ("head", (0, 1)), "id_delimiter" ]) address_dict = request_data.get('address') engine = request_data.get('engine') name = request_data.get('name') namespace = request_data.get('namespace') address = storage.StorageTableMeta.create_address( storage_engine=engine, address_dict=address_dict) in_serialized = request_data.get( "in_serialized", 1 if engine in { storage.StorageEngine.STANDALONE, storage.StorageEngine.EGGROLL, storage.StorageEngine.MYSQL } else 0) destroy = (int(request_data.get("drop", 0)) == 1) data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) if data_table_meta: if destroy: data_table_meta.destroy_metas() else: return get_json_result( retcode=100, retmsg='The data table already exists.' 'If you still want to continue uploading, please add the parameter -drop.' '1 means to add again after deleting the table') id_name = request_data.get("id_name") feature_name = request_data.get("feature_name") schema = None if id_name and feature_name: schema = {'header': feature_name, 'sid': id_name} with storage.Session.build( storage_engine=engine, options=request_data.get("options")) as storage_session: storage_session.create_table( address=address, name=name, namespace=namespace, partitions=request_data.get('partitions', None), hava_head=request_data.get("head"), id_delimiter=request_data.get("id_delimiter"), in_serialized=in_serialized, schema=schema) return get_json_result(data={"table_name": name, "namespace": namespace})
def get_predict_conf(): request_data = request.json required_parameters = ['model_id', 'model_version'] check_config(request_data, required_parameters) model_dir = os.path.join(get_project_base_directory(), 'model_local_cache') model_fp_list = glob.glob( model_dir + f"/guest#*#{request_data['model_id']}/{request_data['model_version']}") if model_fp_list: fp = model_fp_list[0] pipeline_model = PipelinedModel(model_id=fp.split('/')[-2], model_version=fp.split('/')[-1]) pipeline = pipeline_model.read_component_model('pipeline', 'pipeline')['Pipeline'] predict_dsl = json_loads(pipeline.inference_dsl) train_runtime_conf = json_loads(pipeline.train_runtime_conf) parser = schedule_utils.get_dsl_parser_by_version( train_runtime_conf.get('dsl_version', '1')) predict_conf = parser.generate_predict_conf_template( predict_dsl=predict_dsl, train_conf=train_runtime_conf, model_id=request_data['model_id'], model_version=request_data['model_version']) else: predict_conf = '' if predict_conf: if request_data.get("filename"): os.makedirs(TEMP_DIRECTORY, exist_ok=True) temp_filepath = os.path.join(TEMP_DIRECTORY, request_data.get("filename")) with open(temp_filepath, "w") as fout: fout.write(json_dumps(predict_conf, indent=4)) return send_file(open(temp_filepath, "rb"), as_attachment=True, attachment_filename=request_data.get("filename")) else: return get_json_result(data=predict_conf) return error_response( 210, "No model found, please check if arguments are specified correctly.")
def submit_job(): work_mode = request.json.get('job_runtime_conf', {}).get('job_parameters', {}).get('work_mode', None) detect_utils.check_config({'work_mode': work_mode}, required_arguments=[('work_mode', (WorkMode.CLUSTER, WorkMode.STANDALONE))]) job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = DAGScheduler.submit( request.json) return get_json_result(retcode=0, retmsg='success', job_id=job_id, data={ 'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'model_info': model_info, 'board_url': board_url, 'logs_directory': logs_directory })
def download_upload(data_func): request_config = request.json _job_id = generate_job_id() stat_logger.info('generated job_id {}, body {}'.format(_job_id, request_config)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) module = data_func required_arguments = ['work_mode', 'namespace', 'table_name'] if module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif module == 'download': required_arguments.extend(['output_path']) else: raise Exception('can not support this operating: {}'.format(module)) detect_utils.check_config(request_config, required_arguments=required_arguments) if module == "upload": if not os.path.isabs(request_config['file']): request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"]) try: conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module, role=request_config.get('local', {}).get("role"), party_id=request_config.get('local', {}).get("party_id", '')) file_utils.dump_json_conf(request_config, conf_file_path) progs = ["python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-j", _job_id, "-c", conf_file_path ] try: p = run_subprocess(config_dir=_job_dir, process_cmd=progs) except Exception as e: stat_logger.exception(e) p = None return get_json_result(retcode=(0 if p else 101), job_id=_job_id, data={'table_name': request_config['table_name'], 'namespace': request_config['namespace'], 'pid': p.pid if p else ''}) except Exception as e: stat_logger.exception(e) return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)
def check_pipeline_job_runtime_conf(runtime_conf: typing.Dict): detect_utils.check_config(runtime_conf, ['initiator', 'job_parameters', 'role']) detect_utils.check_config(runtime_conf['initiator'], ['role', 'party_id']) detect_utils.check_config(runtime_conf['job_parameters'], [('work_mode', RuntimeConfig.WORK_MODE)]) # deal party id runtime_conf['initiator']['party_id'] = int( runtime_conf['initiator']['party_id']) for r in runtime_conf['role'].keys(): for i in range(len(runtime_conf['role'][r])): runtime_conf['role'][r][i] = int(runtime_conf['role'][r][i])
def deploy(): request_data = request.json require_parameters = ['model_id', 'model_version'] check_config(request_data, require_parameters) model_id = request_data.get("model_id") model_version = request_data.get("model_version") retcode, retmsg, model_info = model_utils.query_model_info_from_file( model_id=model_id, model_version=model_version, to_dict=True) if not model_info: raise Exception( f'Deploy model failed, no model {model_id} {model_version} found.') else: for key, value in model_info.items(): version_check = model_utils.compare_version( value.get('f_fate_version'), '1.5.0') if version_check == 'lt': continue else: init_role = key.split('/')[-2].split('#')[0] init_party_id = key.split('/')[-2].split('#')[1] model_init_role = value.get('f_initiator_role') if value.get( 'f_initiator_role') else value.get( 'f_train_runtime_conf', {}).get('initiator', {}).get( 'role', '') model_init_party_id = value.get( 'f_initiator_role_party_id') if value.get( 'f_initiator_role_party_id') else value.get( 'f_train_runtime_conf', {}).get( 'initiator', {}).get('party_id', '') if (init_role == model_init_role) and (init_party_id == str(model_init_party_id)): break else: raise Exception( "Deploy model failed, can not found model of initiator role or the fate version of model is older than 1.5.0" ) # distribute federated deploy task _job_id = job_utils.generate_job_id() request_data['child_model_version'] = _job_id initiator_party_id = model_init_party_id initiator_role = model_init_role request_data['initiator'] = { 'role': initiator_role, 'party_id': initiator_party_id } deploy_status = True deploy_status_info = {} deploy_status_msg = 'success' deploy_status_info['detail'] = {} for role_name, role_partys in value.get("f_train_runtime_conf", {}).get('role', {}).items(): if role_name not in ['arbiter', 'host', 'guest']: continue deploy_status_info[role_name] = deploy_status_info.get( role_name, {}) deploy_status_info['detail'][role_name] = {} adapter = JobRuntimeConfigAdapter( value.get("f_train_runtime_conf", {})) work_mode = adapter.get_job_work_mode() for _party_id in role_partys: request_data['local'] = { 'role': role_name, 'party_id': _party_id } try: response = federated_api( job_id=_job_id, method='POST', endpoint='/model/deploy/do', src_party_id=initiator_party_id, dest_party_id=_party_id, src_role=initiator_role, json_body=request_data, federated_mode=FederatedMode.MULTIPLE if work_mode else FederatedMode.SINGLE) deploy_status_info[role_name][_party_id] = response[ 'retcode'] detail = {_party_id: {}} detail[_party_id]['retcode'] = response['retcode'] detail[_party_id]['retmsg'] = response['retmsg'] deploy_status_info['detail'][role_name].update(detail) if response['retcode']: deploy_status = False deploy_status_msg = 'failed' except Exception as e: stat_logger.exception(e) deploy_status = False deploy_status_msg = 'failed' deploy_status_info[role_name][_party_id] = 100 deploy_status_info['model_id'] = request_data['model_id'] deploy_status_info['model_version'] = _job_id return get_json_result(retcode=(0 if deploy_status else 101), retmsg=deploy_status_msg, data=deploy_status_info)
def operate_model(model_operation): request_config = request.json or request.form.to_dict() job_id = job_utils.generate_job_id() if model_operation not in [ ModelOperation.STORE, ModelOperation.RESTORE, ModelOperation.EXPORT, ModelOperation.IMPORT ]: raise Exception( 'Can not support this operating now: {}'.format(model_operation)) required_arguments = ["model_id", "model_version", "role", "party_id"] check_config(request_config, required_arguments=required_arguments) request_config["model_id"] = gen_party_model_id( model_id=request_config["model_id"], role=request_config["role"], party_id=request_config["party_id"]) if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]: if model_operation == ModelOperation.IMPORT: try: file = request.files.get('file') file_path = os.path.join(TEMP_DIRECTORY, file.filename) # if not os.path.exists(file_path): # raise Exception('The file is obtained from the fate flow client machine, but it does not exist, ' # 'please check the path: {}'.format(file_path)) try: os.makedirs(os.path.dirname(file_path), exist_ok=True) file.save(file_path) except Exception as e: shutil.rmtree(file_path) raise e request_config['file'] = file_path model = pipelined_model.PipelinedModel( model_id=request_config["model_id"], model_version=request_config["model_version"]) model.unpack_model(file_path) pipeline = model.read_component_model('pipeline', 'pipeline')['Pipeline'] train_runtime_conf = json_loads(pipeline.train_runtime_conf) permitted_party_id = [] for key, value in train_runtime_conf.get('role', {}).items(): for v in value: permitted_party_id.extend([v, str(v)]) if request_config["party_id"] not in permitted_party_id: shutil.rmtree(model.model_path) raise Exception( "party id {} is not in model roles, please check if the party id is valid." ) try: adapter = JobRuntimeConfigAdapter(train_runtime_conf) job_parameters = adapter.get_common_parameters().to_dict() with DB.connection_context(): db_model = MLModel.get_or_none( MLModel.f_job_id == job_parameters.get( "model_version"), MLModel.f_role == request_config["role"]) if not db_model: model_info = model_utils.gather_model_info_data(model) model_info['imported'] = 1 model_info['job_id'] = model_info['f_model_version'] model_info['size'] = model.calculate_model_file_size() model_info['role'] = request_config["model_id"].split( '#')[0] model_info['party_id'] = request_config[ "model_id"].split('#')[1] if model_utils.compare_version( model_info['f_fate_version'], '1.5.1') == 'lt': model_info['roles'] = model_info.get( 'f_train_runtime_conf', {}).get('role', {}) model_info['initiator_role'] = model_info.get( 'f_train_runtime_conf', {}).get('initiator', {}).get('role') model_info['initiator_party_id'] = model_info.get( 'f_train_runtime_conf', {}).get('initiator', {}).get('party_id') model_info[ 'work_mode'] = adapter.get_job_work_mode() model_info['parent'] = False if model_info.get( 'f_inference_dsl') else True model_utils.save_model_info(model_info) else: stat_logger.info( f'job id: {job_parameters.get("model_version")}, ' f'role: {request_config["role"]} model info already existed in database.' ) except peewee.IntegrityError as e: stat_logger.exception(e) operation_record(request_config, "import", "success") return get_json_result() except Exception: operation_record(request_config, "import", "failed") raise else: try: model = pipelined_model.PipelinedModel( model_id=request_config["model_id"], model_version=request_config["model_version"]) if model.exists(): archive_file_path = model.packaging_model() operation_record(request_config, "export", "success") return send_file(archive_file_path, attachment_filename=os.path.basename( archive_file_path), as_attachment=True) else: operation_record(request_config, "export", "failed") res = error_response( response_code=210, retmsg="Model {} {} is not exist.".format( request_config.get("model_id"), request_config.get("model_version"))) return res except Exception as e: operation_record(request_config, "export", "failed") stat_logger.exception(e) return error_response(response_code=210, retmsg=str(e)) else: data = {} job_dsl, job_runtime_conf = gen_model_operation_job_config( request_config, model_operation) submit_result = DAGScheduler.submit( { 'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf }, job_id=job_id) data.update(submit_result) operation_record(data=job_runtime_conf, oper_type=model_operation, oper_status='') return get_json_result(job_id=job_id, data=data)
def migrate_model_process(): request_config = request.json _job_id = job_utils.generate_job_id() initiator_party_id = request_config['migrate_initiator']['party_id'] initiator_role = request_config['migrate_initiator']['role'] if not request_config.get("unify_model_version"): request_config["unify_model_version"] = _job_id migrate_status = True migrate_status_info = {} migrate_status_msg = 'success' migrate_status_info['detail'] = {} require_arguments = [ "migrate_initiator", "role", "migrate_role", "model_id", "model_version", "execute_party", "job_parameters" ] check_config(request_config, require_arguments) try: if compare_roles(request_config.get("migrate_role"), request_config.get("role")): return get_json_result( retcode=100, retmsg= "The config of previous roles is the same with that of migrate roles. " "There is no need to migrate model. Migration process aborting." ) except Exception as e: return get_json_result(retcode=100, retmsg=str(e)) local_template = {"role": "", "party_id": "", "migrate_party_id": ""} res_dict = {} for role_name, role_partys in request_config.get("migrate_role").items(): for offset, party_id in enumerate(role_partys): local_res = deepcopy(local_template) local_res["role"] = role_name local_res["party_id"] = request_config.get("role").get( role_name)[offset] local_res["migrate_party_id"] = party_id if not res_dict.get(role_name): res_dict[role_name] = {} res_dict[role_name][local_res["party_id"]] = local_res for role_name, role_partys in request_config.get("execute_party").items(): migrate_status_info[role_name] = migrate_status_info.get(role_name, {}) migrate_status_info['detail'][role_name] = {} for party_id in role_partys: request_config["local"] = res_dict.get(role_name).get(party_id) try: response = federated_api( job_id=_job_id, method='POST', endpoint='/model/migrate/do', src_party_id=initiator_party_id, dest_party_id=party_id, src_role=initiator_role, json_body=request_config, federated_mode=request_config['job_parameters'] ['federated_mode']) migrate_status_info[role_name][party_id] = response['retcode'] detail = {party_id: {}} detail[party_id]['retcode'] = response['retcode'] detail[party_id]['retmsg'] = response['retmsg'] migrate_status_info['detail'][role_name].update(detail) except Exception as e: stat_logger.exception(e) migrate_status = False migrate_status_msg = 'failed' migrate_status_info[role_name][party_id] = 100 return get_json_result(job_id=_job_id, retcode=(0 if migrate_status else 101), retmsg=migrate_status_msg, data=migrate_status_info)
def call_fun(func, config_data, dsl_path, config_path): ip = server_conf.get(SERVERS).get(ROLE).get('host') if ip in ['localhost', '127.0.0.1']: ip = get_lan_ip() http_port = server_conf.get(SERVERS).get(ROLE).get('http.port') server_url = "http://{}:{}/{}".format(ip, http_port, API_VERSION) if func in JOB_OPERATE_FUNC: if func == 'submit_job': if not config_path: raise Exception('the following arguments are required: {}'.format('runtime conf path')) dsl_data = {} if dsl_path or config_data.get('job_parameters', {}).get('job_type', '') == 'predict': if dsl_path: dsl_path = os.path.abspath(dsl_path) with open(dsl_path, 'r') as f: dsl_data = json.load(f) else: raise Exception('the following arguments are required: {}'.format('dsl path')) post_data = {'job_dsl': dsl_data, 'job_runtime_conf': config_data} response = requests.post("/".join([server_url, "job", func.rstrip('_job')]), json=post_data) try: if response.json()['retcode'] == 999: start_cluster_standalone_job_server() response = requests.post("/".join([server_url, "job", func.rstrip('_job')]), json=post_data) except: pass elif func == 'data_view_query' or func == 'clean_queue': response = requests.post("/".join([server_url, "job", func.replace('_', '/')]), json=config_data) else: if func != 'query_job': detect_utils.check_config(config=config_data, required_arguments=['job_id']) post_data = config_data response = requests.post("/".join([server_url, "job", func.rstrip('_job')]), json=post_data) if func == 'query_job': response = response.json() if response['retcode'] == 0: for i in range(len(response['data'])): del response['data'][i]['f_runtime_conf'] del response['data'][i]['f_dsl'] elif func in JOB_FUNC: if func == 'job_config': detect_utils.check_config(config=config_data, required_arguments=['job_id', 'role', 'party_id', 'output_path']) response = requests.post("/".join([server_url, func.replace('_', '/')]), json=config_data) response_data = response.json() if response_data['retcode'] == 0: job_id = response_data['data']['job_id'] download_directory = os.path.join(config_data['output_path'], 'job_{}_config'.format(job_id)) os.makedirs(download_directory, exist_ok=True) for k, v in response_data['data'].items(): if k == 'job_id': continue with open('{}/{}.json'.format(download_directory, k), 'w') as fw: json.dump(v, fw, indent=4) del response_data['data']['dsl'] del response_data['data']['runtime_conf'] response_data['directory'] = download_directory response_data['retmsg'] = 'download successfully, please check {} directory'.format(download_directory) response = response_data elif func == 'job_log': detect_utils.check_config(config=config_data, required_arguments=['job_id', 'output_path']) job_id = config_data['job_id'] tar_file_name = 'job_{}_log.tar.gz'.format(job_id) extract_dir = os.path.join(config_data['output_path'], 'job_{}_log'.format(job_id)) with closing(requests.get("/".join([server_url, func.replace('_', '/')]), json=config_data, stream=True)) as response: if response.status_code == 200: download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) response = {'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'.format(extract_dir)} else: response = response.json() elif func in TASK_OPERATE_FUNC: response = requests.post("/".join([server_url, "job", "task", func.rstrip('_task')]), json=config_data) elif func in TRACKING_FUNC: if func != 'component_metric_delete': detect_utils.check_config(config=config_data, required_arguments=['job_id', 'component_name', 'role', 'party_id']) if func == 'component_output_data': detect_utils.check_config(config=config_data, required_arguments=['output_path']) tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(config_data['job_id'], config_data['component_name'], config_data['role'], config_data['party_id']) extract_dir = os.path.join(config_data['output_path'], tar_file_name.replace('.tar.gz', '')) with closing(requests.get("/".join([server_url, "tracking", func.replace('_', '/'), 'download']), json=config_data, stream=True)) as response: if response.status_code == 200: try: download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) response = {'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'.format(extract_dir)} except: response = {'retcode': 100, 'retmsg': 'download failed, please check if the parameters are correct'} else: response = response.json() else: response = requests.post("/".join([server_url, "tracking", func.replace('_', '/')]), json=config_data) elif func in DATA_FUNC: if func == 'upload' and config_data.get('use_local_data', 1) != 0: file_name = config_data.get('file') if not os.path.isabs(file_name): file_name = os.path.join(file_utils.get_project_base_directory(), file_name) if os.path.exists(file_name): with open(file_name, 'rb') as fp: data = MultipartEncoder( fields={'file': (os.path.basename(file_name), fp, 'application/octet-stream')} ) tag = [0] def read_callback(monitor): if config_data.get('verbose') == 1: sys.stdout.write("\r UPLOADING:{0}{1}".format("|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len))) sys.stdout.flush() if monitor.bytes_read /monitor.len == 1: tag[0] += 1 if tag[0] == 2: sys.stdout.write('\n') data = MultipartEncoderMonitor(data, read_callback) response = requests.post("/".join([server_url, "data", func.replace('_', '/')]), data=data, params=config_data, headers={'Content-Type': data.content_type}) else: raise Exception('The file is obtained from the fate flow client machine, but it does not exist, ' 'please check the path: {}'.format(file_name)) else: response = requests.post("/".join([server_url, "data", func.replace('_', '/')]), json=config_data) try: if response.json()['retcode'] == 999: start_cluster_standalone_job_server() response = requests.post("/".join([server_url, "data", func]), json=config_data) except: pass elif func in TABLE_FUNC: if func == "table_info": detect_utils.check_config(config=config_data, required_arguments=['namespace', 'table_name']) response = requests.post("/".join([server_url, "table", func]), json=config_data) else: response = requests.post("/".join([server_url, "table", func.lstrip('table_')]), json=config_data) elif func in MODEL_FUNC: if func == "import": file_path = config_data["file"] if not os.path.isabs(file_path): file_path = os.path.join(file_utils.get_project_base_directory(), file_path) if os.path.exists(file_path): files = {'file': open(file_path, 'rb')} else: raise Exception('The file is obtained from the fate flow client machine, but it does not exist, ' 'please check the path: {}'.format(file_path)) response = requests.post("/".join([server_url, "model", func]), data=config_data, files=files) elif func == "export": with closing(requests.get("/".join([server_url, "model", func]), json=config_data, stream=True)) as response: if response.status_code == 200: archive_file_name = re.findall("filename=(.+)", response.headers["Content-Disposition"])[0] os.makedirs(config_data["output_path"], exist_ok=True) archive_file_path = os.path.join(config_data["output_path"], archive_file_name) with open(archive_file_path, 'wb') as fw: for chunk in response.iter_content(1024): if chunk: fw.write(chunk) response = {'retcode': 0, 'file': archive_file_path, 'retmsg': 'download successfully, please check {}'.format(archive_file_path)} else: response = response.json() else: response = requests.post("/".join([server_url, "model", func]), json=config_data) elif func in PERMISSION_FUNC: detect_utils.check_config(config=config_data, required_arguments=['src_party_id', 'src_role']) response = requests.post("/".join([server_url, "permission", func.replace('_', '/')]), json=config_data) return response.json() if isinstance(response, requests.models.Response) else response
def call_fun(func, config_data, dsl_path, config_path): ip = server_conf.get(SERVERS).get(ROLE).get('host') if ip in ['localhost', '127.0.0.1']: ip = get_lan_ip() http_port = server_conf.get(SERVERS).get(ROLE).get('http.port') server_url = "http://{}:{}/{}".format(ip, http_port, API_VERSION) if func in JOB_OPERATE_FUNC: if func == 'submit_job': if not config_path: raise Exception( 'the following arguments are required: {}'.format( 'runtime conf path')) dsl_data = {} if dsl_path or config_data.get('job_parameters', {}).get( 'job_type', '') == 'predict': if dsl_path: dsl_path = os.path.abspath(dsl_path) with open(dsl_path, 'r') as f: dsl_data = json.load(f) else: raise Exception( 'the following arguments are required: {}'.format( 'dsl path')) post_data = {'job_dsl': dsl_data, 'job_runtime_conf': config_data} response = requests.post("/".join( [server_url, "job", func.rstrip('_job')]), json=post_data) try: if response.json()['retcode'] == 999: start_cluster_standalone_job_server() response = requests.post("/".join( [server_url, "job", func.rstrip('_job')]), json=post_data) except: pass elif func == 'data_view_query': response = requests.post("/".join( [server_url, "job", func.replace('_', '/')]), json=config_data) else: if func != 'query_job': detect_utils.check_config(config=config_data, required_arguments=['job_id']) post_data = config_data response = requests.post("/".join( [server_url, "job", func.rstrip('_job')]), json=post_data) if func == 'query_job': response = response.json() if response['retcode'] == 0: for i in range(len(response['data'])): del response['data'][i]['f_runtime_conf'] del response['data'][i]['f_dsl'] elif func in JOB_FUNC: if func == 'job_config': detect_utils.check_config(config=config_data, required_arguments=[ 'job_id', 'role', 'party_id', 'output_path' ]) response = requests.post("/".join( [server_url, func.replace('_', '/')]), json=config_data) response_data = response.json() if response_data['retcode'] == 0: job_id = response_data['data']['job_id'] download_directory = os.path.join( config_data['output_path'], 'job_{}_config'.format(job_id)) os.makedirs(download_directory, exist_ok=True) for k, v in response_data['data'].items(): if k == 'job_id': continue with open('{}/{}.json'.format(download_directory, k), 'w') as fw: json.dump(v, fw, indent=4) del response_data['data']['dsl'] del response_data['data']['runtime_conf'] response_data['directory'] = download_directory response_data[ 'retmsg'] = 'download successfully, please check {} directory'.format( download_directory) response = response_data elif func == 'job_log': detect_utils.check_config( config=config_data, required_arguments=['job_id', 'output_path']) job_id = config_data['job_id'] tar_file_name = 'job_{}_log.tar.gz'.format(job_id) extract_dir = os.path.join(config_data['output_path'], 'job_{}_log'.format(job_id)) with closing( requests.get("/".join([server_url, func.replace('_', '/')]), json=config_data, stream=True)) as response: if response.status_code == 200: download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) response = { 'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'. format(extract_dir) } else: response = response.json() elif func in TASK_OPERATE_FUNC: response = requests.post("/".join( [server_url, "job", "task", func.rstrip('_task')]), json=config_data) elif func in TRACKING_FUNC: if func != 'component_metric_delete': detect_utils.check_config(config=config_data, required_arguments=[ 'job_id', 'component_name', 'role', 'party_id' ]) if func == 'component_output_data': detect_utils.check_config(config=config_data, required_arguments=['output_path']) tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format( config_data['job_id'], config_data['component_name'], config_data['role'], config_data['party_id']) extract_dir = os.path.join(config_data['output_path'], tar_file_name.replace('.tar.gz', '')) with closing( requests.get("/".join([ server_url, "tracking", func.replace('_', '/'), 'download' ]), json=config_data, stream=True)) as response: if response.status_code == 200: download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) response = { 'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'. format(extract_dir) } else: response = response.json() else: response = requests.post("/".join( [server_url, "tracking", func.replace('_', '/')]), json=config_data) elif func in DATA_FUNC: response = requests.post("/".join( [server_url, "data", func.replace('_', '/')]), json=config_data) try: if response.json()['retcode'] == 999: start_cluster_standalone_job_server() response = requests.post("/".join([server_url, "data", func]), json=config_data) except: pass elif func in TABLE_FUNC: if func == "table_info": detect_utils.check_config( config=config_data, required_arguments=['namespace', 'table_name']) response = requests.post("/".join([server_url, "table", func]), json=config_data) else: response = requests.post("/".join( [server_url, "table", func.lstrip('table_')]), json=config_data) elif func in MODEL_FUNC: if func == "version": detect_utils.check_config(config=config_data, required_arguments=['namespace']) response = requests.post("/".join([server_url, "model", func]), json=config_data) elif func in PERMISSION_FUNC: detect_utils.check_config( config=config_data, required_arguments=['src_party_id', 'src_role']) response = requests.post("/".join( [server_url, "permission", func.replace('_', '/')]), json=config_data) return response.json() if isinstance( response, requests.models.Response) else response
def call_fun(func, config_data, dsl_path, config_path): ip = server_conf.get(SERVERS).get(ROLE).get('host') http_port = server_conf.get(SERVERS).get(ROLE).get('http.port') local_url = "http://{}:{}/{}".format(ip, http_port, API_VERSION) if func in JOB_OPERATE_FUNC: if func == 'submit_job': if not config_path: raise Exception( 'the following arguments are required: {}'.format( 'runtime conf path')) dsl_data = {} if dsl_path or config_data.get('job_parameters', {}).get( 'job_type', '') == 'predict': if dsl_path: dsl_path = os.path.abspath(dsl_path) with open(dsl_path, 'r') as f: dsl_data = json.load(f) else: raise Exception( 'the following arguments are required: {}'.format( 'dsl path')) post_data = {'job_dsl': dsl_data, 'job_runtime_conf': config_data} else: if func != 'query_job': detect_utils.check_config(config=config_data, required_arguments=['job_id']) post_data = config_data response = requests.post("/".join( [local_url, "job", func.rstrip('_job')]), json=post_data) if func == 'query_job': response = response.json() if response['retcode'] == 0: for i in range(len(response['data'])): del response['data'][i]['f_runtime_conf'] del response['data'][i]['f_dsl'] elif func in JOB_FUNC: if func == 'job_config': detect_utils.check_config(config=config_data, required_arguments=[ 'job_id', 'role', 'party_id', 'output_path' ]) response = requests.post("/".join( [local_url, func.replace('_', '/')]), json=config_data) response_data = response.json() if response_data['retcode'] == 0: job_id = response_data['data']['job_id'] download_directory = os.path.join( config_data['output_path'], 'job_{}_config'.format(job_id)) os.makedirs(download_directory, exist_ok=True) for k, v in response_data['data'].items(): if k == 'job_id': continue with open('{}/{}.json'.format(download_directory, k), 'w') as fw: json.dump(v, fw, indent=4) del response_data['data']['dsl'] del response_data['data']['runtime_conf'] response_data['directory'] = download_directory response_data[ 'retmsg'] = 'download successfully, please check {} directory'.format( download_directory) response = response_data elif func == 'job_log': detect_utils.check_config( config=config_data, required_arguments=['job_id', 'output_path']) with closing( requests.get("/".join([local_url, func.replace('_', '/')]), json=config_data, stream=True)) as response: job_id = config_data['job_id'] tar_file_name = 'job_{}_log.tar.gz'.format(job_id) with open(tar_file_name, 'wb') as fw: for chunk in response.iter_content(1024): if chunk: fw.write(chunk) extract_dir = os.path.join(config_data['output_path'], 'job_{}_log'.format(job_id)) tar = tarfile.open(tar_file_name, "r:gz") file_names = tar.getnames() for file_name in file_names: tar.extract(file_name, extract_dir) tar.close() os.remove(tar_file_name) response = { 'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'.format( extract_dir) } elif func in TASK_OPERATE_FUNC: response = requests.post("/".join( [local_url, "job", "task", func.rstrip('_task')]), json=config_data) elif func in TRACKING_FUNC: detect_utils.check_config(config=config_data, required_arguments=[ 'job_id', 'component_name', 'role', 'party_id' ]) if func == 'component_output_data': detect_utils.check_config(config=config_data, required_arguments=['output_path']) tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format( config_data['job_id'], config_data['component_name'], config_data['role'], config_data['party_id']) extract_dir = os.path.join(config_data['output_path'], tar_file_name.replace('.tar.gz', '')) with closing( requests.get("/".join([ local_url, "tracking", func.replace('_', '/'), 'download' ]), json=config_data, stream=True)) as res: if res.status_code == 200: with open(tar_file_name, 'wb') as fw: for chunk in res.iter_content(1024): if chunk: fw.write(chunk) tar = tarfile.open(tar_file_name, "r:gz") file_names = tar.getnames() for file_name in file_names: tar.extract(file_name, extract_dir) tar.close() os.remove(tar_file_name) response = { 'retcode': 0, 'directory': extract_dir, 'retmsg': 'download successfully, please check {} directory'. format(extract_dir) } else: response = res.json() else: response = requests.post("/".join( [local_url, "tracking", func.replace('_', '/')]), json=config_data) elif func in DATA_FUNC: response = requests.post("/".join([local_url, "data", func]), json=config_data) elif func in TABLE_FUNC: detect_utils.check_config( config=config_data, required_arguments=['namespace', 'table_name']) response = requests.post("/".join([local_url, "table", func]), json=config_data) elif func in MODEL_FUNC: if func == "version": detect_utils.check_config(config=config_data, required_arguments=['namespace']) response = requests.post("/".join([local_url, "model", func]), json=config_data) return response.json() if isinstance( response, requests.models.Response) else response
def download_upload(access_module): job_id = generate_job_id() if access_module == "upload" and USE_LOCAL_DATA and not ( request.json and request.json.get("use_local_data") == 0): file = request.files['file'] filename = os.path.join(get_job_directory(job_id), 'fate_upload_tmp', file.filename) os.makedirs(os.path.dirname(filename), exist_ok=True) try: file.save(filename) except Exception as e: shutil.rmtree(os.path.join(get_job_directory(job_id), 'tmp')) raise e request_config = request.args.to_dict() request_config['file'] = filename else: request_config = request.json required_arguments = ['work_mode', 'namespace', 'table_name'] if access_module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif access_module == 'download': required_arguments.extend(['output_path']) elif access_module == 'download_test': required_arguments.extend(['output_path']) else: raise Exception( 'can not support this operating: {}'.format(access_module)) detect_utils.check_config(request_config, required_arguments=required_arguments) data = {} if access_module == "upload": data['table_name'] = request_config["table_name"] data['namespace'] = request_config["namespace"] if WORK_MODE != 0: data_table = session.get_data_table( name=request_config["table_name"], namespace=request_config["namespace"]) count = data_table.count() if count and int(request_config.get('drop', 2)) == 2: return get_json_result( retcode=100, retmsg='The data table already exists, table data count:{}.' 'If you still want to continue uploading, please add the parameter -drop. ' '0 means not to delete and continue uploading, ' '1 means to upload again after deleting the table'.format( count)) elif count and int(request_config.get('drop', 2)) == 1: data_table.destroy() job_dsl, job_runtime_conf = gen_data_access_job_config( request_config, access_module) job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job( { 'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf }, job_id=job_id) data.update({ 'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'board_url': board_url, 'logs_directory': logs_directory }) return get_json_result(job_id=job_id, data=data)
def download_upload(data_func): request_config = request.json _job_id = generate_job_id() stat_logger.info('generated job_id {}, body {}'.format( _job_id, request_config)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) module = data_func required_arguments = ['work_mode', 'namespace', 'table_name'] if module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif module == 'download': required_arguments.extend(['output_path']) else: raise Exception('can not support this operating: {}'.format(module)) detect_utils.check_config(request_config, required_arguments=required_arguments) job_work_mode = request_config['work_mode'] # todo: The current code here is redundant with job_app/submit_job, the next version of this function will be implemented by job_app/submit_job if job_work_mode != RuntimeConfig.WORK_MODE: if RuntimeConfig.WORK_MODE == WorkMode.CLUSTER and job_work_mode == WorkMode.STANDALONE: # use cluster standalone job server to execute standalone job return request_execute_server( request=request, execute_host='{}:{}'.format( request.remote_addr, CLUSTER_STANDALONE_JOB_SERVER_PORT)) else: raise Exception( 'server run on standalone can not support cluster mode job') if module == "upload": if not os.path.isabs(request_config['file']): request_config["file"] = os.path.join( file_utils.get_project_base_directory(), request_config["file"]) try: conf_file_path = new_runtime_conf( job_dir=_job_dir, method=data_func, module=module, role=request_config.get('local', {}).get("role"), party_id=request_config.get('local', {}).get("party_id", '')) file_utils.dump_json_conf(request_config, conf_file_path) progs = [ "python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-j", _job_id, "-c", conf_file_path ] try: p = run_subprocess(config_dir=_job_dir, process_cmd=progs) except Exception as e: stat_logger.exception(e) p = None return get_json_result(retcode=(0 if p else 101), job_id=_job_id, data={ 'table_name': request_config['table_name'], 'namespace': request_config['namespace'], 'pid': p.pid if p else '' }) except Exception as e: stat_logger.exception(e) return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)