def cancel_job(job_id, role, party_id): JobController.cancel_job(job_id=job_id, role=role, party_id=int(party_id), job_initiator=request.json.get( 'job_initiator', {})) return get_json_result(retcode=0, retmsg='success')
def save_pipeline(job_id, role, party_id, model_id, model_version): JobController.save_pipeline(job_id=job_id, role=role, party_id=party_id, model_id=base64_decode(model_id), model_version=base64_decode(model_version)) return get_json_result(retcode=0, retmsg='success')
def update_job(): job_info = request.json jobs = JobSaver.query_job(job_id=job_info['job_id'], party_id=job_info['party_id'], role=job_info['role']) if not jobs: return get_json_result(retcode=101, retmsg='find job failed') else: JobSaver.update_job( job_info={ 'description': job_info.get('notes', ''), 'job_id': job_info['job_id'], 'role': job_info['role'], 'party_id': job_info['party_id'] }) return get_json_result(retcode=0, retmsg='success')
def query_storage_connector(): request_data = request.json connector = StorageConnector( connector_name=request_data.get("connector_name")) return get_json_result(retcode=0, retmsg='success', data=connector.get_info())
def rerun_job(): job_id = request.json.get("job_id") jobs = JobSaver.query_job(job_id=job_id) if jobs: status_code, response = FederatedScheduler.request_rerun_job( job=jobs[0], command_body=request.json) if status_code == FederatedSchedulingStatusCode.SUCCESS: return get_json_result(retcode=RetCode.SUCCESS, retmsg="rerun job success") else: return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg="rerun job failed:\n{}".format( json_dumps(response))) else: return get_json_result(retcode=RetCode.DATA_ERROR, retmsg="can not found job")
def table_delete(): request_data = request.json table_name = request_data.get('table_name') namespace = request_data.get('namespace') with storage.Session.build(name=table_name, namespace=namespace) as storage_session: table = storage_session.get_table() if table: table.destroy() data = {'table_name': table_name, 'namespace': namespace} try: table.close() except Exception as e: stat_logger.exception(e) return get_json_result(data=data) else: return get_json_result(retcode=101, retmsg='no find table')
def submit_job(): work_mode = request.json.get('job_runtime_conf', {}).get('job_parameters', {}).get('work_mode', None) detect_utils.check_config({'work_mode': work_mode}, required_arguments=[('work_mode', (WorkMode.CLUSTER, WorkMode.STANDALONE))]) if work_mode == RuntimeConfig.WORK_MODE: job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job( request.json) return get_json_result(retcode=0, retmsg='success', job_id=job_id, data={ 'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'model_info': model_info, 'board_url': board_url, 'logs_directory': logs_directory }) else: if RuntimeConfig.WORK_MODE == WorkMode.CLUSTER and work_mode == WorkMode.STANDALONE: # use cluster standalone job server to execute standalone job return request_execute_server( request=request, execute_host='{}:{}'.format( request.remote_addr, CLUSTER_STANDALONE_JOB_SERVER_PORT)) else: raise Exception( 'server run on standalone can not support cluster mode job')
def component_output_data_table(): output_data_infos = Tracker.query_output_data_infos(**request.json) if output_data_infos: return get_json_result(retcode=0, retmsg='success', data=[{ 'table_name': output_data_info.f_table_name, 'table_namespace': output_data_info.f_table_namespace, "data_name": output_data_info.f_data_name } for output_data_info in output_data_infos]) else: return get_json_result( retcode=100, retmsg='No found table, please check if the parameters are correct' )
def get_component_summary(): request_data = request.json try: required_params = ["job_id", "component_name", "role", "party_id"] detect_utils.check_config(request_data, required_params) tracker = Tracker(job_id=request_data["job_id"], component_name=request_data["component_name"], role=request_data["role"], party_id=request_data["party_id"], task_id=request_data.get("task_id", None), task_version=request_data.get("task_version", None)) summary = tracker.read_summary_from_db() if summary: if request_data.get("filename"): temp_filepath = os.path.join(TEMP_DIRECTORY, request_data.get("filename")) with open(temp_filepath, "w") as fout: fout.write(json.dumps(summary, indent=4)) return send_file( open(temp_filepath, "rb"), as_attachment=True, attachment_filename=request_data.get("filename")) else: return get_json_result(data=summary) return error_response( 210, "No component summary found, please check if arguments are specified correctly." ) except Exception as e: stat_logger.exception(e) return error_response(210, str(e))
def validate_component_param(): if not request.json or not isinstance(request.json, dict): return error_response(400, 'bad request') required_keys = [ 'component_name', 'component_module_name', ] config_keys = ['role'] dsl_version = int(request.json.get('dsl_version', 0)) parser_class = get_dsl_parser_by_version(dsl_version) if dsl_version == 1: config_keys += ['role_parameters', 'algorithm_parameters'] elif dsl_version == 2: config_keys += ['component_parameters'] else: return error_response(400, 'unsupported dsl_version') try: check_config(request.json, required_keys + config_keys) except Exception as e: return error_response(400, str(e)) try: parser_class.validate_component_param( get_federatedml_setting_conf_directory(), {i: request.json[i] for i in config_keys}, *[request.json[i] for i in required_keys]) except Exception as e: return error_response(400, str(e)) return get_json_result()
def get_config(): kwargs = {} job_configuration = None for i in ('job_id', 'role', 'party_id'): if request.json.get(i) is None: return error_response(400, f"'{i}' is required.") kwargs[i] = str(request.json[i]) for i in ('component_name', 'task_id', 'task_version'): if request.json.get(i) is None: break kwargs[i] = str(request.json[i]) else: try: job_configuration = job_utils.get_task_using_job_conf(**kwargs) except Exception: pass if job_configuration is None: job_configuration = job_utils.get_job_configuration( kwargs['job_id'], kwargs['role'], kwargs['party_id']) if job_configuration is None: return error_response(404, 'Job not found.') return get_json_result(data=job_configuration.to_dict())
def publish_model_online(): request_config = request.json if not request_config.get('servings'): # get my party all servings request_config['servings'] = SERVINGS online_status = publish_model.publish_online(config_data=request_config) return get_json_result(retcode=(0 if online_status else 101))
def kill_job(job_id, role, party_id): JobController.kill_job(job_id=job_id, role=role, party_id=int(party_id), job_initiator=request.json.get('job_initiator', {}), timeout=request.json.get('timeout', False), component_name=request.json.get('component_name', '') ) return get_json_result(retcode=0, retmsg='success')
def submit_job(): submit_result = DAGScheduler.submit(JobConfigurationBase(**request.json)) return get_json_result(retcode=submit_result["code"], retmsg=submit_result["message"], job_id=submit_result["job_id"], data=submit_result if submit_result["code"] == RetCode.SUCCESS else None)
def bind_model_service(): request_config = request.json if not request_config.get('servings'): # get my party all servings request_config['servings'] = CenterConfig.get_settings( path=SERVING_PATH, servings_zk_path=SERVINGS_ZK_PATH, use_zk=USE_CONFIGURATION_CENTER, hosts=ZOOKEEPER_HOSTS, server_conf_path=SERVER_CONF_PATH) if not request_config.get('service_id'): return get_json_result(retcode=101, retmsg='no service id') bind_status, service_id = publish_model.bind_model_service( config_data=request_config) return get_json_result(retcode=(0 if bind_status else 101), retmsg='service id is {}'.format(service_id))
def task_status(job_id, component_name, task_id, task_version, role, party_id, status): task_info = {} task_info.update({ "job_id": job_id, "task_id": task_id, "task_version": task_version, "role": role, "party_id": party_id, "status": status }) if TaskController.update_task_status(task_info=task_info): return get_json_result(retcode=0, retmsg='success') else: return get_json_result(retcode=RetCode.NOT_EFFECTIVE, retmsg="update job status does not take effect")
def save_metric_meta(job_id, component_name, task_id, role, party_id): request_data = request.json tracker = Tracking(job_id=job_id, component_name=component_name, task_id=task_id, role=role, party_id=party_id) metric_meta = deserialize_b64(request_data['metric_meta']) tracker.save_metric_meta(metric_namespace=request_data['metric_namespace'], metric_name=request_data['metric_name'], metric_meta=metric_meta, job_level=request_data['job_level']) return get_json_result()
def stop_job(job_id, role, party_id, stop_status): kill_status, kill_details = JobController.stop_jobs( job_id=job_id, stop_status=stop_status, role=role, party_id=party_id) return get_json_result( retcode=RetCode.SUCCESS if kill_status else RetCode.EXCEPTION_ERROR, retmsg='success' if kill_status else 'failed', data=kill_details)
def download_upload(access_module): request_config = request.json required_arguments = ['work_mode', 'namespace', 'table_name'] if access_module == 'upload': required_arguments.extend(['file', 'head', 'partition']) elif access_module == 'download': required_arguments.extend(['output_path']) else: raise Exception( 'can not support this operating: {}'.format(access_module)) detect_utils.check_config(request_config, required_arguments=required_arguments) data = {} if access_module == "upload": data['table_name'] = request_config["table_name"] data['namespace'] = request_config["namespace"] job_dsl, job_runtime_conf = gen_data_access_job_config( request_config, access_module) job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job( { 'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf }) data.update({ 'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path, 'board_url': board_url, 'logs_directory': logs_directory }) return get_json_result(job_id=job_id, data=data)
def report_task(job_id, component_name, task_id, task_version, role, party_id): task_info = {} task_info.update(request.json) task_info.update({ "job_id": job_id, "task_id": task_id, "task_version": task_version, "role": role, "party_id": party_id, }) TaskController.update_task(task_info=task_info) if task_info.get("party_status"): if not TaskController.update_task_status(task_info=task_info): return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg="update task status failed") return get_json_result(retcode=0, retmsg='success')
def table_delete(): request_data = request.json table_name = request_data.get('table_name') namespace = request_data.get('namespace') data = None sess = Session() table = sess.get_table(name=table_name, namespace=namespace, ignore_disable=True) if table: table.destroy() data = {'table_name': table_name, 'namespace': namespace} sess.destroy_all_sessions() if data: return get_json_result(data=data) return get_json_result(retcode=101, retmsg='no find table')
def task_status(job_id, component_name, task_id, task_version, role, party_id, status): task_info = {} task_info.update({ "job_id": job_id, "task_id": task_id, "task_version": task_version, "role": role, "party_id": party_id, "status": status }) if TaskController.update_task_status(task_info=task_info): return get_json_result(retcode=0, retmsg='success') else: return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg="update task status failed")
def get_url(): request_data = request.json detect_utils.check_config( config=request_data, required_arguments=['job_id', 'role', 'party_id']) jobs = JobSaver.query_job(job_id=request_data.get('job_id'), role=request_data.get('role'), party_id=request_data.get('party_id')) if jobs: board_urls = [] for job in jobs: board_url = job_utils.get_board_url(job.f_job_id, job.f_role, job.f_party_id) board_urls.append(board_url) return get_json_result(data={'board_url': board_urls}) else: return get_json_result(retcode=101, retmsg='no found job')
def get_predict_dsl(): request_data = request.json request_data['query_filters'] = ['inference_dsl'] retcode, retmsg, data = model_utils.query_model_info_from_file( **request_data) if data: for d in data: if d.get("f_role") in {"guest", "host"}: _data = d break else: return error_response( 210, "can not found guest or host model, please get predict dsl on guest or host." ) if request_data.get("filename"): os.makedirs(TEMP_DIRECTORY, exist_ok=True) temp_filepath = os.path.join(TEMP_DIRECTORY, request_data.get("filename")) with open(temp_filepath, "w") as fout: fout.write(json_dumps(_data['f_inference_dsl'], indent=4)) return send_file(open(temp_filepath, "rb"), as_attachment=True, attachment_filename=request_data.get("filename")) else: return get_json_result(data=_data['f_inference_dsl']) return error_response( 210, "No model found, please check if arguments are specified correctly.")
def job_status(job_id, role, party_id): JobController.update_job_status(job_id=job_id, role=role, party_id=int(party_id), job_info=request.json, create=False) return get_json_result(retcode=0, retmsg='success')
def homo_convert(): request_config = request.json or request.form.to_dict() retcode, retmsg, res_data = publish_model.convert_homo_model( request_config) operation_record(request.json, "homo_convert", "success" if not retcode else "failed") return get_json_result(retcode=retcode, retmsg=retmsg, data=res_data)
def clean(job_id, role, party_id, roles, party_ids): JobController.clean_job(job_id=job_id, role=role, party_id=party_id, roles=roles, party_ids=party_ids) return get_json_result(retcode=0, retmsg='success')
def set_fate_server_info(): # manager federated_id = request.json.get("federatedId") manager_conf = conf_utils.get_base_config("fatemanager", {}) manager_conf["federatedId"] = federated_id conf_utils.update_config("fatemanager", manager_conf) return get_json_result(data={"federatedId": federated_id})
def dsl_generator(): data = request.json cpn_str = data.get("cpn_str", "") try: if not cpn_str: raise Exception("Component list should not be empty.") if isinstance(cpn_str, list): cpn_list = cpn_str else: if (cpn_str.find("/") and cpn_str.find("\\")) != -1: raise Exception( "Component list string should not contain '/' or '\\'.") cpn_str = cpn_str.replace(" ", "").replace("\n", "").strip(",[]") cpn_list = cpn_str.split(",") train_dsl = json_loads(data.get("train_dsl")) parser = schedule_utils.get_dsl_parser_by_version( data.get("version", "1")) predict_dsl = parser.deploy_component(cpn_list, train_dsl) if data.get("filename"): os.makedirs(TEMP_DIRECTORY, exist_ok=True) temp_filepath = os.path.join(TEMP_DIRECTORY, data.get("filename")) with open(temp_filepath, "w") as fout: fout.write(json.dumps(predict_dsl, indent=4)) return send_file(open(temp_filepath, 'rb'), as_attachment=True, attachment_filename=data.get("filename")) return get_json_result(data=predict_dsl) except Exception as e: stat_logger.exception(e) return error_response( 210, "DSL generating failed. For more details, " "please check logs/fate_flow/fate_flow_stat.log.")
def do_load_model(): request_data = request.json request_data["servings"] = CenterConfig.get_settings(path=SERVING_PATH, servings_zk_path=SERVINGS_ZK_PATH, use_zk=USE_CONFIGURATION_CENTER, hosts=ZOOKEEPER_HOSTS, server_conf_path=SERVER_CONF_PATH) load_status = publish_model.load_model(config_data=request_data) return get_json_result(retcode=(0 if load_status else 101))