def operate_model(model_operation):
request_config = request.json or request.form.to_dict()
job_id = generate_job_id()
if model_operation not in [ModelOperation.STORE, ModelOperation.RESTORE, ModelOperation.EXPORT, ModelOperation.IMPORT]:
raise Exception('Can not support this operating now: {}'.format(model_operation))
required_arguments = ["model_id", "model_version", "role", "party_id"]
check_config(request_config, required_arguments=required_arguments)
request_config["model_id"] = gen_party_model_id(model_id=request_config["model_id"], role=request_config["role"], party_id=request_config["party_id"])
if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]:
if model_operation == ModelOperation.IMPORT:
file = request.files.get('file')
file_path = os.path.join(TEMP_DIRECTORY, file.filename)
try:
os.makedirs(os.path.dirname(file_path), exist_ok=True)
file.save(file_path)
except Exception as e:
shutil.rmtree(file_path)
raise e
request_config['file'] = file_path
model = pipelined_model.PipelinedModel(model_id=request_config["model_id"], model_version=request_config["model_version"])
model.unpack_model(file_path)
return get_json_result()
else:
model = pipelined_model.PipelinedModel(model_id=request_config["model_id"], model_version=request_config["model_version"])
archive_file_path = model.packaging_model()
return send_file(archive_file_path, attachment_filename=os.path.basename(archive_file_path), as_attachment=True)
else:
data = {}
job_dsl, job_runtime_conf = gen_model_operation_job_config(request_config, model_operation)
job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job(
{'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf}, job_id=job_id)
data.update({'job_dsl_path': job_dsl_path, 'job_runtime_conf_path': job_runtime_conf_path,
'board_url': board_url, 'logs_directory': logs_directory})
return get_json_result(job_id=job_id, data=data)
def load_model():
request_config = request.json
_job_id = generate_job_id()
initiator_party_id = request_config['initiator']['party_id']
initiator_role = request_config['initiator']['role']
publish_model.generate_publish_model_info(request_config)
load_status = True
load_status_info = {}
load_status_msg = 'success'
for role_name, role_partys in request_config.get("role").items():
if role_name == 'arbiter':
continue
load_status_info[role_name] = load_status_info.get(role_name, {})
for _party_id in role_partys:
request_config['local'] = {'role': role_name, 'party_id': _party_id}
try:
response = federated_api(job_id=_job_id,
method='POST',
endpoint='/{}/model/load/do'.format(API_VERSION),
src_party_id=initiator_party_id,
dest_party_id=_party_id,
src_role = initiator_role,
json_body=request_config,
work_mode=request_config['job_parameters']['work_mode'])
load_status_info[role_name][_party_id] = response['retcode']
except Exception as e:
stat_logger.exception(e)
load_status = False
load_status_msg = 'failed'
load_status_info[role_name][_party_id] = 100
return get_json_result(job_id=_job_id, retcode=(0 if load_status else 101), retmsg=load_status_msg,
data=load_status_info)
def submit_job(job_data):
job_id = generate_job_id()
schedule_logger.info('submit job, job_id {}, body {}'.format(job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_utils.check_pipeline_job_runtime_conf(job_runtime_conf)
job_parameters = job_runtime_conf['job_parameters']
job_initiator = job_runtime_conf['initiator']
job_type = job_parameters.get('job_type', '')
if job_type != 'predict':
# generate job model info
job_parameters['model_id'] = '#'.join([dtable_utils.all_party_key(job_runtime_conf['role']), 'model'])
job_parameters['model_version'] = job_id
train_runtime_conf = {}
else:
detect_utils.check_config(job_parameters, ['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
job_tracker = Tracking(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'],
model_id=job_parameters['model_id'], model_version=job_parameters['model_version'])
pipeline_model = job_tracker.get_output_model('pipeline')
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf)
job_dsl_path, job_runtime_conf_path = save_job_conf(job_id=job_id,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf)
job = Job()
job.f_job_id = job_id
job.f_roles = json_dumps(job_runtime_conf['role'])
job.f_work_mode = job_parameters['work_mode']
job.f_initiator_party_id = job_initiator['party_id']
job.f_dsl = json_dumps(job_dsl)
job.f_runtime_conf = json_dumps(job_runtime_conf)
job.f_train_runtime_conf = json_dumps(train_runtime_conf)
job.f_run_ip = ''
job.f_status = JobStatus.WAITING
job.f_progress = 0
job.f_create_time = current_timestamp()
# save job info
TaskScheduler.distribute_job(job=job, roles=job_runtime_conf['role'], job_initiator=job_initiator)
# push into queue
RuntimeConfig.JOB_QUEUE.put_event({
'job_id': job_id,
"initiator_role": job_initiator['role'],
"initiator_party_id": job_initiator['party_id']
}
)
schedule_logger.info(
'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters['model_id']))
board_url = BOARD_DASHBOARD_URL.format(job_id, job_initiator['role'], job_initiator['party_id'])
return job_id, job_dsl_path, job_runtime_conf_path, {'model_id': job_parameters['model_id'],
'model_version': job_parameters[
'model_version']}, board_url
def download_upload(access_module):
job_id = job_utils.generate_job_id()
if access_module == "upload" and UPLOAD_DATA_FROM_CLIENT and not (request.json and request.json.get("use_local_data") == 0):
file = request.files['file']
filename = os.path.join(job_utils.get_job_directory(job_id), 'fate_upload_tmp', file.filename)
os.makedirs(os.path.dirname(filename), exist_ok=True)
try:
file.save(filename)
except Exception as e:
shutil.rmtree(os.path.join(job_utils.get_job_directory(job_id), 'fate_upload_tmp'))
raise e
job_config = request.args.to_dict()
if "namespace" in job_config and "table_name" in job_config:
pass
else:
# higher than version 1.5.1, support eggroll run parameters
job_config = json_loads(list(job_config.keys())[0])
job_config['file'] = filename
else:
job_config = request.json
required_arguments = ['work_mode', 'namespace', 'table_name']
if access_module == 'upload':
required_arguments.extend(['file', 'head', 'partition'])
elif access_module == 'download':
required_arguments.extend(['output_path'])
else:
raise Exception('can not support this operating: {}'.format(access_module))
detect_utils.check_config(job_config, required_arguments=required_arguments)
data = {}
# compatibility
if "table_name" in job_config:
job_config["name"] = job_config["table_name"]
if "backend" not in job_config:
job_config["backend"] = 0
for _ in ["work_mode", "backend", "head", "partition", "drop"]:
if _ in job_config:
job_config[_] = int(job_config[_])
if access_module == "upload":
if job_config.get('drop', 0) == 1:
job_config["destroy"] = True
else:
job_config["destroy"] = False
data['table_name'] = job_config["table_name"]
data['namespace'] = job_config["namespace"]
data_table_meta = storage.StorageTableMeta(name=job_config["table_name"], namespace=job_config["namespace"])
if data_table_meta and not job_config["destroy"]:
return get_json_result(retcode=100,
retmsg='The data table already exists.'
'If you still want to continue uploading, please add the parameter -drop.'
' 0 means not to delete and continue uploading, '
'1 means to upload again after deleting the table')
job_dsl, job_runtime_conf = gen_data_access_job_config(job_config, access_module)
submit_result = DAGScheduler.submit({'job_dsl': job_dsl, 'job_runtime_conf': job_runtime_conf}, job_id=job_id)
data.update(submit_result)
return get_json_result(job_id=job_id, data=data)
def start_proxy(role):
request_config = request.json or request.form.to_dict()
_job_id = job_utils.generate_job_id()
if role in ['marketplace']:
response = proxy_api(role, _job_id, request_config)
else:
response = federated_api(job_id=_job_id,
method='POST',
endpoint='/forward/{}/do'.format(role),
src_party_id=request_config.get('header').get('src_party_id'),
dest_party_id=request_config.get('header').get('dest_party_id'),
src_role=None,
json_body=request_config,
federated_mode=FederatedMode.MULTIPLE)
return jsonify(response)
def test_queue_put(self):
job_id = generate_job_id()
event = {
'job_id': job_id,
"initiator_role": 'loacl',
"initiator_party_id": 0
}
# queue put
job_queue.put_event(event)
# queue qsize
n = job_queue.qsize()
if n:
# queue get
job_event = job_queue.get()
self.assertIsNotNone(job_event)
def download_upload(data_func):
request_config = request.json
_job_id = generate_job_id()
stat_logger.info('generated job_id {}, body {}'.format(_job_id, request_config))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
module = data_func
required_arguments = ['work_mode', 'namespace', 'table_name']
if module == 'upload':
required_arguments.extend(['file', 'head', 'partition'])
elif module == 'download':
required_arguments.extend(['output_path'])
else:
raise Exception('can not support this operating: {}'.format(module))
detect_utils.check_config(request_config, required_arguments=required_arguments)
if module == "upload":
if not os.path.isabs(request_config['file']):
request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"])
try:
conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module,
role=request_config.get('local', {}).get("role"),
party_id=request_config.get('local', {}).get("party_id", ''))
file_utils.dump_json_conf(request_config, conf_file_path)
progs = ["python3",
os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]),
"-j", _job_id,
"-c", conf_file_path
]
try:
p = run_subprocess(config_dir=_job_dir, process_cmd=progs)
except Exception as e:
stat_logger.exception(e)
p = None
return get_json_result(retcode=(0 if p else 101), job_id=_job_id,
data={'table_name': request_config['table_name'],
'namespace': request_config['namespace'], 'pid': p.pid if p else ''})
except Exception as e:
stat_logger.exception(e)
return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)
def operate_model(model_operation):
request_config = request.json or request.form.to_dict()
job_id = job_utils.generate_job_id()
if model_operation not in [
ModelOperation.STORE, ModelOperation.RESTORE,
ModelOperation.EXPORT, ModelOperation.IMPORT
]:
raise Exception(
'Can not support this operating now: {}'.format(model_operation))
required_arguments = ["model_id", "model_version", "role", "party_id"]
check_config(request_config, required_arguments=required_arguments)
request_config["model_id"] = gen_party_model_id(
model_id=request_config["model_id"],
role=request_config["role"],
party_id=request_config["party_id"])
if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]:
if model_operation == ModelOperation.IMPORT:
try:
file = request.files.get('file')
file_path = os.path.join(TEMP_DIRECTORY, file.filename)
# if not os.path.exists(file_path):
# raise Exception('The file is obtained from the fate flow client machine, but it does not exist, '
# 'please check the path: {}'.format(file_path))
try:
os.makedirs(os.path.dirname(file_path), exist_ok=True)
file.save(file_path)
except Exception as e:
shutil.rmtree(file_path)
raise e
request_config['file'] = file_path
model = pipelined_model.PipelinedModel(
model_id=request_config["model_id"],
model_version=request_config["model_version"])
model.unpack_model(file_path)
pipeline = model.read_component_model('pipeline',
'pipeline')['Pipeline']
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
permitted_party_id = []
for key, value in train_runtime_conf.get('role', {}).items():
for v in value:
permitted_party_id.extend([v, str(v)])
if request_config["party_id"] not in permitted_party_id:
shutil.rmtree(model.model_path)
raise Exception(
"party id {} is not in model roles, please check if the party id is valid."
)
try:
adapter = JobRuntimeConfigAdapter(train_runtime_conf)
job_parameters = adapter.get_common_parameters().to_dict()
with DB.connection_context():
db_model = MLModel.get_or_none(
MLModel.f_job_id == job_parameters.get(
"model_version"),
MLModel.f_role == request_config["role"])
if not db_model:
model_info = model_utils.gather_model_info_data(model)
model_info['imported'] = 1
model_info['job_id'] = model_info['f_model_version']
model_info['size'] = model.calculate_model_file_size()
model_info['role'] = request_config["model_id"].split(
'#')[0]
model_info['party_id'] = request_config[
"model_id"].split('#')[1]
if model_utils.compare_version(
model_info['f_fate_version'], '1.5.1') == 'lt':
model_info['roles'] = model_info.get(
'f_train_runtime_conf', {}).get('role', {})
model_info['initiator_role'] = model_info.get(
'f_train_runtime_conf',
{}).get('initiator', {}).get('role')
model_info['initiator_party_id'] = model_info.get(
'f_train_runtime_conf',
{}).get('initiator', {}).get('party_id')
model_info[
'work_mode'] = adapter.get_job_work_mode()
model_info['parent'] = False if model_info.get(
'f_inference_dsl') else True
model_utils.save_model_info(model_info)
else:
stat_logger.info(
f'job id: {job_parameters.get("model_version")}, '
f'role: {request_config["role"]} model info already existed in database.'
)
except peewee.IntegrityError as e:
stat_logger.exception(e)
operation_record(request_config, "import", "success")
return get_json_result()
except Exception:
operation_record(request_config, "import", "failed")
raise
else:
try:
model = pipelined_model.PipelinedModel(
model_id=request_config["model_id"],
model_version=request_config["model_version"])
if model.exists():
archive_file_path = model.packaging_model()
operation_record(request_config, "export", "success")
return send_file(archive_file_path,
attachment_filename=os.path.basename(
archive_file_path),
as_attachment=True)
else:
operation_record(request_config, "export", "failed")
res = error_response(
response_code=210,
retmsg="Model {} {} is not exist.".format(
request_config.get("model_id"),
request_config.get("model_version")))
return res
except Exception as e:
operation_record(request_config, "export", "failed")
stat_logger.exception(e)
return error_response(response_code=210, retmsg=str(e))
else:
data = {}
job_dsl, job_runtime_conf = gen_model_operation_job_config(
request_config, model_operation)
submit_result = DAGScheduler.submit(
{
'job_dsl': job_dsl,
'job_runtime_conf': job_runtime_conf
},
job_id=job_id)
data.update(submit_result)
operation_record(data=job_runtime_conf,
oper_type=model_operation,
oper_status='')
return get_json_result(job_id=job_id, data=data)
def migrate_model_process():
request_config = request.json
_job_id = job_utils.generate_job_id()
initiator_party_id = request_config['migrate_initiator']['party_id']
initiator_role = request_config['migrate_initiator']['role']
if not request_config.get("unify_model_version"):
request_config["unify_model_version"] = _job_id
migrate_status = True
migrate_status_info = {}
migrate_status_msg = 'success'
migrate_status_info['detail'] = {}
require_arguments = [
"migrate_initiator", "role", "migrate_role", "model_id",
"model_version", "execute_party", "job_parameters"
]
check_config(request_config, require_arguments)
try:
if compare_roles(request_config.get("migrate_role"),
request_config.get("role")):
return get_json_result(
retcode=100,
retmsg=
"The config of previous roles is the same with that of migrate roles. "
"There is no need to migrate model. Migration process aborting."
)
except Exception as e:
return get_json_result(retcode=100, retmsg=str(e))
local_template = {"role": "", "party_id": "", "migrate_party_id": ""}
res_dict = {}
for role_name, role_partys in request_config.get("migrate_role").items():
for offset, party_id in enumerate(role_partys):
local_res = deepcopy(local_template)
local_res["role"] = role_name
local_res["party_id"] = request_config.get("role").get(
role_name)[offset]
local_res["migrate_party_id"] = party_id
if not res_dict.get(role_name):
res_dict[role_name] = {}
res_dict[role_name][local_res["party_id"]] = local_res
for role_name, role_partys in request_config.get("execute_party").items():
migrate_status_info[role_name] = migrate_status_info.get(role_name, {})
migrate_status_info['detail'][role_name] = {}
for party_id in role_partys:
request_config["local"] = res_dict.get(role_name).get(party_id)
try:
response = federated_api(
job_id=_job_id,
method='POST',
endpoint='/model/migrate/do',
src_party_id=initiator_party_id,
dest_party_id=party_id,
src_role=initiator_role,
json_body=request_config,
federated_mode=request_config['job_parameters']
['federated_mode'])
migrate_status_info[role_name][party_id] = response['retcode']
detail = {party_id: {}}
detail[party_id]['retcode'] = response['retcode']
detail[party_id]['retmsg'] = response['retmsg']
migrate_status_info['detail'][role_name].update(detail)
except Exception as e:
stat_logger.exception(e)
migrate_status = False
migrate_status_msg = 'failed'
migrate_status_info[role_name][party_id] = 100
return get_json_result(job_id=_job_id,
retcode=(0 if migrate_status else 101),
retmsg=migrate_status_msg,
data=migrate_status_info)
def download_upload(access_module):
job_id = generate_job_id()
if access_module == "upload" and USE_LOCAL_DATA and not (
request.json and request.json.get("use_local_data") == 0):
file = request.files['file']
filename = os.path.join(get_job_directory(job_id), 'fate_upload_tmp',
file.filename)
os.makedirs(os.path.dirname(filename), exist_ok=True)
try:
file.save(filename)
except Exception as e:
shutil.rmtree(os.path.join(get_job_directory(job_id), 'tmp'))
raise e
request_config = request.args.to_dict()
request_config['file'] = filename
else:
request_config = request.json
required_arguments = ['work_mode', 'namespace', 'table_name']
if access_module == 'upload':
required_arguments.extend(['file', 'head', 'partition'])
elif access_module == 'download':
required_arguments.extend(['output_path'])
elif access_module == 'download_test':
required_arguments.extend(['output_path'])
else:
raise Exception(
'can not support this operating: {}'.format(access_module))
detect_utils.check_config(request_config,
required_arguments=required_arguments)
data = {}
if access_module == "upload":
data['table_name'] = request_config["table_name"]
data['namespace'] = request_config["namespace"]
if WORK_MODE != 0:
data_table = session.get_data_table(
name=request_config["table_name"],
namespace=request_config["namespace"])
count = data_table.count()
if count and int(request_config.get('drop', 2)) == 2:
return get_json_result(
retcode=100,
retmsg='The data table already exists, table data count:{}.'
'If you still want to continue uploading, please add the parameter -drop. '
'0 means not to delete and continue uploading, '
'1 means to upload again after deleting the table'.format(
count))
elif count and int(request_config.get('drop', 2)) == 1:
data_table.destroy()
job_dsl, job_runtime_conf = gen_data_access_job_config(
request_config, access_module)
job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job(
{
'job_dsl': job_dsl,
'job_runtime_conf': job_runtime_conf
},
job_id=job_id)
data.update({
'job_dsl_path': job_dsl_path,
'job_runtime_conf_path': job_runtime_conf_path,
'board_url': board_url,
'logs_directory': logs_directory
})
return get_json_result(job_id=job_id, data=data)
def submit(cls, job_data, job_id=None):
if not job_id:
job_id = job_utils.generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_initiator = job_runtime_conf['initiator']
job_parameters = RunParameters(**job_runtime_conf['job_parameters'])
cls.backend_compatibility(job_parameters=job_parameters)
job_utils.check_job_runtime_conf(job_runtime_conf)
if job_parameters.job_type != 'predict':
# generate job model info
job_parameters.model_id = model_utils.gen_model_id(job_runtime_conf['role'])
job_parameters.model_version = job_id
train_runtime_conf = {}
else:
detect_utils.check_config(job_parameters.to_dict(), ['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
tracker = Tracker(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'],
model_id=job_parameters.model_id, model_version=job_parameters.model_version)
pipeline_model = tracker.get_output_model('pipeline')
if not job_dsl:
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf)
path_dict = job_utils.save_job_conf(job_id=job_id,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job = Job()
job.f_job_id = job_id
job.f_dsl = job_dsl
job_runtime_conf["job_parameters"] = job_parameters.to_dict()
job.f_runtime_conf = job_runtime_conf
job.f_train_runtime_conf = train_runtime_conf
job.f_roles = job_runtime_conf['role']
job.f_work_mode = job_parameters.work_mode
job.f_initiator_role = job_initiator['role']
job.f_initiator_party_id = job_initiator['party_id']
initiator_role = job_initiator['role']
initiator_party_id = job_initiator['party_id']
if initiator_party_id not in job_runtime_conf['role'][initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(initiator_party_id))
raise Exception("initiator party id error {}".format(initiator_party_id))
dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
cls.adapt_job_parameters(job_parameters=job_parameters)
# update runtime conf
job_runtime_conf["job_parameters"] = job_parameters.to_dict()
job.f_runtime_conf = job_runtime_conf
status_code, response = FederatedScheduler.create_job(job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
raise Exception("create job failed: {}".format(response))
if job_parameters.work_mode == WorkMode.CLUSTER:
# Save the status information of all participants in the initiator for scheduling
for role, party_ids in job_runtime_conf["role"].items():
for party_id in party_ids:
if role == job_initiator['role'] and party_id == job_initiator['party_id']:
continue
JobController.initialize_tasks(job_id, role, party_id, False, job_initiator, job_parameters, dsl_parser)
# push into queue
try:
JobQueue.create_event(job_id=job_id, initiator_role=initiator_role, initiator_party_id=initiator_party_id)
except Exception as e:
raise Exception(f'push job into queue failed:\n{e}')
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters.model_id))
board_url = "http://{}:{}{}".format(
ServiceUtils.get_item("fateboard", "host"),
ServiceUtils.get_item("fateboard", "port"),
FATE_BOARD_DASHBOARD_ENDPOINT).format(job_id, job_initiator['role'], job_initiator['party_id'])
logs_directory = job_utils.get_job_log_directory(job_id)
return job_id, path_dict['job_dsl_path'], path_dict['job_runtime_conf_path'], logs_directory, \
{'model_id': job_parameters.model_id, 'model_version': job_parameters.model_version}, board_url
def submit_job(job_data):
job_id = generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_utils.check_pipeline_job_runtime_conf(job_runtime_conf)
job_parameters = job_runtime_conf['job_parameters']
job_initiator = job_runtime_conf['initiator']
job_type = job_parameters.get('job_type', '')
if job_type != 'predict':
# generate job model info
job_parameters['model_id'] = '#'.join([dtable_utils.all_party_key(job_runtime_conf['role']), 'model'])
job_parameters['model_version'] = job_id
train_runtime_conf = {}
else:
detect_utils.check_config(job_parameters, ['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
job_tracker = Tracking(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'],
model_id=job_parameters['model_id'], model_version=job_parameters['model_version'])
pipeline_model = job_tracker.get_output_model('pipeline')
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf)
path_dict = save_job_conf(job_id=job_id,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job = Job()
job.f_job_id = job_id
job.f_roles = json_dumps(job_runtime_conf['role'])
job.f_work_mode = job_parameters['work_mode']
job.f_initiator_party_id = job_initiator['party_id']
job.f_dsl = json_dumps(job_dsl)
job.f_runtime_conf = json_dumps(job_runtime_conf)
job.f_train_runtime_conf = json_dumps(train_runtime_conf)
job.f_run_ip = ''
job.f_status = JobStatus.WAITING
job.f_progress = 0
job.f_create_time = current_timestamp()
initiator_role = job_initiator['role']
initiator_party_id = job_initiator['party_id']
if initiator_party_id not in job_runtime_conf['role'][initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(initiator_party_id))
raise Exception("initiator party id error {}".format(initiator_party_id))
get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
TaskScheduler.distribute_job(job=job, roles=job_runtime_conf['role'], job_initiator=job_initiator)
# push into queue
job_event = job_utils.job_event(job_id, initiator_role, initiator_party_id)
try:
RuntimeConfig.JOB_QUEUE.put_event(job_event)
except Exception as e:
raise Exception('push job into queue failed')
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters['model_id']))
board_url = BOARD_DASHBOARD_URL.format(job_id, job_initiator['role'], job_initiator['party_id'])
logs_directory = get_job_log_directory(job_id)
return job_id, path_dict['job_dsl_path'], path_dict['job_runtime_conf_path'], logs_directory, \
{'model_id': job_parameters['model_id'],'model_version': job_parameters['model_version']}, board_url
def submit(cls, job_data, job_id=None):
if not job_id:
job_id = job_utils.generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(
job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_utils.check_job_runtime_conf(job_runtime_conf)
authentication_utils.check_constraint(job_runtime_conf, job_dsl)
job_initiator = job_runtime_conf['initiator']
conf_adapter = JobRuntimeConfigAdapter(job_runtime_conf)
common_job_parameters = conf_adapter.get_common_parameters()
if common_job_parameters.job_type != 'predict':
# generate job model info
common_job_parameters.model_id = model_utils.gen_model_id(
job_runtime_conf['role'])
common_job_parameters.model_version = job_id
train_runtime_conf = {}
else:
# check predict job parameters
detect_utils.check_config(common_job_parameters.to_dict(),
['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
tracker = Tracker(
job_id=job_id,
role=job_initiator['role'],
party_id=job_initiator['party_id'],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version)
pipeline_model = tracker.get_output_model('pipeline')
train_runtime_conf = json_loads(
pipeline_model['Pipeline'].train_runtime_conf)
if not model_utils.check_if_deployed(
role=job_initiator['role'],
party_id=job_initiator['party_id'],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version):
raise Exception(
f"Model {common_job_parameters.model_id} {common_job_parameters.model_version} has not been deployed yet."
)
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
job = Job()
job.f_job_id = job_id
job.f_dsl = job_dsl
job.f_train_runtime_conf = train_runtime_conf
job.f_roles = job_runtime_conf['role']
job.f_work_mode = common_job_parameters.work_mode
job.f_initiator_role = job_initiator['role']
job.f_initiator_party_id = job_initiator['party_id']
job.f_role = job_initiator['role']
job.f_party_id = job_initiator['party_id']
path_dict = job_utils.save_job_conf(
job_id=job_id,
role=job.f_initiator_role,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
job_runtime_conf_on_party={},
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
if job.f_initiator_party_id not in job_runtime_conf['role'][
job.f_initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(
job.f_initiator_party_id))
raise Exception("initiator party id error {}".format(
job.f_initiator_party_id))
# create common parameters on initiator
JobController.backend_compatibility(
job_parameters=common_job_parameters)
JobController.adapt_job_parameters(
role=job.f_initiator_role,
job_parameters=common_job_parameters,
create_initiator_baseline=True)
job.f_runtime_conf = conf_adapter.update_common_parameters(
common_parameters=common_job_parameters)
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=job.f_dsl,
runtime_conf=job.f_runtime_conf,
train_runtime_conf=job.f_train_runtime_conf)
# initiator runtime conf as template
job.f_runtime_conf_on_party = job.f_runtime_conf.copy()
job.f_runtime_conf_on_party[
"job_parameters"] = common_job_parameters.to_dict()
if common_job_parameters.work_mode == WorkMode.CLUSTER:
# Save the status information of all participants in the initiator for scheduling
for role, party_ids in job.f_roles.items():
for party_id in party_ids:
if role == job.f_initiator_role and party_id == job.f_initiator_party_id:
continue
JobController.initialize_tasks(job_id, role, party_id,
False, job.f_initiator_role,
job.f_initiator_party_id,
common_job_parameters,
dsl_parser)
status_code, response = FederatedScheduler.create_job(job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
job.f_status = JobStatus.FAILED
job.f_tag = "submit_failed"
FederatedScheduler.sync_job_status(job=job)
raise Exception("create job failed", response)
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(
job.f_job_id, common_job_parameters.model_id))
logs_directory = job_utils.get_job_log_directory(job_id)
submit_result = {
"job_id":
job_id,
"model_info": {
"model_id": common_job_parameters.model_id,
"model_version": common_job_parameters.model_version
},
"logs_directory":
logs_directory,
"board_url":
job_utils.get_board_url(job_id, job_initiator['role'],
job_initiator['party_id'])
}
submit_result.update(path_dict)
return submit_result
def operate_model(model_operation):
request_config = request.json or request.form.to_dict()
job_id = job_utils.generate_job_id()
if model_operation not in [
ModelOperation.STORE, ModelOperation.RESTORE,
ModelOperation.EXPORT, ModelOperation.IMPORT
]:
raise Exception(
'Can not support this operating now: {}'.format(model_operation))
required_arguments = ["model_id", "model_version", "role", "party_id"]
check_config(request_config, required_arguments=required_arguments)
request_config["model_id"] = gen_party_model_id(
model_id=request_config["model_id"],
role=request_config["role"],
party_id=request_config["party_id"])
if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]:
if model_operation == ModelOperation.IMPORT:
try:
file = request.files.get('file')
file_path = os.path.join(TEMP_DIRECTORY, file.filename)
# if not os.path.exists(file_path):
# raise Exception('The file is obtained from the fate flow client machine, but it does not exist, '
# 'please check the path: {}'.format(file_path))
try:
os.makedirs(os.path.dirname(file_path), exist_ok=True)
file.save(file_path)
except Exception as e:
shutil.rmtree(file_path)
raise e
request_config['file'] = file_path
model = pipelined_model.PipelinedModel(
model_id=request_config["model_id"],
model_version=request_config["model_version"])
model.unpack_model(file_path)
pipeline = model.read_component_model('pipeline',
'pipeline')['Pipeline']
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
permitted_party_id = []
for key, value in train_runtime_conf.get('role', {}).items():
for v in value:
permitted_party_id.extend([v, str(v)])
if request_config["party_id"] not in permitted_party_id:
shutil.rmtree(model.model_path)
raise Exception(
"party id {} is not in model roles, please check if the party id is valid."
)
try:
with DB.connection_context():
model = MLModel.get_or_none(
MLModel.f_job_id == train_runtime_conf[
"job_parameters"]["model_version"],
MLModel.f_role == request_config["role"])
if not model:
MLModel.create(
f_role=request_config["role"],
f_party_id=request_config["party_id"],
f_roles=train_runtime_conf["role"],
f_job_id=train_runtime_conf["job_parameters"]
["model_version"],
f_model_id=train_runtime_conf["job_parameters"]
["model_id"],
f_model_version=train_runtime_conf[
"job_parameters"]["model_version"],
f_initiator_role=train_runtime_conf[
"initiator"]["role"],
f_initiator_party_id=train_runtime_conf[
"initiator"]["party_id"],
f_runtime_conf=train_runtime_conf,
f_work_mode=train_runtime_conf[
"job_parameters"]["work_mode"],
f_dsl=json_loads(pipeline.train_dsl),
f_imported=1,
f_job_status='complete')
else:
stat_logger.info(
f'job id: {train_runtime_conf["job_parameters"]["model_version"]}, '
f'role: {request_config["role"]} model info already existed in database.'
)
except peewee.IntegrityError as e:
stat_logger.exception(e)
operation_record(request_config, "import", "success")
return get_json_result()
except Exception:
operation_record(request_config, "import", "failed")
raise
else:
try:
model = pipelined_model.PipelinedModel(
model_id=request_config["model_id"],
model_version=request_config["model_version"])
if model.exists():
archive_file_path = model.packaging_model()
operation_record(request_config, "export", "success")
return send_file(archive_file_path,
attachment_filename=os.path.basename(
archive_file_path),
as_attachment=True)
else:
operation_record(request_config, "export", "failed")
res = error_response(
response_code=210,
retmsg="Model {} {} is not exist.".format(
request_config.get("model_id"),
request_config.get("model_version")))
return res
except Exception as e:
operation_record(request_config, "export", "failed")
stat_logger.exception(e)
return error_response(response_code=210, retmsg=str(e))
else:
data = {}
job_dsl, job_runtime_conf = gen_model_operation_job_config(
request_config, model_operation)
job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = DAGScheduler.submit(
{
'job_dsl': job_dsl,
'job_runtime_conf': job_runtime_conf
},
job_id=job_id)
data.update({
'job_dsl_path': job_dsl_path,
'job_runtime_conf_path': job_runtime_conf_path,
'board_url': board_url,
'logs_directory': logs_directory
})
operation_record(data=job_runtime_conf,
oper_type=model_operation,
oper_status='')
return get_json_result(job_id=job_id, data=data)
def submit(cls, submit_job_conf: JobConfigurationBase, job_id: str = None):
if not job_id:
job_id = job_utils.generate_job_id()
submit_result = {"job_id": job_id}
schedule_logger(job_id).info(
f"submit job, body {submit_job_conf.to_dict()}")
try:
dsl = submit_job_conf.dsl
runtime_conf = deepcopy(submit_job_conf.runtime_conf)
job_utils.check_job_runtime_conf(runtime_conf)
authentication_utils.check_constraint(runtime_conf, dsl)
job_initiator = runtime_conf["initiator"]
conf_adapter = JobRuntimeConfigAdapter(runtime_conf)
common_job_parameters = conf_adapter.get_common_parameters()
if common_job_parameters.job_type != "predict":
# generate job model info
conf_version = schedule_utils.get_conf_version(runtime_conf)
if conf_version != 2:
raise Exception(
"only the v2 version runtime conf is supported")
common_job_parameters.model_id = model_utils.gen_model_id(
runtime_conf["role"])
common_job_parameters.model_version = job_id
train_runtime_conf = {}
else:
# check predict job parameters
detect_utils.check_config(common_job_parameters.to_dict(),
["model_id", "model_version"])
# get inference dsl from pipeline model as job dsl
tracker = Tracker(
job_id=job_id,
role=job_initiator["role"],
party_id=job_initiator["party_id"],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version)
pipeline_model = tracker.get_pipeline_model()
train_runtime_conf = json_loads(
pipeline_model.train_runtime_conf)
if not model_utils.check_if_deployed(
role=job_initiator["role"],
party_id=job_initiator["party_id"],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version):
raise Exception(
f"Model {common_job_parameters.model_id} {common_job_parameters.model_version} has not been deployed yet."
)
dsl = json_loads(pipeline_model.inference_dsl)
# dsl = ProviderManager.fill_fate_flow_provider(dsl)
job = Job()
job.f_job_id = job_id
job.f_dsl = dsl
job.f_train_runtime_conf = train_runtime_conf
job.f_roles = runtime_conf["role"]
job.f_initiator_role = job_initiator["role"]
job.f_initiator_party_id = job_initiator["party_id"]
job.f_role = job_initiator["role"]
job.f_party_id = job_initiator["party_id"]
path_dict = job_utils.save_job_conf(
job_id=job_id,
role=job.f_initiator_role,
party_id=job.f_initiator_party_id,
dsl=dsl,
runtime_conf=runtime_conf,
runtime_conf_on_party={},
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
if job.f_initiator_party_id not in runtime_conf["role"][
job.f_initiator_role]:
msg = f"initiator party id {job.f_initiator_party_id} not in roles {runtime_conf['role']}"
schedule_logger(job_id).info(msg)
raise Exception(msg)
# create common parameters on initiator
JobController.create_common_job_parameters(
job_id=job.f_job_id,
initiator_role=job.f_initiator_role,
common_job_parameters=common_job_parameters)
job.f_runtime_conf = conf_adapter.update_common_parameters(
common_parameters=common_job_parameters)
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=job.f_dsl,
runtime_conf=job.f_runtime_conf,
train_runtime_conf=job.f_train_runtime_conf)
# initiator runtime conf as template
job.f_runtime_conf_on_party = job.f_runtime_conf.copy()
job.f_runtime_conf_on_party[
"job_parameters"] = common_job_parameters.to_dict()
# inherit job
job.f_inheritance_info = common_job_parameters.inheritance_info
job.f_inheritance_status = JobInheritanceStatus.WAITING if common_job_parameters.inheritance_info else JobInheritanceStatus.PASS
if job.f_inheritance_info:
inheritance_jobs = JobSaver.query_job(
job_id=job.f_inheritance_info.get("job_id"),
role=job_initiator["role"],
party_id=job_initiator["party_id"])
inheritance_tasks = JobSaver.query_task(
job_id=job.f_inheritance_info.get("job_id"),
role=job_initiator["role"],
party_id=job_initiator["party_id"],
only_latest=True)
job_utils.check_job_inheritance_parameters(
job, inheritance_jobs, inheritance_tasks)
status_code, response = FederatedScheduler.create_job(job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
job.f_status = JobStatus.FAILED
job.f_tag = "submit_failed"
FederatedScheduler.sync_job_status(job=job)
raise Exception("create job failed", response)
else:
need_run_components = {}
for role in response:
need_run_components[role] = {}
for party, res in response[role].items():
need_run_components[role][party] = [
name for name, value in response[role][party]
["data"]["components"].items()
if value["need_run"] is True
]
if common_job_parameters.federated_mode == FederatedMode.MULTIPLE:
# create the task holder in db to record information of all participants in the initiator for scheduling
for role, party_ids in job.f_roles.items():
for party_id in party_ids:
if role == job.f_initiator_role and party_id == job.f_initiator_party_id:
continue
if not need_run_components[role][party_id]:
continue
JobController.initialize_tasks(
job_id=job_id,
role=role,
party_id=party_id,
run_on_this_party=False,
initiator_role=job.f_initiator_role,
initiator_party_id=job.f_initiator_party_id,
job_parameters=common_job_parameters,
dsl_parser=dsl_parser,
components=need_run_components[role][party_id])
job.f_status = JobStatus.WAITING
status_code, response = FederatedScheduler.sync_job_status(
job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
raise Exception("set job to waiting status failed")
schedule_logger(job_id).info(
f"submit job successfully, job id is {job.f_job_id}, model id is {common_job_parameters.model_id}"
)
logs_directory = job_utils.get_job_log_directory(job_id)
result = {
"code":
RetCode.SUCCESS,
"message":
"success",
"model_info": {
"model_id": common_job_parameters.model_id,
"model_version": common_job_parameters.model_version
},
"logs_directory":
logs_directory,
"board_url":
job_utils.get_board_url(job_id, job_initiator["role"],
job_initiator["party_id"])
}
warn_parameter = JobRuntimeConfigAdapter(
submit_job_conf.runtime_conf).check_removed_parameter()
if warn_parameter:
result[
"message"] = f"[WARN]{warn_parameter} is removed,it does not take effect!"
submit_result.update(result)
submit_result.update(path_dict)
except Exception as e:
submit_result["code"] = RetCode.OPERATING_ERROR
submit_result["message"] = exception_to_trace_string(e)
schedule_logger(job_id).exception(e)
return submit_result
def load_model():
request_config = request.json
if request_config.get('job_id', None):
retcode, retmsg, res_data = model_utils.query_model_info(
model_version=request_config['job_id'], role='guest')
if res_data:
model_info = res_data[0]
request_config['initiator'] = {}
request_config['initiator']['party_id'] = str(
model_info.get('f_initiator_party_id'))
request_config['initiator']['role'] = model_info.get(
'f_initiator_role')
runtime_conf = model_info.get(
'f_runtime_conf', {}) if model_info.get(
'f_runtime_conf', {}) else model_info.get(
'f_train_runtime_conf', {})
adapter = JobRuntimeConfigAdapter(runtime_conf)
job_parameters = adapter.get_common_parameters().to_dict()
request_config[
'job_parameters'] = job_parameters if job_parameters else model_info.get(
'f_train_runtime_conf', {}).get('job_parameters')
roles = runtime_conf.get('role')
request_config['role'] = roles if roles else model_info.get(
'f_train_runtime_conf', {}).get('role')
for key, value in request_config['role'].items():
for i, v in enumerate(value):
value[i] = str(v)
request_config.pop('job_id')
else:
return get_json_result(
retcode=101,
retmsg="model with version {} can not be found in database. "
"Please check if the model version is valid.".format(
request_config.get('job_id')))
_job_id = job_utils.generate_job_id()
initiator_party_id = request_config['initiator']['party_id']
initiator_role = request_config['initiator']['role']
publish_model.generate_publish_model_info(request_config)
load_status = True
load_status_info = {}
load_status_msg = 'success'
load_status_info['detail'] = {}
if "federated_mode" not in request_config['job_parameters']:
if request_config["job_parameters"][
"work_mode"] == WorkMode.STANDALONE:
request_config['job_parameters'][
"federated_mode"] = FederatedMode.SINGLE
elif request_config["job_parameters"]["work_mode"] == WorkMode.CLUSTER:
request_config['job_parameters'][
"federated_mode"] = FederatedMode.MULTIPLE
for role_name, role_partys in request_config.get("role").items():
if role_name == 'arbiter':
continue
load_status_info[role_name] = load_status_info.get(role_name, {})
load_status_info['detail'][role_name] = {}
for _party_id in role_partys:
request_config['local'] = {
'role': role_name,
'party_id': _party_id
}
try:
response = federated_api(
job_id=_job_id,
method='POST',
endpoint='/model/load/do',
src_party_id=initiator_party_id,
dest_party_id=_party_id,
src_role=initiator_role,
json_body=request_config,
federated_mode=request_config['job_parameters']
['federated_mode'])
load_status_info[role_name][_party_id] = response['retcode']
detail = {_party_id: {}}
detail[_party_id]['retcode'] = response['retcode']
detail[_party_id]['retmsg'] = response['retmsg']
load_status_info['detail'][role_name].update(detail)
if response['retcode']:
load_status = False
load_status_msg = 'failed'
except Exception as e:
stat_logger.exception(e)
load_status = False
load_status_msg = 'failed'
load_status_info[role_name][_party_id] = 100
return get_json_result(job_id=_job_id,
retcode=(0 if load_status else 101),
retmsg=load_status_msg,
data=load_status_info)
def deploy():
request_data = request.json
require_parameters = ['model_id', 'model_version']
check_config(request_data, require_parameters)
model_id = request_data.get("model_id")
model_version = request_data.get("model_version")
retcode, retmsg, model_info = model_utils.query_model_info_from_file(
model_id=model_id, model_version=model_version, to_dict=True)
if not model_info:
raise Exception(
f'Deploy model failed, no model {model_id} {model_version} found.')
else:
for key, value in model_info.items():
version_check = model_utils.compare_version(
value.get('f_fate_version'), '1.5.0')
if version_check == 'lt':
continue
else:
init_role = key.split('/')[-2].split('#')[0]
init_party_id = key.split('/')[-2].split('#')[1]
model_init_role = value.get('f_initiator_role') if value.get(
'f_initiator_role') else value.get(
'f_train_runtime_conf', {}).get('initiator', {}).get(
'role', '')
model_init_party_id = value.get(
'f_initiator_role_party_id') if value.get(
'f_initiator_role_party_id') else value.get(
'f_train_runtime_conf', {}).get(
'initiator', {}).get('party_id', '')
if (init_role
== model_init_role) and (init_party_id
== str(model_init_party_id)):
break
else:
raise Exception(
"Deploy model failed, can not found model of initiator role or the fate version of model is older than 1.5.0"
)
# distribute federated deploy task
_job_id = job_utils.generate_job_id()
request_data['child_model_version'] = _job_id
initiator_party_id = model_init_party_id
initiator_role = model_init_role
request_data['initiator'] = {
'role': initiator_role,
'party_id': initiator_party_id
}
deploy_status = True
deploy_status_info = {}
deploy_status_msg = 'success'
deploy_status_info['detail'] = {}
for role_name, role_partys in value.get("f_train_runtime_conf",
{}).get('role', {}).items():
if role_name not in ['arbiter', 'host', 'guest']:
continue
deploy_status_info[role_name] = deploy_status_info.get(
role_name, {})
deploy_status_info['detail'][role_name] = {}
adapter = JobRuntimeConfigAdapter(
value.get("f_train_runtime_conf", {}))
work_mode = adapter.get_job_work_mode()
for _party_id in role_partys:
request_data['local'] = {
'role': role_name,
'party_id': _party_id
}
try:
response = federated_api(
job_id=_job_id,
method='POST',
endpoint='/model/deploy/do',
src_party_id=initiator_party_id,
dest_party_id=_party_id,
src_role=initiator_role,
json_body=request_data,
federated_mode=FederatedMode.MULTIPLE
if work_mode else FederatedMode.SINGLE)
deploy_status_info[role_name][_party_id] = response[
'retcode']
detail = {_party_id: {}}
detail[_party_id]['retcode'] = response['retcode']
detail[_party_id]['retmsg'] = response['retmsg']
deploy_status_info['detail'][role_name].update(detail)
if response['retcode']:
deploy_status = False
deploy_status_msg = 'failed'
except Exception as e:
stat_logger.exception(e)
deploy_status = False
deploy_status_msg = 'failed'
deploy_status_info[role_name][_party_id] = 100
deploy_status_info['model_id'] = request_data['model_id']
deploy_status_info['model_version'] = _job_id
return get_json_result(retcode=(0 if deploy_status else 101),
retmsg=deploy_status_msg,
data=deploy_status_info)
def download_upload(data_func):
request_config = request.json
_job_id = generate_job_id()
stat_logger.info('generated job_id {}, body {}'.format(
_job_id, request_config))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
module = data_func
required_arguments = ['work_mode', 'namespace', 'table_name']
if module == 'upload':
required_arguments.extend(['file', 'head', 'partition'])
elif module == 'download':
required_arguments.extend(['output_path'])
else:
raise Exception('can not support this operating: {}'.format(module))
detect_utils.check_config(request_config,
required_arguments=required_arguments)
job_work_mode = request_config['work_mode']
# todo: The current code here is redundant with job_app/submit_job, the next version of this function will be implemented by job_app/submit_job
if job_work_mode != RuntimeConfig.WORK_MODE:
if RuntimeConfig.WORK_MODE == WorkMode.CLUSTER and job_work_mode == WorkMode.STANDALONE:
# use cluster standalone job server to execute standalone job
return request_execute_server(
request=request,
execute_host='{}:{}'.format(
request.remote_addr, CLUSTER_STANDALONE_JOB_SERVER_PORT))
else:
raise Exception(
'server run on standalone can not support cluster mode job')
if module == "upload":
if not os.path.isabs(request_config['file']):
request_config["file"] = os.path.join(
file_utils.get_project_base_directory(),
request_config["file"])
try:
conf_file_path = new_runtime_conf(
job_dir=_job_dir,
method=data_func,
module=module,
role=request_config.get('local', {}).get("role"),
party_id=request_config.get('local', {}).get("party_id", ''))
file_utils.dump_json_conf(request_config, conf_file_path)
progs = [
"python3",
os.path.join(file_utils.get_project_base_directory(),
JOB_MODULE_CONF[module]["module_path"]), "-j",
_job_id, "-c", conf_file_path
]
try:
p = run_subprocess(config_dir=_job_dir, process_cmd=progs)
except Exception as e:
stat_logger.exception(e)
p = None
return get_json_result(retcode=(0 if p else 101),
job_id=_job_id,
data={
'table_name': request_config['table_name'],
'namespace': request_config['namespace'],
'pid': p.pid if p else ''
})
except Exception as e:
stat_logger.exception(e)
return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)
