def submit_job():
work_mode = request.json.get('job_runtime_conf',
{}).get('job_parameters',
{}).get('work_mode', None)
detect_utils.check_config({'work_mode': work_mode},
required_arguments=[('work_mode',
(WorkMode.CLUSTER,
WorkMode.STANDALONE))])
if work_mode == RuntimeConfig.WORK_MODE:
job_id, job_dsl_path, job_runtime_conf_path, logs_directory, model_info, board_url = JobController.submit_job(
request.json)
return get_json_result(retcode=0,
retmsg='success',
job_id=job_id,
data={
'job_dsl_path': job_dsl_path,
'job_runtime_conf_path':
job_runtime_conf_path,
'model_info': model_info,
'board_url': board_url,
'logs_directory': logs_directory
})
else:
if RuntimeConfig.WORK_MODE == WorkMode.CLUSTER and work_mode == WorkMode.STANDALONE:
# use cluster standalone job server to execute standalone job
return request_execute_server(
request=request,
execute_host='{}:{}'.format(
request.remote_addr, CLUSTER_STANDALONE_JOB_SERVER_PORT))
else:
raise Exception(
'server run on standalone can not support cluster mode job')
def _wrapper(*args, **kwargs):
job_server = set()
jobs = query_job(job_id=request.json.get('job_id', None))
for job in jobs:
if job.f_run_ip:
job_server.add(job.f_run_ip)
if len(job_server) == 1:
execute_host = job_server.pop()
if execute_host != RuntimeConfig.JOB_SERVER_HOST:
if routing_type == 0:
return api_utils.request_execute_server(
request=request, execute_host=execute_host)
else:
return redirect('http://{}{}'.format(
execute_host, url_for(request.endpoint)),
code=307)
return func(*args, **kwargs)
def download_upload(data_func):
request_config = request.json
_job_id = generate_job_id()
stat_logger.info('generated job_id {}, body {}'.format(
_job_id, request_config))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
module = data_func
required_arguments = ['work_mode', 'namespace', 'table_name']
if module == 'upload':
required_arguments.extend(['file', 'head', 'partition'])
elif module == 'download':
required_arguments.extend(['output_path'])
else:
raise Exception('can not support this operating: {}'.format(module))
detect_utils.check_config(request_config,
required_arguments=required_arguments)
job_work_mode = request_config['work_mode']
# todo: The current code here is redundant with job_app/submit_job, the next version of this function will be implemented by job_app/submit_job
if job_work_mode != RuntimeConfig.WORK_MODE:
if RuntimeConfig.WORK_MODE == WorkMode.CLUSTER and job_work_mode == WorkMode.STANDALONE:
# use cluster standalone job server to execute standalone job
return request_execute_server(
request=request,
execute_host='{}:{}'.format(
request.remote_addr, CLUSTER_STANDALONE_JOB_SERVER_PORT))
else:
raise Exception(
'server run on standalone can not support cluster mode job')
if module == "upload":
if not os.path.isabs(request_config['file']):
request_config["file"] = os.path.join(
file_utils.get_project_base_directory(),
request_config["file"])
try:
conf_file_path = new_runtime_conf(
job_dir=_job_dir,
method=data_func,
module=module,
role=request_config.get('local', {}).get("role"),
party_id=request_config.get('local', {}).get("party_id", ''))
file_utils.dump_json_conf(request_config, conf_file_path)
progs = [
"python3",
os.path.join(file_utils.get_project_base_directory(),
JOB_MODULE_CONF[module]["module_path"]), "-j",
_job_id, "-c", conf_file_path
]
try:
p = run_subprocess(config_dir=_job_dir, process_cmd=progs)
except Exception as e:
stat_logger.exception(e)
p = None
return get_json_result(retcode=(0 if p else 101),
job_id=_job_id,
data={
'table_name': request_config['table_name'],
'namespace': request_config['namespace'],
'pid': p.pid if p else ''
})
except Exception as e:
stat_logger.exception(e)
return get_json_result(retcode=-104, retmsg="failed", job_id=_job_id)