def run_job(self, job_id, config):
default_runtime_dict = file_utils.load_json_conf('workflow/conf/default_runtime_conf.json')
setting_conf = file_utils.load_json_conf('workflow/conf/setting_conf.json')
_job_dir = get_job_directory(job_id=job_id)
os.makedirs(_job_dir, exist_ok=True)
ParameterOverride.override_parameter(default_runtime_dict, setting_conf, config, _job_dir)
logger.info('job_id {} parameters overrode {}'.format(config, _job_dir))
channel, stub = get_proxy_data_channel()
for runtime_conf_path in glob.glob(os.path.join(_job_dir, '**', 'runtime_conf.json'), recursive=True):
runtime_conf = file_utils.load_json_conf(os.path.abspath(runtime_conf_path))
_role = runtime_conf['local']['role']
_party_id = runtime_conf['local']['party_id']
_method = 'POST'
_module = runtime_conf['module']
_url = '/workflow/{}/{}/{}'.format(job_id, _module, _role)
_packet = wrap_grpc_packet(runtime_conf, _method, _url, _party_id, job_id)
logger.info(
'Starting workflow job_id:{} party_id:{} role:{} method:{} url:{}'.format(job_id, _party_id,
_role, _method,
_url))
try:
_return = stub.unaryCall(_packet)
logger.info("Grpc unary response: {}".format(_return))
except grpc.RpcError as e:
msg = 'job_id:{} party_id:{} role:{} method:{} url:{} Failed to start workflow'.format(job_id,
_party_id,
_role, _method,
_url)
logger.exception(msg)
return get_json_result(-101, 'UnaryCall submit to remote manager failed')
def download_data(data_func):
_data = request.json
_job_id = generate_job_id()
logger.info('generated job_id {}, body {}'.format(_job_id, _data))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
_download_module = os.path.join(file_utils.get_project_base_directory(),
"arch/api/utils/download.py")
_upload_module = os.path.join(file_utils.get_project_base_directory(),
"arch/api/utils/upload.py")
if data_func == "download":
_module = _download_module
else:
_module = _upload_module
try:
if data_func == "download":
progs = [
"python3", _module, "-j", _job_id, "-c",
os.path.abspath(_data.get("config_path"))
]
else:
progs = [
"python3", _module, "-c",
os.path.abspath(_data.get("config_path"))
]
logger.info('Starting progs: {}'.format(progs))
std_log = open(os.path.join(_job_dir, 'std.log'), 'w')
task_pid_path = os.path.join(_job_dir, 'pids')
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
else:
startupinfo = None
p = subprocess.Popen(progs,
stdout=std_log,
stderr=std_log,
startupinfo=startupinfo)
os.makedirs(task_pid_path, exist_ok=True)
with open(os.path.join(task_pid_path, data_func + ".pid"), 'w') as f:
f.truncate()
f.write(str(p.pid) + "\n")
f.flush()
return get_json_result(0, "success, job_id {}".format(_job_id))
except Exception as e:
print(e)
return get_json_result(-104, "failed, job_id {}".format(_job_id))
def start_workflow(job_id, module, role):
_data = request.json
_job_dir = get_job_directory(job_id)
_party_id = str(_data['local']['party_id'])
_method = _data['WorkFlowParam']['method']
conf_path_dir = os.path.join(_job_dir, _method, module, role, _party_id)
os.makedirs(conf_path_dir, exist_ok=True)
conf_file_path = os.path.join(conf_path_dir, 'runtime_conf.json')
with open(conf_file_path, 'w+') as f:
f.truncate()
f.write(json.dumps(_data, indent=4))
f.flush()
if os.name == 'nt':
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
else:
startupinfo = None
task_pid_path = os.path.join(_job_dir, 'pids')
std_log = open(os.path.join(_job_dir, role + '.std.log'), 'w')
progs = [
"python3",
os.path.join(file_utils.get_project_base_directory(),
_data['CodePath']), "-j", job_id, "-c",
os.path.abspath(conf_file_path)
]
logger.info('Starting progs: {}'.format(" ".join(progs)))
p = subprocess.Popen(progs,
stdout=std_log,
stderr=std_log,
startupinfo=startupinfo)
os.makedirs(task_pid_path, exist_ok=True)
with open(os.path.join(task_pid_path, role + ".pid"), 'w') as f:
f.truncate()
f.write(str(p.pid) + "\n")
f.flush()
job_data = dict()
job_data["begin_date"] = datetime.datetime.now()
job_data["status"] = "ready"
with open(conf_file_path) as fr:
config = json.load(fr)
job_data.update(config)
job_data["my_role"] = config.get("local", {}).get("role")
save_job_info(job_id=job_id, **job_data)
update_job_queue(job_id=job_id, update_data={"status": "ready"})
return get_json_result(msg="success, pid is %s" % p.pid)
def start_workflow(job_id, module, role):
_config = request.json
_job_dir = get_job_directory(job_id)
_party_id = str(_config['local']['party_id'])
_method = _config['WorkFlowParam']['method']
default_runtime_dict = file_utils.load_json_conf(
'workflow/conf/default_runtime_conf.json')
fill_runtime_conf_table_info(runtime_conf=_config,
default_runtime_conf=default_runtime_dict)
conf_file_path = new_runtime_conf(job_dir=_job_dir,
method=_method,
module=module,
role=role,
party_id=_party_id)
with open(conf_file_path, 'w+') as f:
f.truncate()
f.write(json.dumps(_config, indent=4))
f.flush()
progs = [
"python3",
os.path.join(file_utils.get_project_base_directory(),
_config['CodePath']), "-j", job_id, "-c",
os.path.abspath(conf_file_path)
]
p = run_subprocess(job_dir=_job_dir, job_role=role, progs=progs)
job_status = "start"
job_data = dict()
job_data["begin_date"] = datetime.datetime.now()
job_data["status"] = job_status
job_data.update(_config)
job_data["pid"] = p.pid
job_data["all_party"] = json.dumps(_config.get("role", {}))
job_data["initiator"] = _config.get("JobParam", {}).get("initiator")
save_job_info(job_id=job_id,
role=_config.get("local", {}).get("role"),
party_id=_config.get("local", {}).get("party_id"),
save_info=job_data,
create=True)
update_job_queue(job_id=job_id,
role=role,
party_id=_party_id,
save_data={
"status": job_status,
"pid": p.pid
})
return get_json_result(data={'pid': p.pid}, job_id=job_id)
def stop_job(job_id):
_job_dir = get_job_directory(job_id)
for runtime_conf_path in glob.glob(os.path.join(_job_dir, '**', 'runtime_conf.json'), recursive=True):
runtime_conf = file_utils.load_json_conf(os.path.abspath(runtime_conf_path))
_role = runtime_conf['local']['role']
_party_id = runtime_conf['local']['party_id']
_url = '/workflow/{}'.format(job_id)
_method = 'DELETE'
_packet = wrap_grpc_packet({}, _method, _url, _party_id, job_id)
channel, stub = get_proxy_data_channel()
try:
_return = stub.unaryCall(_packet)
logger.info("Grpc unary response: {}".format(_return))
except grpc.RpcError as e:
msg = 'job_id:{} party_id:{} role:{} method:{} url:{} Failed to start workflow'.format(job_id,
_party_id,
_role, _method,
_url)
logger.exception(msg)
return get_json_result(-101, 'UnaryCall stop to remote manager failed')
return get_json_result()
def stop_workflow(job_id, role, party_id):
_job_dir = get_job_directory(job_id)
task_pid_path = os.path.join(_job_dir, 'pids')
if os.path.isdir(task_pid_path):
for pid_file in os.listdir(task_pid_path):
try:
if not pid_file.endswith('.pid'):
continue
with open(os.path.join(task_pid_path, pid_file), 'r') as f:
pids = f.read().split('\n')
for pid in pids:
try:
if len(pid) == 0:
continue
logger.debug(
"terminating process pid:{} {}".format(
pid, pid_file))
p = psutil.Process(int(pid))
for child in p.children(recursive=True):
child.kill()
p.kill()
except NoSuchProcess:
continue
except Exception as e:
logger.exception("error")
continue
federated_api(job_id=job_id,
method='POST',
url='/job/jobStatus/{}/{}/{}'.format(
job_id, role, party_id),
party_id=party_id,
json_body={
'status': 'failed',
'stopJob': True
})
clean_job(job_id=job_id)
return get_json_result(job_id=job_id)
def download_upload(data_func):
request_config = request.json
_job_id = generate_job_id()
logger.info('generated job_id {}, body {}'.format(_job_id, request_config))
_job_dir = get_job_directory(_job_id)
os.makedirs(_job_dir, exist_ok=True)
module = data_func
if module == "upload":
if not os.path.isabs(request_config.get("file", "")):
request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"])
try:
request_config["work_mode"] = request_config.get('work_mode', WORK_MODE)
table_name, namespace = dtable_utils.get_table_info(config=request_config, create=(True if module == 'upload' else False))
if not table_name or not namespace:
return get_json_result(status=102, msg='no table name and namespace')
request_config['table_name'] = table_name
request_config['namespace'] = namespace
conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module,
role=request_config.get('local', {}).get("role"),
party_id=request_config.get('local', {}).get("party_id", PARTY_ID))
file_utils.dump_json_conf(request_config, conf_file_path)
if module == "download":
progs = ["python3",
os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]),
"-j", _job_id,
"-c", conf_file_path
]
else:
progs = ["python3",
os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]),
"-c", conf_file_path
]
p = run_subprocess(job_dir=_job_dir, job_role=data_func, progs=progs)
return get_json_result(job_id=_job_id, data={'pid': p.pid, 'table_name': request_config['table_name'], 'namespace': request_config['namespace']})
except Exception as e:
logger.exception(e)
return get_json_result(status=-104, msg="failed", job_id=_job_id)
def get_runtime_conf(job_id):
_job_dir = get_job_directory(job_id)
response_data = []
shutil.copytree(_job_dir, job_id)
return get_json_result(data=response_data, job_id=job_id)
