def run_job(self, job_id, config): default_runtime_dict = file_utils.load_json_conf('workflow/conf/default_runtime_conf.json') setting_conf = file_utils.load_json_conf('workflow/conf/setting_conf.json') _job_dir = get_job_directory(job_id=job_id) os.makedirs(_job_dir, exist_ok=True) ParameterOverride.override_parameter(default_runtime_dict, setting_conf, config, _job_dir) logger.info('job_id {} parameters overrode {}'.format(config, _job_dir)) channel, stub = get_proxy_data_channel() for runtime_conf_path in glob.glob(os.path.join(_job_dir, '**', 'runtime_conf.json'), recursive=True): runtime_conf = file_utils.load_json_conf(os.path.abspath(runtime_conf_path)) _role = runtime_conf['local']['role'] _party_id = runtime_conf['local']['party_id'] _method = 'POST' _module = runtime_conf['module'] _url = '/workflow/{}/{}/{}'.format(job_id, _module, _role) _packet = wrap_grpc_packet(runtime_conf, _method, _url, _party_id, job_id) logger.info( 'Starting workflow job_id:{} party_id:{} role:{} method:{} url:{}'.format(job_id, _party_id, _role, _method, _url)) try: _return = stub.unaryCall(_packet) logger.info("Grpc unary response: {}".format(_return)) except grpc.RpcError as e: msg = 'job_id:{} party_id:{} role:{} method:{} url:{} Failed to start workflow'.format(job_id, _party_id, _role, _method, _url) logger.exception(msg) return get_json_result(-101, 'UnaryCall submit to remote manager failed')
def download_data(data_func): _data = request.json _job_id = generate_job_id() logger.info('generated job_id {}, body {}'.format(_job_id, _data)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) _download_module = os.path.join(file_utils.get_project_base_directory(), "arch/api/utils/download.py") _upload_module = os.path.join(file_utils.get_project_base_directory(), "arch/api/utils/upload.py") if data_func == "download": _module = _download_module else: _module = _upload_module try: if data_func == "download": progs = [ "python3", _module, "-j", _job_id, "-c", os.path.abspath(_data.get("config_path")) ] else: progs = [ "python3", _module, "-c", os.path.abspath(_data.get("config_path")) ] logger.info('Starting progs: {}'.format(progs)) std_log = open(os.path.join(_job_dir, 'std.log'), 'w') task_pid_path = os.path.join(_job_dir, 'pids') if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = subprocess.SW_HIDE else: startupinfo = None p = subprocess.Popen(progs, stdout=std_log, stderr=std_log, startupinfo=startupinfo) os.makedirs(task_pid_path, exist_ok=True) with open(os.path.join(task_pid_path, data_func + ".pid"), 'w') as f: f.truncate() f.write(str(p.pid) + "\n") f.flush() return get_json_result(0, "success, job_id {}".format(_job_id)) except Exception as e: print(e) return get_json_result(-104, "failed, job_id {}".format(_job_id))
def start_workflow(job_id, module, role): _data = request.json _job_dir = get_job_directory(job_id) _party_id = str(_data['local']['party_id']) _method = _data['WorkFlowParam']['method'] conf_path_dir = os.path.join(_job_dir, _method, module, role, _party_id) os.makedirs(conf_path_dir, exist_ok=True) conf_file_path = os.path.join(conf_path_dir, 'runtime_conf.json') with open(conf_file_path, 'w+') as f: f.truncate() f.write(json.dumps(_data, indent=4)) f.flush() if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = subprocess.SW_HIDE else: startupinfo = None task_pid_path = os.path.join(_job_dir, 'pids') std_log = open(os.path.join(_job_dir, role + '.std.log'), 'w') progs = [ "python3", os.path.join(file_utils.get_project_base_directory(), _data['CodePath']), "-j", job_id, "-c", os.path.abspath(conf_file_path) ] logger.info('Starting progs: {}'.format(" ".join(progs))) p = subprocess.Popen(progs, stdout=std_log, stderr=std_log, startupinfo=startupinfo) os.makedirs(task_pid_path, exist_ok=True) with open(os.path.join(task_pid_path, role + ".pid"), 'w') as f: f.truncate() f.write(str(p.pid) + "\n") f.flush() job_data = dict() job_data["begin_date"] = datetime.datetime.now() job_data["status"] = "ready" with open(conf_file_path) as fr: config = json.load(fr) job_data.update(config) job_data["my_role"] = config.get("local", {}).get("role") save_job_info(job_id=job_id, **job_data) update_job_queue(job_id=job_id, update_data={"status": "ready"}) return get_json_result(msg="success, pid is %s" % p.pid)
def start_workflow(job_id, module, role): _config = request.json _job_dir = get_job_directory(job_id) _party_id = str(_config['local']['party_id']) _method = _config['WorkFlowParam']['method'] default_runtime_dict = file_utils.load_json_conf( 'workflow/conf/default_runtime_conf.json') fill_runtime_conf_table_info(runtime_conf=_config, default_runtime_conf=default_runtime_dict) conf_file_path = new_runtime_conf(job_dir=_job_dir, method=_method, module=module, role=role, party_id=_party_id) with open(conf_file_path, 'w+') as f: f.truncate() f.write(json.dumps(_config, indent=4)) f.flush() progs = [ "python3", os.path.join(file_utils.get_project_base_directory(), _config['CodePath']), "-j", job_id, "-c", os.path.abspath(conf_file_path) ] p = run_subprocess(job_dir=_job_dir, job_role=role, progs=progs) job_status = "start" job_data = dict() job_data["begin_date"] = datetime.datetime.now() job_data["status"] = job_status job_data.update(_config) job_data["pid"] = p.pid job_data["all_party"] = json.dumps(_config.get("role", {})) job_data["initiator"] = _config.get("JobParam", {}).get("initiator") save_job_info(job_id=job_id, role=_config.get("local", {}).get("role"), party_id=_config.get("local", {}).get("party_id"), save_info=job_data, create=True) update_job_queue(job_id=job_id, role=role, party_id=_party_id, save_data={ "status": job_status, "pid": p.pid }) return get_json_result(data={'pid': p.pid}, job_id=job_id)
def stop_job(job_id): _job_dir = get_job_directory(job_id) for runtime_conf_path in glob.glob(os.path.join(_job_dir, '**', 'runtime_conf.json'), recursive=True): runtime_conf = file_utils.load_json_conf(os.path.abspath(runtime_conf_path)) _role = runtime_conf['local']['role'] _party_id = runtime_conf['local']['party_id'] _url = '/workflow/{}'.format(job_id) _method = 'DELETE' _packet = wrap_grpc_packet({}, _method, _url, _party_id, job_id) channel, stub = get_proxy_data_channel() try: _return = stub.unaryCall(_packet) logger.info("Grpc unary response: {}".format(_return)) except grpc.RpcError as e: msg = 'job_id:{} party_id:{} role:{} method:{} url:{} Failed to start workflow'.format(job_id, _party_id, _role, _method, _url) logger.exception(msg) return get_json_result(-101, 'UnaryCall stop to remote manager failed') return get_json_result()
def stop_workflow(job_id, role, party_id): _job_dir = get_job_directory(job_id) task_pid_path = os.path.join(_job_dir, 'pids') if os.path.isdir(task_pid_path): for pid_file in os.listdir(task_pid_path): try: if not pid_file.endswith('.pid'): continue with open(os.path.join(task_pid_path, pid_file), 'r') as f: pids = f.read().split('\n') for pid in pids: try: if len(pid) == 0: continue logger.debug( "terminating process pid:{} {}".format( pid, pid_file)) p = psutil.Process(int(pid)) for child in p.children(recursive=True): child.kill() p.kill() except NoSuchProcess: continue except Exception as e: logger.exception("error") continue federated_api(job_id=job_id, method='POST', url='/job/jobStatus/{}/{}/{}'.format( job_id, role, party_id), party_id=party_id, json_body={ 'status': 'failed', 'stopJob': True }) clean_job(job_id=job_id) return get_json_result(job_id=job_id)
def download_upload(data_func): request_config = request.json _job_id = generate_job_id() logger.info('generated job_id {}, body {}'.format(_job_id, request_config)) _job_dir = get_job_directory(_job_id) os.makedirs(_job_dir, exist_ok=True) module = data_func if module == "upload": if not os.path.isabs(request_config.get("file", "")): request_config["file"] = os.path.join(file_utils.get_project_base_directory(), request_config["file"]) try: request_config["work_mode"] = request_config.get('work_mode', WORK_MODE) table_name, namespace = dtable_utils.get_table_info(config=request_config, create=(True if module == 'upload' else False)) if not table_name or not namespace: return get_json_result(status=102, msg='no table name and namespace') request_config['table_name'] = table_name request_config['namespace'] = namespace conf_file_path = new_runtime_conf(job_dir=_job_dir, method=data_func, module=module, role=request_config.get('local', {}).get("role"), party_id=request_config.get('local', {}).get("party_id", PARTY_ID)) file_utils.dump_json_conf(request_config, conf_file_path) if module == "download": progs = ["python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-j", _job_id, "-c", conf_file_path ] else: progs = ["python3", os.path.join(file_utils.get_project_base_directory(), JOB_MODULE_CONF[module]["module_path"]), "-c", conf_file_path ] p = run_subprocess(job_dir=_job_dir, job_role=data_func, progs=progs) return get_json_result(job_id=_job_id, data={'pid': p.pid, 'table_name': request_config['table_name'], 'namespace': request_config['namespace']}) except Exception as e: logger.exception(e) return get_json_result(status=-104, msg="failed", job_id=_job_id)
def get_runtime_conf(job_id): _job_dir = get_job_directory(job_id) response_data = [] shutil.copytree(_job_dir, job_id) return get_json_result(data=response_data, job_id=job_id)