def get_process_dirs(cls,
worker_name: WorkerName,
job_id=None,
role=None,
party_id=None,
task: Task = None):
worker_id = base_utils.new_unique_id()
party_id = str(party_id)
if task:
config_dir = job_utils.get_job_directory(job_id, role, party_id,
task.f_component_name,
task.f_task_id,
str(task.f_task_version),
worker_name.value,
worker_id)
log_dir = job_utils.get_job_log_directory(job_id, role, party_id,
task.f_component_name)
elif job_id and role and party_id:
config_dir = job_utils.get_job_directory(job_id, role, party_id,
worker_name.value,
worker_id)
log_dir = job_utils.get_job_log_directory(job_id, role, party_id,
worker_name.value,
worker_id)
else:
config_dir = job_utils.get_general_worker_directory(
worker_name.value, worker_id)
log_dir = job_utils.get_general_worker_log_directory(
worker_name.value, worker_id)
os.makedirs(config_dir, exist_ok=True)
return worker_id, config_dir, log_dir
def start_inheriting_job(cls, job):
JobSaver.update_job(
job_info={
"job_id": job.f_job_id,
"role": job.f_role,
"party_id": job.f_party_id,
"inheritance_status": JobInheritanceStatus.RUNNING
})
conf_dir = job_utils.get_job_directory(job_id=job.f_job_id)
os.makedirs(conf_dir, exist_ok=True)
process_cmd = [
sys.executable or 'python3',
sys.modules[JobInherit.__module__].__file__,
'--job_id',
job.f_job_id,
'--role',
job.f_role,
'--party_id',
job.f_party_id,
]
log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job.f_job_id),
"job_inheritance")
p = process_utils.run_subprocess(job_id=job.f_job_id,
config_dir=conf_dir,
process_cmd=process_cmd,
log_dir=log_dir,
process_name="job_inheritance")
def upload_file_block(self, file_list, data_head, table_list):
if data_head:
self.update_table_meta(data_head)
upload_process = []
for block_index, block_file in enumerate(file_list):
task_dir = os.path.join(
job_utils.get_job_directory(job_id=self.tracker.job_id),
self.tracker.role, str(self.tracker.party_id),
self.tracker.component_name, 'upload')
os.makedirs(task_dir, exist_ok=True)
process_cmd = [
sys.executable or 'python3',
sys.modules[upload_utils.UploadFile.__module__].__file__,
'--session_id', self.session_id, '--storage',
self.storage_engine, '--file', block_file, '--namespace',
table_list[block_index].get("namespace"), '--name',
table_list[block_index].get("name"), '--partitions',
self.parameters.get('partition')
]
LOGGER.info(process_cmd)
job_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=self.tracker.job_id),
self.tracker.role, str(self.tracker.party_id))
task_log_dir = os.path.join(job_log_dir,
self.tracker.component_name,
f'block_{block_index}')
p = process_utils.run_subprocess(job_id=self.tracker.job_id,
config_dir=task_dir,
process_cmd=process_cmd,
log_dir=task_log_dir)
upload_process.append(p)
self.check_upload_process(upload_process)
self.union_table(table_list)
def start_task(job_id, component_name, task_id, role, party_id,
task_config):
schedule_logger.info('job {} {} {} {} task subprocess is ready'.format(
job_id, component_name, role, party_id, task_config))
task_process_start_status = False
try:
task_dir = os.path.join(job_utils.get_job_directory(job_id=job_id),
role, party_id, component_name)
os.makedirs(task_dir, exist_ok=True)
task_config_path = os.path.join(task_dir, 'task_config.json')
with open(task_config_path, 'w') as fw:
json.dump(task_config, fw)
process_cmd = [
'python3', sys.modules[TaskExecutor.__module__].__file__, '-j',
job_id, '-n', component_name, '-t', task_id, '-r', role, '-p',
party_id, '-c', task_config_path
]
task_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job_id), role, party_id,
component_name)
schedule_logger.info(
'job {} {} {} {} task subprocess start'.format(
job_id, component_name, role, party_id, task_config))
p = job_utils.run_subprocess(config_dir=task_dir,
process_cmd=process_cmd,
log_dir=task_log_dir)
if p:
task_process_start_status = True
except Exception as e:
schedule_logger.exception(e)
finally:
schedule_logger.info(
'job {} component {} on {} {} start task subprocess {}'.format(
job_id, component_name, role, party_id,
'success' if task_process_start_status else 'failed'))
def component_info_log():
job_id = request.json.get('job_id', '')
role = request.json.get('role')
party_id = request.json.get('party_id')
job_log_dir = job_utils.get_job_log_directory(job_id=job_id)
file_name = os.path.join(job_log_dir, role, str(party_id), 'INFO.log')
if os.path.exists(file_name):
return send_file(open(file_name, 'rb'), attachment_filename='{}_{}_{}_INFO.log'.format(job_id, role, party_id), as_attachment=True)
else:
response = make_response("no find log file")
response.status = '500'
return response
def check_job_log_dir():
job_id = str(request.json['job_id'])
job_log_dir = job_utils.get_job_log_directory(job_id=job_id)
if not os.path.exists(job_log_dir):
abort(
error_response(
404,
f"Log file path: '{job_log_dir}' not found. Please check if the job id is valid."
))
return job_id, job_log_dir
def job_log():
job_id = request.json.get('job_id', '')
memory_file = io.BytesIO()
tar = tarfile.open(fileobj=memory_file, mode='w:gz')
job_log_dir = job_utils.get_job_log_directory(job_id=job_id)
for root, dir, files in os.walk(job_log_dir):
for file in files:
full_path = os.path.join(root, file)
rel_path = os.path.relpath(full_path, job_log_dir)
tar.add(full_path, rel_path)
tar.close()
memory_file.seek(0)
return send_file(memory_file, attachment_filename='job_{}_log.tar.gz'.format(job_id), as_attachment=True)
def run(job_id, component_name, task_id, task_version, role, party_id,
task_parameters_path, task_info, **kwargs):
process_cmd = [
sys.executable,
sys.modules[TaskExecutor.__module__].__file__,
'-j',
job_id,
'-n',
component_name,
'-t',
task_id,
'-v',
task_version,
'-r',
role,
'-p',
party_id,
'-c',
task_parameters_path,
'--run_ip',
RuntimeConfig.JOB_SERVER_HOST,
'--job_server',
'{}:{}'.format(RuntimeConfig.JOB_SERVER_HOST,
RuntimeConfig.HTTP_PORT),
]
task_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job_id), role, party_id,
component_name)
task_job_dir = os.path.join(job_utils.get_job_directory(job_id=job_id),
role, party_id, component_name)
schedule_logger(job_id).info(
'job {} task {} {} on {} {} executor subprocess is ready'.format(
job_id, task_id, task_version, role, party_id))
task_dir = os.path.dirname(task_parameters_path)
p = job_utils.run_subprocess(job_id=job_id,
config_dir=task_dir,
process_cmd=process_cmd,
log_dir=task_log_dir,
job_dir=task_job_dir)
task_info["run_pid"] = p.pid
return p
def job_log():
job_id = request.json.get('job_id', '')
job_log_dir = job_utils.get_job_log_directory(job_id=job_id)
if os.path.exists(job_log_dir):
memory_file = io.BytesIO()
tar = tarfile.open(fileobj=memory_file, mode='w:gz')
for root, dir, files in os.walk(job_log_dir):
for file in files:
full_path = os.path.join(root, file)
rel_path = os.path.relpath(full_path, job_log_dir)
tar.add(full_path, rel_path)
tar.close()
memory_file.seek(0)
return send_file(
memory_file,
attachment_filename='job_{}_log.tar.gz'.format(job_id),
as_attachment=True)
else:
return error_response(
210,
"Log file path: {} not found. Please check if the job id is valid."
.format(job_log_dir))
def run_task():
task = Task()
task.f_create_time = current_timestamp()
try:
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--job_id', required=True, type=str, help="job id")
parser.add_argument('-n', '--component_name', required=True, type=str,
help="component name")
parser.add_argument('-t', '--task_id', required=True, type=str, help="task id")
parser.add_argument('-r', '--role', required=True, type=str, help="role")
parser.add_argument('-p', '--party_id', required=True, type=str, help="party id")
parser.add_argument('-c', '--config', required=True, type=str, help="task config")
parser.add_argument('--processors_per_node', help="processors_per_node", type=int)
parser.add_argument('--job_server', help="job server", type=str)
args = parser.parse_args()
schedule_logger(args.job_id).info('enter task process')
schedule_logger(args.job_id).info(args)
# init function args
if args.job_server:
RuntimeConfig.init_config(HTTP_PORT=args.job_server.split(':')[1])
RuntimeConfig.set_process_role(ProcessRole.EXECUTOR)
job_id = args.job_id
component_name = args.component_name
task_id = args.task_id
role = args.role
party_id = int(args.party_id)
executor_pid = os.getpid()
task_config = file_utils.load_json_conf(args.config)
job_parameters = task_config['job_parameters']
job_initiator = task_config['job_initiator']
job_args = task_config['job_args']
task_input_dsl = task_config['input']
task_output_dsl = task_config['output']
component_parameters = TaskExecutor.get_parameters(job_id, component_name, role, party_id)
task_parameters = task_config['task_parameters']
module_name = task_config['module_name']
TaskExecutor.monkey_patch()
except Exception as e:
traceback.print_exc()
schedule_logger().exception(e)
task.f_status = TaskStatus.FAILED
return
try:
job_log_dir = os.path.join(job_utils.get_job_log_directory(job_id=job_id), role, str(party_id))
task_log_dir = os.path.join(job_log_dir, component_name)
log_utils.LoggerFactory.set_directory(directory=task_log_dir, parent_log_dir=job_log_dir,
append_to_parent_log=True, force=True)
task.f_job_id = job_id
task.f_component_name = component_name
task.f_task_id = task_id
task.f_role = role
task.f_party_id = party_id
task.f_operator = 'python_operator'
tracker = Tracking(job_id=job_id, role=role, party_id=party_id, component_name=component_name,
task_id=task_id,
model_id=job_parameters['model_id'],
model_version=job_parameters['model_version'],
component_module_name=module_name)
task.f_start_time = current_timestamp()
task.f_run_ip = get_lan_ip()
task.f_run_pid = executor_pid
run_class_paths = component_parameters.get('CodePath').split('/')
run_class_package = '.'.join(run_class_paths[:-2]) + '.' + run_class_paths[-2].replace('.py', '')
run_class_name = run_class_paths[-1]
task.f_status = TaskStatus.RUNNING
TaskExecutor.sync_task_status(job_id=job_id, component_name=component_name, task_id=task_id, role=role,
party_id=party_id, initiator_party_id=job_initiator.get('party_id', None),
initiator_role=job_initiator.get('role', None),
task_info=task.to_json())
# init environment, process is shared globally
RuntimeConfig.init_config(WORK_MODE=job_parameters['work_mode'],
BACKEND=job_parameters.get('backend', 0))
if args.processors_per_node and args.processors_per_node > 0 and RuntimeConfig.BACKEND == Backend.EGGROLL:
session_options = {"eggroll.session.processors.per.node": args.processors_per_node}
else:
session_options = {}
session.init(job_id=job_utils.generate_session_id(task_id, role, party_id),
mode=RuntimeConfig.WORK_MODE,
backend=RuntimeConfig.BACKEND,
options=session_options)
federation.init(job_id=task_id, runtime_conf=component_parameters)
schedule_logger().info('run {} {} {} {} {} task'.format(job_id, component_name, task_id, role, party_id))
schedule_logger().info(component_parameters)
schedule_logger().info(task_input_dsl)
task_run_args = TaskExecutor.get_task_run_args(job_id=job_id, role=role, party_id=party_id,
task_id=task_id,
job_args=job_args,
job_parameters=job_parameters,
task_parameters=task_parameters,
input_dsl=task_input_dsl,
if_save_as_task_input_data=job_parameters.get("save_as_task_input_data", SAVE_AS_TASK_INPUT_DATA_SWITCH)
)
run_object = getattr(importlib.import_module(run_class_package), run_class_name)()
run_object.set_tracker(tracker=tracker)
run_object.set_taskid(taskid=task_id)
run_object.run(component_parameters, task_run_args)
output_data = run_object.save_data()
tracker.save_output_data_table(output_data, task_output_dsl.get('data')[0] if task_output_dsl.get('data') else 'component')
output_model = run_object.export_model()
# There is only one model output at the current dsl version.
tracker.save_output_model(output_model, task_output_dsl['model'][0] if task_output_dsl.get('model') else 'default')
task.f_status = TaskStatus.COMPLETE
except Exception as e:
task.f_status = TaskStatus.FAILED
schedule_logger().exception(e)
finally:
sync_success = False
try:
task.f_end_time = current_timestamp()
task.f_elapsed = task.f_end_time - task.f_start_time
task.f_update_time = current_timestamp()
TaskExecutor.sync_task_status(job_id=job_id, component_name=component_name, task_id=task_id, role=role,
party_id=party_id,
initiator_party_id=job_initiator.get('party_id', None),
initiator_role=job_initiator.get('role', None),
task_info=task.to_json())
sync_success = True
except Exception as e:
traceback.print_exc()
schedule_logger().exception(e)
schedule_logger().info('task {} {} {} start time: {}'.format(task_id, role, party_id, timestamp_to_date(task.f_start_time)))
schedule_logger().info('task {} {} {} end time: {}'.format(task_id, role, party_id, timestamp_to_date(task.f_end_time)))
schedule_logger().info('task {} {} {} takes {}s'.format(task_id, role, party_id, int(task.f_elapsed)/1000))
schedule_logger().info(
'finish {} {} {} {} {} {} task'.format(job_id, component_name, task_id, role, party_id, task.f_status if sync_success else TaskStatus.FAILED))
print('finish {} {} {} {} {} {} task'.format(job_id, component_name, task_id, role, party_id, task.f_status if sync_success else TaskStatus.FAILED))
def run_task(job_id, component_name, task_id, role, party_id, task_config):
schedule_logger(job_id).info(
'job {} {} {} {} task subprocess is ready'.format(job_id, component_name, role, party_id, task_config))
task_process_start_status = False
try:
task_dir = os.path.join(job_utils.get_job_directory(job_id=job_id), role, party_id, component_name)
os.makedirs(task_dir, exist_ok=True)
task_config_path = os.path.join(task_dir, 'task_config.json')
with open(task_config_path, 'w') as fw:
json.dump(task_config, fw)
try:
backend = task_config['job_parameters']['backend']
except KeyError:
backend = 0
schedule_logger(job_id).warning("failed to get backend, set as 0")
backend = Backend(backend)
if backend.is_eggroll():
process_cmd = [
'python3', sys.modules[TaskExecutor.__module__].__file__,
'-j', job_id,
'-n', component_name,
'-t', task_id,
'-r', role,
'-p', party_id,
'-c', task_config_path,
'--processors_per_node', str(task_config['job_parameters'].get("processors_per_node", 0)),
'--job_server', '{}:{}'.format(get_lan_ip(), HTTP_PORT),
]
elif backend.is_spark():
if "SPARK_HOME" not in os.environ:
raise EnvironmentError("SPARK_HOME not found")
spark_home = os.environ["SPARK_HOME"]
# additional configs
spark_submit_config = task_config['job_parameters'].get("spark_submit_config", dict())
deploy_mode = spark_submit_config.get("deploy-mode", "client")
if deploy_mode not in ["client"]:
raise ValueError(f"deploy mode {deploy_mode} not supported")
spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit")
process_cmd = [spark_submit_cmd, f'--name={task_id}#{role}']
for k, v in spark_submit_config.items():
if k != "conf":
process_cmd.append(f'--{k}={v}')
if "conf" in spark_submit_config:
for ck, cv in spark_submit_config["conf"].items():
process_cmd.append(f'--conf')
process_cmd.append(f'{ck}={cv}')
process_cmd.extend([
sys.modules[TaskExecutor.__module__].__file__,
'-j', job_id,
'-n', component_name,
'-t', task_id,
'-r', role,
'-p', party_id,
'-c', task_config_path,
'--job_server',
'{}:{}'.format(get_lan_ip(), HTTP_PORT),
])
else:
raise ValueError(f"${backend} supported")
task_log_dir = os.path.join(job_utils.get_job_log_directory(job_id=job_id), role, party_id, component_name)
schedule_logger(job_id).info(
'job {} {} {} {} task subprocess start'.format(job_id, component_name, role, party_id, task_config))
p = job_utils.run_subprocess(config_dir=task_dir, process_cmd=process_cmd, log_dir=task_log_dir)
if p:
task_process_start_status = True
except Exception as e:
schedule_logger(job_id).exception(e)
finally:
schedule_logger(job_id).info(
'job {} component {} on {} {} start task subprocess {}'.format(job_id, component_name, role, party_id,
'success' if task_process_start_status else 'failed'))
def start_task(cls, job_id, component_name, task_id, task_version, role,
party_id):
"""
Start task, update status and party status
:param job_id:
:param component_name:
:param task_id:
:param task_version:
:param role:
:param party_id:
:return:
"""
schedule_logger(job_id).info(
'try to start job {} task {} {} on {} {} executor subprocess'.
format(job_id, task_id, task_version, role, party_id))
task_executor_process_start_status = False
task_info = {
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
}
try:
task_dir = os.path.join(job_utils.get_job_directory(job_id=job_id),
role, party_id, component_name, task_id,
task_version)
os.makedirs(task_dir, exist_ok=True)
task_parameters_path = os.path.join(task_dir,
'task_parameters.json')
run_parameters_dict = job_utils.get_job_parameters(
job_id, role, party_id)
with open(task_parameters_path, 'w') as fw:
fw.write(json_dumps(run_parameters_dict))
run_parameters = RunParameters(**run_parameters_dict)
schedule_logger(job_id=job_id).info(
f"use computing engine {run_parameters.computing_engine}")
if run_parameters.computing_engine in {
ComputingEngine.EGGROLL, ComputingEngine.STANDALONE
}:
process_cmd = [
sys.executable,
sys.modules[TaskExecutor.__module__].__file__,
'-j',
job_id,
'-n',
component_name,
'-t',
task_id,
'-v',
task_version,
'-r',
role,
'-p',
party_id,
'-c',
task_parameters_path,
'--run_ip',
RuntimeConfig.JOB_SERVER_HOST,
'--job_server',
'{}:{}'.format(RuntimeConfig.JOB_SERVER_HOST,
RuntimeConfig.HTTP_PORT),
]
elif run_parameters.computing_engine == ComputingEngine.SPARK:
if "SPARK_HOME" not in os.environ:
raise EnvironmentError("SPARK_HOME not found")
spark_home = os.environ["SPARK_HOME"]
# additional configs
spark_submit_config = run_parameters.spark_run
deploy_mode = spark_submit_config.get("deploy-mode", "client")
if deploy_mode not in ["client"]:
raise ValueError(
f"deploy mode {deploy_mode} not supported")
spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit")
process_cmd = [spark_submit_cmd, f'--name={task_id}#{role}']
for k, v in spark_submit_config.items():
if k != "conf":
process_cmd.append(f'--{k}={v}')
if "conf" in spark_submit_config:
for ck, cv in spark_submit_config["conf"].items():
process_cmd.append(f'--conf')
process_cmd.append(f'{ck}={cv}')
process_cmd.extend([
sys.modules[TaskExecutor.__module__].__file__,
'-j',
job_id,
'-n',
component_name,
'-t',
task_id,
'-v',
task_version,
'-r',
role,
'-p',
party_id,
'-c',
task_parameters_path,
'--run_ip',
RuntimeConfig.JOB_SERVER_HOST,
'--job_server',
'{}:{}'.format(RuntimeConfig.JOB_SERVER_HOST,
RuntimeConfig.HTTP_PORT),
])
else:
raise ValueError(
f"${run_parameters.computing_engine} is not supported")
task_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job_id), role, party_id,
component_name)
schedule_logger(job_id).info(
'job {} task {} {} on {} {} executor subprocess is ready'.
format(job_id, task_id, task_version, role, party_id))
p = job_utils.run_subprocess(job_id=job_id,
config_dir=task_dir,
process_cmd=process_cmd,
log_dir=task_log_dir)
if p:
task_info["party_status"] = TaskStatus.RUNNING
#task_info["run_pid"] = p.pid
task_info["start_time"] = current_timestamp()
task_executor_process_start_status = True
else:
task_info["party_status"] = TaskStatus.FAILED
except Exception as e:
schedule_logger(job_id).exception(e)
task_info["party_status"] = TaskStatus.FAILED
finally:
try:
cls.update_task(task_info=task_info)
cls.update_task_status(task_info=task_info)
except Exception as e:
schedule_logger(job_id).exception(e)
schedule_logger(job_id).info(
'job {} task {} {} on {} {} executor subprocess start {}'.
format(
job_id, task_id, task_version, role, party_id, "success"
if task_executor_process_start_status else "failed"))
def start_task(job_id, component_name, task_id, role, party_id,
task_config):
schedule_logger(job_id).info(
'job {} {} {} {} task subprocess is ready'.format(
job_id, component_name, role, party_id, task_config))
task_process_start_status = False
try:
task_dir = os.path.join(job_utils.get_job_directory(job_id=job_id),
role, party_id, component_name)
os.makedirs(task_dir, exist_ok=True)
task_config_path = os.path.join(task_dir, 'task_config.json')
with open(task_config_path, 'w') as fw:
json.dump(task_config, fw)
try:
backend = task_config['job_parameters']['backend']
except KeyError:
backend = 0
schedule_logger(job_id).warning(
"failed to get backend, set as 0")
backend = Backend(backend)
if backend.is_eggroll() or backend.is_eggroll2():
process_cmd = [
'python3',
sys.modules[TaskExecutor.__module__].__file__,
'-j',
job_id,
'-n',
component_name,
'-t',
task_id,
'-r',
role,
'-p',
party_id,
'-c',
task_config_path,
'--job_server',
'{}:{}'.format(task_config['job_server']['ip'],
task_config['job_server']['http_port']),
]
elif backend.is_spark():
if "SPARK_HOME" not in os.environ:
raise EnvironmentError("SPARK_HOME not found")
spark_submit_config = task_config['job_parameters'].get(
"spark_submit_config", dict())
deploy_mode = spark_submit_config.get("deploy-mode", "client")
queue = spark_submit_config.get("queue", "default")
driver_memory = spark_submit_config.get("driver-memory", "1g")
num_executors = spark_submit_config.get("num-executors", 2)
executor_memory = spark_submit_config.get(
"executor-memory", "1g")
executor_cores = spark_submit_config.get("executor-cores", 1)
if deploy_mode not in ["client"]:
raise ValueError(
f"deploy mode {deploy_mode} not supported")
spark_home = os.environ["SPARK_HOME"]
spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit")
process_cmd = [
spark_submit_cmd,
f'--name={task_id}#{role}',
f'--deploy-mode={deploy_mode}',
f'--queue={queue}',
f'--driver-memory={driver_memory}',
f'--num-executors={num_executors}',
f'--executor-memory={executor_memory}',
f'--executor-cores={executor_cores}',
sys.modules[TaskExecutor.__module__].__file__,
'-j',
job_id,
'-n',
component_name,
'-t',
task_id,
'-r',
role,
'-p',
party_id,
'-c',
task_config_path,
'--job_server',
'{}:{}'.format(task_config['job_server']['ip'],
task_config['job_server']['http_port']),
]
else:
raise ValueError(f"${backend} supported")
task_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job_id), role, party_id,
component_name)
schedule_logger(job_id).info(
'job {} {} {} {} task subprocess start'.format(
job_id, component_name, role, party_id, task_config))
p = job_utils.run_subprocess(config_dir=task_dir,
process_cmd=process_cmd,
log_dir=task_log_dir)
if p:
task_process_start_status = True
except Exception as e:
schedule_logger(job_id).exception(e)
finally:
schedule_logger(job_id).info(
'job {} component {} on {} {} start task subprocess {}'.format(
job_id, component_name, role, party_id,
'success' if task_process_start_status else 'failed'))
def submit(cls, job_data, job_id=None):
if not job_id:
job_id = job_utils.generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(
job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_utils.check_job_runtime_conf(job_runtime_conf)
authentication_utils.check_constraint(job_runtime_conf, job_dsl)
job_initiator = job_runtime_conf['initiator']
conf_adapter = JobRuntimeConfigAdapter(job_runtime_conf)
common_job_parameters = conf_adapter.get_common_parameters()
if common_job_parameters.job_type != 'predict':
# generate job model info
common_job_parameters.model_id = model_utils.gen_model_id(
job_runtime_conf['role'])
common_job_parameters.model_version = job_id
train_runtime_conf = {}
else:
# check predict job parameters
detect_utils.check_config(common_job_parameters.to_dict(),
['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
tracker = Tracker(
job_id=job_id,
role=job_initiator['role'],
party_id=job_initiator['party_id'],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version)
pipeline_model = tracker.get_output_model('pipeline')
train_runtime_conf = json_loads(
pipeline_model['Pipeline'].train_runtime_conf)
if not model_utils.check_if_deployed(
role=job_initiator['role'],
party_id=job_initiator['party_id'],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version):
raise Exception(
f"Model {common_job_parameters.model_id} {common_job_parameters.model_version} has not been deployed yet."
)
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
job = Job()
job.f_job_id = job_id
job.f_dsl = job_dsl
job.f_train_runtime_conf = train_runtime_conf
job.f_roles = job_runtime_conf['role']
job.f_work_mode = common_job_parameters.work_mode
job.f_initiator_role = job_initiator['role']
job.f_initiator_party_id = job_initiator['party_id']
job.f_role = job_initiator['role']
job.f_party_id = job_initiator['party_id']
path_dict = job_utils.save_job_conf(
job_id=job_id,
role=job.f_initiator_role,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
job_runtime_conf_on_party={},
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
if job.f_initiator_party_id not in job_runtime_conf['role'][
job.f_initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(
job.f_initiator_party_id))
raise Exception("initiator party id error {}".format(
job.f_initiator_party_id))
# create common parameters on initiator
JobController.backend_compatibility(
job_parameters=common_job_parameters)
JobController.adapt_job_parameters(
role=job.f_initiator_role,
job_parameters=common_job_parameters,
create_initiator_baseline=True)
job.f_runtime_conf = conf_adapter.update_common_parameters(
common_parameters=common_job_parameters)
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=job.f_dsl,
runtime_conf=job.f_runtime_conf,
train_runtime_conf=job.f_train_runtime_conf)
# initiator runtime conf as template
job.f_runtime_conf_on_party = job.f_runtime_conf.copy()
job.f_runtime_conf_on_party[
"job_parameters"] = common_job_parameters.to_dict()
if common_job_parameters.work_mode == WorkMode.CLUSTER:
# Save the status information of all participants in the initiator for scheduling
for role, party_ids in job.f_roles.items():
for party_id in party_ids:
if role == job.f_initiator_role and party_id == job.f_initiator_party_id:
continue
JobController.initialize_tasks(job_id, role, party_id,
False, job.f_initiator_role,
job.f_initiator_party_id,
common_job_parameters,
dsl_parser)
status_code, response = FederatedScheduler.create_job(job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
job.f_status = JobStatus.FAILED
job.f_tag = "submit_failed"
FederatedScheduler.sync_job_status(job=job)
raise Exception("create job failed", response)
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(
job.f_job_id, common_job_parameters.model_id))
logs_directory = job_utils.get_job_log_directory(job_id)
submit_result = {
"job_id":
job_id,
"model_info": {
"model_id": common_job_parameters.model_id,
"model_version": common_job_parameters.model_version
},
"logs_directory":
logs_directory,
"board_url":
job_utils.get_board_url(job_id, job_initiator['role'],
job_initiator['party_id'])
}
submit_result.update(path_dict)
return submit_result
def submit(cls, submit_job_conf: JobConfigurationBase, job_id: str = None):
if not job_id:
job_id = job_utils.generate_job_id()
submit_result = {"job_id": job_id}
schedule_logger(job_id).info(
f"submit job, body {submit_job_conf.to_dict()}")
try:
dsl = submit_job_conf.dsl
runtime_conf = deepcopy(submit_job_conf.runtime_conf)
job_utils.check_job_runtime_conf(runtime_conf)
authentication_utils.check_constraint(runtime_conf, dsl)
job_initiator = runtime_conf["initiator"]
conf_adapter = JobRuntimeConfigAdapter(runtime_conf)
common_job_parameters = conf_adapter.get_common_parameters()
if common_job_parameters.job_type != "predict":
# generate job model info
conf_version = schedule_utils.get_conf_version(runtime_conf)
if conf_version != 2:
raise Exception(
"only the v2 version runtime conf is supported")
common_job_parameters.model_id = model_utils.gen_model_id(
runtime_conf["role"])
common_job_parameters.model_version = job_id
train_runtime_conf = {}
else:
# check predict job parameters
detect_utils.check_config(common_job_parameters.to_dict(),
["model_id", "model_version"])
# get inference dsl from pipeline model as job dsl
tracker = Tracker(
job_id=job_id,
role=job_initiator["role"],
party_id=job_initiator["party_id"],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version)
pipeline_model = tracker.get_pipeline_model()
train_runtime_conf = json_loads(
pipeline_model.train_runtime_conf)
if not model_utils.check_if_deployed(
role=job_initiator["role"],
party_id=job_initiator["party_id"],
model_id=common_job_parameters.model_id,
model_version=common_job_parameters.model_version):
raise Exception(
f"Model {common_job_parameters.model_id} {common_job_parameters.model_version} has not been deployed yet."
)
dsl = json_loads(pipeline_model.inference_dsl)
# dsl = ProviderManager.fill_fate_flow_provider(dsl)
job = Job()
job.f_job_id = job_id
job.f_dsl = dsl
job.f_train_runtime_conf = train_runtime_conf
job.f_roles = runtime_conf["role"]
job.f_initiator_role = job_initiator["role"]
job.f_initiator_party_id = job_initiator["party_id"]
job.f_role = job_initiator["role"]
job.f_party_id = job_initiator["party_id"]
path_dict = job_utils.save_job_conf(
job_id=job_id,
role=job.f_initiator_role,
party_id=job.f_initiator_party_id,
dsl=dsl,
runtime_conf=runtime_conf,
runtime_conf_on_party={},
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
if job.f_initiator_party_id not in runtime_conf["role"][
job.f_initiator_role]:
msg = f"initiator party id {job.f_initiator_party_id} not in roles {runtime_conf['role']}"
schedule_logger(job_id).info(msg)
raise Exception(msg)
# create common parameters on initiator
JobController.create_common_job_parameters(
job_id=job.f_job_id,
initiator_role=job.f_initiator_role,
common_job_parameters=common_job_parameters)
job.f_runtime_conf = conf_adapter.update_common_parameters(
common_parameters=common_job_parameters)
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=job.f_dsl,
runtime_conf=job.f_runtime_conf,
train_runtime_conf=job.f_train_runtime_conf)
# initiator runtime conf as template
job.f_runtime_conf_on_party = job.f_runtime_conf.copy()
job.f_runtime_conf_on_party[
"job_parameters"] = common_job_parameters.to_dict()
# inherit job
job.f_inheritance_info = common_job_parameters.inheritance_info
job.f_inheritance_status = JobInheritanceStatus.WAITING if common_job_parameters.inheritance_info else JobInheritanceStatus.PASS
if job.f_inheritance_info:
inheritance_jobs = JobSaver.query_job(
job_id=job.f_inheritance_info.get("job_id"),
role=job_initiator["role"],
party_id=job_initiator["party_id"])
inheritance_tasks = JobSaver.query_task(
job_id=job.f_inheritance_info.get("job_id"),
role=job_initiator["role"],
party_id=job_initiator["party_id"],
only_latest=True)
job_utils.check_job_inheritance_parameters(
job, inheritance_jobs, inheritance_tasks)
status_code, response = FederatedScheduler.create_job(job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
job.f_status = JobStatus.FAILED
job.f_tag = "submit_failed"
FederatedScheduler.sync_job_status(job=job)
raise Exception("create job failed", response)
else:
need_run_components = {}
for role in response:
need_run_components[role] = {}
for party, res in response[role].items():
need_run_components[role][party] = [
name for name, value in response[role][party]
["data"]["components"].items()
if value["need_run"] is True
]
if common_job_parameters.federated_mode == FederatedMode.MULTIPLE:
# create the task holder in db to record information of all participants in the initiator for scheduling
for role, party_ids in job.f_roles.items():
for party_id in party_ids:
if role == job.f_initiator_role and party_id == job.f_initiator_party_id:
continue
if not need_run_components[role][party_id]:
continue
JobController.initialize_tasks(
job_id=job_id,
role=role,
party_id=party_id,
run_on_this_party=False,
initiator_role=job.f_initiator_role,
initiator_party_id=job.f_initiator_party_id,
job_parameters=common_job_parameters,
dsl_parser=dsl_parser,
components=need_run_components[role][party_id])
job.f_status = JobStatus.WAITING
status_code, response = FederatedScheduler.sync_job_status(
job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
raise Exception("set job to waiting status failed")
schedule_logger(job_id).info(
f"submit job successfully, job id is {job.f_job_id}, model id is {common_job_parameters.model_id}"
)
logs_directory = job_utils.get_job_log_directory(job_id)
result = {
"code":
RetCode.SUCCESS,
"message":
"success",
"model_info": {
"model_id": common_job_parameters.model_id,
"model_version": common_job_parameters.model_version
},
"logs_directory":
logs_directory,
"board_url":
job_utils.get_board_url(job_id, job_initiator["role"],
job_initiator["party_id"])
}
warn_parameter = JobRuntimeConfigAdapter(
submit_job_conf.runtime_conf).check_removed_parameter()
if warn_parameter:
result[
"message"] = f"[WARN]{warn_parameter} is removed,it does not take effect!"
submit_result.update(result)
submit_result.update(path_dict)
except Exception as e:
submit_result["code"] = RetCode.OPERATING_ERROR
submit_result["message"] = exception_to_trace_string(e)
schedule_logger(job_id).exception(e)
return submit_result
def submit(cls, job_data, job_id=None):
if not job_id:
job_id = job_utils.generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_initiator = job_runtime_conf['initiator']
job_parameters = RunParameters(**job_runtime_conf['job_parameters'])
cls.backend_compatibility(job_parameters=job_parameters)
job_utils.check_job_runtime_conf(job_runtime_conf)
if job_parameters.job_type != 'predict':
# generate job model info
job_parameters.model_id = model_utils.gen_model_id(job_runtime_conf['role'])
job_parameters.model_version = job_id
train_runtime_conf = {}
else:
detect_utils.check_config(job_parameters.to_dict(), ['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
tracker = Tracker(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'],
model_id=job_parameters.model_id, model_version=job_parameters.model_version)
pipeline_model = tracker.get_output_model('pipeline')
if not job_dsl:
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf)
path_dict = job_utils.save_job_conf(job_id=job_id,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job = Job()
job.f_job_id = job_id
job.f_dsl = job_dsl
job_runtime_conf["job_parameters"] = job_parameters.to_dict()
job.f_runtime_conf = job_runtime_conf
job.f_train_runtime_conf = train_runtime_conf
job.f_roles = job_runtime_conf['role']
job.f_work_mode = job_parameters.work_mode
job.f_initiator_role = job_initiator['role']
job.f_initiator_party_id = job_initiator['party_id']
initiator_role = job_initiator['role']
initiator_party_id = job_initiator['party_id']
if initiator_party_id not in job_runtime_conf['role'][initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(initiator_party_id))
raise Exception("initiator party id error {}".format(initiator_party_id))
dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
cls.adapt_job_parameters(job_parameters=job_parameters)
# update runtime conf
job_runtime_conf["job_parameters"] = job_parameters.to_dict()
job.f_runtime_conf = job_runtime_conf
status_code, response = FederatedScheduler.create_job(job=job)
if status_code != FederatedSchedulingStatusCode.SUCCESS:
raise Exception("create job failed: {}".format(response))
if job_parameters.work_mode == WorkMode.CLUSTER:
# Save the status information of all participants in the initiator for scheduling
for role, party_ids in job_runtime_conf["role"].items():
for party_id in party_ids:
if role == job_initiator['role'] and party_id == job_initiator['party_id']:
continue
JobController.initialize_tasks(job_id, role, party_id, False, job_initiator, job_parameters, dsl_parser)
# push into queue
try:
JobQueue.create_event(job_id=job_id, initiator_role=initiator_role, initiator_party_id=initiator_party_id)
except Exception as e:
raise Exception(f'push job into queue failed:\n{e}')
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters.model_id))
board_url = "http://{}:{}{}".format(
ServiceUtils.get_item("fateboard", "host"),
ServiceUtils.get_item("fateboard", "port"),
FATE_BOARD_DASHBOARD_ENDPOINT).format(job_id, job_initiator['role'], job_initiator['party_id'])
logs_directory = job_utils.get_job_log_directory(job_id)
return job_id, path_dict['job_dsl_path'], path_dict['job_runtime_conf_path'], logs_directory, \
{'model_id': job_parameters.model_id, 'model_version': job_parameters.model_version}, board_url
def start_task(cls, job_id, component_name, task_id, task_version, role,
party_id, **kwargs):
"""
Start task, update status and party status
:param job_id:
:param component_name:
:param task_id:
:param task_version:
:param role:
:param party_id:
:return:
"""
job_dsl = job_utils.get_job_dsl(job_id, role, party_id)
PrivilegeAuth.authentication_component(
job_dsl,
src_party_id=kwargs.get('src_party_id'),
src_role=kwargs.get('src_role'),
party_id=party_id,
component_name=component_name)
schedule_logger(job_id).info(
f"try to start task {task_id} {task_version} on {role} {party_id} executor subprocess"
)
task_executor_process_start_status = False
task_info = {
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
}
is_failed = False
try:
task = JobSaver.query_task(task_id=task_id,
task_version=task_version,
role=role,
party_id=party_id)[0]
run_parameters_dict = job_utils.get_job_parameters(
job_id, role, party_id)
run_parameters_dict["src_user"] = kwargs.get("src_user")
run_parameters = RunParameters(**run_parameters_dict)
config_dir = job_utils.get_task_directory(job_id, role, party_id,
component_name, task_id,
task_version)
os.makedirs(config_dir, exist_ok=True)
run_parameters_path = os.path.join(config_dir,
'task_parameters.json')
with open(run_parameters_path, 'w') as fw:
fw.write(json_dumps(run_parameters_dict))
schedule_logger(job_id).info(
f"use computing engine {run_parameters.computing_engine}")
task_info["engine_conf"] = {
"computing_engine": run_parameters.computing_engine
}
backend_engine = build_engine(run_parameters.computing_engine)
run_info = backend_engine.run(
task=task,
run_parameters=run_parameters,
run_parameters_path=run_parameters_path,
config_dir=config_dir,
log_dir=job_utils.get_job_log_directory(
job_id, role, party_id, component_name),
cwd_dir=job_utils.get_job_directory(job_id, role, party_id,
component_name),
user_name=kwargs.get("user_id"))
task_info.update(run_info)
task_info["start_time"] = current_timestamp()
task_executor_process_start_status = True
except Exception as e:
schedule_logger(job_id).exception(e)
is_failed = True
finally:
try:
cls.update_task(task_info=task_info)
task_info["party_status"] = TaskStatus.RUNNING
cls.update_task_status(task_info=task_info)
if is_failed:
task_info["party_status"] = TaskStatus.FAILED
cls.update_task_status(task_info=task_info)
except Exception as e:
schedule_logger(job_id).exception(e)
schedule_logger(job_id).info(
"task {} {} on {} {} executor subprocess start {}".format(
task_id, task_version, role, party_id, "success"
if task_executor_process_start_status else "failed"))
def run_task():
task = Task()
task.f_create_time = current_timestamp()
try:
parser = argparse.ArgumentParser()
parser.add_argument('-j',
'--job_id',
required=True,
type=str,
help="job id")
parser.add_argument('-n',
'--component_name',
required=True,
type=str,
help="component name")
parser.add_argument('-t',
'--task_id',
required=True,
type=str,
help="task id")
parser.add_argument('-r',
'--role',
required=True,
type=str,
help="role")
parser.add_argument('-p',
'--party_id',
required=True,
type=str,
help="party id")
parser.add_argument('-c',
'--config',
required=True,
type=str,
help="task config")
parser.add_argument('--job_server', help="job server", type=str)
args = parser.parse_args()
schedule_logger.info('enter task process')
schedule_logger.info(args)
# init function args
if args.job_server:
RuntimeConfig.init_config(
HTTP_PORT=args.job_server.split(':')[1])
job_id = args.job_id
component_name = args.component_name
task_id = args.task_id
role = args.role
party_id = int(args.party_id)
task_config = file_utils.load_json_conf(args.config)
job_parameters = task_config['job_parameters']
job_initiator = task_config['job_initiator']
job_args = task_config['job_args']
task_input_dsl = task_config['input']
task_output_dsl = task_config['output']
parameters = task_config['parameters']
module_name = task_config['module_name']
except Exception as e:
schedule_logger.exception(e)
task.f_status = TaskStatus.FAILED
return
try:
# init environment, process is shared globally
RuntimeConfig.init_config(WORK_MODE=job_parameters['work_mode'])
storage.init_storage(job_id=task_id,
work_mode=RuntimeConfig.WORK_MODE)
federation.init(job_id=task_id, runtime_conf=parameters)
job_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job_id), role,
str(party_id))
task_log_dir = os.path.join(job_log_dir, component_name)
log_utils.LoggerFactory.set_directory(directory=task_log_dir,
parent_log_dir=job_log_dir,
append_to_parent_log=True,
force=True)
task.f_job_id = job_id
task.f_component_name = component_name
task.f_task_id = task_id
task.f_role = role
task.f_party_id = party_id
task.f_operator = 'python_operator'
tracker = Tracking(job_id=job_id,
role=role,
party_id=party_id,
component_name=component_name,
task_id=task_id,
model_id=job_parameters['model_id'],
model_version=job_parameters['model_version'],
module_name=module_name)
task.f_start_time = current_timestamp()
task.f_run_ip = get_lan_ip()
task.f_run_pid = os.getpid()
run_class_paths = parameters.get('CodePath').split('/')
run_class_package = '.'.join(
run_class_paths[:-2]) + '.' + run_class_paths[-2].replace(
'.py', '')
run_class_name = run_class_paths[-1]
task_run_args = TaskExecutor.get_task_run_args(
job_id=job_id,
role=role,
party_id=party_id,
job_parameters=job_parameters,
job_args=job_args,
input_dsl=task_input_dsl)
run_object = getattr(importlib.import_module(run_class_package),
run_class_name)()
run_object.set_tracker(tracker=tracker)
run_object.set_taskid(taskid=task_id)
task.f_status = TaskStatus.RUNNING
TaskExecutor.sync_task_status(job_id=job_id,
component_name=component_name,
task_id=task_id,
role=role,
party_id=party_id,
initiator_party_id=job_initiator.get(
'party_id', None),
task_info=task.to_json())
schedule_logger.info('run {} {} {} {} {} task'.format(
job_id, component_name, task_id, role, party_id))
schedule_logger.info(parameters)
schedule_logger.info(task_input_dsl)
run_object.run(parameters, task_run_args)
if task_output_dsl:
if task_output_dsl.get('data', []):
output_data = run_object.save_data()
tracker.save_output_data_table(
output_data,
task_output_dsl.get('data')[0])
if task_output_dsl.get('model', []):
output_model = run_object.export_model()
# There is only one model output at the current dsl version.
tracker.save_output_model(output_model,
task_output_dsl['model'][0])
task.f_status = TaskStatus.SUCCESS
except Exception as e:
schedule_logger.exception(e)
task.f_status = TaskStatus.FAILED
finally:
try:
task.f_end_time = current_timestamp()
task.f_elapsed = task.f_end_time - task.f_start_time
task.f_update_time = current_timestamp()
TaskExecutor.sync_task_status(
job_id=job_id,
component_name=component_name,
task_id=task_id,
role=role,
party_id=party_id,
initiator_party_id=job_initiator.get('party_id', None),
task_info=task.to_json())
except Exception as e:
schedule_logger.exception(e)
schedule_logger.info('finish {} {} {} {} {} {} task'.format(
job_id, component_name, task_id, role, party_id, task.f_status))
print('finish {} {} {} {} {} {} task'.format(job_id, component_name,
task_id, role, party_id,
task.f_status))
def run_task(cls):
task_info = {}
try:
parser = argparse.ArgumentParser()
parser.add_argument('-j',
'--job_id',
required=True,
type=str,
help="job id")
parser.add_argument('-n',
'--component_name',
required=True,
type=str,
help="component name")
parser.add_argument('-t',
'--task_id',
required=True,
type=str,
help="task id")
parser.add_argument('-v',
'--task_version',
required=True,
type=int,
help="task version")
parser.add_argument('-r',
'--role',
required=True,
type=str,
help="role")
parser.add_argument('-p',
'--party_id',
required=True,
type=int,
help="party id")
parser.add_argument('-c',
'--config',
required=True,
type=str,
help="task parameters")
parser.add_argument('--run_ip', help="run ip", type=str)
parser.add_argument('--job_server', help="job server", type=str)
args = parser.parse_args()
schedule_logger(args.job_id).info('enter task process')
schedule_logger(args.job_id).info(args)
# init function args
if args.job_server:
RuntimeConfig.init_config(
JOB_SERVER_HOST=args.job_server.split(':')[0],
HTTP_PORT=args.job_server.split(':')[1])
RuntimeConfig.set_process_role(ProcessRole.EXECUTOR)
job_id = args.job_id
component_name = args.component_name
task_id = args.task_id
task_version = args.task_version
role = args.role
party_id = args.party_id
executor_pid = os.getpid()
task_info.update({
"job_id": job_id,
"component_name": component_name,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
"run_ip": args.run_ip,
"run_pid": executor_pid
})
start_time = current_timestamp()
job_conf = job_utils.get_job_conf(job_id, role)
job_dsl = job_conf["job_dsl_path"]
job_runtime_conf = job_conf["job_runtime_conf_path"]
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=job_conf["train_runtime_conf_path"],
pipeline_dsl=job_conf["pipeline_dsl_path"])
party_index = job_runtime_conf["role"][role].index(party_id)
job_args_on_party = TaskExecutor.get_job_args_on_party(
dsl_parser, job_runtime_conf, role, party_id)
component = dsl_parser.get_component_info(
component_name=component_name)
component_parameters = component.get_role_parameters()
component_parameters_on_party = component_parameters[role][
party_index] if role in component_parameters else {}
module_name = component.get_module()
task_input_dsl = component.get_input()
task_output_dsl = component.get_output()
component_parameters_on_party[
'output_data_name'] = task_output_dsl.get('data')
task_parameters = RunParameters(
**file_utils.load_json_conf(args.config))
job_parameters = task_parameters
if job_parameters.assistant_role:
TaskExecutor.monkey_patch()
except Exception as e:
traceback.print_exc()
schedule_logger().exception(e)
task_info["party_status"] = TaskStatus.FAILED
return
try:
job_log_dir = os.path.join(
job_utils.get_job_log_directory(job_id=job_id), role,
str(party_id))
task_log_dir = os.path.join(job_log_dir, component_name)
log.LoggerFactory.set_directory(directory=task_log_dir,
parent_log_dir=job_log_dir,
append_to_parent_log=True,
force=True)
tracker = Tracker(job_id=job_id,
role=role,
party_id=party_id,
component_name=component_name,
task_id=task_id,
task_version=task_version,
model_id=job_parameters.model_id,
model_version=job_parameters.model_version,
component_module_name=module_name,
job_parameters=job_parameters)
tracker_client = TrackerClient(
job_id=job_id,
role=role,
party_id=party_id,
component_name=component_name,
task_id=task_id,
task_version=task_version,
model_id=job_parameters.model_id,
model_version=job_parameters.model_version,
component_module_name=module_name,
job_parameters=job_parameters)
run_class_paths = component_parameters_on_party.get(
'CodePath').split('/')
run_class_package = '.'.join(
run_class_paths[:-2]) + '.' + run_class_paths[-2].replace(
'.py', '')
run_class_name = run_class_paths[-1]
task_info["party_status"] = TaskStatus.RUNNING
cls.report_task_update_to_driver(task_info=task_info)
# init environment, process is shared globally
RuntimeConfig.init_config(
WORK_MODE=job_parameters.work_mode,
COMPUTING_ENGINE=job_parameters.computing_engine,
FEDERATION_ENGINE=job_parameters.federation_engine,
FEDERATED_MODE=job_parameters.federated_mode)
if RuntimeConfig.COMPUTING_ENGINE == ComputingEngine.EGGROLL:
session_options = task_parameters.eggroll_run.copy()
else:
session_options = {}
sess = session.Session(
computing_type=job_parameters.computing_engine,
federation_type=job_parameters.federation_engine)
computing_session_id = job_utils.generate_session_id(
task_id, task_version, role, party_id)
sess.init_computing(computing_session_id=computing_session_id,
options=session_options)
federation_session_id = job_utils.generate_task_version_id(
task_id, task_version)
component_parameters_on_party[
"job_parameters"] = job_parameters.to_dict()
sess.init_federation(
federation_session_id=federation_session_id,
runtime_conf=component_parameters_on_party,
service_conf=job_parameters.engines_address.get(
EngineType.FEDERATION, {}))
sess.as_default()
schedule_logger().info('Run {} {} {} {} {} task'.format(
job_id, component_name, task_id, role, party_id))
schedule_logger().info("Component parameters on party {}".format(
component_parameters_on_party))
schedule_logger().info("Task input dsl {}".format(task_input_dsl))
task_run_args = cls.get_task_run_args(
job_id=job_id,
role=role,
party_id=party_id,
task_id=task_id,
task_version=task_version,
job_args=job_args_on_party,
job_parameters=job_parameters,
task_parameters=task_parameters,
input_dsl=task_input_dsl,
)
if module_name in {"Upload", "Download", "Reader", "Writer"}:
task_run_args["job_parameters"] = job_parameters
run_object = getattr(importlib.import_module(run_class_package),
run_class_name)()
run_object.set_tracker(tracker=tracker_client)
run_object.set_task_version_id(
task_version_id=job_utils.generate_task_version_id(
task_id, task_version))
# add profile logs
profile.profile_start()
run_object.run(component_parameters_on_party, task_run_args)
profile.profile_ends()
output_data = run_object.save_data()
if not isinstance(output_data, list):
output_data = [output_data]
for index in range(0, len(output_data)):
data_name = task_output_dsl.get(
'data')[index] if task_output_dsl.get(
'data') else '{}'.format(index)
persistent_table_namespace, persistent_table_name = tracker.save_output_data(
computing_table=output_data[index],
output_storage_engine=job_parameters.storage_engine,
output_storage_address=job_parameters.engines_address.get(
EngineType.STORAGE, {}))
if persistent_table_namespace and persistent_table_name:
tracker.log_output_data_info(
data_name=data_name,
table_namespace=persistent_table_namespace,
table_name=persistent_table_name)
output_model = run_object.export_model()
# There is only one model output at the current dsl version.
tracker.save_output_model(
output_model, task_output_dsl['model'][0]
if task_output_dsl.get('model') else 'default')
task_info["party_status"] = TaskStatus.SUCCESS
except Exception as e:
task_info["party_status"] = TaskStatus.FAILED
schedule_logger().exception(e)
finally:
try:
task_info["end_time"] = current_timestamp()
task_info["elapsed"] = task_info["end_time"] - start_time
cls.report_task_update_to_driver(task_info=task_info)
except Exception as e:
task_info["party_status"] = TaskStatus.FAILED
traceback.print_exc()
schedule_logger().exception(e)
schedule_logger().info('task {} {} {} start time: {}'.format(
task_id, role, party_id, timestamp_to_date(start_time)))
schedule_logger().info('task {} {} {} end time: {}'.format(
task_id, role, party_id, timestamp_to_date(task_info["end_time"])))
schedule_logger().info('task {} {} {} takes {}s'.format(
task_id, role, party_id,
int(task_info["elapsed"]) / 1000))
schedule_logger().info('Finish {} {} {} {} {} {} task {}'.format(
job_id, component_name, task_id, task_version, role, party_id,
task_info["party_status"]))
print('Finish {} {} {} {} {} {} task {}'.format(
job_id, component_name, task_id, task_version, role, party_id,
task_info["party_status"]))
return task_info
def submit_job(job_data):
job_id = generate_job_id()
schedule_logger(job_id).info('submit job, job_id {}, body {}'.format(job_id, job_data))
job_dsl = job_data.get('job_dsl', {})
job_runtime_conf = job_data.get('job_runtime_conf', {})
job_utils.check_pipeline_job_runtime_conf(job_runtime_conf)
job_parameters = job_runtime_conf['job_parameters']
job_initiator = job_runtime_conf['initiator']
job_type = job_parameters.get('job_type', '')
if job_type != 'predict':
# generate job model info
job_parameters['model_id'] = '#'.join([dtable_utils.all_party_key(job_runtime_conf['role']), 'model'])
job_parameters['model_version'] = job_id
train_runtime_conf = {}
else:
detect_utils.check_config(job_parameters, ['model_id', 'model_version'])
# get inference dsl from pipeline model as job dsl
job_tracker = Tracking(job_id=job_id, role=job_initiator['role'], party_id=job_initiator['party_id'],
model_id=job_parameters['model_id'], model_version=job_parameters['model_version'])
pipeline_model = job_tracker.get_output_model('pipeline')
job_dsl = json_loads(pipeline_model['Pipeline'].inference_dsl)
train_runtime_conf = json_loads(pipeline_model['Pipeline'].train_runtime_conf)
path_dict = save_job_conf(job_id=job_id,
job_dsl=job_dsl,
job_runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf,
pipeline_dsl=None)
job = Job()
job.f_job_id = job_id
job.f_roles = json_dumps(job_runtime_conf['role'])
job.f_work_mode = job_parameters['work_mode']
job.f_initiator_party_id = job_initiator['party_id']
job.f_dsl = json_dumps(job_dsl)
job.f_runtime_conf = json_dumps(job_runtime_conf)
job.f_train_runtime_conf = json_dumps(train_runtime_conf)
job.f_run_ip = ''
job.f_status = JobStatus.WAITING
job.f_progress = 0
job.f_create_time = current_timestamp()
initiator_role = job_initiator['role']
initiator_party_id = job_initiator['party_id']
if initiator_party_id not in job_runtime_conf['role'][initiator_role]:
schedule_logger(job_id).info("initiator party id error:{}".format(initiator_party_id))
raise Exception("initiator party id error {}".format(initiator_party_id))
get_job_dsl_parser(dsl=job_dsl,
runtime_conf=job_runtime_conf,
train_runtime_conf=train_runtime_conf)
TaskScheduler.distribute_job(job=job, roles=job_runtime_conf['role'], job_initiator=job_initiator)
# push into queue
job_event = job_utils.job_event(job_id, initiator_role, initiator_party_id)
try:
RuntimeConfig.JOB_QUEUE.put_event(job_event)
except Exception as e:
raise Exception('push job into queue failed')
schedule_logger(job_id).info(
'submit job successfully, job id is {}, model id is {}'.format(job.f_job_id, job_parameters['model_id']))
board_url = BOARD_DASHBOARD_URL.format(job_id, job_initiator['role'], job_initiator['party_id'])
logs_directory = get_job_log_directory(job_id)
return job_id, path_dict['job_dsl_path'], path_dict['job_runtime_conf_path'], logs_directory, \
{'model_id': job_parameters['model_id'],'model_version': job_parameters['model_version']}, board_url
