def clean_task(job_id, component_name, task_id, task_version, role, party_id,
content_type):
TaskController.clean_task(job_id=job_id,
task_id=task_id,
task_version=task_version,
role=role,
party_id=int(party_id),
content_type=content_type)
return get_json_result(retcode=0, retmsg='success')
def update_task(job_id, component_name, task_id, task_version, role, party_id):
task_info = {}
task_info.update(request.json)
task_info.update({
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
})
TaskController.update_task(task_info=task_info)
return get_json_result(retcode=0, retmsg='success')
def initialize_task_holder_for_scheduling(cls, role, party_id, components,
common_task_info, provider_info):
for component_name in components:
task_info = {}
task_info.update(common_task_info)
task_info["component_name"] = component_name
task_info["component_module"] = ""
task_info["provider_info"] = provider_info
task_info["component_parameters"] = {}
TaskController.create_task(
role=role,
party_id=party_id,
run_on_this_party=common_task_info["run_on_this_party"],
task_info=task_info)
def report_task(job_id, component_name, task_id, task_version, role, party_id):
task_info = {}
task_info.update(request.json)
task_info.update({
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
})
TaskController.update_task(task_info=task_info)
if task_info.get("party_status"):
if not TaskController.update_task_status(task_info=task_info):
return get_json_result(retcode=RetCode.OPERATING_ERROR,
retmsg="update task status failed")
return get_json_result(retcode=0, retmsg='success')
def _run(self):
result = {}
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=self.args.dsl,
runtime_conf=self.args.runtime_conf,
train_runtime_conf=self.args.train_runtime_conf,
pipeline_dsl=self.args.pipeline_dsl)
provider = ComponentProvider(**self.args.config["provider"])
common_task_info = self.args.config["common_task_info"]
log_msg = f"initialize the components: {self.args.config['components']}"
LOGGER.info(
start_log(log_msg,
role=self.args.role,
party_id=self.args.party_id))
for component_name in self.args.config["components"]:
result[component_name] = {}
task_info = {}
task_info.update(common_task_info)
parameters, user_specified_parameters = ProviderManager.get_component_parameters(
dsl_parser=dsl_parser,
component_name=component_name,
role=self.args.role,
party_id=self.args.party_id,
provider=provider)
if parameters:
task_info = {}
task_info.update(common_task_info)
task_info["component_name"] = component_name
task_info["component_module"] = parameters["module"]
task_info["provider_info"] = provider.to_dict()
task_info["component_parameters"] = parameters
TaskController.create_task(
role=self.args.role,
party_id=self.args.party_id,
run_on_this_party=common_task_info["run_on_this_party"],
task_info=task_info)
result[component_name]["need_run"] = True
else:
# The party does not need to run, pass
result[component_name]["need_run"] = False
LOGGER.info(
successful_log(log_msg,
role=self.args.role,
party_id=self.args.party_id))
return result
def report_task(job_id, component_name, task_id, task_version, role, party_id):
task_info = {}
task_info.update(request.json)
task_info.update({
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
})
TaskController.update_task(task_info=task_info)
if task_info.get("party_status"):
if not TaskController.update_task_status(task_info=task_info):
return get_json_result(
retcode=RetCode.NOT_EFFECTIVE,
retmsg="update job status does not take effect")
return get_json_result(retcode=0, retmsg='success')
def initialize_tasks(cls,
job_id,
role,
party_id,
run_on_this_party,
initiator_role,
initiator_party_id,
job_parameters: RunParameters,
dsl_parser,
component_name=None,
task_version=None):
common_task_info = {}
common_task_info["job_id"] = job_id
common_task_info["initiator_role"] = initiator_role
common_task_info["initiator_party_id"] = initiator_party_id
common_task_info["role"] = role
common_task_info["party_id"] = party_id
common_task_info["federated_mode"] = job_parameters.federated_mode
common_task_info[
"federated_status_collect_type"] = job_parameters.federated_status_collect_type
if task_version:
common_task_info["task_version"] = task_version
if not component_name:
components = dsl_parser.get_topology_components()
else:
components = [
dsl_parser.get_component_info(component_name=component_name)
]
for component in components:
component_parameters = component.get_role_parameters()
for parameters_on_party in component_parameters.get(
common_task_info["role"], []):
if parameters_on_party.get(
'local',
{}).get('party_id') == common_task_info["party_id"]:
task_info = {}
task_info.update(common_task_info)
task_info["component_name"] = component.get_name()
TaskController.create_task(
role=role,
party_id=party_id,
run_on_this_party=run_on_this_party,
task_info=task_info)
def stop_task(job_id, component_name, task_id, task_version, role, party_id,
stop_status):
tasks = JobSaver.query_task(job_id=job_id,
task_id=task_id,
task_version=task_version,
role=role,
party_id=int(party_id))
kill_status = True
for task in tasks:
kill_status = kill_status & TaskController.stop_task(
task=task, stop_status=stop_status)
return get_json_result(
retcode=RetCode.SUCCESS if kill_status else RetCode.EXCEPTION_ERROR,
retmsg='success' if kill_status else 'failed')
def collect_task(job_id, component_name, task_id, task_version, role,
party_id):
task_info = TaskController.collect_task(job_id=job_id,
component_name=component_name,
task_id=task_id,
task_version=task_version,
role=role,
party_id=party_id)
if task_info:
return get_json_result(retcode=RetCode.SUCCESS,
retmsg="success",
data=task_info)
else:
return get_json_result(retcode=RetCode.OPERATING_ERROR,
retmsg="query task failed")
def task_status(job_id, component_name, task_id, task_version, role, party_id,
status):
task_info = {}
task_info.update({
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
"status": status
})
if TaskController.update_task_status(task_info=task_info):
return get_json_result(retcode=0, retmsg='success')
else:
return get_json_result(retcode=RetCode.OPERATING_ERROR,
retmsg="update task status failed")
def task_status(job_id, component_name, task_id, task_version, role, party_id,
status):
task_info = {}
task_info.update({
"job_id": job_id,
"task_id": task_id,
"task_version": task_version,
"role": role,
"party_id": party_id,
"status": status
})
if TaskController.update_task_status(task_info=task_info):
return get_json_result(retcode=0, retmsg='success')
else:
return get_json_result(retcode=RetCode.NOT_EFFECTIVE,
retmsg="update job status does not take effect")
def stop_job(cls, job, stop_status):
tasks = JobSaver.query_task(job_id=job.f_job_id,
role=job.f_role,
party_id=job.f_party_id,
reverse=True)
kill_status = True
kill_details = {}
for task in tasks:
kill_task_status = TaskController.stop_task(
task=task, stop_status=stop_status)
kill_status = kill_status & kill_task_status
kill_details[
task.f_task_id] = 'success' if kill_task_status else 'failed'
if kill_status:
job_info = job.to_human_model_dict(only_primary_with=["status"])
job_info["status"] = stop_status
JobController.update_job_status(job_info)
return kill_status, kill_details
def start_task(job_id, component_name, task_id, task_version, role, party_id):
TaskController.start_task(job_id, component_name, task_id, task_version,
role, party_id, **request.json)
return get_json_result(retcode=0, retmsg='success')