def schedule_running_job(cls, job, force_sync_status=False):
schedule_logger(job_id=job.f_job_id).info("scheduling job {}".format(
job.f_job_id))
dsl_parser = schedule_utils.get_job_dsl_parser(
dsl=job.f_dsl,
runtime_conf=job.f_runtime_conf_on_party,
train_runtime_conf=job.f_train_runtime_conf)
task_scheduling_status_code, tasks = TaskScheduler.schedule(
job=job, dsl_parser=dsl_parser, canceled=job.f_cancel_signal)
tasks_status = [task.f_status for task in tasks]
new_job_status = cls.calculate_job_status(
task_scheduling_status_code=task_scheduling_status_code,
tasks_status=tasks_status)
if new_job_status == JobStatus.WAITING and job.f_cancel_signal:
new_job_status = JobStatus.CANCELED
total, finished_count = cls.calculate_job_progress(
tasks_status=tasks_status)
new_progress = float(finished_count) / total * 100
schedule_logger(job_id=job.f_job_id).info(
"Job {} status is {}, calculate by task status list: {}".format(
job.f_job_id, new_job_status, tasks_status))
if new_job_status != job.f_status or new_progress != job.f_progress:
# Make sure to update separately, because these two fields update with anti-weight logic
if int(new_progress) - job.f_progress > 0:
job.f_progress = new_progress
FederatedScheduler.sync_job(job=job,
update_fields=["progress"])
cls.update_job_on_initiator(initiator_job=job,
update_fields=["progress"])
if new_job_status != job.f_status:
job.f_status = new_job_status
if EndStatus.contains(job.f_status):
FederatedScheduler.save_pipelined_model(job=job)
FederatedScheduler.sync_job_status(job=job)
cls.update_job_on_initiator(initiator_job=job,
update_fields=["status"])
if EndStatus.contains(job.f_status):
cls.finish(job=job, end_status=job.f_status)
if force_sync_status:
FederatedScheduler.sync_job_status(job=job)
schedule_logger(job_id=job.f_job_id).info(
"finish scheduling job {}".format(job.f_job_id))
def update_task_status(cls, task_info):
update_status = JobSaver.update_task_status(task_info=task_info)
if update_status and EndStatus.contains(task_info.get("status")):
ResourceManager.return_task_resource(task_info=task_info)
cls.clean_task(job_id=task_info["job_id"],
task_id=task_info["task_id"],
task_version=task_info["task_version"],
role=task_info["role"],
party_id=task_info["party_id"],
content_type="table")
cls.report_task_to_initiator(task_info=task_info)
return update_status
def schedule(cls, job, dsl_parser, canceled=False):
schedule_logger(job_id=job.f_job_id).info("scheduling job {} tasks".format(job.f_job_id))
initiator_tasks_group = JobSaver.get_tasks_asc(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id)
waiting_tasks = []
for initiator_task in initiator_tasks_group.values():
# collect all party task party status
if job.f_runtime_conf_on_party["job_parameters"]["federated_status_collect_type"] == FederatedCommunicationType.PULL:
tasks_on_all_party = JobSaver.query_task(task_id=initiator_task.f_task_id, task_version=initiator_task.f_task_version)
tasks_status_on_all = set([task.f_status for task in tasks_on_all_party])
if len(tasks_status_on_all) > 1 or TaskStatus.RUNNING in tasks_status_on_all:
cls.collect_task_of_all_party(job=job, task=initiator_task)
new_task_status = cls.federated_task_status(job_id=initiator_task.f_job_id, task_id=initiator_task.f_task_id, task_version=initiator_task.f_task_version)
task_status_have_update = False
if new_task_status != initiator_task.f_status:
task_status_have_update = True
initiator_task.f_status = new_task_status
FederatedScheduler.sync_task_status(job=job, task=initiator_task)
if initiator_task.f_status == TaskStatus.WAITING:
waiting_tasks.append(initiator_task)
elif task_status_have_update and EndStatus.contains(initiator_task.f_status):
FederatedScheduler.stop_task(job=job, task=initiator_task, stop_status=initiator_task.f_status)
scheduling_status_code = SchedulingStatusCode.NO_NEXT
if not canceled:
for waiting_task in waiting_tasks:
for component in dsl_parser.get_upstream_dependent_components(component_name=waiting_task.f_component_name):
dependent_task = initiator_tasks_group[
JobSaver.task_key(task_id=job_utils.generate_task_id(job_id=job.f_job_id, component_name=component.get_name()),
role=job.f_role,
party_id=job.f_party_id
)
]
if dependent_task.f_status != TaskStatus.SUCCESS:
# can not start task
break
else:
# all upstream dependent tasks have been successful, can start this task
scheduling_status_code = SchedulingStatusCode.HAVE_NEXT
status_code = cls.start_task(job=job, task=waiting_task)
if status_code == SchedulingStatusCode.NO_RESOURCE:
# wait for the next round of scheduling
schedule_logger(job_id=job.f_job_id).info(f"job {waiting_task.f_job_id} task {waiting_task.f_task_id} can not apply resource, wait for the next round of scheduling")
break
elif status_code == SchedulingStatusCode.FAILED:
scheduling_status_code = SchedulingStatusCode.FAILED
break
else:
schedule_logger(job_id=job.f_job_id).info("have cancel signal, pass start job {} tasks".format(job.f_job_id))
schedule_logger(job_id=job.f_job_id).info("finish scheduling job {} tasks".format(job.f_job_id))
return scheduling_status_code, initiator_tasks_group.values()
def update_job_status(cls, job_info):
schedule_logger(job_id=job_info["job_id"]).info("try to update job {} status to {}".format(job_info["job_id"], job_info.get("status")))
update_status = cls.update_status(Job, job_info)
if update_status:
schedule_logger(job_id=job_info["job_id"]).info("update job {} status successfully".format(job_info["job_id"]))
if EndStatus.contains(job_info.get("status")):
new_job_info = {}
for k in ["job_id", "role", "party_id"]:
new_job_info[k] = job_info[k]
new_job_info["tag"] = "job_end"
cls.update_entity_table(Job, new_job_info)
else:
schedule_logger(job_id=job_info["job_id"]).info("update job {} status does not take effect".format(job_info["job_id"]))
return update_status
def update_status(cls, entity_model, entity_info):
query_filters = []
primary_keys = entity_model.get_primary_keys_name()
for p_k in primary_keys:
query_filters.append(
operator.attrgetter(p_k)(entity_model) == entity_info[
p_k.lstrip("f").lstrip("_")])
objs = entity_model.select().where(*query_filters)
if objs:
obj = objs[0]
else:
raise Exception("can not found the obj to update")
update_filters = query_filters[:]
update_info = {"job_id": entity_info["job_id"]}
for status_field in cls.STATUS_FIELDS:
if entity_info.get(status_field) and hasattr(
entity_model, f"f_{status_field}"):
if status_field in ["status", "party_status"]:
update_info[status_field] = entity_info[status_field]
old_status = getattr(obj, f"f_{status_field}")
new_status = update_info[status_field]
if_pass = False
if isinstance(obj, Task):
if TaskStatus.StateTransitionRule.if_pass(
src_status=old_status, dest_status=new_status):
if_pass = True
elif isinstance(obj, Job):
if JobStatus.StateTransitionRule.if_pass(
src_status=old_status, dest_status=new_status):
if_pass = True
if EndStatus.contains(
new_status) and new_status not in {
JobStatus.SUCCESS, JobStatus.CANCELED
}:
update_filters.append(Job.f_rerun_signal == False)
if if_pass:
update_filters.append(
operator.attrgetter(f"f_{status_field}")(type(obj))
== old_status)
else:
# not allow update status
update_info.pop(status_field)
return cls.execute_update(old_obj=obj,
model=entity_model,
update_info=update_info,
update_filters=update_filters)
def schedule_rerun_job(cls, job):
if EndStatus.contains(job.f_status):
job.f_status = JobStatus.WAITING
job.f_ready_signal = False
job.f_ready_time = None
job.f_rerun_signal = False
job.f_progress = 0
job.f_end_time = None
job.f_elapsed = None
schedule_logger(job_id=job.f_job_id).info(f"job {job.f_job_id} has been finished, set waiting to rerun")
status, response = FederatedScheduler.sync_job_status(job=job)
if status == FederatedSchedulingStatusCode.SUCCESS:
cls.rerun_signal(job_id=job.f_job_id, set_or_reset=False)
FederatedScheduler.sync_job(job=job, update_fields=["ready_signal", "ready_time", "rerun_signal", "progress", "end_time", "elapsed"])
schedule_logger(job_id=job.f_job_id).info(f"job {job.f_job_id} set waiting to rerun successfully")
else:
schedule_logger(job_id=job.f_job_id).info(f"job {job.f_job_id} set waiting to rerun failed")
else:
cls.rerun_signal(job_id=job.f_job_id, set_or_reset=False)
cls.schedule_running_job(job)
def test_tracking(self):
with open(
os.path.join(file_utils.get_python_base_directory(),
self.dsl_path), 'r') as f:
dsl_data = json.load(f)
with open(
os.path.join(file_utils.get_python_base_directory(),
self.config_path), 'r') as f:
config_data = json.load(f)
config_data['job_parameters']['work_mode'] = WORK_MODE
config_data["initiator"]["party_id"] = self.guest_party_id
config_data["role"] = {
"guest": [self.guest_party_id],
"host": [self.host_party_id],
"arbiter": [self.host_party_id]
}
response = requests.post("/".join([self.server_url, 'job', 'submit']),
json={
'job_dsl': dsl_data,
'job_runtime_conf': config_data
})
self.assertTrue(response.status_code in [200, 201])
self.assertTrue(int(response.json()['retcode']) == 0)
job_id = response.json()['jobId']
job_info = {'f_status': 'running'}
for i in range(60):
response = requests.post("/".join(
[self.server_url, 'job', 'query']),
json={
'job_id': job_id,
'role': 'guest'
})
self.assertTrue(response.status_code in [200, 201])
job_info = response.json()['data'][0]
if EndStatus.contains(job_info['f_status']):
break
time.sleep(self.sleep_time)
print('waiting job run success, the job has been running for {}s'.
format((i + 1) * self.sleep_time))
self.assertTrue(job_info['f_status'] == JobStatus.SUCCESS)
os.makedirs(self.success_job_dir, exist_ok=True)
with open(os.path.join(self.success_job_dir, job_id), 'w') as fw:
json.dump(job_info, fw)
self.assertTrue(
os.path.exists(os.path.join(self.success_job_dir, job_id)))
# test_component_parameters
test_component(self, 'component/parameters')
# test_component_metric_all
test_component(self, 'component/metric/all')
# test_component_metric
test_component(self, 'component/metrics')
# test_component_output_model
test_component(self, 'component/output/model')
# test_component_output_data_download
test_component(self, 'component/output/data')
# test_component_output_data_download
test_component(self, 'component/output/data/download')
# test_job_data_view
test_component(self, 'job/data_view')
def update_job_status(cls, job_info):
update_status = JobSaver.update_job_status(job_info=job_info)
if update_status and EndStatus.contains(job_info.get("status")):
ResourceManager.return_job_resource(job_id=job_info["job_id"], role=job_info["role"], party_id=job_info["party_id"])
return update_status