def set_schedule(self, activity_id, service_act, process_id, version, parent_data):
wait_callback = service_act.service.interval is None
schedule = self.create(id="%s%s" % (activity_id, version), activity_id=activity_id, service_act=service_act,
process_id=process_id, wait_callback=wait_callback, version=version)
data_service.set_schedule_data(schedule.id, parent_data)
if not wait_callback:
count_down = service_act.service.interval.next()
valve.send(signals, 'schedule_ready', sender=ScheduleService, process_id=process_id,
schedule_id=schedule.id,
countdown=count_down)
return schedule
def schedule(process_id, schedule_id):
"""
调度服务主函数
:param process_id: 被调度的节点所属的 PipelineProcess
:param schedule_id: 调度 ID
:return:
"""
with schedule_exception_handler(process_id, schedule_id):
ScheduleService.objects.filter(id=schedule_id).update(
is_scheduling=True)
sched_service = ScheduleService.objects.get(id=schedule_id)
service_act = sched_service.service_act
act_id = sched_service.activity_id
version = sched_service.version
if not Status.objects.filter(id=act_id, version=version).exists():
# forced failed
logger.warning(
'schedule service failed, schedule(%s - %s) had been forced exit.'
% (act_id, version))
sched_service.destroy()
return
# get data
parent_data = get_schedule_parent_data(sched_service.id)
if parent_data is None:
raise exceptions.DataRetrieveError(
'child process(%s) retrieve parent_data error, sched_id: %s' %
(process_id, schedule_id))
# schedule
ex_data = None
success = False
try:
success = service_act.schedule(parent_data,
sched_service.callback_data)
if success is None:
success = True
except Exception as e:
if service_act.error_ignorable:
success = True
service_act.ignore_error()
service_act.finish_schedule()
ex_data = traceback.format_exc(e)
logging.error(ex_data)
sched_service.schedule_times += 1
set_schedule_data(sched_service.id, parent_data)
# schedule failed
if not success:
if not Status.objects.transit(
id=act_id, version=version, to_state=states.FAILED).result:
# forced failed
logger.warning(
'FAILED transit failed, schedule(%s - %s) had been forced exit.'
% (act_id, version))
sched_service.destroy()
return
if service_act.timeout:
signals.service_activity_timeout_monitor_end.send(
sender=service_act.__class__,
node_id=service_act.id,
version=version)
logger.info('node %s %s timeout monitor revoke' %
(service_act.id, version))
Data.objects.write_node_data(service_act, ex_data=ex_data)
process = PipelineProcess.objects.get(id=sched_service.process_id)
process.adjust_status()
# send activity error signal
try:
service_act.schedule_fail()
except Exception as e:
logger.error('schedule_fail handler fail: %s' %
traceback.format_exc(e))
signals.service_schedule_fail.send(sender=ScheduleService,
activity_shell=service_act,
schedule_service=sched_service,
ex_data=ex_data)
valve.send(signals,
'activity_failed',
sender=process.root_pipeline,
pipeline_id=process.root_pipeline_id,
pipeline_activity_id=service_act.id)
return
# schedule execute finished or callback finished
if service_act.is_schedule_done() or sched_service.wait_callback:
error_ignorable = not service_act.get_result_bit()
if not Status.objects.transit(id=act_id,
version=version,
to_state=states.FINISHED).result:
# forced failed
logger.warning(
'FINISHED transit failed, schedule(%s - %s) had been forced exit.'
% (act_id, version))
sched_service.destroy()
return
if service_act.timeout:
signals.service_activity_timeout_monitor_end.send(
sender=service_act.__class__,
node_id=service_act.id,
version=version)
logger.info('node %s %s timeout monitor revoke' %
(service_act.id, version))
Data.objects.write_node_data(service_act)
if error_ignorable:
s = Status.objects.get(id=act_id)
s.error_ignorable = True
s.save()
# sync parent data
with transaction.atomic():
process = PipelineProcess.objects.select_for_update().get(
id=sched_service.process_id)
if not process.is_alive:
logger.warning('schedule(%s - %s) revoked.' %
(act_id, version))
sched_service.destroy()
return
process.top_pipeline.data.update_outputs(
parent_data.get_outputs())
# extract outputs
process.top_pipeline.context.extract_output(service_act)
process.save(save_snapshot=True)
# clear temp data
delete_parent_data(sched_service.id)
# save schedule service
sched_service.finish()
signals.service_schedule_success.send(
sender=ScheduleService,
activity_shell=service_act,
schedule_service=sched_service)
valve.send(signals,
'wake_from_schedule',
sender=ScheduleService,
process_id=sched_service.process_id,
activity_id=sched_service.activity_id)
else:
sched_service.set_next_schedule()
def schedule(process_id, schedule_id):
"""
调度服务主函数
:param process_id: 被调度的节点所属的 PipelineProcess
:param schedule_id: 调度 ID
:return:
"""
with schedule_exception_handler(process_id, schedule_id):
ScheduleService.objects.filter(id=schedule_id).update(
is_scheduling=True)
sched_service = ScheduleService.objects.get(id=schedule_id)
service_act = sched_service.service_act
act_id = sched_service.activity_id
version = sched_service.version
if not Status.objects.filter(id=act_id, version=version).exists():
# forced failed
logger.warning('schedule(%s - %s) forced exit.' %
(act_id, version))
sched_service.destroy()
return
# get data
parent_data = get_schedule_parent_data(sched_service.id)
# schedule
ex_data = None
result = False
try:
result = service_act.schedule(parent_data,
sched_service.callback_data)
except Exception as e:
if service_act.error_ignorable:
result = True
service_act.ignore_error()
service_act.service.finish_schedule()
else:
# send activity error signal
process = PipelineProcess.objects.get(
id=sched_service.process_id)
pipeline_inst = PipelineInstance.objects.get(
instance_id=process.root_pipeline_id)
valve.send(signals,
'activity_failed',
sender=process.root_pipeline,
pipeline_instance=pipeline_inst,
pipeline_activity_id=service_act.id)
ex_data = traceback.format_exc(e)
logging.error(ex_data)
sched_service.schedule_times += 1
set_schedule_data(sched_service.id, parent_data)
# schedule failed
if result is False:
Data.objects.write_node_data(service_act, ex_data=ex_data)
if not Status.objects.transit(
id=act_id, version=version, to_state=states.FAILED):
# forced failed
logger.warning('schedule(%s - %s) forced exit.' %
(act_id, version))
sched_service.destroy()
return
process = PipelineProcess.objects.get(id=sched_service.process_id)
process.adjust_status()
# send activity error signal
pipeline_inst = PipelineInstance.objects.get(
instance_id=process.root_pipeline_id)
valve.send(signals,
'activity_failed',
sender=process.root_pipeline,
pipeline_instance=pipeline_inst,
pipeline_activity_id=service_act.id)
return
# schedule execute finished or callback finished
if service_act.schedule_done() or sched_service.wait_callback:
# write node data and transit status
Data.objects.write_node_data(service_act)
if not Status.objects.transit(
id=act_id, version=version, to_state=states.FINISHED):
# forced failed
logger.warning('schedule(%s - %s) forced exit.' %
(act_id, version))
sched_service.destroy()
return
# sync parent data
with transaction.atomic():
process = PipelineProcess.objects.select_for_update().get(
id=sched_service.process_id)
if not process.is_alive:
logger.warning('schedule(%s - %s) revoked.' %
(act_id, version))
sched_service.destroy()
return
process.top_pipeline.data.update_outputs(
parent_data.get_outputs())
# extract outputs
process.top_pipeline.context().extract_output(service_act)
process.save(save_snapshot=True)
# clear temp data
delete_parent_data(sched_service.id)
# save schedule service
sched_service.finish()
valve.send(signals,
'wake_from_schedule',
sender=ScheduleService,
process_id=sched_service.process_id,
activity_id=sched_service.activity_id)
else:
sched_service.set_next_schedule()