def create_stream_workflow() -> Workflow:
workflow = Workflow()
for i in range(3):
job = TestScheduler.create_job(i, 5)
workflow.add_job(job)
deps = []
deps.append(
JobControlEdge(target_node_id='0_job',
source_node_id='2_job',
met_config=MetConfig(
event_key=generate_job_status_key('0_job'),
event_value=State.FINISHED.value)))
deps.append(
JobControlEdge(target_node_id='1_job',
source_node_id='2_job',
met_config=MetConfig(
event_key=generate_job_status_key('1_job'),
event_value=State.FINISHED.value)))
workflow.add_edges("2_job", deps)
workflow.workflow_id = 1
job = TestScheduler.create_job(3, 1)
workflow.add_job(job)
dep2 = []
dep2.append(
JobControlEdge(target_node_id='0_job',
source_node_id='3_job',
met_config=MetConfig(event_key='key1',
event_value='value1')))
workflow.add_edges('3_job', dep2)
return workflow
def create_restart_job_workflow() -> Workflow:
workflow = Workflow()
for i in range(2):
job = TestScheduler.create_periodic_job(i, 4)
workflow.add_job(job)
job = TestScheduler.create_periodic_job(2, 20)
job.job_config.periodic_config = None
workflow.add_job(job)
deps = []
deps.append(
JobControlEdge(target_node_id='0_job',
source_node_id='2_job',
met_config=MetConfig(
event_key=generate_job_status_key('0_job'),
event_value=State.FINISHED.value,
action=TaskAction.RESTART)))
deps.append(
JobControlEdge(target_node_id='1_job',
source_node_id='2_job',
met_config=MetConfig(
event_key=generate_job_status_key('1_job'),
event_value=State.FINISHED.value,
action=TaskAction.RESTART)))
workflow.add_edges("2_job", deps)
workflow.workflow_id = 1
return workflow
def create_workflow() -> Workflow:
workflow = Workflow()
workflow.project_desc = ProjectDesc()
workflow.project_desc.project_name = "workflow_1"
for i in range(6):
job = TestDAGGenerator.create_dummy_job(i)
if i == 2:
job.job_config = SendEventJobConfig('localhost:50051', 'key_1',
'value_1',
UNDEFINED_EVENT_TYPE)
elif i == 3:
job.job_config = SendEventJobConfig('localhost:50051', 'key_2',
'value_2',
UNDEFINED_EVENT_TYPE)
elif i == 5:
job.job_config = SendEventJobConfig('localhost:50051', 'key_2',
'value_2',
"STOP_SCHEDULER_CMD")
workflow.add_job(job)
dependencies = [
JobControlEdge(target_node_id='0_job',
source_node_id='2_job',
met_config=MetConfig(
event_key=generate_job_status_key('0_job'),
event_value=State.FINISHED.value)),
JobControlEdge(target_node_id='1_job',
source_node_id='2_job',
met_config=MetConfig(
event_key=generate_job_status_key('1_job'),
event_value=State.FINISHED.value))
]
workflow.add_edges("2_job", dependencies)
dependencies = [
JobControlEdge(target_node_id='2_job',
source_node_id='4_job',
met_config=MetConfig(
event_key='key_1',
event_value='value_1',
event_type=UNDEFINED_EVENT_TYPE)),
JobControlEdge(target_node_id='3_job',
source_node_id='4_job',
met_config=MetConfig(
event_key='key_2',
event_value='value_2',
event_type=UNDEFINED_EVENT_TYPE))
]
workflow.add_edges("4_job", dependencies)
dependencies = [
JobControlEdge(target_node_id='4_job',
source_node_id='5_job',
met_config=MetConfig(
event_key=generate_job_status_key('5_job'),
event_value=State.FINISHED.value))
]
workflow.add_edges("5_job", dependencies)
workflow.workflow_id = 1
return workflow
def update_kv_store(self, event: Union[BaseEvent, JobStatusEvent]):
if isinstance(event, BaseEvent):
self.kv_store.update(event.key, event.value)
elif isinstance(event, JobStatusEvent):
if event.job_id not in self.workflow.jobs:
return
job = self.workflow.jobs[event.job_id]
self.log.info("workflow_id {} job_name {} status {}".format(
event.workflow_id, job.job_name, event.status))
key = generate_job_status_key(event.job_id)
self.kv_store.update(key, event.status)
job = self.workflow.jobs[event.job_id]
if State.RUNNING == State(event.status):
self.stat_lock.acquire()
try:
self.starting_jobs.discard(event.job_id)
self.running_jobs.add(event.job_id)
job.status = State.RUNNING
self.client.update_job(job_name=job.job_name,
job_state=State.RUNNING)
finally:
self.stat_lock.release()
elif State.FINISHED == State(event.status):
self.stat_lock.acquire()
try:
self.finished_jobs.add(event.job_id)
self.starting_jobs.discard(event.job_id)
self.running_jobs.discard(event.job_id)
job.status = State.FINISHED
end_time = round(time.time())
job.end_time = end_time
self.client.update_job(job_name=job.job_name,
job_state=State.FINISHED)
self.client.update_job(job_name=job.job_name,
end_time=end_time)
if job.instance_id in self.listen_job_handlers:
self.listener_manager.stop_job_handler_listening(
self.listen_job_handlers[job.instance_id])
del self.listen_job_handlers[job.instance_id]
finally:
self.stat_lock.release()
elif State.FAILED == State(event.status):
self.stat_lock.acquire()
try:
self.failed_jobs.add(event.job_id)
self.running_jobs.discard(event.job_id)
self.starting_jobs.discard(event.job_id)
job.status = State.FAILED
end_time = round(time.time())
job.end_time = end_time
self.client.update_job(job_name=job.job_name,
job_state=State.FAILED)
self.client.update_job(job_name=job.job_name,
end_time=end_time)
if job.instance_id in self.listen_job_handlers:
self.listener_manager.stop_job_handler_listening(
self.listen_job_handlers[job.instance_id])
del self.listen_job_handlers[job.instance_id]
finally:
self.stat_lock.release()
else:
pass