def proto_to_execution_meta(
workflow_execution_proto: WorkflowExecutionProto
) -> WorkflowExecutionMeta:
execution_state = State(
StateProto.Name(workflow_execution_proto.execution_state))
properties = workflow_execution_proto.properties
if properties == {}:
properties = None
return WorkflowExecutionMeta(
uuid=workflow_execution_proto.uuid,
name=workflow_execution_proto.name,
execution_state=execution_state,
properties=properties,
project_id=workflow_execution_proto.project_id.value
if workflow_execution_proto.HasField('project_id') else None,
start_time=workflow_execution_proto.start_time.value
if workflow_execution_proto.HasField('start_time') else None,
end_time=workflow_execution_proto.end_time.value
if workflow_execution_proto.HasField('end_time') else None,
log_uri=workflow_execution_proto.log_uri.value
if workflow_execution_proto.HasField('log_uri') else None,
workflow_json=workflow_execution_proto.workflow_json.value
if workflow_execution_proto.HasField('workflow_json') else None,
signature=workflow_execution_proto.signature.value
if workflow_execution_proto.HasField('signature') else None)
def updateJob(self, request, context):
properties = None if request.properties == {} else request.properties
job_state = None if request.job_state == 0 else State(StateProto.Name(request.job_state))
job = self.store.update_job(job_name=request.name, job_state=job_state,
properties=properties,
job_id=request.job_id.value if request.HasField('job_id') else None,
workflow_execution_id=request.workflow_execution_id.value
if request.HasField('workflow_execution_id') else None,
end_time=request.end_time.value if request.HasField('end_time') else None,
log_uri=request.log_uri.value if request.HasField('log_uri') else None,
signature=request.signature.value if request.HasField('signature') else None)
return _wrap_meta_response(MetaToProto.job_meta_to_proto(job))
def result_to_job_meta(job_result) -> JobMeta:
properties = job_result.properties
if properties is not None:
properties = ast.literal_eval(properties)
state = State(job_result.job_state)
return JobMeta(uuid=job_result.uuid,
name=job_result.name,
job_id=job_result.job_id,
properties=properties,
start_time=job_result.start_time,
end_time=job_result.end_time,
job_state=state,
log_uri=job_result.log_uri,
workflow_execution_id=job_result.workflow_execution_id,
signature=job_result.signature)
def updateWorkflowExecution(self, request, context):
properties = None if request.properties == {} else request.properties
execution_state = None if request.execution_state == 0 else State(StateProto.Name(request.execution_state))
workflow_execution = self.store.update_workflow_execution(execution_name=request.name,
execution_state=execution_state,
project_id=request.project_id.value if request.HasField(
'project_id') else None,
properties=properties,
end_time=request.end_time.value if request.HasField(
'end_time') else None,
log_uri=request.log_uri_value if request.HasField(
'log_uri') else None,
workflow_json=request.workjson.value if request.HasField(
'workflow_json') else None,
signature=request.signature.value if request.HasField(
'signature') else None)
return _wrap_meta_response(MetaToProto.workflow_execution_meta_to_proto(workflow_execution))
def result_to_workflow_execution_meta(
execution_result) -> WorkflowExecutionMeta:
properties = execution_result.properties
if properties is not None:
properties = ast.literal_eval(properties)
execution_state = State(execution_result.execution_state)
return WorkflowExecutionMeta(
uuid=execution_result.uuid,
name=execution_result.name,
properties=properties,
start_time=execution_result.start_time,
end_time=execution_result.end_time,
execution_state=execution_state,
log_uri=execution_result.log_uri,
project_id=execution_result.project_id,
workflow_json=execution_result.workflow_json,
signature=execution_result.signature)
def transform_job_meta(job_proto):
job_state = State(StateProto.Name(job_proto.job_state))
properties = job_proto.properties
if properties == {}:
properties = None
return JobMeta(name=job_proto.name,
job_state=job_state,
properties=properties,
workflow_execution_id=job_proto.workflow_execution_id.value
if job_proto.HasField('workflow_execution_id') else None,
job_id=job_proto.job_id.value
if job_proto.HasField('job_id') else None,
start_time=job_proto.start_time.value
if job_proto.HasField('start_time') else None,
end_time=job_proto.end_time.value
if job_proto.HasField('end_time') else None,
log_uri=job_proto.log_uri.value
if job_proto.HasField('log_uri') else None,
signature=job_proto.signature.value
if job_proto.HasField('signature') else None)
def update_kv_store(self, event: Union[BaseEvent, JobStatusEvent]):
if isinstance(event, BaseEvent):
self.kv_store.update(event.key, event.value)
elif isinstance(event, JobStatusEvent):
if event.job_id not in self.workflow.jobs:
return
job = self.workflow.jobs[event.job_id]
self.log.info("workflow_id {} job_name {} status {}".format(
event.workflow_id, job.job_name, event.status))
key = generate_job_status_key(event.job_id)
self.kv_store.update(key, event.status)
job = self.workflow.jobs[event.job_id]
if State.RUNNING == State(event.status):
self.stat_lock.acquire()
try:
self.starting_jobs.discard(event.job_id)
self.running_jobs.add(event.job_id)
job.status = State.RUNNING
self.client.update_job(job_name=job.job_name,
job_state=State.RUNNING)
finally:
self.stat_lock.release()
elif State.FINISHED == State(event.status):
self.stat_lock.acquire()
try:
self.finished_jobs.add(event.job_id)
self.starting_jobs.discard(event.job_id)
self.running_jobs.discard(event.job_id)
job.status = State.FINISHED
end_time = round(time.time())
job.end_time = end_time
self.client.update_job(job_name=job.job_name,
job_state=State.FINISHED)
self.client.update_job(job_name=job.job_name,
end_time=end_time)
if job.instance_id in self.listen_job_handlers:
self.listener_manager.stop_job_handler_listening(
self.listen_job_handlers[job.instance_id])
del self.listen_job_handlers[job.instance_id]
finally:
self.stat_lock.release()
elif State.FAILED == State(event.status):
self.stat_lock.acquire()
try:
self.failed_jobs.add(event.job_id)
self.running_jobs.discard(event.job_id)
self.starting_jobs.discard(event.job_id)
job.status = State.FAILED
end_time = round(time.time())
job.end_time = end_time
self.client.update_job(job_name=job.job_name,
job_state=State.FAILED)
self.client.update_job(job_name=job.job_name,
end_time=end_time)
if job.instance_id in self.listen_job_handlers:
self.listener_manager.stop_job_handler_listening(
self.listen_job_handlers[job.instance_id])
del self.listen_job_handlers[job.instance_id]
finally:
self.stat_lock.release()
else:
pass