def capture_input(self):
t0 = time()
self.task_id = t0
self.task_obj = {
"id": self.task_id,
"startTime": t0,
"wf_execution": self.wfexec_id
}
if self.parent_cycle_iteration:
self.task_obj[
"parent_cycle_iteration"] = self.parent_cycle_iteration
if self.parent_cycle_name:
self.task_obj["parent_cycle_name"] = self.parent_cycle_name
obj = {
"task": self.task_obj,
"dt": self.data_transformation_name,
"type": "Input",
"values": self.input_args
}
retrospective_json = {
"prov_obj": obj,
"dataflow_name": self.workflow_name,
"act_type": Task.ACT_TYPE
}
prov_utils.append_log(retrospective_json, self.log_dir,
self.workflow_name, self.wfexec_id)
return self
def _capture_output(self,
values: dict,
stdout: str = None,
stderr: str = None):
'''
:param task_id: Identifier of the task created in collect_in
:param dt: Data Transformation key string
:param type: i (input) or o (output)
:param values: dict containing the expected arguments to save
:param stdout: Optional argument for stdout msgs
:param stderr: Optional argument for stderr msgs
'''
self.task_obj["endTime"] = time()
self.task_obj["status"] = "FINISHED"
if stdout:
self.task_obj["stdout"] = stdout
if stderr:
self.task_obj["stderr"] = stderr
obj = {
"task": self.task_obj,
"dt": self.data_transformation_name,
"type": "Output",
"values": values
}
retrospective_json = {
"prov_obj": obj,
"dataflow_name": self.workflow_name,
"act_type": Task.ACT_TYPE
}
prov_utils.append_log(retrospective_json, self.log_dir,
self.workflow_name, self.wfexec_id)
def begin(workflow_name: str, log_dir: str) -> float:
wf_start_time = time()
obj = {"wf_execution": wf_start_time, "startTime": wf_start_time}
retrospective_json = {
"prov_obj": obj,
"dataflow_name": workflow_name,
"act_type": Workflow.ACT_TYPE
}
if not os.path.exists(log_dir):
os.makedirs(log_dir)
prov_utils.append_log(retrospective_json, log_dir, workflow_name,
str(wf_start_time))
return wf_start_time
def end(workflow_name: str, wf_start_time: float, log_dir: str):
obj = {
"wf_execution": wf_start_time,
"starTime": wf_start_time,
"endTime": time(),
"status": "FINISHED"
}
retrospective_json = {
"prov_obj": obj,
"dataflow_name": workflow_name,
"act_type": Workflow.ACT_TYPE
}
prov_utils.append_log(retrospective_json, log_dir, workflow_name,
str(wf_start_time))
def capture_input(self):
t0 = time()
self.prov_obj = {
"iteration_id": self.iteration_id,
"startTime": t0,
"wf_execution": self.wfexec_id
}
obj = {
"cycle": self.prov_obj,
"cycle_name": self.cycle_name,
"type": "Input",
"values": self.input_args
}
retrospective_json = {
"prov_obj": obj,
"dataflow_name": self.workflow_name,
"act_type": CycleIteration.ACT_TYPE
}
prov_utils.append_log(retrospective_json, self.log_dir,
self.workflow_name, self.wfexec_id)
return self