def test_get_tasks(self):
in_task = InputTask()
inc_task = Inc()
pow_task = Pow()
task_names = ['InputTask', 'Inc', 'Pow']
workflow_tasks = [in_task, inc_task, pow_task]
eow = LinearWorkflow(*workflow_tasks)
returned_tasks = eow.get_tasks()
# check if tasks are present
for task_name in task_names:
self.assertIn(task_name, returned_tasks.keys())
# check if tasks still work
arguments_dict = {
in_task: {'val': 2},
inc_task: {'d': 2},
pow_task: {'n': 3}
}
res_workflow = eow.execute(arguments_dict)
res_workflow_value = [res_workflow[key] for key in res_workflow.keys()][0]
for idx, task in enumerate(workflow_tasks):
if idx == 0:
res_tasks_value = task.execute(**arguments_dict[task])
else:
res_tasks_value = task.execute(res_tasks_value, **arguments_dict[task])
self.assertEqual(res_workflow_value, res_tasks_value)
def test_linear_workflow(self):
in_task = InputTask()
in_task_name = 'My input task'
inc_task = Inc()
pow_task = Pow()
eow = LinearWorkflow((in_task, in_task_name), inc_task, inc_task,
pow_task)
res = eow.execute({
in_task: {
'val': 2
},
inc_task: {
'd': 2
}, # Note that this will assign value only to one instance of Inc task
pow_task: {
'n': 3
}
})
self.assertEqual(res[pow_task], (2 + 2 + 1)**3)
task_map = eow.get_tasks()
self.assertTrue(
in_task_name in task_map,
"A task with name '{}' should be amongst tasks".format(
in_task_name))
self.assertEqual(
task_map[in_task_name], in_task,
"A task with name '{}' should map into {}".format(
in_task_name, in_task))
def test_get_tasks(self):
in_task = InputTask()
inc_task = Inc()
task_names = ['InputTask', 'Inc', 'Inc_1', 'Inc_2']
eow = LinearWorkflow(in_task, inc_task, inc_task, inc_task)
returned_tasks = eow.get_tasks()
# check if tasks are present
self.assertEqual(sorted(task_names), sorted(returned_tasks))
# check if tasks still work
arguments_dict = {in_task: {'val': 2}, inc_task: {'d': 2}}
res_workflow = eow.execute(arguments_dict)
res_workflow_value = list(res_workflow.values())
res_tasks_values = []
for idx, task in enumerate(returned_tasks.values()):
res_tasks_values = [
task.execute(*res_tasks_values, **arguments_dict.get(task, {}))
]
self.assertEqual(res_workflow_value, res_tasks_values)
def get_exec_args(workflow: LinearWorkflow, eopatch_list: List[str], config: PostProcessConfig) -> List[dict]:
""" Utility function to get execution arguments """
exec_args = []
tasks = workflow.get_tasks()
load_bbox = LoadTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}', features=[FeatureType.BBOX])
for name in tqdm(eopatch_list):
single_exec_dict = {}
try:
eop = load_bbox.execute(eopatch_folder=name)
for task_name, task in tasks.items():
if isinstance(task, ExportToTiff):
single_exec_dict[task] = dict(filename=f'{name}-{eop.bbox.crs.epsg}.tiff')
if isinstance(task, (LoadTask, SaveTask)):
single_exec_dict[task] = dict(eopatch_folder=name)
exec_args.append(single_exec_dict)
except ResourceNotFound as exc:
print(f'{name} - {exc}')
return exec_args
