def test_from_json_dict_invalids(self):
json_dict = json.loads('{"header": {}}')
with self.assertRaises(ValueError) as cm:
Workflow.from_json_dict(json_dict)
self.assertEqual(
str(cm.exception),
'missing mandatory property "qualified_name" in Workflow-JSON')
def create_example_3_steps_workflow(cls):
step1 = OpStep(op1, node_id='op1')
step2 = OpStep(op2, node_id='op2')
step3 = OpStep(op3, node_id='op3')
workflow = Workflow(OpMetaInfo('myWorkflow', inputs=OrderedDict(p={}), outputs=OrderedDict(q={})))
workflow.add_steps(step1, step2, step3)
step1.inputs.x.source = workflow.inputs.p
step2.inputs.a.source = step1.outputs.y
step3.inputs.u.source = step1.outputs.y
step3.inputs.v.source = step2.outputs.b
workflow.outputs.q.source = step3.outputs.w
return step1, step2, step3, workflow
def create_example_3_steps_workflow(cls):
step1 = OpStep(op1, node_id='op1')
step2 = OpStep(op2, node_id='op2')
step3 = OpStep(op3, node_id='op3')
workflow = Workflow(OpMetaInfo('myWorkflow', inputs=OrderedDict(p={}), outputs=OrderedDict(q={})))
workflow.add_steps(step1, step2, step3)
step1.inputs.x.source = workflow.inputs.p
step2.inputs.a.source = step1.outputs.y
step3.inputs.u.source = step1.outputs.y
step3.inputs.v.source = step2.outputs.b
workflow.outputs.q.source = step3.outputs.w
return step1, step2, step3, workflow
def execute_workflow(self,
res_name: str = None,
monitor: Monitor = Monitor.NONE):
self._assert_open()
if not res_name:
steps = self.workflow.sorted_steps
else:
res_step = self.workflow.find_node(res_name)
if res_step is None:
raise WorkspaceError('Resource "%s" not found' % res_name)
steps = self.workflow.find_steps_to_compute(res_step.id)
Workflow.invoke_steps(steps,
value_cache=self._resource_cache,
monitor=monitor)
return steps[-1].get_output_value()
def test_repr_svg(self):
step1 = OpStep(Op1, node_id='op1')
step2 = OpStep(Op2, node_id='op2')
step3 = OpStep(Op3, node_id='op3')
workflow = Workflow(OpMetaInfo('my_workflow', input_dict=OrderedDict(p={}), output_dict=OrderedDict(q={})))
workflow.add_steps(step1, step2, step3)
step1.input.x.source = workflow.input.p
step2.input.a.source = step1.output.y
step3.input.u.source = step1.output.y
step3.input.v.source = step2.output.b
workflow.output.q.source = step3.output.w
workflow_json = workflow._repr_svg_()
# print('\n\n%s\n\n' % workflow_json)
self.assertIsNotNone(workflow_json)
def test_to_json_dict(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
actual_json_dict = step.to_json_dict()
expected_json_text = """
{
"id": "jojo_87",
"workflow": "%s",
"inputs": {
"p": {}
},
"outputs": {
"q": {}
}
}
""" % resource
actual_json_text = json.dumps(actual_json_dict, indent=4)
expected_json_dict = json.loads(expected_json_text)
actual_json_dict = json.loads(actual_json_text)
self.assertEqual(actual_json_dict, expected_json_dict,
msg='\n%sexpected:\n%s\n%s\nbut got:\n%s\n' %
(120 * '-', expected_json_text,
120 * '-', actual_json_text))
def test_to_json_dict(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
actual_json_dict = step.to_json_dict()
expected_json_text = """
{
"id": "jojo_87",
"workflow": "%s",
"inputs": {
"p": {}
},
"outputs": {
"q": {}
}
}
""" % resource
actual_json_text = json.dumps(actual_json_dict, indent=4)
expected_json_dict = json.loads(expected_json_text)
actual_json_dict = json.loads(actual_json_text)
self.assertEqual(
actual_json_dict,
expected_json_dict,
msg='\n%sexpected:\n%s\n%s\nbut got:\n%s\n' %
(120 * '-', expected_json_text, 120 * '-', actual_json_text))
def test_resolve_source_ref(self):
step1 = OpStep(op1, node_id='myop1')
step2 = OpStep(op2, node_id='myop2')
step2.inputs.a._source_ref = ('myop1', 'y')
g = Workflow(OpMetaInfo('myWorkflow',
has_monitor=True,
inputs=OrderedDict(x={}),
outputs=OrderedDict(b={})))
g.add_steps(step1, step2)
step2.inputs.a.update_source()
self.assertEqual(step2.inputs.a._source_ref, ('myop1', 'y'))
self.assertIs(step2.inputs.a.source, step1.outputs.y)
self.assertIs(step2.inputs.a.value, None)
def test_execute_empty_workflow(self):
ws = Workspace(
'/path',
Workflow(
OpMetaInfo('workspace_workflow',
header=dict(description='Test!'))))
ws.execute_workflow()
def test_workspace_is_part_of_context(self):
def some_op(ctx: dict) -> dict:
return dict(ctx)
from cate.core.op import OP_REGISTRY
try:
op_reg = OP_REGISTRY.add_op(some_op)
op_reg.op_meta_info.inputs['ctx']['context'] = True
ws = Workspace(
'/path',
Workflow(
OpMetaInfo('workspace_workflow',
header=dict(description='Test!'))))
ws.set_resource(op_reg.op_meta_info.qualified_name, {},
res_name='new_ctx')
ws.execute_workflow('new_ctx')
self.assertTrue('new_ctx' in ws.resource_cache)
self.assertTrue('workspace' in ws.resource_cache['new_ctx'])
self.assertIs(ws.resource_cache['new_ctx']['workspace'], ws)
finally:
OP_REGISTRY.remove_op(some_op)
def test_set_resource_is_reentrant(self):
from concurrent.futures import ThreadPoolExecutor
ws = Workspace(
'/path',
Workflow(
OpMetaInfo('workspace_workflow',
header=dict(description='Test!'))))
def set_resource_and_execute():
res_name = ws.set_resource('cate.ops.utility.no_op',
op_kwargs=dict(
num_steps=dict(value=10),
step_duration=dict(value=0.05)))
ws.execute_workflow(res_name=res_name)
return res_name
num_res = 5
res_names = []
with ThreadPoolExecutor(max_workers=2 * num_res) as executor:
for i in range(num_res):
res_names.append(executor.submit(set_resource_and_execute))
actual_res_names = {f.result() for f in res_names}
expected_res_names = {'res_%s' % (i + 1) for i in range(num_res)}
self.assertEqual(actual_res_names, expected_res_names)
def test_resolve_source_ref(self):
step1 = OpStep(op1, node_id='myop1')
step2 = OpStep(op2, node_id='myop2')
step2.inputs.a._source_ref = ('myop1', 'y')
g = Workflow(OpMetaInfo('myWorkflow',
has_monitor=True,
inputs=OrderedDict(x={}),
outputs=OrderedDict(b={})))
g.add_steps(step1, step2)
step2.inputs.a.update_source()
self.assertEqual(step2.inputs.a._source_ref, ('myop1', 'y'))
self.assertIs(step2.inputs.a.source, step1.outputs.y)
self.assertIs(step2.inputs.a.value, None)
def test_set_and_execute_step(self):
ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!'))))
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='X')
ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="precipitation"), res_name='Y')
self.assertEqual(ws.resource_cache, {})
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="temperature"), res_name='Y',
overwrite=True)
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
self.assertIs(ws.resource_cache['Y'], UNDEFINED)
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_2), res_name='X', overwrite=True)
self.assertIn('X', ws.resource_cache)
self.assertIs(ws.resource_cache['X'], UNDEFINED)
self.assertIn('Y', ws.resource_cache)
self.assertIs(ws.resource_cache['Y'], UNDEFINED)
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
def test_set_step_and_run_op(self):
ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!'))))
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='X')
ws.execute_workflow('X')
self.assertIsNotNone(ws.workflow)
self.assertEqual(len(ws.workflow.steps), 1)
self.assertIn('X', ws.resource_cache)
op_name = '_extract_point'
op_args = mk_op_kwargs(ds='@X', point='10.22, 34.52', indexers=dict(time='2014-09-11'), should_return=True)
op_result = ws.run_op(op_name, op_args)
self.assertEqual(len(op_result), 4)
self.assertAlmostEqual(op_result['lat'], 34.5)
self.assertAlmostEqual(op_result['lon'], 10.2)
self.assertAlmostEqual(op_result['precipitation'], 5.5)
self.assertAlmostEqual(op_result['temperature'], 32.9)
# without asking for return data
op_args = mk_op_kwargs(ds='@X', point='10.22, 34.52', indexers=dict(time='2014-09-11'))
op_result = ws.run_op(op_name, op_args)
self.assertIsNone(op_result)
# with a non existing operator name
with self.assertRaises(ValidationError) as we:
ws.run_op("not_existing_op", {})
self.assertEqual('Unknown operation "not_existing_op"', str(we.exception))
def test_example(self):
expected_json_text = """{
"schema_version": 1,
"qualified_name": "workspace_workflow",
"header": {
"description": "Test!"
},
"inputs": {},
"outputs": {},
"steps": [
{
"id": "p",
"op": "cate.ops.io.read_netcdf",
"inputs": {
"file": {
"value": "%s"
}
}
},
{
"id": "ts",
"op": "cate.ops.timeseries.tseries_mean",
"inputs": {
"ds": "p",
"var": {
"value": "precipitation"
}
}
}
]
}
""" % NETCDF_TEST_FILE_1.replace('\\', '\\\\')
expected_json_dict = json.loads(expected_json_text)
ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!'))))
# print("wf_1: " + json.dumps(ws.workflow.to_json_dict(), indent=' '))
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='p')
# print("wf_2: " + json.dumps(ws.workflow.to_json_dict(), indent=' '))
ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@p", var="precipitation"), res_name='ts')
# print("wf_3: " + json.dumps(ws.workflow.to_json_dict(), indent=' '))
self.maxDiff = None
self.assertEqual(ws.workflow.to_json_dict(), expected_json_dict)
with self.assertRaises(ValueError) as e:
ws.set_resource('cate.ops.timeseries.tseries_point',
mk_op_kwargs(ds="@p", point="iih!", var="precipitation"), res_name='ts2',
validate_args=True)
self.assertEqual(str(e.exception),
"Input 'point' for operation 'cate.ops.timeseries.tseries_point': "
"Value cannot be converted into a 'PointLike': "
"Invalid geometry WKT format.")
ws2 = Workspace.from_json_dict(ws.to_json_dict())
self.assertEqual(ws2.base_dir, ws.base_dir)
self.assertEqual(ws2.workflow.op_meta_info.qualified_name, ws.workflow.op_meta_info.qualified_name)
self.assertEqual(len(ws2.workflow.steps), len(ws.workflow.steps))
def test_invoke_as_part_of_workflow(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
workflow = Workflow(
OpMetaInfo('contains_jojo_87',
has_monitor=True,
inputs=OrderedDict(x={}),
outputs=OrderedDict(y={})))
workflow.add_step(step)
step.inputs.p.source = workflow.inputs.x
workflow.outputs.y.source = step.outputs.q
value_cache = ValueCache()
workflow.inputs.x.value = 4
workflow.invoke(context=dict(value_cache=value_cache))
output_value = workflow.outputs.y.value
self.assertEqual(output_value, 2 * (4 + 1) + 3 * (2 * (4 + 1)))
self.assertEqual(
value_cache, {
'jojo_87._child': {
'op1': {
'y': 5
},
'op2': {
'b': 10
},
'op3': {
'w': 40
}
}
})
def test_routing(self):
from cate.core.workflow import Workflow
import os.path
workflow = Workflow.load(
os.path.join(os.path.dirname(__file__), 'workflows',
'four_steps_chain.json'))
actual_svg = workflow._repr_svg_()
self._write_svg_html(actual_svg)
def open(cls, base_dir: str) -> 'Workspace':
if not os.path.isdir(cls.get_workspace_dir(base_dir)):
raise WorkspaceError('Not a valid workspace: %s' % base_dir)
try:
workflow_file = cls.get_workflow_file(base_dir)
workflow = Workflow.load(workflow_file)
return Workspace(base_dir, workflow)
except (IOError, OSError) as e:
raise WorkspaceError(e)
def test_invoke(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
value_cache = {}
step.inputs.p.value = 3
step.invoke(context=dict(value_cache=value_cache))
output_value = step.outputs.q.value
self.assertEqual(output_value, 2 * (3 + 1) + 3 * (2 * (3 + 1)))
self.assertEqual(value_cache, {'op1': {'y': 4}, 'op2': {'b': 8}, 'op3': {'w': 32}})
def test_invoke(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
value_cache = {}
step.inputs.p.value = 3
step.invoke(context=dict(value_cache=value_cache))
output_value = step.outputs.q.value
self.assertEqual(output_value, 2 * (3 + 1) + 3 * (2 * (3 + 1)))
self.assertEqual(value_cache, {'op1': {'y': 4}, 'op2': {'b': 8}, 'op3': {'w': 32}})
def test_workspace_can_create_new_res_names(self):
ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!'))))
res_name_1 = ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value='A'))
res_name_2 = ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value='B'))
res_name_3 = ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value='C'))
self.assertEqual(res_name_1, 'res_1')
self.assertEqual(res_name_2, 'res_2')
self.assertEqual(res_name_3, 'res_3')
self.assertIsNotNone(ws.workflow.find_node(res_name_1))
self.assertIsNotNone(ws.workflow.find_node(res_name_2))
self.assertIsNotNone(ws.workflow.find_node(res_name_3))
def test_init(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
self.assertEqual(step.id, 'jojo_87')
self.assertEqual(step.resource, resource)
self.assertEqual(str(step), 'jojo_87 = cool_workflow(p=None) -> (q) [WorkflowStep]')
self.assertEqual(repr(step), "WorkflowStep(Workflow('cool_workflow'), '%s', node_id='jojo_87')" % resource)
self.assertIsNotNone(step.workflow)
self.assertIn('p', step.workflow.inputs)
self.assertIn('q', step.workflow.outputs)
def test_init(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
self.assertEqual(step.id, 'jojo_87')
self.assertEqual(step.resource, resource)
self.assertEqual(str(step), 'jojo_87 = cool_workflow(p=None) -> (q) [WorkflowStep]')
self.assertEqual(repr(step), "WorkflowStep(Workflow('cool_workflow'), '%s', node_id='jojo_87')" % resource)
self.assertIsNotNone(step.workflow)
self.assertIn('p', step.workflow.inputs)
self.assertIn('q', step.workflow.outputs)
def test_set_and_rename_and_execute_step(self):
ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!'))))
self.assertEqual(ws.user_data, {})
ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value=1), res_name='X')
ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value="@X"), res_name='Y')
ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value="@X"), res_name='Z')
self.assertEqual(len(ws.workflow.steps), 3)
self.assertEqual(ws.resource_cache, {})
value = ws.execute_workflow('Y')
self.assertEqual(value, 1)
self.assertEqual(ws.resource_cache.get('X'), 1)
self.assertEqual(ws.resource_cache.get('Y'), 1)
self.assertEqual(ws.resource_cache.get('Z'), None)
value = ws.execute_workflow('Z')
self.assertEqual(value, 1)
self.assertEqual(ws.resource_cache.get('X'), 1)
self.assertEqual(ws.resource_cache.get('Y'), 1)
self.assertEqual(ws.resource_cache.get('Z'), 1)
ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value=9), res_name='X', overwrite=True)
self.assertEqual(len(ws.workflow.steps), 3)
self.assertEqual(ws.resource_cache.get('X'), UNDEFINED)
self.assertEqual(ws.resource_cache.get('Y'), UNDEFINED)
self.assertEqual(ws.resource_cache.get('Z'), UNDEFINED)
ws.execute_workflow()
self.assertEqual(ws.resource_cache.get('X'), 9)
self.assertEqual(ws.resource_cache.get('Y'), 9)
self.assertEqual(ws.resource_cache.get('Z'), 9)
ws.rename_resource('X', 'A')
self.assertIsNone(ws.workflow.find_node('X'))
self.assertIsNotNone(ws.workflow.find_node('A'))
self.assertEqual(ws.resource_cache.get('X', '--'), '--')
self.assertEqual(ws.resource_cache.get('A'), 9)
self.assertEqual(ws.resource_cache.get('Y'), 9)
self.assertEqual(ws.resource_cache.get('Z'), 9)
ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value=5), res_name='A', overwrite=True)
self.assertEqual(ws.resource_cache.get('X', '--'), '--')
self.assertEqual(ws.resource_cache.get('A'), UNDEFINED)
self.assertEqual(ws.resource_cache.get('Y'), UNDEFINED)
self.assertEqual(ws.resource_cache.get('Z'), UNDEFINED)
ws.execute_workflow()
self.assertEqual(ws.resource_cache.get('X', '--'), '--')
self.assertEqual(ws.resource_cache.get('A'), 5)
self.assertEqual(ws.resource_cache.get('Y'), 5)
self.assertEqual(ws.resource_cache.get('Z'), 5)
def test_invoke_from_workflow(self):
resource = get_resource('workflows/one_expr.json')
workflow = Workflow.load(resource)
a = 1.5
b = -2.6
c = 4.3
workflow.inputs.a.value = a
workflow.inputs.b.value = b
workflow.inputs.c.value = c
workflow.invoke()
output_value_x = workflow.outputs.x.value
output_value_y = workflow.outputs.y.value
self.assertEqual(output_value_x, 1 + 2 * a)
self.assertEqual(output_value_y, 3 * b**2 + 4 * c**3)
def test_invoke_from_workflow(self):
resource = get_resource('workflows/one_expr.json')
workflow = Workflow.load(resource)
a = 1.5
b = -2.6
c = 4.3
workflow.inputs.a.value = a
workflow.inputs.b.value = b
workflow.inputs.c.value = c
workflow.invoke()
output_value_x = workflow.outputs.x.value
output_value_y = workflow.outputs.y.value
self.assertEqual(output_value_x, 1 + 2 * a)
self.assertEqual(output_value_y, 3 * b ** 2 + 4 * c ** 3)
def test_invoke_with_context_inputs(self):
def some_op(context, workflow, workflow_id, step, step_id, invalid):
return dict(context=context,
workflow=workflow,
workflow_id=workflow_id,
step=step,
step_id=step_id,
invalid=invalid)
from cate.core.op import OP_REGISTRY
try:
op_reg = OP_REGISTRY.add_op(some_op)
op_reg.op_meta_info.inputs['context']['context'] = True
op_reg.op_meta_info.inputs['workflow']['context'] = 'workflow'
op_reg.op_meta_info.inputs['workflow_id']['context'] = 'workflow.id'
op_reg.op_meta_info.inputs['step']['context'] = 'step'
op_reg.op_meta_info.inputs['step_id']['context'] = 'step.id'
op_reg.op_meta_info.inputs['invalid']['context'] = 'gnarz[8]'
step = OpStep(op_reg, node_id='test_step')
workflow = Workflow(OpMetaInfo('test_workflow'))
workflow.add_step(step)
workflow.invoke()
output = step.outputs['return'].value
self.assertIsInstance(output, dict)
self.assertIsInstance(output.get('context'), dict)
self.assertIs(output.get('workflow'), workflow)
self.assertEqual(output.get('workflow_id'), 'test_workflow')
self.assertIs(output.get('step'), step)
self.assertEqual(output.get('step_id'), 'test_step')
self.assertEqual(output.get('invalid', 1), None)
finally:
OP_REGISTRY.remove_op(some_op)
def test_set_and_execute_step(self):
ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!'))))
with self.assertRaises(ValidationError) as we:
ws.set_resource("not_existing_op", {})
self.assertEqual('Unknown operation "not_existing_op"', str(we.exception))
with self.assertRaises(ValidationError) as we:
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(location=NETCDF_TEST_FILE_1), res_name='X')
self.assertEqual('"location" is not an input of operation "cate.ops.io.read_netcdf"', str(we.exception))
with self.assertRaises(ValidationError) as we:
ws.set_resource('cate.ops.io.read_netcdf', {'file': {'foo': 'bar'}}, res_name='X')
self.assertEqual('Illegal argument for input "file" of operation "cate.ops.io.read_netcdf', str(we.exception))
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='X')
ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="precipitation"), res_name='Y')
self.assertEqual(ws.resource_cache, {})
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="temperature"), res_name='Y',
overwrite=True)
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
self.assertIs(ws.resource_cache['Y'], UNDEFINED)
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_2), res_name='X', overwrite=True)
self.assertIn('X', ws.resource_cache)
self.assertIs(ws.resource_cache['X'], UNDEFINED)
self.assertIn('Y', ws.resource_cache)
self.assertIs(ws.resource_cache['Y'], UNDEFINED)
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
def test_set_and_execute_step(self):
ws = Workspace(
'/path',
Workflow(
OpMetaInfo('workspace_workflow',
header_dict=dict(description='Test!'))))
ws.set_resource('X', 'cate.ops.io.read_netcdf',
["file=%s" % NETCDF_TEST_FILE_1])
ws.set_resource('Y', 'cate.ops.timeseries.tseries_mean',
["ds=X", "var=precipitation"])
self.assertEqual(ws.resource_cache, {})
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
ws.set_resource('Y',
'cate.ops.timeseries.tseries_mean',
["ds=X", "var=temperature"],
overwrite=True)
self.assertIn('X', ws.resource_cache)
self.assertNotIn('Y', ws.resource_cache)
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
ws.set_resource('X',
'cate.ops.io.read_netcdf',
["file=%s" % NETCDF_TEST_FILE_2],
overwrite=True)
self.assertNotIn('X', ws.resource_cache)
self.assertNotIn('Y', ws.resource_cache)
ws.execute_workflow('Y')
self.assertIn('X', ws.resource_cache)
self.assertIn('Y', ws.resource_cache)
def test_invoke_as_part_of_workflow(self):
resource = get_resource('workflows/three_ops.json')
workflow = Workflow.load(resource)
step = WorkflowStep(workflow, resource, node_id='jojo_87')
workflow = Workflow(OpMetaInfo('contains_jojo_87',
has_monitor=True,
inputs=OrderedDict(x={}),
outputs=OrderedDict(y={})))
workflow.add_step(step)
step.inputs.p.source = workflow.inputs.x
workflow.outputs.y.source = step.outputs.q
value_cache = ValueCache()
workflow.inputs.x.value = 4
workflow.invoke(context=dict(value_cache=value_cache))
output_value = workflow.outputs.y.value
self.assertEqual(output_value, 2 * (4 + 1) + 3 * (2 * (4 + 1)))
self.assertEqual(value_cache, {'jojo_87._child': {'op1': {'y': 5}, 'op2': {'b': 10}, 'op3': {'w': 40}}})
def test_invoke_with_context_inputs(self):
def some_op(context, workflow, workflow_id, step, step_id, invalid):
return dict(context=context,
workflow=workflow,
workflow_id=workflow_id,
step=step,
step_id=step_id,
invalid=invalid)
from cate.core.op import OP_REGISTRY
try:
op_reg = OP_REGISTRY.add_op(some_op)
op_reg.op_meta_info.inputs['context']['context'] = True
op_reg.op_meta_info.inputs['workflow']['context'] = 'workflow'
op_reg.op_meta_info.inputs['workflow_id'][
'context'] = 'workflow.id'
op_reg.op_meta_info.inputs['step']['context'] = 'step'
op_reg.op_meta_info.inputs['step_id']['context'] = 'step.id'
op_reg.op_meta_info.inputs['invalid']['context'] = 'gnarz[8]'
step = OpStep(op_reg, node_id='test_step')
workflow = Workflow(OpMetaInfo('test_workflow'))
workflow.add_step(step)
workflow.invoke()
output = step.outputs['return'].value
self.assertIsInstance(output, dict)
self.assertIsInstance(output.get('context'), dict)
self.assertIs(output.get('workflow'), workflow)
self.assertEqual(output.get('workflow_id'), 'test_workflow')
self.assertIs(output.get('step'), step)
self.assertEqual(output.get('step_id'), 'test_step')
self.assertEqual(output.get('invalid', 1), None)
finally:
OP_REGISTRY.remove_op(some_op)
def test_to_json_dict(self):
def dataset_op() -> xr.Dataset:
periods = 5
temperature_data = (15 + 8 * np.random.randn(periods, 2, 2)).round(decimals=1)
temperature_attrs = {'a': np.array([1, 2, 3]), 'comment': 'hot', '_FillValue': np.nan}
precipitation_data = (10 * np.random.rand(periods, 2, 2)).round(decimals=1)
precipitation_attrs = {'x': True, 'comment': 'wet', '_FillValue': -1.0}
ds = xr.Dataset(
data_vars={
'temperature': (('time', 'lat', 'lon'), temperature_data, temperature_attrs),
'precipitation': (('time', 'lat', 'lon'), precipitation_data, precipitation_attrs)
},
coords={
'lon': np.array([12, 13]),
'lat': np.array([50, 51]),
'time': pd.date_range('2014-09-06', periods=periods)
},
attrs={
'history': 'a b c'
})
return ds
def scalar_dataset_op() -> xr.Dataset:
ds = xr.Dataset(
data_vars={
'temperature': (('time', 'lat', 'lon'), [[[15.2]]]),
'precipitation': (('time', 'lat', 'lon'), [[[10.1]]])
},
coords={
'lon': [12.],
'lat': [50.],
'time': [pd.to_datetime('2014-09-06')],
},
attrs={
'history': 'a b c'
})
return ds
def empty_dataset_op() -> xr.Dataset:
ds = xr.Dataset(
data_vars={
'temperature': (('time', 'lat', 'lon'), np.ndarray(shape=(0, 0, 0), dtype=np.float32)),
'precipitation': (('time', 'lat', 'lon'), np.ndarray(shape=(0, 0, 0), dtype=np.float32))
},
coords={
'lon': np.ndarray(shape=(0,), dtype=np.float32),
'lat': np.ndarray(shape=(0,), dtype=np.float32),
'time': np.ndarray(shape=(0,), dtype=np.datetime64),
},
attrs={
'history': 'a b c'
})
return ds
def data_frame_op() -> pd.DataFrame:
data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, 5, 5, 9, 1, 2, 7, 6]}
time = pd.date_range('2000-01-01', freq='MS', periods=12)
return pd.DataFrame(data=data, index=time, dtype=float, columns=['A', 'B'])
def scalar_data_frame_op() -> pd.DataFrame:
data = {'A': [1.3],
'B': [5.9]}
return pd.DataFrame(data=data, dtype=float, columns=['A', 'B'])
def empty_data_frame_op() -> pd.DataFrame:
data = {'A': [],
'B': []}
return pd.DataFrame(data=data, dtype=float, columns=['A', 'B'])
def geo_data_frame_op() -> gpd.GeoDataFrame:
data = {'name': ['A', 'B', 'C'],
'lat': [45, 46, 47.5],
'lon': [-120, -121.2, -122.9]}
df = pd.DataFrame(data, columns=['name', 'lat', 'lon'])
geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])]
return gpd.GeoDataFrame(df, geometry=geometry)
def scalar_geo_data_frame_op() -> gpd.GeoDataFrame:
data = {'name': [2000 * 'A'],
'lat': [45],
'lon': [-120]}
df = pd.DataFrame(data, columns=['name', 'lat', 'lon'])
geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])]
return gpd.GeoDataFrame(df, geometry=geometry)
def int_op() -> int:
return 394852
def str_op() -> str:
return 'Hi!'
from cate.core.op import OP_REGISTRY
try:
OP_REGISTRY.add_op(dataset_op)
OP_REGISTRY.add_op(data_frame_op)
OP_REGISTRY.add_op(geo_data_frame_op)
OP_REGISTRY.add_op(scalar_dataset_op)
OP_REGISTRY.add_op(scalar_data_frame_op)
OP_REGISTRY.add_op(scalar_geo_data_frame_op)
OP_REGISTRY.add_op(empty_dataset_op)
OP_REGISTRY.add_op(empty_data_frame_op)
OP_REGISTRY.add_op(int_op)
OP_REGISTRY.add_op(str_op)
workflow = Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))
workflow.add_step(OpStep(dataset_op, node_id='ds'))
workflow.add_step(OpStep(data_frame_op, node_id='df'))
workflow.add_step(OpStep(geo_data_frame_op, node_id='gdf'))
workflow.add_step(OpStep(scalar_dataset_op, node_id='scalar_ds'))
workflow.add_step(OpStep(scalar_data_frame_op, node_id='scalar_df'))
workflow.add_step(OpStep(scalar_geo_data_frame_op, node_id='scalar_gdf'))
workflow.add_step(OpStep(empty_dataset_op, node_id='empty_ds'))
workflow.add_step(OpStep(empty_data_frame_op, node_id='empty_df'))
workflow.add_step(OpStep(int_op, node_id='i'))
workflow.add_step(OpStep(str_op, node_id='s'))
ws = Workspace('/path', workflow)
ws.execute_workflow()
d_ws = ws.to_json_dict()
# import pprint
# pprint.pprint(d_ws)
d_wf = d_ws.get('workflow')
self.assertIsNotNone(d_wf)
l_res = d_ws.get('resources')
self.assertIsNotNone(l_res)
self.assertEqual(len(l_res), 10)
res_ds = l_res[0]
self.assertEqual(res_ds.get('name'), 'ds')
self.assertEqual(res_ds.get('dataType'), 'xarray.core.dataset.Dataset')
self.assertEqual(res_ds.get('dimSizes'), dict(lat=2, lon=2, time=5))
self.assertEqual(res_ds.get('attributes'), {'history': 'a b c'})
res_ds_vars = res_ds.get('variables')
self.assertIsNotNone(res_ds_vars)
self.assertEqual(len(res_ds_vars), 2)
res_ds_var_1 = res_ds_vars[0]
self.assertEqual(res_ds_var_1.get('name'), 'precipitation')
self.assertEqual(res_ds_var_1.get('dataType'), 'float64')
self.assertEqual(res_ds_var_1.get('numDims'), 3)
self.assertEqual(res_ds_var_1.get('shape'), (5, 2, 2))
self.assertEqual(res_ds_var_1.get('chunkSizes'), None)
self.assertEqual(res_ds_var_1.get('isYFlipped'), True)
self.assertEqual(res_ds_var_1.get('isFeatureAttribute'), None)
self.assertEqual(res_ds_var_1.get('attributes'), dict(x=True, comment='wet', _FillValue=-1.))
res_ds_var_2 = res_ds_vars[1]
self.assertEqual(res_ds_var_2.get('name'), 'temperature')
self.assertEqual(res_ds_var_2.get('dataType'), 'float64')
self.assertEqual(res_ds_var_2.get('numDims'), 3)
self.assertEqual(res_ds_var_2.get('shape'), (5, 2, 2))
self.assertEqual(res_ds_var_2.get('chunkSizes'), None)
self.assertEqual(res_ds_var_2.get('isYFlipped'), True)
self.assertEqual(res_ds_var_2.get('isFeatureAttribute'), None)
self.assertEqual(res_ds_var_2.get('attributes'), dict(a=[1, 2, 3], comment='hot', _FillValue=np.nan))
res_df = l_res[1]
self.assertEqual(res_df.get('name'), 'df')
self.assertEqual(res_df.get('dataType'), 'pandas.core.frame.DataFrame')
self.assertEqual(res_df.get('attributes'), {'num_rows': 12, 'num_columns': 2})
res_df_vars = res_df.get('variables')
self.assertIsNotNone(res_df_vars)
self.assertEqual(len(res_df_vars), 2)
res_df_var_1 = res_df_vars[0]
self.assertEqual(res_df_var_1.get('name'), 'A')
self.assertEqual(res_df_var_1.get('dataType'), 'float64')
self.assertEqual(res_df_var_1.get('numDims'), 1)
self.assertEqual(res_df_var_1.get('shape'), (12,))
self.assertEqual(res_df_var_1.get('isYFlipped'), None)
self.assertEqual(res_df_var_1.get('isFeatureAttribute'), True)
self.assertIsNone(res_df_var_1.get('attributes'))
res_df_var_2 = res_df_vars[1]
self.assertEqual(res_df_var_2.get('name'), 'B')
self.assertEqual(res_df_var_2.get('dataType'), 'float64')
self.assertEqual(res_df_var_2.get('numDims'), 1)
self.assertEqual(res_df_var_2.get('shape'), (12,))
self.assertEqual(res_df_var_2.get('isYFlipped'), None)
self.assertEqual(res_df_var_2.get('isFeatureAttribute'), True)
self.assertIsNone(res_df_var_2.get('attributes'))
res_gdf = l_res[2]
self.assertEqual(res_gdf.get('name'), 'gdf')
self.assertEqual(res_gdf.get('dataType'), 'geopandas.geodataframe.GeoDataFrame')
self.assertEqual(res_gdf.get('attributes'), {'num_rows': 3, 'num_columns': 4, 'geom_type': 'Point'})
res_gdf_vars = res_gdf.get('variables')
self.assertIsNotNone(res_gdf_vars)
self.assertEqual(len(res_gdf_vars), 4)
res_gdf_var_1 = res_gdf_vars[0]
self.assertEqual(res_gdf_var_1.get('name'), 'name')
self.assertEqual(res_gdf_var_1.get('dataType'), 'object')
self.assertEqual(res_gdf_var_1.get('numDims'), 1)
self.assertEqual(res_gdf_var_1.get('shape'), (3,))
self.assertEqual(res_gdf_var_1.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_1.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_1.get('attributes'))
res_gdf_var_2 = res_gdf_vars[1]
self.assertEqual(res_gdf_var_2.get('name'), 'lat')
self.assertEqual(res_gdf_var_2.get('dataType'), 'float64')
self.assertEqual(res_gdf_var_2.get('numDims'), 1)
self.assertEqual(res_gdf_var_2.get('shape'), (3,))
self.assertEqual(res_gdf_var_2.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_2.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_2.get('attributes'))
res_gdf_var_3 = res_gdf_vars[2]
self.assertEqual(res_gdf_var_3.get('name'), 'lon')
self.assertEqual(res_gdf_var_3.get('dataType'), 'float64')
self.assertEqual(res_gdf_var_3.get('numDims'), 1)
self.assertEqual(res_gdf_var_3.get('shape'), (3,))
self.assertEqual(res_gdf_var_3.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_3.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_3.get('attributes'))
res_gdf_var_4 = res_gdf_vars[3]
self.assertEqual(res_gdf_var_4.get('name'), 'geometry')
self.assertEqual(res_gdf_var_4.get('dataType'), 'geometry')
self.assertEqual(res_gdf_var_4.get('numDims'), 1)
self.assertEqual(res_gdf_var_4.get('shape'), (3,))
self.assertEqual(res_gdf_var_4.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_4.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_4.get('attributes'))
res_scalar_ds = l_res[3]
res_scalar_ds_vars = res_scalar_ds.get('variables')
self.assertIsNotNone(res_scalar_ds_vars)
self.assertEqual(len(res_scalar_ds_vars), 2)
scalar_values = {res_scalar_ds_vars[0].get('name'): res_scalar_ds_vars[0].get('value'),
res_scalar_ds_vars[1].get('name'): res_scalar_ds_vars[1].get('value')}
self.assertEqual(scalar_values, {'temperature': 15.2, 'precipitation': 10.1})
res_scalar_df = l_res[4]
res_scalar_df_vars = res_scalar_df.get('variables')
self.assertIsNotNone(res_scalar_df_vars)
self.assertEqual(len(res_scalar_df_vars), 2)
scalar_values = {res_scalar_df_vars[0].get('name'): res_scalar_df_vars[0].get('value'),
res_scalar_df_vars[1].get('name'): res_scalar_df_vars[1].get('value')}
self.assertEqual(scalar_values, {'A': 1.3, 'B': 5.9})
res_scalar_gdf = l_res[5]
res_scalar_gdf_vars = res_scalar_gdf.get('variables')
self.assertIsNotNone(res_scalar_gdf_vars)
self.assertEqual(len(res_scalar_gdf_vars), 4)
scalar_values = {res_scalar_gdf_vars[0].get('name'): res_scalar_gdf_vars[0].get('value'),
res_scalar_gdf_vars[1].get('name'): res_scalar_gdf_vars[1].get('value'),
res_scalar_gdf_vars[2].get('name'): res_scalar_gdf_vars[2].get('value'),
res_scalar_gdf_vars[3].get('name'): res_scalar_gdf_vars[3].get('value')}
self.assertEqual(scalar_values, {'name': (1000 * 'A') + '...',
'lat': 45,
'lon': -120,
'geometry': 'POINT (-120 45)'})
res_empty_ds = l_res[6]
res_empty_ds_vars = res_empty_ds.get('variables')
self.assertIsNotNone(res_empty_ds_vars)
self.assertEqual(len(res_empty_ds_vars), 2)
scalar_values = {res_empty_ds_vars[0].get('name'): res_empty_ds_vars[0].get('value'),
res_empty_ds_vars[1].get('name'): res_empty_ds_vars[1].get('value')}
self.assertEqual(scalar_values, {'temperature': None, 'precipitation': None})
res_empty_df = l_res[7]
res_empty_df_vars = res_empty_df.get('variables')
self.assertIsNotNone(res_empty_df_vars)
self.assertEqual(len(res_empty_df_vars), 2)
scalar_values = {res_empty_df_vars[0].get('name'): res_empty_df_vars[0].get('value'),
res_empty_df_vars[1].get('name'): res_empty_df_vars[1].get('value')}
self.assertEqual(scalar_values, {'A': None, 'B': None})
res_int = l_res[8]
self.assertEqual(res_int.get('name'), 'i')
self.assertEqual(res_int.get('dataType'), 'int')
self.assertIsNone(res_int.get('attributes'))
self.assertIsNone(res_int.get('variables'))
res_str = l_res[9]
self.assertEqual(res_str.get('name'), 's')
self.assertEqual(res_str.get('dataType'), 'str')
self.assertIsNone(res_str.get('attributes'))
self.assertIsNone(res_str.get('variables'))
finally:
OP_REGISTRY.remove_op(dataset_op)
OP_REGISTRY.remove_op(data_frame_op)
OP_REGISTRY.remove_op(geo_data_frame_op)
OP_REGISTRY.remove_op(scalar_dataset_op)
OP_REGISTRY.remove_op(scalar_data_frame_op)
OP_REGISTRY.remove_op(scalar_geo_data_frame_op)
OP_REGISTRY.remove_op(empty_dataset_op)
OP_REGISTRY.remove_op(empty_data_frame_op)
OP_REGISTRY.remove_op(int_op)
OP_REGISTRY.remove_op(str_op)
def test_to_json_dict(self):
def dataset_op() -> xr.Dataset:
periods = 5
temperature_data = (15 + 8 * np.random.randn(periods, 2, 2)).round(decimals=1)
temperature_attrs = {'a': np.array([1, 2, 3]), 'comment': 'hot', '_FillValue': np.nan}
precipitation_data = (10 * np.random.rand(periods, 2, 2)).round(decimals=1)
precipitation_attrs = {'x': True, 'comment': 'wet', '_FillValue': -1.0}
ds = xr.Dataset(
data_vars={
'temperature': (('time', 'lat', 'lon'), temperature_data, temperature_attrs),
'precipitation': (('time', 'lat', 'lon'), precipitation_data, precipitation_attrs)
},
coords={
'lon': np.array([12, 13]),
'lat': np.array([50, 51]),
'time': pd.date_range('2014-09-06', periods=periods)
},
attrs={
'history': 'a b c'
})
return ds
def scalar_dataset_op() -> xr.Dataset:
ds = xr.Dataset(
data_vars={
'temperature': (('time', 'lat', 'lon'), [[[15.2]]]),
'precipitation': (('time', 'lat', 'lon'), [[[10.1]]])
},
coords={
'lon': [12.],
'lat': [50.],
'time': [pd.to_datetime('2014-09-06')],
},
attrs={
'history': 'a b c'
})
return ds
def empty_dataset_op() -> xr.Dataset:
ds = xr.Dataset(
data_vars={
'temperature': (('time', 'lat', 'lon'), np.ndarray(shape=(0, 0, 0), dtype=np.float32)),
'precipitation': (('time', 'lat', 'lon'), np.ndarray(shape=(0, 0, 0), dtype=np.float32))
},
coords={
'lon': np.ndarray(shape=(0,), dtype=np.float32),
'lat': np.ndarray(shape=(0,), dtype=np.float32),
'time': np.ndarray(shape=(0,), dtype=np.datetime64),
},
attrs={
'history': 'a b c'
})
return ds
def data_frame_op() -> pd.DataFrame:
data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, 5, 5, 9, 1, 2, 7, 6]}
time = pd.date_range('2000-01-01', freq='MS', periods=12)
return pd.DataFrame(data=data, index=time, dtype=float, columns=['A', 'B'])
def scalar_data_frame_op() -> pd.DataFrame:
data = {'A': [1.3],
'B': [5.9]}
return pd.DataFrame(data=data, dtype=float, columns=['A', 'B'])
def empty_data_frame_op() -> pd.DataFrame:
data = {'A': [],
'B': []}
return pd.DataFrame(data=data, dtype=float, columns=['A', 'B'])
def geo_data_frame_op() -> gpd.GeoDataFrame:
data = {'name': ['A', 'B', 'C'],
'lat': [45, 46, 47.5],
'lon': [-120, -121.2, -122.9]}
df = pd.DataFrame(data, columns=['name', 'lat', 'lon'])
geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])]
return gpd.GeoDataFrame(df, geometry=geometry)
def scalar_geo_data_frame_op() -> gpd.GeoDataFrame:
data = {'name': [2000 * 'A'],
'lat': [45],
'lon': [-120]}
df = pd.DataFrame(data, columns=['name', 'lat', 'lon'])
geometry = [Point(xy) for xy in zip(df['lon'], df['lat'])]
return gpd.GeoDataFrame(df, geometry=geometry)
def int_op() -> int:
return 394852
def str_op() -> str:
return 'Hi!'
from cate.core.op import OP_REGISTRY
try:
OP_REGISTRY.add_op(dataset_op)
OP_REGISTRY.add_op(data_frame_op)
OP_REGISTRY.add_op(geo_data_frame_op)
OP_REGISTRY.add_op(scalar_dataset_op)
OP_REGISTRY.add_op(scalar_data_frame_op)
OP_REGISTRY.add_op(scalar_geo_data_frame_op)
OP_REGISTRY.add_op(empty_dataset_op)
OP_REGISTRY.add_op(empty_data_frame_op)
OP_REGISTRY.add_op(int_op)
OP_REGISTRY.add_op(str_op)
workflow = Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))
workflow.add_step(OpStep(dataset_op, node_id='ds'))
workflow.add_step(OpStep(data_frame_op, node_id='df'))
workflow.add_step(OpStep(geo_data_frame_op, node_id='gdf'))
workflow.add_step(OpStep(scalar_dataset_op, node_id='scalar_ds'))
workflow.add_step(OpStep(scalar_data_frame_op, node_id='scalar_df'))
workflow.add_step(OpStep(scalar_geo_data_frame_op, node_id='scalar_gdf'))
workflow.add_step(OpStep(empty_dataset_op, node_id='empty_ds'))
workflow.add_step(OpStep(empty_data_frame_op, node_id='empty_df'))
workflow.add_step(OpStep(int_op, node_id='i'))
workflow.add_step(OpStep(str_op, node_id='s'))
ws = Workspace('/path', workflow)
ws.execute_workflow()
d_ws = ws.to_json_dict()
# import pprint
# pprint.pprint(d_ws)
d_wf = d_ws.get('workflow')
self.assertIsNotNone(d_wf)
l_res = d_ws.get('resources')
self.assertIsNotNone(l_res)
self.assertEqual(len(l_res), 10)
res_ds = l_res[0]
self.assertEqual(res_ds.get('name'), 'ds')
self.assertEqual(res_ds.get('dataType'), 'xarray.core.dataset.Dataset')
self.assertEqual(res_ds.get('dimSizes'), dict(lat=2, lon=2, time=5))
self.assertEqual(res_ds.get('attributes'), {'history': 'a b c'})
res_ds_vars = res_ds.get('variables')
self.assertIsNotNone(res_ds_vars)
self.assertEqual(len(res_ds_vars), 2)
res_ds_var_1 = res_ds_vars[0]
self.assertEqual(res_ds_var_1.get('name'), 'precipitation')
self.assertEqual(res_ds_var_1.get('dataType'), 'float64')
self.assertEqual(res_ds_var_1.get('numDims'), 3)
self.assertEqual(res_ds_var_1.get('shape'), (5, 2, 2))
self.assertEqual(res_ds_var_1.get('chunkSizes'), None)
self.assertEqual(res_ds_var_1.get('isYFlipped'), True)
self.assertEqual(res_ds_var_1.get('isFeatureAttribute'), None)
self.assertEqual(res_ds_var_1.get('attributes'), dict(x=True, comment='wet', _FillValue=-1.))
res_ds_var_2 = res_ds_vars[1]
self.assertEqual(res_ds_var_2.get('name'), 'temperature')
self.assertEqual(res_ds_var_2.get('dataType'), 'float64')
self.assertEqual(res_ds_var_2.get('numDims'), 3)
self.assertEqual(res_ds_var_2.get('shape'), (5, 2, 2))
self.assertEqual(res_ds_var_2.get('chunkSizes'), None)
self.assertEqual(res_ds_var_2.get('isYFlipped'), True)
self.assertEqual(res_ds_var_2.get('isFeatureAttribute'), None)
self.assertEqual(res_ds_var_2.get('attributes'), dict(a=[1, 2, 3], comment='hot', _FillValue=np.nan))
res_df = l_res[1]
self.assertEqual(res_df.get('name'), 'df')
self.assertEqual(res_df.get('dataType'), 'pandas.core.frame.DataFrame')
self.assertEqual(res_df.get('attributes'), {'num_rows': 12, 'num_columns': 2})
res_df_vars = res_df.get('variables')
self.assertIsNotNone(res_df_vars)
self.assertEqual(len(res_df_vars), 2)
res_df_var_1 = res_df_vars[0]
self.assertEqual(res_df_var_1.get('name'), 'A')
self.assertEqual(res_df_var_1.get('dataType'), 'float64')
self.assertEqual(res_df_var_1.get('numDims'), 1)
self.assertEqual(res_df_var_1.get('shape'), (12,))
self.assertEqual(res_df_var_1.get('isYFlipped'), None)
self.assertEqual(res_df_var_1.get('isFeatureAttribute'), True)
self.assertIsNone(res_df_var_1.get('attributes'))
res_df_var_2 = res_df_vars[1]
self.assertEqual(res_df_var_2.get('name'), 'B')
self.assertEqual(res_df_var_2.get('dataType'), 'float64')
self.assertEqual(res_df_var_2.get('numDims'), 1)
self.assertEqual(res_df_var_2.get('shape'), (12,))
self.assertEqual(res_df_var_2.get('isYFlipped'), None)
self.assertEqual(res_df_var_2.get('isFeatureAttribute'), True)
self.assertIsNone(res_df_var_2.get('attributes'))
res_gdf = l_res[2]
self.assertEqual(res_gdf.get('name'), 'gdf')
self.assertEqual(res_gdf.get('dataType'), 'geopandas.geodataframe.GeoDataFrame')
self.assertEqual(res_gdf.get('attributes'), {'num_rows': 3, 'num_columns': 4, 'geom_type': 'Point'})
res_gdf_vars = res_gdf.get('variables')
self.assertIsNotNone(res_gdf_vars)
self.assertEqual(len(res_gdf_vars), 4)
res_gdf_var_1 = res_gdf_vars[0]
self.assertEqual(res_gdf_var_1.get('name'), 'name')
self.assertEqual(res_gdf_var_1.get('dataType'), 'object')
self.assertEqual(res_gdf_var_1.get('numDims'), 1)
self.assertEqual(res_gdf_var_1.get('shape'), (3,))
self.assertEqual(res_gdf_var_1.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_1.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_1.get('attributes'))
res_gdf_var_2 = res_gdf_vars[1]
self.assertEqual(res_gdf_var_2.get('name'), 'lat')
self.assertEqual(res_gdf_var_2.get('dataType'), 'float64')
self.assertEqual(res_gdf_var_2.get('numDims'), 1)
self.assertEqual(res_gdf_var_2.get('shape'), (3,))
self.assertEqual(res_gdf_var_2.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_2.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_2.get('attributes'))
res_gdf_var_3 = res_gdf_vars[2]
self.assertEqual(res_gdf_var_3.get('name'), 'lon')
self.assertEqual(res_gdf_var_3.get('dataType'), 'float64')
self.assertEqual(res_gdf_var_3.get('numDims'), 1)
self.assertEqual(res_gdf_var_3.get('shape'), (3,))
self.assertEqual(res_gdf_var_3.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_3.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_3.get('attributes'))
res_gdf_var_4 = res_gdf_vars[3]
self.assertEqual(res_gdf_var_4.get('name'), 'geometry')
self.assertEqual(res_gdf_var_4.get('dataType'), 'object')
self.assertEqual(res_gdf_var_4.get('numDims'), 1)
self.assertEqual(res_gdf_var_4.get('shape'), (3,))
self.assertEqual(res_gdf_var_4.get('isYFlipped'), None)
self.assertEqual(res_gdf_var_4.get('isFeatureAttribute'), True)
self.assertIsNone(res_gdf_var_4.get('attributes'))
res_scalar_ds = l_res[3]
res_scalar_ds_vars = res_scalar_ds.get('variables')
self.assertIsNotNone(res_scalar_ds_vars)
self.assertEqual(len(res_scalar_ds_vars), 2)
scalar_values = {res_scalar_ds_vars[0].get('name'): res_scalar_ds_vars[0].get('value'),
res_scalar_ds_vars[1].get('name'): res_scalar_ds_vars[1].get('value')}
self.assertEqual(scalar_values, {'temperature': 15.2, 'precipitation': 10.1})
res_scalar_df = l_res[4]
res_scalar_df_vars = res_scalar_df.get('variables')
self.assertIsNotNone(res_scalar_df_vars)
self.assertEqual(len(res_scalar_df_vars), 2)
scalar_values = {res_scalar_df_vars[0].get('name'): res_scalar_df_vars[0].get('value'),
res_scalar_df_vars[1].get('name'): res_scalar_df_vars[1].get('value')}
self.assertEqual(scalar_values, {'A': 1.3, 'B': 5.9})
res_scalar_gdf = l_res[5]
res_scalar_gdf_vars = res_scalar_gdf.get('variables')
self.assertIsNotNone(res_scalar_gdf_vars)
self.assertEqual(len(res_scalar_gdf_vars), 4)
scalar_values = {res_scalar_gdf_vars[0].get('name'): res_scalar_gdf_vars[0].get('value'),
res_scalar_gdf_vars[1].get('name'): res_scalar_gdf_vars[1].get('value'),
res_scalar_gdf_vars[2].get('name'): res_scalar_gdf_vars[2].get('value'),
res_scalar_gdf_vars[3].get('name'): res_scalar_gdf_vars[3].get('value')}
self.assertEqual(scalar_values, {'name': (1000 * 'A') + '...',
'lat': 45,
'lon': -120,
'geometry': 'POINT (-120 45)'})
res_empty_ds = l_res[6]
res_empty_ds_vars = res_empty_ds.get('variables')
self.assertIsNotNone(res_empty_ds_vars)
self.assertEqual(len(res_empty_ds_vars), 2)
scalar_values = {res_empty_ds_vars[0].get('name'): res_empty_ds_vars[0].get('value'),
res_empty_ds_vars[1].get('name'): res_empty_ds_vars[1].get('value')}
self.assertEqual(scalar_values, {'temperature': None, 'precipitation': None})
res_empty_df = l_res[7]
res_empty_df_vars = res_empty_df.get('variables')
self.assertIsNotNone(res_empty_df_vars)
self.assertEqual(len(res_empty_df_vars), 2)
scalar_values = {res_empty_df_vars[0].get('name'): res_empty_df_vars[0].get('value'),
res_empty_df_vars[1].get('name'): res_empty_df_vars[1].get('value')}
self.assertEqual(scalar_values, {'A': None, 'B': None})
res_int = l_res[8]
self.assertEqual(res_int.get('name'), 'i')
self.assertEqual(res_int.get('dataType'), 'int')
self.assertIsNone(res_int.get('attributes'))
self.assertIsNone(res_int.get('variables'))
res_str = l_res[9]
self.assertEqual(res_str.get('name'), 's')
self.assertEqual(res_str.get('dataType'), 'str')
self.assertIsNone(res_str.get('attributes'))
self.assertIsNone(res_str.get('variables'))
finally:
OP_REGISTRY.remove_op(dataset_op)
OP_REGISTRY.remove_op(data_frame_op)
OP_REGISTRY.remove_op(geo_data_frame_op)
OP_REGISTRY.remove_op(scalar_dataset_op)
OP_REGISTRY.remove_op(scalar_data_frame_op)
OP_REGISTRY.remove_op(scalar_geo_data_frame_op)
OP_REGISTRY.remove_op(empty_dataset_op)
OP_REGISTRY.remove_op(empty_data_frame_op)
OP_REGISTRY.remove_op(int_op)
OP_REGISTRY.remove_op(str_op)
def test_from_json_dict(self):
workflow_json_text = """
{
"qualified_name": "my_workflow",
"header": {
"description": "My workflow is not too bad."
},
"inputs": {
"p": {"description": "Input 'p'"}
},
"outputs": {
"q": {"source": "op3.w", "description": "Output 'q'"}
},
"steps": [
{
"id": "op1",
"op": "test.core.test_workflow.op1",
"inputs": {
"x": { "source": ".p" }
}
},
{
"id": "op2",
"op": "test.core.test_workflow.op2",
"inputs": {
"a": {"source": "op1"}
}
},
{
"id": "op3",
"persistent": true,
"op": "test.core.test_workflow.op3",
"inputs": {
"u": {"source": "op1.y"},
"v": {"source": "op2.b"}
}
}
]
}
"""
workflow_json_dict = json.loads(workflow_json_text)
workflow = Workflow.from_json_dict(workflow_json_dict)
self.assertIsNotNone(workflow)
self.assertEqual(workflow.id, "my_workflow")
self.assertEqual(workflow.op_meta_info.qualified_name, workflow.id)
self.assertEqual(workflow.op_meta_info.header, dict(description="My workflow is not too bad."))
self.assertEqual(len(workflow.op_meta_info.inputs), 1)
self.assertEqual(len(workflow.op_meta_info.outputs), 1)
self.assertEqual(workflow.op_meta_info.inputs['p'], dict(description="Input 'p'"))
self.assertEqual(workflow.op_meta_info.outputs['q'], dict(source="op3.w", description="Output 'q'"))
self.assertEqual(len(workflow.inputs), 1)
self.assertEqual(len(workflow.outputs), 1)
self.assertIn('p', workflow.inputs)
self.assertIn('q', workflow.outputs)
self.assertEqual(len(workflow.steps), 3)
step1 = workflow.steps[0]
step2 = workflow.steps[1]
step3 = workflow.steps[2]
self.assertEqual(step1.id, 'op1')
self.assertEqual(step2.id, 'op2')
self.assertEqual(step3.id, 'op3')
self.assertIs(step1.inputs.x.source, workflow.inputs.p)
self.assertIs(step2.inputs.a.source, step1.outputs.y)
self.assertIs(step3.inputs.u.source, step1.outputs.y)
self.assertIs(step3.inputs.v.source, step2.outputs.b)
self.assertIs(workflow.outputs.q.source, step3.outputs.w)
self.assertEqual(step1.persistent, False)
self.assertEqual(step2.persistent, False)
self.assertEqual(step3.persistent, True)
def test_from_json_dict_empty(self):
json_dict = json.loads('{"qualified_name": "hello"}')
workflow = Workflow.from_json_dict(json_dict)
self.assertEqual(workflow.id, 'hello')
def test_from_json_dict_invalids(self):
json_dict = json.loads('{"header": {}}')
with self.assertRaises(ValueError) as cm:
Workflow.from_json_dict(json_dict)
self.assertEqual(str(cm.exception), 'missing mandatory property "qualified_name" in Workflow-JSON')
def test_to_json_dict(self):
step1 = OpStep(op1, node_id='op1')
step2 = OpStep(op2, node_id='op2')
step3 = OpStep(op3, node_id='op3')
workflow = Workflow(OpMetaInfo('my_workflow', inputs=OrderedDict(p={}), outputs=OrderedDict(q={})))
workflow.add_steps(step1, step2, step3)
step1.inputs.x.source = workflow.inputs.p
step2.inputs.a.source = step1.outputs.y
step3.inputs.u.source = step1.outputs.y
step3.inputs.v.source = step2.outputs.b
workflow.outputs.q.source = step3.outputs.w
step2.persistent = True
workflow_dict = workflow.to_json_dict()
expected_json_text = """
{
"schema_version": 1,
"qualified_name": "my_workflow",
"header": {},
"inputs": {
"p": {}
},
"outputs": {
"q": {
"source": "op3.w"
}
},
"steps": [
{
"id": "op1",
"op": "test.core.test_workflow.op1",
"inputs": {
"x": "my_workflow.p"
}
},
{
"id": "op2",
"persistent": true,
"op": "test.core.test_workflow.op2",
"inputs": {
"a": "op1.y"
}
},
{
"id": "op3",
"op": "test.core.test_workflow.op3",
"inputs": {
"v": "op2.b",
"u": "op1.y"
}
}
]
}
"""
actual_json_text = json.dumps(workflow_dict, indent=4)
expected_json_obj = json.loads(expected_json_text)
actual_json_obj = json.loads(actual_json_text)
self.assertEqual(actual_json_obj, expected_json_obj,
msg='\nexpected:\n%s\n%s\nbut got:\n%s\n%s\n' %
(120 * '-', expected_json_text,
120 * '-', actual_json_text))
def test_to_json_dict(self):
def dataset_op() -> xr.Dataset:
periods = 5
temperature_data = (15 + 8 * np.random.randn(periods, 2, 2)).round(
decimals=1)
temperature_attrs = {
'a': np.array([1, 2, 3]),
'comment': 'hot',
'_FillValue': np.nan
}
precipitation_data = (10 * np.random.rand(periods, 2, 2)).round(
decimals=1)
precipitation_attrs = {
'x': True,
'comment': 'wet',
'_FillValue': -1.0
}
ds = xr.Dataset(data_vars={
'temperature':
(('time', 'lat', 'lon'), temperature_data, temperature_attrs),
'precipitation': (('time', 'lat', 'lon'), precipitation_data,
precipitation_attrs)
},
coords={
'lon':
np.array([12, 13]),
'lat':
np.array([50, 51]),
'time':
pd.date_range('2014-09-06', periods=periods)
},
attrs={'history': 'a b c'})
return ds
def data_frame_op() -> pd.DataFrame:
data = {
'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, 5, 5, 9, 1, 2, 7, 6]
}
time = pd.date_range('2000-01-01', freq='MS', periods=12)
return pd.DataFrame(data=data, index=time, dtype=float)
def int_op() -> int:
return 394852
def str_op() -> str:
return 'Hi!'
from cate.core.op import OP_REGISTRY
try:
OP_REGISTRY.add_op(dataset_op)
OP_REGISTRY.add_op(data_frame_op)
OP_REGISTRY.add_op(int_op)
OP_REGISTRY.add_op(str_op)
workflow = Workflow(
OpMetaInfo('workspace_workflow',
header=dict(description='Test!')))
workflow.add_step(OpStep(dataset_op, node_id='ds'))
workflow.add_step(OpStep(data_frame_op, node_id='df'))
workflow.add_step(OpStep(int_op, node_id='i'))
workflow.add_step(OpStep(str_op, node_id='s'))
ws = Workspace('/path', workflow)
ws.execute_workflow()
d_ws = ws.to_json_dict()
# import pprint
# pprint.pprint(d_ws)
d_wf = d_ws.get('workflow')
self.assertIsNotNone(d_wf)
l_res = d_ws.get('resources')
self.assertIsNotNone(l_res)
self.assertEqual(len(l_res), 4)
res_1 = l_res[0]
self.assertEqual(res_1.get('name'), 'ds')
self.assertEqual(res_1.get('dataType'),
'xarray.core.dataset.Dataset')
self.assertEqual(res_1.get('dimSizes'), dict(lat=2, lon=2, time=5))
self.assertEqual(res_1.get('attributes'), {'history': 'a b c'})
res_1_vars = res_1.get('variables')
self.assertIsNotNone(res_1_vars)
self.assertEqual(len(res_1_vars), 2)
var_1 = res_1_vars[0]
self.assertEqual(var_1.get('name'), 'precipitation')
self.assertEqual(var_1.get('dataType'), 'float64')
self.assertEqual(var_1.get('numDims'), 3)
self.assertEqual(var_1.get('shape'), (5, 2, 2))
self.assertEqual(var_1.get('chunkSizes'), None)
self.assertEqual(var_1.get('isYFlipped'), True)
self.assertEqual(var_1.get('isFeatureAttribute'), None)
self.assertEqual(var_1.get('attributes'),
dict(x=True, comment='wet', _FillValue=-1.))
var_2 = res_1_vars[1]
self.assertEqual(var_2.get('name'), 'temperature')
self.assertEqual(var_2.get('dataType'), 'float64')
self.assertEqual(var_2.get('numDims'), 3)
self.assertEqual(var_2.get('shape'), (5, 2, 2))
self.assertEqual(var_2.get('chunkSizes'), None)
self.assertEqual(var_2.get('isYFlipped'), True)
self.assertEqual(var_2.get('isFeatureAttribute'), None)
self.assertEqual(
var_2.get('attributes'),
dict(a=[1, 2, 3], comment='hot', _FillValue=np.nan))
res_2 = l_res[1]
self.assertEqual(res_2.get('name'), 'df')
self.assertEqual(res_2.get('dataType'),
'pandas.core.frame.DataFrame')
self.assertIsNone(res_2.get('attributes'))
res_2_vars = res_2.get('variables')
self.assertIsNotNone(res_2_vars)
self.assertEqual(len(res_2_vars), 2)
var_1 = res_2_vars[0]
self.assertEqual(var_1.get('name'), 'A')
self.assertEqual(var_1.get('dataType'), 'float64')
self.assertEqual(var_1.get('numDims'), 1)
self.assertEqual(var_1.get('shape'), (12, ))
self.assertEqual(var_1.get('isYFlipped'), None)
self.assertEqual(var_1.get('isFeatureAttribute'), None)
self.assertIsNone(var_1.get('attributes'))
var_2 = res_2_vars[1]
self.assertEqual(var_2.get('name'), 'B')
self.assertEqual(var_2.get('dataType'), 'float64')
self.assertEqual(var_2.get('numDims'), 1)
self.assertEqual(var_2.get('shape'), (12, ))
self.assertEqual(var_2.get('isYFlipped'), None)
self.assertEqual(var_2.get('isFeatureAttribute'), None)
self.assertIsNone(var_2.get('attributes'))
res_3 = l_res[2]
self.assertEqual(res_3.get('name'), 'i')
self.assertEqual(res_3.get('dataType'), 'int')
self.assertIsNone(res_3.get('attributes'))
self.assertIsNone(res_3.get('variables'))
res_4 = l_res[3]
self.assertEqual(res_4.get('name'), 's')
self.assertEqual(res_4.get('dataType'), 'str')
self.assertIsNone(res_4.get('attrs'))
self.assertIsNone(res_4.get('variables'))
finally:
OP_REGISTRY.remove_op(dataset_op)
OP_REGISTRY.remove_op(data_frame_op)
OP_REGISTRY.remove_op(int_op)
OP_REGISTRY.remove_op(str_op)
def test_sort_steps(self):
step1, step2, step3, _ = self.create_example_3_steps_workflow()
self.assertEqual(Workflow.sort_steps([]), [])
self.assertEqual(Workflow.sort_steps([step1]), [step1])
self.assertEqual(Workflow.sort_steps([step1, step2]), [step1, step2])
self.assertEqual(Workflow.sort_steps([step1, step3]), [step1, step3])
self.assertEqual(Workflow.sort_steps([step2, step1]), [step1, step2])
self.assertEqual(Workflow.sort_steps([step3, step1]), [step1, step3])
self.assertEqual(Workflow.sort_steps([step2, step3]), [step2, step3])
self.assertEqual(Workflow.sort_steps([step3, step2]), [step2, step3])
self.assertEqual(Workflow.sort_steps([step1, step2, step3]), [step1, step2, step3])
self.assertEqual(Workflow.sort_steps([step2, step1, step3]), [step1, step2, step3])
self.assertEqual(Workflow.sort_steps([step3, step2, step1]), [step1, step2, step3])
self.assertEqual(Workflow.sort_steps([step1, step3, step2]), [step1, step2, step3])
def from_json_dict(cls, json_dict):
base_dir = json_dict.get('base_dir', None)
workflow_json = json_dict.get('workflow', {})
is_modified = json_dict.get('is_modified', False)
workflow = Workflow.from_json_dict(workflow_json)
return Workspace(base_dir, workflow, is_modified=is_modified)
def test_sort_steps(self):
step1, step2, step3, _ = self.create_example_3_steps_workflow()
self.assertEqual(Workflow.sort_steps([]), [])
self.assertEqual(Workflow.sort_steps([step1]), [step1])
self.assertEqual(Workflow.sort_steps([step1, step2]), [step1, step2])
self.assertEqual(Workflow.sort_steps([step1, step3]), [step1, step3])
self.assertEqual(Workflow.sort_steps([step2, step1]), [step1, step2])
self.assertEqual(Workflow.sort_steps([step3, step1]), [step1, step3])
self.assertEqual(Workflow.sort_steps([step2, step3]), [step2, step3])
self.assertEqual(Workflow.sort_steps([step3, step2]), [step2, step3])
self.assertEqual(Workflow.sort_steps([step1, step2, step3]),
[step1, step2, step3])
self.assertEqual(Workflow.sort_steps([step2, step1, step3]),
[step1, step2, step3])
self.assertEqual(Workflow.sort_steps([step3, step2, step1]),
[step1, step2, step3])
self.assertEqual(Workflow.sort_steps([step1, step3, step2]),
[step1, step2, step3])
def test_from_json_dict(self):
workflow_json_text = """
{
"qualified_name": "my_workflow",
"header": {
"description": "My workflow is not too bad."
},
"inputs": {
"p": {"description": "Input 'p'"}
},
"outputs": {
"q": {"source": "op3.w", "description": "Output 'q'"}
},
"steps": [
{
"id": "op1",
"op": "tests.core.test_workflow.op1",
"inputs": {
"x": { "source": ".p" }
}
},
{
"id": "op2",
"op": "tests.core.test_workflow.op2",
"inputs": {
"a": {"source": "op1"}
}
},
{
"id": "op3",
"persistent": true,
"op": "tests.core.test_workflow.op3",
"inputs": {
"u": {"source": "op1.y"},
"v": {"source": "op2.b"}
}
}
]
}
"""
workflow_json_dict = json.loads(workflow_json_text)
workflow = Workflow.from_json_dict(workflow_json_dict)
self.assertIsNotNone(workflow)
self.assertEqual(workflow.id, "my_workflow")
self.assertEqual(workflow.op_meta_info.qualified_name, workflow.id)
self.assertEqual(workflow.op_meta_info.header,
dict(description="My workflow is not too bad."))
self.assertEqual(len(workflow.op_meta_info.inputs), 1)
self.assertEqual(len(workflow.op_meta_info.outputs), 1)
self.assertEqual(workflow.op_meta_info.inputs['p'],
dict(description="Input 'p'"))
self.assertEqual(workflow.op_meta_info.outputs['q'],
dict(source="op3.w", description="Output 'q'"))
self.assertEqual(len(workflow.inputs), 1)
self.assertEqual(len(workflow.outputs), 1)
self.assertIn('p', workflow.inputs)
self.assertIn('q', workflow.outputs)
self.assertEqual(len(workflow.steps), 3)
step1 = workflow.steps[0]
step2 = workflow.steps[1]
step3 = workflow.steps[2]
self.assertEqual(step1.id, 'op1')
self.assertEqual(step2.id, 'op2')
self.assertEqual(step3.id, 'op3')
self.assertIs(step1.inputs.x.source, workflow.inputs.p)
self.assertIs(step2.inputs.a.source, step1.outputs.y)
self.assertIs(step3.inputs.u.source, step1.outputs.y)
self.assertIs(step3.inputs.v.source, step2.outputs.b)
self.assertIs(workflow.outputs.q.source, step3.outputs.w)
self.assertEqual(step1.persistent, False)
self.assertEqual(step2.persistent, False)
self.assertEqual(step3.persistent, True)
def test_from_json_dict_empty(self):
json_dict = json.loads('{"qualified_name": "hello"}')
workflow = Workflow.from_json_dict(json_dict)
self.assertEqual(workflow.id, 'hello')
def test_to_json_dict(self):
step1 = OpStep(op1, node_id='op1')
step2 = OpStep(op2, node_id='op2')
step3 = OpStep(op3, node_id='op3')
workflow = Workflow(
OpMetaInfo('my_workflow',
inputs=OrderedDict(p={}),
outputs=OrderedDict(q={})))
workflow.add_steps(step1, step2, step3)
step1.inputs.x.source = workflow.inputs.p
step2.inputs.a.source = step1.outputs.y
step3.inputs.u.source = step1.outputs.y
step3.inputs.v.source = step2.outputs.b
workflow.outputs.q.source = step3.outputs.w
step2.persistent = True
workflow_dict = workflow.to_json_dict()
expected_json_text = """
{
"schema_version": 1,
"qualified_name": "my_workflow",
"header": {},
"inputs": {
"p": {}
},
"outputs": {
"q": {
"source": "op3.w"
}
},
"steps": [
{
"id": "op1",
"op": "tests.core.test_workflow.op1",
"inputs": {
"x": "my_workflow.p"
}
},
{
"id": "op2",
"persistent": true,
"op": "tests.core.test_workflow.op2",
"inputs": {
"a": "op1.y"
}
},
{
"id": "op3",
"op": "tests.core.test_workflow.op3",
"inputs": {
"v": "op2.b",
"u": "op1.y"
}
}
]
}
"""
actual_json_text = json.dumps(workflow_dict, indent=4)
expected_json_obj = json.loads(expected_json_text)
actual_json_obj = json.loads(actual_json_text)
self.assertEqual(
actual_json_obj,
expected_json_obj,
msg='\nexpected:\n%s\n%s\nbut got:\n%s\n%s\n' %
(120 * '-', expected_json_text, 120 * '-', actual_json_text))