def test_utilities(self): self.assertEqual(mk_op_arg(1), {'value': 1}) self.assertEqual(mk_op_arg('2'), {'value': '2'}) self.assertEqual(mk_op_arg('a'), {'value': 'a'}) self.assertEqual(mk_op_arg('@b'), {'source': 'b'}) self.assertEqual(mk_op_args(), []) self.assertEqual(mk_op_args(1, '2', 'a', '@b'), [{ 'value': 1 }, { 'value': '2' }, { 'value': 'a' }, { 'source': 'b' }]) self.assertEqual(mk_op_kwargs(a=1), OrderedDict([('a', {'value': 1})])) self.assertEqual( mk_op_kwargs(a=1, b='@c'), OrderedDict([('a', { 'value': 1 }), ('b', { 'source': 'c' })]))
def test_set_step_and_run_op(self): ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))) ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='X') ws.execute_workflow('X') self.assertIsNotNone(ws.workflow) self.assertEqual(len(ws.workflow.steps), 1) self.assertIn('X', ws.resource_cache) op_name = '_extract_point' op_args = mk_op_kwargs(ds='@X', point='10.22, 34.52', indexers=dict(time='2014-09-11'), should_return=True) op_result = ws.run_op(op_name, op_args) self.assertEqual(len(op_result), 4) self.assertAlmostEqual(op_result['lat'], 34.5) self.assertAlmostEqual(op_result['lon'], 10.2) self.assertAlmostEqual(op_result['precipitation'], 5.5) self.assertAlmostEqual(op_result['temperature'], 32.9) # without asking for return data op_args = mk_op_kwargs(ds='@X', point='10.22, 34.52', indexers=dict(time='2014-09-11')) op_result = ws.run_op(op_name, op_args) self.assertIsNone(op_result) # with a non existing operator name with self.assertRaises(ValidationError) as we: ws.run_op("not_existing_op", {}) self.assertEqual('Unknown operation "not_existing_op"', str(we.exception))
def test_set_and_execute_step(self): ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))) ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='X') ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="precipitation"), res_name='Y') self.assertEqual(ws.resource_cache, {}) ws.execute_workflow('Y') self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache) ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="temperature"), res_name='Y', overwrite=True) self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache) self.assertIs(ws.resource_cache['Y'], UNDEFINED) ws.execute_workflow('Y') self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache) ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_2), res_name='X', overwrite=True) self.assertIn('X', ws.resource_cache) self.assertIs(ws.resource_cache['X'], UNDEFINED) self.assertIn('Y', ws.resource_cache) self.assertIs(ws.resource_cache['Y'], UNDEFINED) ws.execute_workflow('Y') self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache)
def test_example(self): expected_json_text = """{ "schema_version": 1, "qualified_name": "workspace_workflow", "header": { "description": "Test!" }, "inputs": {}, "outputs": {}, "steps": [ { "id": "p", "op": "cate.ops.io.read_netcdf", "inputs": { "file": { "value": "%s" } } }, { "id": "ts", "op": "cate.ops.timeseries.tseries_mean", "inputs": { "ds": "p", "var": { "value": "precipitation" } } } ] } """ % NETCDF_TEST_FILE_1.replace('\\', '\\\\') expected_json_dict = json.loads(expected_json_text) ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))) # print("wf_1: " + json.dumps(ws.workflow.to_json_dict(), indent=' ')) ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='p') # print("wf_2: " + json.dumps(ws.workflow.to_json_dict(), indent=' ')) ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@p", var="precipitation"), res_name='ts') # print("wf_3: " + json.dumps(ws.workflow.to_json_dict(), indent=' ')) self.maxDiff = None self.assertEqual(ws.workflow.to_json_dict(), expected_json_dict) with self.assertRaises(ValueError) as e: ws.set_resource('cate.ops.timeseries.tseries_point', mk_op_kwargs(ds="@p", point="iih!", var="precipitation"), res_name='ts2', validate_args=True) self.assertEqual(str(e.exception), "Input 'point' for operation 'cate.ops.timeseries.tseries_point': " "Value cannot be converted into a 'PointLike': " "Invalid geometry WKT format.") ws2 = Workspace.from_json_dict(ws.to_json_dict()) self.assertEqual(ws2.base_dir, ws.base_dir) self.assertEqual(ws2.workflow.op_meta_info.qualified_name, ws.workflow.op_meta_info.qualified_name) self.assertEqual(len(ws2.workflow.steps), len(ws.workflow.steps))
def test_workspace_can_create_new_res_names(self): ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))) res_name_1 = ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value='A')) res_name_2 = ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value='B')) res_name_3 = ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value='C')) self.assertEqual(res_name_1, 'res_1') self.assertEqual(res_name_2, 'res_2') self.assertEqual(res_name_3, 'res_3') self.assertIsNotNone(ws.workflow.find_node(res_name_1)) self.assertIsNotNone(ws.workflow.find_node(res_name_2)) self.assertIsNotNone(ws.workflow.find_node(res_name_3))
def test_utilities(self): self.assertEqual(mk_op_arg(1), {'value': 1}) self.assertEqual(mk_op_arg('2'), {'value': '2'}) self.assertEqual(mk_op_arg('a'), {'value': 'a'}) self.assertEqual(mk_op_arg('@b'), {'source': 'b'}) self.assertEqual(mk_op_args(), []) self.assertEqual(mk_op_args(1, '2', 'a', '@b'), [{'value': 1}, {'value': '2'}, {'value': 'a'}, {'source': 'b'}]) self.assertEqual(mk_op_kwargs(a=1), OrderedDict([('a', {'value': 1})])) self.assertEqual(mk_op_kwargs(a=1, b='@c'), OrderedDict([('a', {'value': 1}), ('b', {'source': 'c'})]))
def test_set_and_rename_and_execute_step(self): ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))) self.assertEqual(ws.user_data, {}) ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value=1), res_name='X') ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value="@X"), res_name='Y') ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value="@X"), res_name='Z') self.assertEqual(len(ws.workflow.steps), 3) self.assertEqual(ws.resource_cache, {}) value = ws.execute_workflow('Y') self.assertEqual(value, 1) self.assertEqual(ws.resource_cache.get('X'), 1) self.assertEqual(ws.resource_cache.get('Y'), 1) self.assertEqual(ws.resource_cache.get('Z'), None) value = ws.execute_workflow('Z') self.assertEqual(value, 1) self.assertEqual(ws.resource_cache.get('X'), 1) self.assertEqual(ws.resource_cache.get('Y'), 1) self.assertEqual(ws.resource_cache.get('Z'), 1) ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value=9), res_name='X', overwrite=True) self.assertEqual(len(ws.workflow.steps), 3) self.assertEqual(ws.resource_cache.get('X'), UNDEFINED) self.assertEqual(ws.resource_cache.get('Y'), UNDEFINED) self.assertEqual(ws.resource_cache.get('Z'), UNDEFINED) ws.execute_workflow() self.assertEqual(ws.resource_cache.get('X'), 9) self.assertEqual(ws.resource_cache.get('Y'), 9) self.assertEqual(ws.resource_cache.get('Z'), 9) ws.rename_resource('X', 'A') self.assertIsNone(ws.workflow.find_node('X')) self.assertIsNotNone(ws.workflow.find_node('A')) self.assertEqual(ws.resource_cache.get('X', '--'), '--') self.assertEqual(ws.resource_cache.get('A'), 9) self.assertEqual(ws.resource_cache.get('Y'), 9) self.assertEqual(ws.resource_cache.get('Z'), 9) ws.set_resource('cate.ops.utility.identity', mk_op_kwargs(value=5), res_name='A', overwrite=True) self.assertEqual(ws.resource_cache.get('X', '--'), '--') self.assertEqual(ws.resource_cache.get('A'), UNDEFINED) self.assertEqual(ws.resource_cache.get('Y'), UNDEFINED) self.assertEqual(ws.resource_cache.get('Z'), UNDEFINED) ws.execute_workflow() self.assertEqual(ws.resource_cache.get('X', '--'), '--') self.assertEqual(ws.resource_cache.get('A'), 5) self.assertEqual(ws.resource_cache.get('Y'), 5) self.assertEqual(ws.resource_cache.get('Z'), 5)
def test_session(self): rel_path = 'TESTOMAT' base_dir = self.new_base_dir(rel_path) workspace_manager = self.new_workspace_manager() workspace1 = workspace_manager.new_workspace( base_dir, description='session workspace') self.assertIsNotNone(workspace1) self.assertEqual(workspace1.base_dir, base_dir) self.assertEqual( workspace1.workflow.op_meta_info.header.get('description', None), 'session workspace') workspace_manager.save_workspace(base_dir) self.assertTrue(os.path.exists(base_dir)) res_name = 'ds' workspace_manager.set_workspace_resource( base_dir, 'cate.ops.io.read_netcdf', dict(file=dict(value=NETCDF_TEST_FILE)), res_name=res_name) workspace2 = workspace_manager.get_workspace(base_dir) self.assertEqual(workspace2.base_dir, base_dir) self.assertEqual(workspace2.workflow.id, workspace1.workflow.id) self.assertEqual(len(workspace2.workflow.steps), 1) sst_step = workspace2.workflow.find_node(res_name) self.assertIsNotNone(sst_step) file_path = os.path.abspath( os.path.join(self._root_dir, 'TESTOMAT', 'precip_and_temp_copy.nc')) workspace_manager.write_workspace_resource(base_dir, res_name, file_path=file_path) self.assertTrue(os.path.isfile(file_path)) run_file_path = os.path.abspath( os.path.join(self._root_dir, 'TESTOMAT', 'precip_and_temp_runcopy.nc')) workspace_manager.run_op_in_workspace( base_dir, 'write_netcdf4', mk_op_kwargs(obj='@ds', file=run_file_path)) self.assertTrue(os.path.isfile(run_file_path)) workspaces = workspace_manager.get_open_workspaces() self.assertIsNotNone(workspaces) self.assertEqual(len(workspaces), 1) self.assertEqual(workspaces[0].base_dir, base_dir) workspace3 = workspace_manager.delete_workspace_resource( base_dir, res_name) self.assertEqual(len(workspace3.workflow.steps), 0) self.del_base_dir(base_dir)
def test_persistence(self): rel_path = 'TESTOMAT' base_dir = self.new_base_dir(rel_path) workspace_manager = self.new_workspace_manager() workspace_manager.new_workspace(base_dir) workspace_manager.save_workspace(base_dir) self.assertTrue(os.path.exists(base_dir)) workspace_manager.set_workspace_resource( base_dir, 'cate.ops.io.read_netcdf', dict(file=dict(value=NETCDF_TEST_FILE)), res_name='ds') workspace1, _ = workspace_manager.set_workspace_resource( base_dir, 'cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds='@ds', var='temperature'), res_name='ts') self.assertEqual(workspace1.base_dir, base_dir) self.assertEqual(len(workspace1.workflow.steps), 2) self.assertFalse(workspace1.workflow.find_node('ds').persistent) self.assertFalse(workspace1.workflow.find_node('ts').persistent) ts_file_path = os.path.abspath( os.path.join(self._root_dir, 'TESTOMAT', '.cate-workspace', 'ts.nc')) self.assertFalse(os.path.isfile(ts_file_path)) workspace3 = workspace_manager.set_workspace_resource_persistence( base_dir, 'ts', True) self.assertFalse(workspace3.workflow.find_node('ds').persistent) self.assertTrue(workspace3.workflow.find_node('ts').persistent) workspace4 = workspace_manager.set_workspace_resource_persistence( base_dir, 'ts', True) self.assertFalse(workspace4.workflow.find_node('ds').persistent) self.assertTrue(workspace4.workflow.find_node('ts').persistent) workspace_manager.save_workspace(base_dir) self.assertTrue(os.path.isfile(ts_file_path)) workspace_manager.close_workspace(base_dir) self.assertEqual(len(workspace_manager.get_open_workspaces()), 0) self.assertTrue(os.path.isfile(ts_file_path)) workspace5 = workspace_manager.open_workspace(base_dir) self.assertEqual(workspace4.workflow.to_json_dict(), workspace5.workflow.to_json_dict()) workspace_manager.set_workspace_resource_persistence( base_dir, 'ts', False) workspace_manager.close_workspace(base_dir) workspace_manager.close_workspace( base_dir) # closing a 2nd time should give no error self.assertFalse(os.path.isfile(ts_file_path)) self.del_base_dir(base_dir)
def test_set_and_execute_step(self): ws = Workspace('/path', Workflow(OpMetaInfo('workspace_workflow', header=dict(description='Test!')))) with self.assertRaises(ValidationError) as we: ws.set_resource("not_existing_op", {}) self.assertEqual('Unknown operation "not_existing_op"', str(we.exception)) with self.assertRaises(ValidationError) as we: ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(location=NETCDF_TEST_FILE_1), res_name='X') self.assertEqual('"location" is not an input of operation "cate.ops.io.read_netcdf"', str(we.exception)) with self.assertRaises(ValidationError) as we: ws.set_resource('cate.ops.io.read_netcdf', {'file': {'foo': 'bar'}}, res_name='X') self.assertEqual('Illegal argument for input "file" of operation "cate.ops.io.read_netcdf', str(we.exception)) ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_1), res_name='X') ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="precipitation"), res_name='Y') self.assertEqual(ws.resource_cache, {}) ws.execute_workflow('Y') self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache) ws.set_resource('cate.ops.timeseries.tseries_mean', mk_op_kwargs(ds="@X", var="temperature"), res_name='Y', overwrite=True) self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache) self.assertIs(ws.resource_cache['Y'], UNDEFINED) ws.execute_workflow('Y') self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache) ws.set_resource('cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE_2), res_name='X', overwrite=True) self.assertIn('X', ws.resource_cache) self.assertIs(ws.resource_cache['X'], UNDEFINED) self.assertIn('Y', ws.resource_cache) self.assertIs(ws.resource_cache['Y'], UNDEFINED) ws.execute_workflow('Y') self.assertIn('X', ws.resource_cache) self.assertIn('Y', ws.resource_cache)
def test_set_workspace_resource(self): base_dir = self.new_base_dir('TESTOMAT') workspace_manager = self.new_workspace_manager() workspace1 = workspace_manager.new_workspace(base_dir) workspace1.save() self.assertTrue(os.path.exists(base_dir)) workspace_manager.set_workspace_resource( base_dir, 'cate.ops.io.read_netcdf', mk_op_kwargs(file=NETCDF_TEST_FILE), res_name='SST') workspace2 = workspace_manager.get_workspace(base_dir) self.assertEqual(workspace2.base_dir, workspace1.base_dir) self.assertEqual(workspace2.workflow.id, workspace1.workflow.id) sst_step = workspace2.workflow.find_node('SST') self.assertIsNotNone(sst_step) self.del_base_dir(base_dir)