class OpTest(TestCase):
def setUp(self):
self.registry = OpRegistry()
def tearDown(self):
self.registry = None
def test_f(self):
def f(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f!"""
return str(a + b + c + u + len(v) + w)
registry = self.registry
added_op_reg = registry.add_op(f)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(f, fail_if_exists=True)
self.assertIs(registry.add_op(f, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.operation, f)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(data_type=float, position=0)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(default_value=3)
expected_inputs['v'] = dict(default_value='A')
expected_inputs['w'] = dict(default_value=4.9)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info, object_to_qualified_name(f),
dict(description='Hi, I am f!'), expected_inputs,
expected_outputs)
removed_op_reg = registry.remove_op(f)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(f, fail_if_not_exists=True)
def test_f_op(self):
@op(registry=self.registry)
def f_op(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op))
self.assertIs(op_reg.operation, f_op)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(default_value=3)
expected_inputs['v'] = dict(default_value='A')
expected_inputs['w'] = dict(default_value=4.9)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op),
dict(description='Hi, I am f_op!'),
expected_inputs, expected_outputs)
def test_f_op_inp_ret(self):
@op_input('a', value_range=[0., 1.], registry=self.registry)
@op_input('v', value_set=['A', 'B', 'C'], registry=self.registry)
@op_return(registry=self.registry)
def f_op_inp_ret(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op_inp_ret!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op_inp_ret, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op_inp_ret))
self.assertIs(op_reg.operation, f_op_inp_ret)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0,
data_type=float,
value_range=[0., 1.])
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(default_value=3)
expected_inputs['v'] = dict(default_value='A',
value_set=['A', 'B', 'C'])
expected_inputs['w'] = dict(default_value=4.9)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op_inp_ret),
dict(description='Hi, I am f_op_inp_ret!'),
expected_inputs, expected_outputs)
def test_C(self):
class C:
"""Hi, I am C!"""
def __call__(self):
return None
registry = self.registry
added_op_reg = registry.add_op(C)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(C, fail_if_exists=True)
self.assertIs(registry.add_op(C, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(C))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.operation, C)
self._assertMetaInfo(op_reg.op_meta_info, object_to_qualified_name(C),
dict(description='Hi, I am C!'), OrderedDict(),
OrderedDict({RETURN: {}}))
removed_op_reg = registry.remove_op(C)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(C))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(C, fail_if_not_exists=True)
def test_C_op(self):
@op(author='Ernie and Bert', registry=self.registry)
class C_op:
"""Hi, I am C_op!"""
def __call__(self):
return None
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(C_op, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(C_op))
self.assertIs(op_reg.operation, C_op)
self._assertMetaInfo(
op_reg.op_meta_info, object_to_qualified_name(C_op),
dict(description='Hi, I am C_op!', author='Ernie and Bert'),
OrderedDict(), OrderedDict({RETURN: {}}))
def _assertMetaInfo(self, op_meta_info: OpMetaInfo, expected_name: str,
expected_header: dict, expected_input: OrderedDict,
expected_output: OrderedDict):
self.assertIsNotNone(op_meta_info)
self.assertEqual(op_meta_info.qualified_name, expected_name)
self.assertEqual(op_meta_info.header, expected_header)
self.assertEqual(OrderedDict(op_meta_info.input), expected_input)
self.assertEqual(OrderedDict(op_meta_info.output), expected_output)
def test_function_validation(self):
@op_input('x',
registry=self.registry,
data_type=float,
value_range=[0.1, 0.9],
default_value=0.5)
@op_input('y', registry=self.registry)
@op_input('a',
registry=self.registry,
data_type=int,
value_set=[1, 4, 5])
@op_return(registry=self.registry, data_type=float)
def f(x, y: float, a=4):
return a * x + y if a != 5 else 'foo'
self.assertEqual(f(y=1, x=8), 33)
self.assertEqual(f(**dict(a=5, x=8, y=1)), 'foo')
op_reg = self.registry.get_op(f)
self.assertEqual(op_reg.op_meta_info.input['x'].get('data_type', None),
float)
self.assertEqual(
op_reg.op_meta_info.input['x'].get('value_range', None),
[0.1, 0.9])
self.assertEqual(
op_reg.op_meta_info.input['x'].get('default_value', None), 0.5)
self.assertEqual(op_reg.op_meta_info.input['x'].get('position', None),
0)
self.assertEqual(op_reg.op_meta_info.input['y'].get('data_type', None),
float)
self.assertEqual(op_reg.op_meta_info.input['y'].get('position', None),
1)
self.assertEqual(op_reg.op_meta_info.input['a'].get('data_type', None),
int)
self.assertEqual(op_reg.op_meta_info.input['a'].get('value_set', None),
[1, 4, 5])
self.assertEqual(
op_reg.op_meta_info.input['a'].get('default_value', None), 4)
self.assertEqual(op_reg.op_meta_info.input['a'].get('position', None),
None)
self.assertEqual(
op_reg.op_meta_info.output[RETURN].get('data_type', None), float)
with self.assertRaises(ValueError) as cm:
result = op_reg(x=0, y=3.)
self.assertEqual(
str(cm.exception),
"input 'x' for operation 'test.core.test_op.f' must be in range [0.1, 0.9]"
)
with self.assertRaises(ValueError) as cm:
result = op_reg(x='A', y=3.)
self.assertEqual(
str(cm.exception),
"input 'x' for operation 'test.core.test_op.f' must be of type 'float', "
"but got type 'str'")
with self.assertRaises(ValueError) as cm:
result = op_reg(x=0.4)
self.assertEqual(
str(cm.exception),
"input 'y' for operation 'test.core.test_op.f' required")
with self.assertRaises(ValueError) as cm:
result = op_reg(x=0.6, y=0.1, a=2)
self.assertEqual(
str(cm.exception),
"input 'a' for operation 'test.core.test_op.f' must be one of [1, 4, 5]"
)
with self.assertRaises(ValueError) as cm:
result = op_reg(x=0.6, y=0.1, a=5)
self.assertEqual(
str(cm.exception),
"output '%s' for operation 'test.core.test_op.f' must be of type <class 'float'>"
% RETURN)
result = op_reg(y=3)
self.assertEqual(result, 4 * 0.5 + 3)
def test_function_invocation(self):
def f(x, a=4):
return a * x
op_reg = self.registry.add_op(f)
result = op_reg(x=2.5)
self.assertEqual(result, 4 * 2.5)
def test_function_invocation_with_monitor(self):
def f(monitor: Monitor, x, a=4):
monitor.start('f', 23)
return_value = a * x
monitor.done()
return return_value
op_reg = self.registry.add_op(f)
monitor = MyMonitor()
result = op_reg(x=2.5, monitor=monitor)
self.assertEqual(result, 4 * 2.5)
self.assertEqual(monitor.total_work, 23)
self.assertEqual(monitor.is_done, True)
def test_class_invocation(self):
@op_input('x', registry=self.registry)
@op_input('a', default_value=4, registry=self.registry)
@op_output('y', registry=self.registry)
class C:
def __call__(self, x, a):
return {'y': x * a}
op_reg = self.registry.get_op(C)
result = op_reg(x=2.5)
self.assertEqual(result, {'y': 4 * 2.5})
def test_class_invocation_with_monitor(self):
@op_input('x', registry=self.registry)
@op_input('a', default_value=4, registry=self.registry)
@op_output('y', registry=self.registry)
class C:
def __call__(self, x, a, monitor: Monitor):
monitor.start('C', 19)
output = {'y': x * a}
monitor.done()
return output
op_reg = self.registry.get_op(C)
monitor = MyMonitor()
result = op_reg(x=2.5, monitor=monitor)
self.assertEqual(result, {'y': 4 * 2.5})
self.assertEqual(monitor.total_work, 19)
self.assertEqual(monitor.is_done, True)
def test_class_invocation_with_start_up(self):
@op_input('x', registry=self.registry)
@op_input('a', default_value=4, registry=self.registry)
@op_output('y', registry=self.registry)
class C:
b = None
@classmethod
def start_up(cls):
C.b = 1.5
@classmethod
def tear_down(cls):
C.b = None
def __call__(self, x, a):
return {'y': x * a + C.b}
op_reg = self.registry.get_op(C)
with self.assertRaisesRegex(
TypeError,
"unsupported operand type\\(s\\) for \\+\\: 'float' and 'NoneType'"
):
# because C.b is None, C.start_up has not been called yet
op_reg(x=2.5)
# Note: this is exemplary code how the framework could call special class methods start_up/tear_down if it
# finds them declared in a given op-class.
# - 'start_up' may be called a single time before instances are created.
# - 'tear_down' may be called and an operation is deregistered and it's 'start_up' has been called.
C.start_up()
result = op_reg(x=2.5)
C.tear_down()
self.assertEqual(result, {'y': 4 * 2.5 + 1.5})
def test_C_op_inp_out(self):
@op_input('a',
data_type=float,
default_value=0.5,
value_range=[0., 1.],
registry=self.registry)
@op_input('b',
data_type=str,
default_value='A',
value_set=['A', 'B', 'C'],
registry=self.registry)
@op_output('x', data_type=float, registry=self.registry)
@op_output('y', data_type=list, registry=self.registry)
class C_op_inp_out:
"""Hi, I am C_op_inp_out!"""
def __call__(self, a, b):
x = 2.5 * a
y = [a, b]
return {'x': x, 'y': y}
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(C_op_inp_out, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(C_op_inp_out))
self.assertIs(op_reg.operation, C_op_inp_out)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0,
data_type=float,
default_value=0.5,
value_range=[0., 1.])
expected_inputs['b'] = dict(position=1,
data_type=str,
default_value='A',
value_set=['A', 'B', 'C'])
expected_outputs = OrderedDict()
expected_outputs['y'] = dict(data_type=list)
expected_outputs['x'] = dict(data_type=float)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(C_op_inp_out),
dict(description='Hi, I am C_op_inp_out!'),
expected_inputs, expected_outputs)
def test_history_op(self):
"""
Test adding operation signature to output history information.
"""
import xarray as xr
from cate import __version__
# Test @op_return
@op(version='0.9', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_op(ds: xr.Dataset, a=1, b='bilinear'):
ret = ds.copy()
return ret
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.input) + '\nProvided input values: '
ret_ds = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue(stamp in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret_ds = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret_ds.attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Test @op_output
@op(version='1.9', registry=self.registry)
@op_output('name1', add_history=True, registry=self.registry)
@op_output('name2', add_history=False, registry=self.registry)
@op_output('name3', registry=self.registry)
def history_named_op(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
ds2 = ds.copy()
ds3 = ds.copy()
return {'name1': ds1, 'name2': ds2, 'name3': ds3}
ds = xr.Dataset()
op_reg = self.registry.get_op(
object_to_qualified_name(history_named_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.input) + '\nProvided input values: '
ret = op_reg(ds=ds, a=2, b='trilinear')
# Check that the dataset was stamped
self.assertTrue(stamp in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Check that none of the other two datasets have been stamped
with self.assertRaises(KeyError):
ret['name2'].attrs['history']
with self.assertRaises(KeyError):
ret['name3'].attrs['history']
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret['name1'].attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Other datasets should have the old history, while 'name1' should be
# updated
self.assertTrue(
ret['name1'].attrs['history'] != ret['name2'].attrs['history'])
self.assertTrue(
ret['name1'].attrs['history'] != ret['name3'].attrs['history'])
self.assertTrue(
ret['name2'].attrs['history'] == ret['name3'].attrs['history'])
# Test missing version
@op(registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_no_version(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
return ds1
ds = xr.Dataset()
op_reg = \
self.registry.get_op(object_to_qualified_name(history_no_version))
with self.assertRaises(ValueError) as err:
ret = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue('Could not add history' in str(err.exception))
# Test not implemented output type stamping
@op(version='1.1', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_wrong_type(ds: xr.Dataset, a=1, b='bilinear'):
return "Joke's on you"
ds = xr.Dataset()
op_reg = \
self.registry.get_op(object_to_qualified_name(history_wrong_type))
with self.assertRaises(NotImplementedError) as err:
ret = op_reg(ds=ds, a=2, b='abc')
self.assertTrue('Adding of operation signature' in str(err.exception))
class OpTest(TestCase):
def setUp(self):
self.registry = OpRegistry()
def tearDown(self):
self.registry = None
def test_new_executable_op_without_ds(self):
op = new_subprocess_op(
OpMetaInfo('make_entropy',
inputs={
'num_steps': {
'data_type': int
},
'period': {
'data_type': float
},
},
outputs={'return': {
'data_type': int
}}), MAKE_ENTROPY_EXE + " {num_steps} {period}")
exit_code = op(num_steps=5, period=0.05)
self.assertEqual(exit_code, 0)
def test_new_executable_op_with_ds_file(self):
op = new_subprocess_op(
OpMetaInfo('filter_ds',
inputs={
'ifile': {
'data_type': FileLike
},
'ofile': {
'data_type': FileLike
},
'var': {
'data_type': VarName
},
},
outputs={'return': {
'data_type': int
}}), FILTER_DS_EXE + " {ifile} {ofile} {var}")
ofile = os.path.join(DIR, 'test_data', 'filter_ds.nc')
if os.path.isfile(ofile):
os.remove(ofile)
exit_code = op(ifile=SOILMOISTURE_NC, ofile=ofile, var='sm')
self.assertEqual(exit_code, 0)
self.assertTrue(os.path.isfile(ofile))
os.remove(ofile)
def test_new_executable_op_with_ds_in_mem(self):
op = new_subprocess_op(
OpMetaInfo('filter_ds',
inputs={
'ds': {
'data_type': xr.Dataset,
'write_to': 'ifile'
},
'var': {
'data_type': VarName
},
},
outputs={
'return': {
'data_type': xr.Dataset,
'read_from': 'ofile'
}
}), FILTER_DS_EXE + " {ifile} {ofile} {var}")
ds = xr.open_dataset(SOILMOISTURE_NC)
ds_out = op(ds=ds, var='sm')
self.assertIsNotNone(ds_out)
self.assertIsNotNone('sm' in ds_out)
def test_new_expression_op(self):
op = new_expression_op(
OpMetaInfo('add_xy',
inputs={
'x': {
'data_type': float
},
'y': {
'data_type': float
},
},
outputs={'return': {
'data_type': float
}}), 'x + y')
z = op(x=1.2, y=2.4)
self.assertEqual(z, 1.2 + 2.4)
op = new_expression_op(
OpMetaInfo('add_xy', inputs={
'x': {},
'y': {},
}), 'x * y')
z = op(x=1.2, y=2.4)
self.assertEqual(z, 1.2 * 2.4)
self.assertIn('return', op.op_meta_info.outputs)
def test_plain_function(self):
def f(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f!"""
return str(a + b + c + u + len(v) + w)
registry = self.registry
added_op_reg = registry.add_op(f)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(f, fail_if_exists=True)
self.assertIs(registry.add_op(f, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.wrapped_op, f)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4,
default_value='A',
data_type=str)
expected_inputs['w'] = dict(position=5,
default_value=4.9,
data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info, object_to_qualified_name(f),
dict(description='Hi, I am f!'), expected_inputs,
expected_outputs)
removed_op_reg = registry.remove_op(f)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(f, fail_if_not_exists=True)
def test_decorated_function(self):
@op(registry=self.registry)
def f_op(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op))
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4,
default_value='A',
data_type=str)
expected_inputs['w'] = dict(position=5,
default_value=4.9,
data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op),
dict(description='Hi, I am f_op!'),
expected_inputs, expected_outputs)
def test_decorated_function_with_inputs_and_outputs(self):
@op_input('a', value_range=[0., 1.], registry=self.registry)
@op_input('v', value_set=['A', 'B', 'C'], registry=self.registry)
@op_return(registry=self.registry)
def f_op_inp_ret(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op_inp_ret!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op_inp_ret, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op_inp_ret))
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0,
data_type=float,
value_range=[0., 1.])
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4,
default_value='A',
data_type=str,
value_set=['A', 'B', 'C'])
expected_inputs['w'] = dict(position=5,
default_value=4.9,
data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op_inp_ret),
dict(description='Hi, I am f_op_inp_ret!'),
expected_inputs, expected_outputs)
def _assertMetaInfo(self, op_meta_info: OpMetaInfo, expected_name: str,
expected_header: dict, expected_input: OrderedDict,
expected_output: OrderedDict):
self.assertIsNotNone(op_meta_info)
self.assertEqual(op_meta_info.qualified_name, expected_name)
self.assertEqual(op_meta_info.header, expected_header)
self.assertEqual(OrderedDict(op_meta_info.inputs), expected_input)
self.assertEqual(OrderedDict(op_meta_info.outputs), expected_output)
def test_function_validation(self):
@op_input('x',
registry=self.registry,
data_type=float,
value_range=[0.1, 0.9],
default_value=0.5)
@op_input('y', registry=self.registry)
@op_input('a',
registry=self.registry,
data_type=int,
value_set=[1, 4, 5])
@op_return(registry=self.registry, data_type=float)
def f(x, y: float, a=4):
return a * x + y if a != 5 else 'foo'
self.assertIs(f, self.registry.get_op(f))
self.assertEqual(f.op_meta_info.inputs['x'].get('data_type', None),
float)
self.assertEqual(f.op_meta_info.inputs['x'].get('value_range', None),
[0.1, 0.9])
self.assertEqual(f.op_meta_info.inputs['x'].get('default_value', None),
0.5)
self.assertEqual(f.op_meta_info.inputs['x'].get('position', None), 0)
self.assertEqual(f.op_meta_info.inputs['y'].get('data_type', None),
float)
self.assertEqual(f.op_meta_info.inputs['y'].get('position', None), 1)
self.assertEqual(f.op_meta_info.inputs['a'].get('data_type', None),
int)
self.assertEqual(f.op_meta_info.inputs['a'].get('value_set', None),
[1, 4, 5])
self.assertEqual(f.op_meta_info.inputs['a'].get('default_value', None),
4)
self.assertEqual(f.op_meta_info.inputs['a'].get('position', None), 2)
self.assertEqual(f.op_meta_info.outputs[RETURN].get('data_type', None),
float)
self.assertEqual(f(y=1, x=0.2), 4 * 0.2 + 1)
self.assertEqual(f(y=3), 4 * 0.5 + 3)
self.assertEqual(f(0.6, y=3, a=1), 1 * 0.6 + 3.0)
with self.assertRaises(ValueError) as cm:
f(y=1, x=8)
self.assertEqual(
str(cm.exception),
"Input 'x' for operation 'test.core.test_op.f' must be in range [0.1, 0.9]."
)
with self.assertRaises(ValueError) as cm:
f(y=None, x=0.2)
self.assertEqual(
str(cm.exception),
"Input 'y' for operation 'test.core.test_op.f' must be given.")
with self.assertRaises(ValueError) as cm:
f(y=0.5, x=0.2, a=2)
self.assertEqual(
str(cm.exception),
"Input 'a' for operation 'test.core.test_op.f' must be one of [1, 4, 5]."
)
with self.assertRaises(ValueError) as cm:
f(x=0, y=3.)
self.assertEqual(
str(cm.exception),
"Input 'x' for operation 'test.core.test_op.f' must be in range [0.1, 0.9]."
)
with self.assertRaises(ValueError) as cm:
f(x='A', y=3.)
self.assertEqual(
str(cm.exception),
"Input 'x' for operation 'test.core.test_op.f' must be of type 'float', "
"but got type 'str'.")
with self.assertRaises(ValueError) as cm:
f(x=0.4)
self.assertEqual(
str(cm.exception),
"Input 'y' for operation 'test.core.test_op.f' must be given.")
with self.assertRaises(ValueError) as cm:
f(x=0.6, y=0.1, a=2)
self.assertEqual(
str(cm.exception),
"Input 'a' for operation 'test.core.test_op.f' must be one of [1, 4, 5]."
)
with self.assertRaises(ValueError) as cm:
f(y=3, a=5)
self.assertEqual(
str(cm.exception),
"Output 'return' for operation 'test.core.test_op.f' must be of type 'float', "
"but got type 'str'.")
def test_function_invocation(self):
def f(x, a=4):
return a * x
op_reg = self.registry.add_op(f)
result = op_reg(x=2.5)
self.assertEqual(result, 4 * 2.5)
def test_function_invocation_with_monitor(self):
def f(monitor: Monitor, x, a=4):
monitor.start('f', 23)
return_value = a * x
monitor.done()
return return_value
op_reg = self.registry.add_op(f)
monitor = MyMonitor()
result = op_reg(x=2.5, monitor=monitor)
self.assertEqual(result, 4 * 2.5)
self.assertEqual(monitor.total_work, 23)
self.assertEqual(monitor.is_done, True)
def test_history_op(self):
"""
Test adding operation signature to output history information.
"""
import xarray as xr
from cate import __version__
# Test @op_return
@op(version='0.9', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_op(ds: xr.Dataset, a=1, b='bilinear'):
ret = ds.copy()
return ret
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.inputs) + '\nProvided input values: '
ret_ds = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue(stamp in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret_ds = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret_ds.attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Test @op_output
@op(version='1.9', registry=self.registry)
@op_output('name1', add_history=True, registry=self.registry)
@op_output('name2', add_history=False, registry=self.registry)
@op_output('name3', registry=self.registry)
def history_named_op(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
ds2 = ds.copy()
ds3 = ds.copy()
return {'name1': ds1, 'name2': ds2, 'name3': ds3}
ds = xr.Dataset()
op_reg = self.registry.get_op(
object_to_qualified_name(history_named_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.inputs) + '\nProvided input values: '
ret = op_reg(ds=ds, a=2, b='trilinear')
# Check that the dataset was stamped
self.assertTrue(stamp in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Check that none of the other two datasets have been stamped
with self.assertRaises(KeyError):
ret['name2'].attrs['history']
with self.assertRaises(KeyError):
ret['name3'].attrs['history']
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret['name1'].attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Other datasets should have the old history, while 'name1' should be
# updated
self.assertTrue(
ret['name1'].attrs['history'] != ret['name2'].attrs['history'])
self.assertTrue(
ret['name1'].attrs['history'] != ret['name3'].attrs['history'])
self.assertTrue(
ret['name2'].attrs['history'] == ret['name3'].attrs['history'])
# Test missing version
@op(registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_no_version(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
return ds1
ds = xr.Dataset()
op_reg = self.registry.get_op(
object_to_qualified_name(history_no_version))
with self.assertRaises(ValueError) as err:
ret = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue('Could not add history' in str(err.exception))
# Test not implemented output type stamping
@op(version='1.1', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_wrong_type(ds: xr.Dataset, a=1, b='bilinear'):
return "Joke's on you"
ds = xr.Dataset()
op_reg = self.registry.get_op(
object_to_qualified_name(history_wrong_type))
with self.assertRaises(NotImplementedError) as err:
ret = op_reg(ds=ds, a=2, b='abc')
self.assertTrue(
'Adding history information to an' in str(err.exception))
class OpTest(TestCase):
def setUp(self):
self.registry = OpRegistry()
def tearDown(self):
self.registry = None
def test_new_executable_op_without_ds(self):
op = new_subprocess_op(OpMetaInfo('make_entropy',
inputs={
'num_steps': {'data_type': int},
'period': {'data_type': float},
},
outputs={
'return': {'data_type': int}
}),
MAKE_ENTROPY_EXE + " {num_steps} {period}")
exit_code = op(num_steps=5, period=0.05)
self.assertEqual(exit_code, 0)
def test_new_executable_op_with_ds_file(self):
op = new_subprocess_op(OpMetaInfo('filter_ds',
inputs={
'ifile': {'data_type': FileLike},
'ofile': {'data_type': FileLike},
'var': {'data_type': VarName},
},
outputs={
'return': {'data_type': int}
}),
FILTER_DS_EXE + " {ifile} {ofile} {var}")
ofile = os.path.join(DIR, 'test_data', 'filter_ds.nc')
if os.path.isfile(ofile):
os.remove(ofile)
exit_code = op(ifile=SOILMOISTURE_NC, ofile=ofile, var='sm')
self.assertEqual(exit_code, 0)
self.assertTrue(os.path.isfile(ofile))
os.remove(ofile)
def test_new_executable_op_with_ds_in_mem(self):
op = new_subprocess_op(OpMetaInfo('filter_ds',
inputs={
'ds': {
'data_type': xr.Dataset,
'write_to': 'ifile'
},
'var': {
'data_type': VarName
},
},
outputs={
'return': {
'data_type': xr.Dataset,
'read_from': 'ofile'
}
}),
FILTER_DS_EXE + " {ifile} {ofile} {var}")
ds = xr.open_dataset(SOILMOISTURE_NC)
ds_out = op(ds=ds, var='sm')
self.assertIsNotNone(ds_out)
self.assertIsNotNone('sm' in ds_out)
def test_new_expression_op(self):
op = new_expression_op(OpMetaInfo('add_xy',
inputs={
'x': {'data_type': float},
'y': {'data_type': float},
},
outputs={
'return': {'data_type': float}
}),
'x + y')
z = op(x=1.2, y=2.4)
self.assertEqual(z, 1.2 + 2.4)
op = new_expression_op(OpMetaInfo('add_xy',
inputs={
'x': {},
'y': {},
}),
'x * y')
z = op(x=1.2, y=2.4)
self.assertEqual(z, 1.2 * 2.4)
self.assertIn('return', op.op_meta_info.outputs)
def test_plain_function(self):
def f(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f!"""
return str(a + b + c + u + len(v) + w)
registry = self.registry
added_op_reg = registry.add_op(f)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(f, fail_if_exists=True)
self.assertIs(registry.add_op(f, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.wrapped_op, f)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4, default_value='A', data_type=str)
expected_inputs['w'] = dict(position=5, default_value=4.9, data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f),
dict(description='Hi, I am f!'),
expected_inputs,
expected_outputs)
removed_op_reg = registry.remove_op(f)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(f, fail_if_not_exists=True)
def test_decorated_function(self):
@op(registry=self.registry)
def f_op(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op))
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4, default_value='A', data_type=str)
expected_inputs['w'] = dict(position=5, default_value=4.9, data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op),
dict(description='Hi, I am f_op!'),
expected_inputs,
expected_outputs)
def test_decorated_function_with_inputs_and_outputs(self):
@op_input('a', value_range=[0., 1.], registry=self.registry)
@op_input('v', value_set=['A', 'B', 'C'], registry=self.registry)
@op_return(registry=self.registry)
def f_op_inp_ret(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op_inp_ret!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op_inp_ret, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op_inp_ret))
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float, value_range=[0., 1.])
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4, default_value='A', data_type=str, value_set=['A', 'B', 'C'])
expected_inputs['w'] = dict(position=5, default_value=4.9, data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op_inp_ret),
dict(description='Hi, I am f_op_inp_ret!'),
expected_inputs,
expected_outputs)
def _assertMetaInfo(self, op_meta_info: OpMetaInfo,
expected_name: str,
expected_header: dict,
expected_input: OrderedDict,
expected_output: OrderedDict):
self.assertIsNotNone(op_meta_info)
self.assertEqual(op_meta_info.qualified_name, expected_name)
self.assertEqual(op_meta_info.header, expected_header)
self.assertEqual(OrderedDict(op_meta_info.inputs), expected_input)
self.assertEqual(OrderedDict(op_meta_info.outputs), expected_output)
def test_function_validation(self):
@op_input('x', registry=self.registry, data_type=float, value_range=[0.1, 0.9], default_value=0.5)
@op_input('y', registry=self.registry)
@op_input('a', registry=self.registry, data_type=int, value_set=[1, 4, 5])
@op_return(registry=self.registry, data_type=float)
def f(x, y: float or None, a=4):
return a * x + y if a != 5 else 'foo'
self.assertIs(f, self.registry.get_op(f))
self.assertEqual(f.op_meta_info.inputs['x'].get('data_type', None), float)
self.assertEqual(f.op_meta_info.inputs['x'].get('value_range', None), [0.1, 0.9])
self.assertEqual(f.op_meta_info.inputs['x'].get('default_value', None), 0.5)
self.assertEqual(f.op_meta_info.inputs['x'].get('position', None), 0)
self.assertEqual(f.op_meta_info.inputs['y'].get('data_type', None), float)
self.assertEqual(f.op_meta_info.inputs['y'].get('position', None), 1)
self.assertEqual(f.op_meta_info.inputs['a'].get('data_type', None), int)
self.assertEqual(f.op_meta_info.inputs['a'].get('value_set', None), [1, 4, 5])
self.assertEqual(f.op_meta_info.inputs['a'].get('default_value', None), 4)
self.assertEqual(f.op_meta_info.inputs['a'].get('position', None), 2)
self.assertEqual(f.op_meta_info.outputs[RETURN].get('data_type', None), float)
self.assertEqual(f(y=1, x=0.2), 4 * 0.2 + 1)
self.assertEqual(f(y=3), 4 * 0.5 + 3)
self.assertEqual(f(0.6, y=3, a=1), 1 * 0.6 + 3.0)
with self.assertRaises(ValueError) as cm:
f(y=1, x=8)
self.assertEqual(str(cm.exception),
"Input 'x' for operation 'test.core.test_op.f' must be in range [0.1, 0.9].")
with self.assertRaises(ValueError) as cm:
f(y=None, x=0.2)
self.assertEqual(str(cm.exception),
"Input 'y' for operation 'test.core.test_op.f' must be given.")
with self.assertRaises(ValueError) as cm:
f(y=0.5, x=0.2, a=2)
self.assertEqual(str(cm.exception),
"Input 'a' for operation 'test.core.test_op.f' must be one of [1, 4, 5].")
with self.assertRaises(ValueError) as cm:
f(x=0, y=3.)
self.assertEqual(str(cm.exception),
"Input 'x' for operation 'test.core.test_op.f' must be in range [0.1, 0.9].")
with self.assertRaises(ValueError) as cm:
f(x='A', y=3.)
self.assertEqual(str(cm.exception),
"Input 'x' for operation 'test.core.test_op.f' must be of type 'float', "
"but got type 'str'.")
with self.assertRaises(ValueError) as cm:
f(x=0.4)
self.assertEqual(str(cm.exception),
"Input 'y' for operation 'test.core.test_op.f' must be given.")
with self.assertRaises(ValueError) as cm:
f(x=0.6, y=0.1, a=2)
self.assertEqual(str(cm.exception),
"Input 'a' for operation 'test.core.test_op.f' must be one of [1, 4, 5].")
with self.assertRaises(ValueError) as cm:
f(y=3, a=5)
self.assertEqual(str(cm.exception),
"Output 'return' for operation 'test.core.test_op.f' must be of type 'float', "
"but got type 'str'.")
def test_function_invocation(self):
def f(x, a=4):
return a * x
op_reg = self.registry.add_op(f)
result = op_reg(x=2.5)
self.assertEqual(result, 4 * 2.5)
def test_function_invocation_with_monitor(self):
def f(monitor: Monitor, x, a=4):
monitor.start('f', 23)
return_value = a * x
monitor.done()
return return_value
op_reg = self.registry.add_op(f)
monitor = MyMonitor()
result = op_reg(x=2.5, monitor=monitor)
self.assertEqual(result, 4 * 2.5)
self.assertEqual(monitor.total_work, 23)
self.assertEqual(monitor.is_done, True)
def test_history_op(self):
"""
Test adding operation signature to output history information.
"""
import xarray as xr
from cate import __version__
# Test @op_return
@op(version='0.9', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_op(ds: xr.Dataset, a=1, b='bilinear'):
ret = ds.copy()
return ret
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.inputs) + '\nProvided input values: '
ret_ds = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue(stamp in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret_ds = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret_ds.attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Test @op_output
@op(version='1.9', registry=self.registry)
@op_output('name1', add_history=True, registry=self.registry)
@op_output('name2', add_history=False, registry=self.registry)
@op_output('name3', registry=self.registry)
def history_named_op(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
ds2 = ds.copy()
ds3 = ds.copy()
return {'name1': ds1, 'name2': ds2, 'name3': ds3}
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_named_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.inputs) + '\nProvided input values: '
ret = op_reg(ds=ds, a=2, b='trilinear')
# Check that the dataset was stamped
self.assertTrue(stamp in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Check that none of the other two datasets have been stamped
with self.assertRaises(KeyError):
ret['name2'].attrs['history']
with self.assertRaises(KeyError):
ret['name3'].attrs['history']
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret['name1'].attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Other datasets should have the old history, while 'name1' should be
# updated
self.assertTrue(ret['name1'].attrs['history']
!= ret['name2'].attrs['history'])
self.assertTrue(ret['name1'].attrs['history']
!= ret['name3'].attrs['history'])
self.assertTrue(ret['name2'].attrs['history']
== ret['name3'].attrs['history'])
# Test missing version
@op(registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_no_version(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
return ds1
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_no_version))
with self.assertRaises(ValueError) as err:
ret = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue('Could not add history' in str(err.exception))
# Test not implemented output type stamping
@op(version='1.1', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_wrong_type(ds: xr.Dataset, a=1, b='bilinear'):
return "Joke's on you"
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_wrong_type))
with self.assertRaises(NotImplementedError) as err:
ret = op_reg(ds=ds, a=2, b='abc')
self.assertTrue('Adding history information to an' in str(err.exception))