def test_C(self):
class C:
"""Hi, I am C!"""
def __call__(self):
return None
registry = self.registry
added_op_reg = registry.add_op(C)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(C, fail_if_exists=True)
self.assertIs(registry.add_op(C, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(C))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.operation, C)
self._assertMetaInfo(op_reg.op_meta_info, object_to_qualified_name(C),
dict(description='Hi, I am C!'), OrderedDict(),
OrderedDict({RETURN: {}}))
removed_op_reg = registry.remove_op(C)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(C))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(C, fail_if_not_exists=True)
def test_f_op(self):
@op(registry=self.registry)
def f_op(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op))
self.assertIs(op_reg.operation, f_op)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(default_value=3)
expected_inputs['v'] = dict(default_value='A')
expected_inputs['w'] = dict(default_value=4.9)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op),
dict(description='Hi, I am f_op!'),
expected_inputs, expected_outputs)
def test_decorated_function_with_inputs_and_outputs(self):
@op_input('a', value_range=[0., 1.], registry=self.registry)
@op_input('v', value_set=['A', 'B', 'C'], registry=self.registry)
@op_return(registry=self.registry)
def f_op_inp_ret(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op_inp_ret!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op_inp_ret, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op_inp_ret))
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0,
data_type=float,
value_range=[0., 1.])
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4,
default_value='A',
data_type=str,
value_set=['A', 'B', 'C'])
expected_inputs['w'] = dict(position=5,
default_value=4.9,
data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op_inp_ret),
dict(description='Hi, I am f_op_inp_ret!'),
expected_inputs, expected_outputs)
def test_decorated_function_with_inputs_and_outputs(self):
@op_input('a', value_range=[0., 1.], registry=self.registry)
@op_input('v', value_set=['A', 'B', 'C'], registry=self.registry)
@op_return(registry=self.registry)
def f_op_inp_ret(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f_op_inp_ret!"""
return str(a + b + c + u + len(v) + w)
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(f_op_inp_ret, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(f_op_inp_ret))
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float, value_range=[0., 1.])
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4, default_value='A', data_type=str, value_set=['A', 'B', 'C'])
expected_inputs['w'] = dict(position=5, default_value=4.9, data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f_op_inp_ret),
dict(description='Hi, I am f_op_inp_ret!'),
expected_inputs,
expected_outputs)
def test_python_primitive_type_names(self):
"""
Python primitive types
"""
self.assertEqual(object_to_qualified_name(bool), 'bool')
self.assertEqual(object_to_qualified_name(int), 'int')
self.assertEqual(object_to_qualified_name(float), 'float')
self.assertEqual(object_to_qualified_name(str), 'str')
def test_python_primitive_type_names(self):
"""
Python primitive types
"""
self.assertEqual(object_to_qualified_name(bool), 'bool')
self.assertEqual(object_to_qualified_name(int), 'int')
self.assertEqual(object_to_qualified_name(float), 'float')
self.assertEqual(object_to_qualified_name(str), 'str')
def to_json_dict(self) -> dict:
json_dict = OrderedDict()
json_dict['name'] = self.name
json_dict['data_type'] = object_to_qualified_name(self.data_type)
# TODO (nf, 20160627): convert self.value to JSON value
json_dict['value'] = self.value
return json_dict
def test_registered(self):
"""
Test the operation when invoked through the OP_REGISTRY
"""
reg_op = OP_REGISTRY.get_op(
object_to_qualified_name(arithmetics.ds_arithmetics))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90)
})
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90)
})
actual = reg_op(ds=dataset, op='+2, -2, *3, /3, *4')
assert_dataset_equal(expected * 4, actual)
def test_registered(self):
"""
Test registered operation execution
"""
reg_op = OP_REGISTRY.get_op(
object_to_qualified_name(temporal_aggregation))
ds = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 366])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 366])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90),
'time':
pd.date_range('2000-01-01', '2000-12-31')
})
ds = adjust_temporal_attrs(ds)
ex = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90),
'time':
pd.date_range('2000-01-01', freq='MS', periods=12)
})
ex.first.attrs['cell_methods'] = 'time: mean within years'
ex.second.attrs['cell_methods'] = 'time: mean within years'
actual = reg_op(ds=ds)
self.assertTrue(actual.broadcast_equals(ex))
def test_registered(self):
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(index.enso_nino34))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': ([datetime(2001, x, 1) for x in range(1, 13)]
+ [datetime(2002, x, 1) for x in range(1, 13)])})
lta = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': [x for x in range(1, 13)]})
lta = 2 * lta
expected_time = ([datetime(2001, x, 1) for x in range(3, 13)]
+ [datetime(2002, x, 1) for x in range(1, 11)])
expected = pd.DataFrame(data=(np.ones([20]) * -1),
columns=['ENSO N3.4 Index'],
index=expected_time)
with create_tmp_file() as tmp_file:
lta.to_netcdf(tmp_file)
actual = reg_op(ds=dataset, var='first', file=tmp_file)
self.assertTrue(expected.equals(actual))
def test_registered(self):
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(
index.enso_nino34))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90),
'time': ([datetime(2001, x, 1) for x in range(1, 13)] +
[datetime(2002, x, 1) for x in range(1, 13)])
})
lta = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90),
'time': [x for x in range(1, 13)]
})
lta = 2 * lta
expected_time = ([datetime(2001, x, 1) for x in range(3, 13)] +
[datetime(2002, x, 1) for x in range(1, 11)])
expected = pd.DataFrame(data=(np.ones([20]) * -1),
columns=['ENSO N3.4 Index'],
index=expected_time)
with create_tmp_file() as tmp_file:
lta.to_netcdf(tmp_file)
actual = reg_op(ds=dataset, var='first', file=tmp_file)
self.assertTrue(expected.equals(actual))
def test_registered(self):
"""
Test nominal execution of ONI Index calculation, as a registered
operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(index.oni))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': ([datetime(2001, x, 1) for x in range(1, 13)]
+ [datetime(2002, x, 1) for x in range(1, 13)])})
lta = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': [x for x in range(1, 13)]})
lta = 2 * lta
expected_time = ([datetime(2001, x, 1) for x in range(2, 13)]
+ [datetime(2002, x, 1) for x in range(1, 12)])
expected = pd.DataFrame(data=(np.ones([22]) * -1),
columns=['ONI Index'],
index=expected_time)
with create_tmp_file() as tmp_file:
lta.to_netcdf(tmp_file)
actual = reg_op(ds=dataset, var='first', file=tmp_file)
self.assertTrue(expected.equals(actual))
def test_init_operation_and_name_are_equivalent(self):
step3 = OpStep(op3)
self.assertIsNotNone(step3.op)
self.assertIsNotNone(step3.op_meta_info)
node31 = OpStep(object_to_qualified_name(op3))
self.assertIs(node31.op, step3.op)
self.assertIs(node31.op_meta_info, step3.op_meta_info)
def test_registered(self):
"""
Test nominal execution through the operations registry.
"""
reg_op = OP_REGISTRY.get_op(
object_to_qualified_name(anomaly.anomaly_internal))
ds = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90),
'time': [datetime(2000, x, 1) for x in range(1, 13)]
})
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])),
'lat':
np.linspace(-88, 88, 45),
'lon':
np.linspace(-178, 178, 90),
'time': [datetime(2000, x, 1) for x in range(1, 13)]
})
actual = reg_op(ds=ds,
time_range='2000-01-01, 2000-04-01',
region='-50, -50, 50, 50')
assert_dataset_equal(expected, actual)
def test_introspect_operation(self):
# noinspection PyUnusedLocal
def f(a: str, b: int = None, c: float = 1, d='A') -> float:
"""
The doc.
:param a: the a str
:param b: the
b int
:param c: the c float
:param d:
the d 'A'
:return: a float
"""
op_meta_info = OpMetaInfo.introspect_operation(f)
self.assertEqual(op_meta_info.qualified_name, object_to_qualified_name(f))
self.assertEqual(op_meta_info.header, dict(description='The doc.'))
self.assertEqual(len(op_meta_info.inputs), 4)
self.assertEqual(len(op_meta_info.outputs), 1)
self.assertIn('a', op_meta_info.inputs)
self.assertIn('b', op_meta_info.inputs)
self.assertIn('c', op_meta_info.inputs)
self.assertIn('d', op_meta_info.inputs)
self.assertIn(RETURN, op_meta_info.outputs)
self.assertEqual(op_meta_info.inputs['a'], dict(position=0, data_type=str, description='the a str'))
self.assertEqual(op_meta_info.inputs['b'], dict(position=1, data_type=int, nullable=True, default_value=None, description='the b int'))
self.assertEqual(op_meta_info.inputs['c'], dict(position=2, data_type=float, default_value=1, description='the c float'))
self.assertEqual(op_meta_info.inputs['d'], dict(position=3, data_type=str, default_value='A', description="the d 'A'"))
self.assertEqual(op_meta_info.outputs[RETURN], dict(data_type=float, description='a float'))
self.assertEqual(op_meta_info.has_monitor, False)
self.assertEqual(op_meta_info.has_named_outputs, False)
self.assertEqual(op_meta_info.can_cache, True)
def test_registered(self):
"""
Test nominal execution of ONI Index calculation, as a registered
operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(index.oni))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': ([datetime(2001, x, 1) for x in range(1, 13)] +
[datetime(2002, x, 1) for x in range(1, 13)])})
lta = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': [x for x in range(1, 13)]})
lta = 2 * lta
expected_time = ([datetime(2001, x, 1) for x in range(2, 13)] +
[datetime(2002, x, 1) for x in range(1, 12)])
expected = pd.DataFrame(data=(np.ones([22]) * -1),
columns=['ONI Index'],
index=expected_time)
with create_tmp_file() as tmp_file:
lta.to_netcdf(tmp_file)
actual = reg_op(ds=dataset, var='first', file=tmp_file)
self.assertTrue(expected.equals(actual))
def test_registered(self):
"""
Test if it runs as an operation registered in the op registry.
"""
reg_op = OP_REGISTRY.get_op(
object_to_qualified_name(subset.subset_temporal_index))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'lat':
np.linspace(-89.5, 89.5, 180),
'lon':
np.linspace(-179.5, 179.5, 360),
'time': [
'2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01',
'2000-05-01', '2000-06-01'
]
})
actual = reg_op(ds=dataset, time_ind_min=2, time_ind_max=4)
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'lat':
np.linspace(-89.5, 89.5, 180),
'lon':
np.linspace(-179.5, 179.5, 360),
'time': ['2000-03-01', '2000-04-01', '2000-05-01']
})
assert_dataset_equal(expected, actual)
def test_registered_compute_with_context(self):
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(arithmetics.compute))
first = np.ones([45, 90, 3])
second = np.ones([45, 90, 3])
lon = np.linspace(-178, 178, 90)
lat = np.linspace(-88, 88, 45)
res_1 = xr.Dataset({
'first': (['lat', 'lon', 'time'], first),
'lat': lat,
'lon': lon
})
res_2 = xr.Dataset({
'second': (['lat', 'lon', 'time'], second),
'lat': lat,
'lon': lon
})
# Note, if executed from a workflow, _ctx will be set by the framework
_ctx = dict(value_cache=dict(res_1=res_1, res_2=res_2))
actual = reg_op(ds=None,
expr="6 * res_1.first - 3 * res_2.second",
var="third",
_ctx=_ctx)
expected = xr.Dataset({
'third': (['lat', 'lon', 'time'], 6 * first - 3 * second),
'lat': lat,
'lon': lon})
assert_dataset_equal(expected, actual)
def test_registered_compute(self):
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(arithmetics.compute))
first = np.ones([45, 90, 3])
second = np.ones([45, 90, 3])
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], first),
'second': (['lat', 'lon', 'time'], second),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90)})
actual = reg_op(ds=dataset,
expr="6 * first - 3 * second",
var="third")
expected = xr.Dataset({
'third': (['lat', 'lon', 'time'], 6 * first - 3 * second),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90)})
assert_dataset_equal(expected, actual)
actual = reg_op(ds=dataset,
expr="6 * first - 3 * second",
var="third",
copy=True)
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], first),
'second': (['lat', 'lon', 'time'], second),
'third': (['lat', 'lon', 'time'], 6 * first - 3 * second),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90)})
assert_dataset_equal(expected, actual)
def test_init_operation_and_name_are_equivalent(self):
step3 = OpStep(op3)
self.assertIsNotNone(step3.op)
self.assertIsNotNone(step3.op_meta_info)
node31 = OpStep(object_to_qualified_name(op3))
self.assertIs(node31.op, step3.op)
self.assertIs(node31.op_meta_info, step3.op_meta_info)
def test_registered(self):
"""
Test if it runs as an operation registered in the op registry.
"""
reg_op = OP_REGISTRY.get_op(
object_to_qualified_name(subset.subset_temporal))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'lat':
np.linspace(-89.5, 89.5, 180),
'lon':
np.linspace(-179.5, 179.5, 360),
'time': [datetime(2000, x, 1) for x in range(1, 7)]
})
actual = reg_op(ds=dataset, time_range='2000-01-10, 2000-04-01')
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'lat':
np.linspace(-89.5, 89.5, 180),
'lon':
np.linspace(-179.5, 179.5, 360),
'time': [datetime(2000, x, 1) for x in range(2, 5)]
})
assert_dataset_equal(expected, actual)
def test_C_op(self):
@op(author='Ernie and Bert', registry=self.registry)
class C_op:
"""Hi, I am C_op!"""
def __call__(self):
return None
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(C_op, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(C_op))
self.assertIs(op_reg.operation, C_op)
self._assertMetaInfo(
op_reg.op_meta_info, object_to_qualified_name(C_op),
dict(description='Hi, I am C_op!', author='Ernie and Bert'),
OrderedDict(), OrderedDict({RETURN: {}}))
def to_json_dict(self) -> dict:
json_dict = OrderedDict()
json_dict['name'] = self.name
json_dict['data_type'] = object_to_qualified_name(self.data_type)
json_dict['dimension_names'] = self.dimension_names
json_dict['attributes'] = [
attribute.to_json_dict() for attribute in self.attributes
]
return json_dict
def test_object_to_qualified_name(self):
self.assertEqual(object_to_qualified_name(float), 'float')
self.assertEqual(object_to_qualified_name(np.dtype('float64')), 'float64')
self.assertEqual(object_to_qualified_name(TestCase), 'unittest.case.TestCase')
self.assertEqual(object_to_qualified_name(ElementTree), 'xml.etree.ElementTree.ElementTree')
self.assertEqual(object_to_qualified_name({}, fail=False), '{}')
with self.assertRaisesRegex(ValueError, "missing attribute '__name__'"):
object_to_qualified_name({}, fail=True)
def test_registered(self):
"""
Test nominal execution as a registered operation
"""
op = OP_REGISTRY.get_op(object_to_qualified_name(identity))
self.assertEqual(op(value=True), True)
self.assertEqual(op(value=42), 42)
self.assertEqual(op(value=3.14), 3.14)
self.assertEqual(op(value='ha'), 'ha')
self.assertEqual(op(value=[3, 4, 5]), [3, 4, 5])
def test_registered(self):
"""
Test nominal execution as a registered operation
"""
op = OP_REGISTRY.get_op(object_to_qualified_name(identity))
self.assertEqual(op(value=True), True)
self.assertEqual(op(value=42), 42)
self.assertEqual(op(value=3.14), 3.14)
self.assertEqual(op(value='ha'), 'ha')
self.assertEqual(op(value=[3, 4, 5]), [3, 4, 5])
def test_registered(self):
"""
Test nominal execution as a registered operation
"""
op = OP_REGISTRY.get_op(object_to_qualified_name(literal))
self.assertEqual(op(value='True'), True)
self.assertEqual(op(value='42'), 42)
self.assertEqual(op(value='3.14'), 3.14)
self.assertEqual(op(value='"ha"'), 'ha')
self.assertEqual(op(value='[3,4,5]'), [3, 4, 5])
def test_registered(self):
"""
Test nominal execution as a registered operation
"""
op = OP_REGISTRY.get_op(object_to_qualified_name(literal))
self.assertEqual(op(value='True'), True)
self.assertEqual(op(value='42'), 42)
self.assertEqual(op(value='3.14'), 3.14)
self.assertEqual(op(value='"ha"'), 'ha')
self.assertEqual(op(value='[3,4,5]'), [3, 4, 5])
def test_C_op_inp_out(self):
@op_input('a',
data_type=float,
default_value=0.5,
value_range=[0., 1.],
registry=self.registry)
@op_input('b',
data_type=str,
default_value='A',
value_set=['A', 'B', 'C'],
registry=self.registry)
@op_output('x', data_type=float, registry=self.registry)
@op_output('y', data_type=list, registry=self.registry)
class C_op_inp_out:
"""Hi, I am C_op_inp_out!"""
def __call__(self, a, b):
x = 2.5 * a
y = [a, b]
return {'x': x, 'y': y}
with self.assertRaises(ValueError):
# must exist
self.registry.add_op(C_op_inp_out, fail_if_exists=True)
op_reg = self.registry.get_op(object_to_qualified_name(C_op_inp_out))
self.assertIs(op_reg.operation, C_op_inp_out)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0,
data_type=float,
default_value=0.5,
value_range=[0., 1.])
expected_inputs['b'] = dict(position=1,
data_type=str,
default_value='A',
value_set=['A', 'B', 'C'])
expected_outputs = OrderedDict()
expected_outputs['y'] = dict(data_type=list)
expected_outputs['x'] = dict(data_type=float)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(C_op_inp_out),
dict(description='Hi, I am C_op_inp_out!'),
expected_inputs, expected_outputs)
def test_registered(self):
"""
Test nominal execution of the function as a registered operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot))
# Test plot
dataset = xr.Dataset({'first': np.random.rand(10)})
with create_tmp_file('remove_me', 'jpg') as tmp_file:
reg_op(ds=dataset, var='first', file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self):
"""
Test as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(harmonize))
dataset = xr.Dataset(
{'first': (['latitude', 'longitude'], [[1, 2, 3], [2, 3, 4]])})
expected = xr.Dataset(
{'first': (['lat', 'lon'], [[1, 2, 3], [2, 3, 4]])})
actual = reg_op(ds=dataset)
assertDatasetEqual(actual, expected)
def test_plain_function(self):
def f(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f!"""
return str(a + b + c + u + len(v) + w)
registry = self.registry
added_op_reg = registry.add_op(f)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(f, fail_if_exists=True)
self.assertIs(registry.add_op(f, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.wrapped_op, f)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4,
default_value='A',
data_type=str)
expected_inputs['w'] = dict(position=5,
default_value=4.9,
data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info, object_to_qualified_name(f),
dict(description='Hi, I am f!'), expected_inputs,
expected_outputs)
removed_op_reg = registry.remove_op(f)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(f, fail_if_not_exists=True)
def test_registered(self):
"""
Test as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(normalize))
dataset = xr.Dataset({'first': (['latitude',
'longitude'], [[1, 2, 3],
[2, 3, 4]])})
expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
[2, 3, 4]])})
actual = reg_op(ds=dataset)
assertDatasetEqual(actual, expected)
def test_registered(self):
"""
Test nominal execution of the function as a registered operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot))
# Test plot
dataset = xr.Dataset({
'first': np.random.rand(10)})
with create_tmp_file('remove_me', 'jpg') as tmp_file:
reg_op(ds=dataset, var='first', file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_plain_function(self):
def f(a: float, b, c, u=3, v='A', w=4.9) -> str:
"""Hi, I am f!"""
return str(a + b + c + u + len(v) + w)
registry = self.registry
added_op_reg = registry.add_op(f)
self.assertIsNotNone(added_op_reg)
with self.assertRaises(ValueError):
registry.add_op(f, fail_if_exists=True)
self.assertIs(registry.add_op(f, fail_if_exists=False), added_op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIs(op_reg, added_op_reg)
self.assertIs(op_reg.wrapped_op, f)
expected_inputs = OrderedDict()
expected_inputs['a'] = dict(position=0, data_type=float)
expected_inputs['b'] = dict(position=1)
expected_inputs['c'] = dict(position=2)
expected_inputs['u'] = dict(position=3, default_value=3, data_type=int)
expected_inputs['v'] = dict(position=4, default_value='A', data_type=str)
expected_inputs['w'] = dict(position=5, default_value=4.9, data_type=float)
expected_outputs = OrderedDict()
expected_outputs[RETURN] = dict(data_type=str)
self._assertMetaInfo(op_reg.op_meta_info,
object_to_qualified_name(f),
dict(description='Hi, I am f!'),
expected_inputs,
expected_outputs)
removed_op_reg = registry.remove_op(f)
self.assertIs(removed_op_reg, op_reg)
op_reg = registry.get_op(object_to_qualified_name(f))
self.assertIsNone(op_reg)
with self.assertRaises(ValueError):
registry.remove_op(f, fail_if_not_exists=True)
def test_cate_cdm_type_names(self):
"""
Cate Common Data Model (CDM) types
"""
self.assertEqual(object_to_qualified_name(np.ndarray), 'numpy.ndarray')
self.assertEqual(object_to_qualified_name(xr.Dataset), 'xarray.core.dataset.Dataset')
self.assertEqual(object_to_qualified_name(xr.DataArray), 'xarray.core.dataarray.DataArray')
self.assertEqual(object_to_qualified_name(gpd.GeoDataFrame), 'geopandas.geodataframe.GeoDataFrame')
self.assertEqual(object_to_qualified_name(gpd.GeoSeries), 'geopandas.geoseries.GeoSeries')
self.assertEqual(object_to_qualified_name(pd.DataFrame), 'pandas.core.frame.DataFrame')
self.assertEqual(object_to_qualified_name(pd.Series), 'pandas.core.series.Series')
def test_object_to_qualified_name(self):
self.assertEqual(object_to_qualified_name(float), 'float')
self.assertEqual(object_to_qualified_name(dtype('float64')), 'float64')
self.assertEqual(object_to_qualified_name(TestCase),
'unittest.case.TestCase')
self.assertEqual(object_to_qualified_name(ElementTree),
'xml.etree.ElementTree.ElementTree')
self.assertEqual(object_to_qualified_name({}, fail=False), '{}')
with self.assertRaisesRegex(ValueError,
"missing attribute '__name__'"):
object_to_qualified_name({}, fail=True)
def test_registered(self):
"""
Test nominal execution of the function as a registered operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_line))
# Test plot
dataset = xr.Dataset({
'first': (['time'], np.random.rand(10)),
'second': (['time'], np.random.rand(10)),
'time': pd.date_range('2000-01-01', periods=10)})
with create_tmp_file('remove_me', 'jpg') as tmp_file:
reg_op(ds=dataset, var_names=['first', 'second'], file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self):
"""
Test the method when run as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_data_frame))
data = {'A': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
'B': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}
df = pd.DataFrame(data=data, index=pd.date_range('2000-01-01',
periods=10))
with create_tmp_file('remove_me', 'png') as tmp_file:
reg_op(df=df, file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self):
"""
Test the method when run as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_data_frame))
data = {'A': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
'B': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}
df = pd.DataFrame(data=data, index=pd.date_range('2000-01-01',
periods=10))
with create_tmp_file('remove_me', 'png') as tmp_file:
reg_op(df=df, file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self):
"""
Test nominal execution of the function as a registered operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_map))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.random.rand(5, 10, 2)),
'second': (['lat', 'lon', 'time'], np.random.rand(5, 10, 2)),
'lat': np.linspace(-89.5, 89.5, 5),
'lon': np.linspace(-179.5, 179.5, 10),
'time': pd.date_range('2000-01-01', periods=2)})
with create_tmp_file('remove_me', 'png') as tmp_file:
reg_op(ds=dataset, file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self):
"""
Test registered operation execution
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(long_term_average))
ds = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': pd.date_range('2000-01-01', freq='MS', periods=24)})
ds = adjust_temporal_attrs(ds)
reg_op(ds=ds)
def test_registered(self):
"""
Test registered operation execution
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(long_term_average))
ds = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': pd.date_range('2000-01-01', freq='MS', periods=24)})
ds = adjust_temporal_attrs(ds)
reg_op(ds=ds)
def test_registered(self):
"""
Test nominal execution of the function as a registered operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_map))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.random.rand(5, 10, 2)),
'second': (['lat', 'lon', 'time'], np.random.rand(5, 10, 2)),
'lat': np.linspace(-89.5, 89.5, 5),
'lon': np.linspace(-179.5, 179.5, 10),
'time': pd.date_range('2000-01-01', periods=2)})
with create_tmp_file('remove_me', 'png') as tmp_file:
reg_op(ds=dataset, file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self):
"""
Test nominal execution of the function as a registered operation.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_line))
# Test plot
dataset = xr.Dataset({
'first': (['time'], np.random.rand(10)),
'second': (['time'], np.random.rand(10)),
'time': pd.date_range('2000-01-01', periods=10)
})
with create_tmp_file('remove_me', 'jpg') as tmp_file:
reg_op(ds=dataset, var_names=['first', 'second'], file=tmp_file)
self.assertTrue(os.path.isfile(tmp_file))
def test_cate_op_api_type_names(self):
"""
Additional Cate types used by operations API.
"""
self.assertEqual(object_to_qualified_name(VarName), 'cate.core.types.VarName')
self.assertEqual(object_to_qualified_name(VarNamesLike), 'cate.core.types.VarNamesLike')
self.assertEqual(object_to_qualified_name(PointLike), 'cate.core.types.PointLike')
self.assertEqual(object_to_qualified_name(PolygonLike), 'cate.core.types.PolygonLike')
self.assertEqual(object_to_qualified_name(GeometryLike), 'cate.core.types.GeometryLike')
self.assertEqual(object_to_qualified_name(TimeRangeLike), 'cate.core.types.TimeRangeLike')
def test_registered(self):
"""
Test operation when run as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(pandas_fillna))
# Test na filling using a given method
data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]}
expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]}
time = pd.date_range('2000-01-01', freq='MS', periods=12)
expected = pd.DataFrame(data=expected, index=time, dtype=float)
df = pd.DataFrame(data=data, index=time, dtype=float)
actual = reg_op(df=df, method='ffill')
self.assertTrue(actual.equals(expected))
def test_introspect_operation_with_monitor(self):
# noinspection PyUnusedLocal
def g(x: float, monitor: Monitor) -> float:
"""The doc."""
op_meta_info = OpMetaInfo.introspect_operation(g)
self.assertEqual(op_meta_info.qualified_name, object_to_qualified_name(g))
self.assertEqual(op_meta_info.header, dict(description='The doc.'))
self.assertEqual(len(op_meta_info.inputs), 1)
self.assertEqual(len(op_meta_info.outputs), 1)
self.assertIn('x', op_meta_info.inputs)
self.assertNotIn(MONITOR, op_meta_info.inputs)
self.assertIn(RETURN, op_meta_info.outputs)
self.assertEqual(op_meta_info.inputs['x'], dict(data_type=float, position=0))
self.assertEqual(op_meta_info.outputs[RETURN], dict(data_type=float))
self.assertEqual(op_meta_info.has_monitor, True)
self.assertEqual(op_meta_info.has_named_outputs, False)
def test_registered(self):
"""
Test if it runs as an operation registered in the op registry.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(subset.subset_spatial))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'lat': np.linspace(-89.5, 89.5, 180),
'lon': np.linspace(-179.5, 179.5, 360)})
actual = reg_op(ds=dataset, region="-20, -10, 20, 10")
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([22, 42, 6])),
'second': (['lat', 'lon', 'time'], np.ones([22, 42, 6])),
'lat': np.linspace(-10.5, 10.5, 22),
'lon': np.linspace(-20.5, 20.5, 42)})
assert_dataset_equal(expected, actual)
def test_registered(self):
"""
Test operation when run as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(pandas_fillna))
# Test na filling using a given method
data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]}
expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6],
'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]}
time = pd.date_range('2000-01-01', freq='MS', periods=12, tz=timezone.utc)
expected = pd.DataFrame(data=expected, index=time, dtype=float)
df = pd.DataFrame(data=data, index=time, dtype=float)
actual = reg_op(df=df, method='ffill')
self.assertTrue(actual.equals(expected))
def test_registered(self):
"""
Test the operation when invoked from the OP_REGISTRY
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(arithmetics.diff))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90)})
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90)})
actual = reg_op(ds=dataset, ds2=dataset * 2)
assert_dataset_equal(expected * -1, actual)
def test_registered(self):
"""
Test execution as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(sel))
ds = new_ds()
sel_ds = reg_op(ds=ds, time='2014-09-06')
self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'})
self.assertEqual(sel_ds.dims['lon'], 4)
self.assertEqual(sel_ds.dims['lat'], 2)
self.assertNotIn('time', sel_ds.dims)
sel_ds = reg_op(ds=ds, point=(34.51, 10.25))
self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'})
self.assertNotIn('lon', sel_ds.dims)
self.assertNotIn('lat', sel_ds.dims)
self.assertEqual(sel_ds.dims['time'], 10)
def test_registered(self):
"""
Test if it runs as an operation registered in the op registry.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(subset.subset_temporal))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'lat': np.linspace(-89.5, 89.5, 180),
'lon': np.linspace(-179.5, 179.5, 360),
'time': [datetime(2000, x, 1) for x in range(1, 7)]})
actual = reg_op(ds=dataset, time_range='2000-01-10, 2000-04-01')
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'lat': np.linspace(-89.5, 89.5, 180),
'lon': np.linspace(-179.5, 179.5, 360),
'time': [datetime(2000, x, 1) for x in range(2, 5)]})
assert_dataset_equal(expected, actual)
def test_registered(self):
"""
Test nominal execution as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(from_dataframe))
time = pd.date_range('2000-01-01', periods=10, tz=timezone.utc)
data = {'A': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
'B': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
'time': time}
df = pd.DataFrame(data)
df = df.set_index('time')
expected = xr.Dataset({
'A': (['time'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
'B': (['time'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
'time': time})
actual = reg_op(df=df)
assert_dataset_equal(expected, actual)
def registered(self):
"""
Test tseries_point as a registered operation
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(tseries_mean))
dataset = xr.Dataset({
'abs': (['lat', 'lon', 'time'], np.ones([4, 8, 6])),
'bbs': (['lat', 'lon', 'time'], np.ones([4, 8, 6])),
'lat': np.linspace(-67.5, 67.5, 4),
'lon': np.linspace(-157.5, 157.5, 8),
'time': ['2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01',
'2000-05-01', '2000-06-01']})
actual = reg_op(ds=dataset, var='*bs')
expected = xr.Dataset({
'abs': (['time'], np.ones([6])),
'bbs': (['time'], np.ones([6])),
'abs_std': (['time'], np.zeros([6])),
'bbs_std': (['time'], np.zeros([6])),
'lat': np.linspace(-67.5, 67.5, 4),
'lon': np.linspace(-157.5, 157.5, 8),
'time': ['2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01',
'2000-05-01', '2000-06-01']})
assertDatasetEqual(expected, actual)
def test_registered(self):
"""
Test registered operation execution
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(temporal_aggregation))
ds = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 366])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 366])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': pd.date_range('2000-01-01', '2000-12-31')})
ds = adjust_temporal_attrs(ds)
ex = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])),
'lat': np.linspace(-88, 88, 45),
'lon': np.linspace(-178, 178, 90),
'time': pd.date_range('2000-01-01', freq='MS', periods=12)})
ex.first.attrs['cell_methods'] = 'time: mean within years'
ex.second.attrs['cell_methods'] = 'time: mean within years'
actual = reg_op(ds=ds)
self.assertTrue(actual.broadcast_equals(ex))
def test_registered(self):
"""
Test if it runs as an operation registered in the op registry.
"""
reg_op = OP_REGISTRY.get_op(object_to_qualified_name(subset.subset_temporal_index))
dataset = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])),
'lat': np.linspace(-89.5, 89.5, 180),
'lon': np.linspace(-179.5, 179.5, 360),
'time': ['2000-01-01',
'2000-02-01',
'2000-03-01',
'2000-04-01',
'2000-05-01',
'2000-06-01']})
actual = reg_op(ds=dataset, time_ind_min=2, time_ind_max=4)
expected = xr.Dataset({
'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])),
'lat': np.linspace(-89.5, 89.5, 180),
'lon': np.linspace(-179.5, 179.5, 360),
'time': ['2000-03-01', '2000-04-01', '2000-05-01']})
assert_dataset_equal(expected, actual)
def test_registered_op(self):
registered_op = _OP_REGISTRY.get_op(object_to_qualified_name(scale_point))
point = registered_op(point_like="2.4, 4.8", factor=0.5)
self.assertEqual(point, ExamplePoint(1.2, 2.4))
def test_history_op(self):
"""
Test adding operation signature to output history information.
"""
import xarray as xr
from cate import __version__
# Test @op_return
@op(version='0.9', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_op(ds: xr.Dataset, a=1, b='bilinear'):
ret = ds.copy()
return ret
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.inputs) + '\nProvided input values: '
ret_ds = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue(stamp in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret_ds = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret_ds.attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret_ds.attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret_ds.attrs['history'])
# Test @op_output
@op(version='1.9', registry=self.registry)
@op_output('name1', add_history=True, registry=self.registry)
@op_output('name2', add_history=False, registry=self.registry)
@op_output('name3', registry=self.registry)
def history_named_op(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
ds2 = ds.copy()
ds3 = ds.copy()
return {'name1': ds1, 'name2': ds2, 'name3': ds3}
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_named_op))
op_meta_info = op_reg.op_meta_info
# This is a partial stamp, as the way a dict is stringified is not
# always the same
stamp = '\nModified with Cate v' + __version__ + ' ' + \
op_meta_info.qualified_name + ' v' + \
op_meta_info.header['version'] + \
' \nDefault input values: ' + \
str(op_meta_info.inputs) + '\nProvided input values: '
ret = op_reg(ds=ds, a=2, b='trilinear')
# Check that the dataset was stamped
self.assertTrue(stamp in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Check that none of the other two datasets have been stamped
with self.assertRaises(KeyError):
ret['name2'].attrs['history']
with self.assertRaises(KeyError):
ret['name3'].attrs['history']
# Double line-break indicates that this is a subsequent stamp entry
stamp2 = '\n\nModified with Cate v' + __version__
ret = op_reg(ds=ret_ds, a=4, b='quadrilinear')
self.assertTrue(stamp2 in ret['name1'].attrs['history'])
# Check that a passed value is found in the stamp
self.assertTrue('quadrilinear' in ret['name1'].attrs['history'])
# Check that a previous passed value is found in the stamp
self.assertTrue('trilinear' in ret['name1'].attrs['history'])
# Other datasets should have the old history, while 'name1' should be
# updated
self.assertTrue(ret['name1'].attrs['history']
!= ret['name2'].attrs['history'])
self.assertTrue(ret['name1'].attrs['history']
!= ret['name3'].attrs['history'])
self.assertTrue(ret['name2'].attrs['history']
== ret['name3'].attrs['history'])
# Test missing version
@op(registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_no_version(ds: xr.Dataset, a=1, b='bilinear'):
ds1 = ds.copy()
return ds1
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_no_version))
with self.assertRaises(ValueError) as err:
ret = op_reg(ds=ds, a=2, b='trilinear')
self.assertTrue('Could not add history' in str(err.exception))
# Test not implemented output type stamping
@op(version='1.1', registry=self.registry)
@op_return(add_history=True, registry=self.registry)
def history_wrong_type(ds: xr.Dataset, a=1, b='bilinear'):
return "Joke's on you"
ds = xr.Dataset()
op_reg = self.registry.get_op(object_to_qualified_name(history_wrong_type))
with self.assertRaises(NotImplementedError) as err:
ret = op_reg(ds=ds, a=2, b='abc')
self.assertTrue('Adding history information to an' in str(err.exception))
