def test_registered(self): """ Test nominal execution of ONI Index calculation, as a registered operation. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(index.oni)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': ([datetime(2001, x, 1) for x in range(1, 13)] + [datetime(2002, x, 1) for x in range(1, 13)])}) lta = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [x for x in range(1, 13)]}) lta = 2 * lta expected_time = ([datetime(2001, x, 1) for x in range(2, 13)] + [datetime(2002, x, 1) for x in range(1, 12)]) expected = pd.DataFrame(data=(np.ones([22]) * -1), columns=['ONI Index'], index=expected_time) with create_tmp_file() as tmp_file: lta.to_netcdf(tmp_file) actual = reg_op(ds=dataset, var='first', file=tmp_file) self.assertTrue(expected.equals(actual))
def test_registered(self): reg_op = OP_REGISTRY.get_op(object_to_qualified_name( index.enso_nino34)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': ([datetime(2001, x, 1) for x in range(1, 13)] + [datetime(2002, x, 1) for x in range(1, 13)]) }) lta = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [x for x in range(1, 13)] }) lta = 2 * lta expected_time = ([datetime(2001, x, 1) for x in range(3, 13)] + [datetime(2002, x, 1) for x in range(1, 11)]) expected = pd.DataFrame(data=(np.ones([20]) * -1), columns=['ENSO N3.4 Index'], index=expected_time) with create_tmp_file() as tmp_file: lta.to_netcdf(tmp_file) actual = reg_op(ds=dataset, var='first', file=tmp_file) self.assertTrue(expected.equals(actual))
def test_registered_compute(self): reg_op = OP_REGISTRY.get_op(object_to_qualified_name(arithmetics.compute)) first = np.ones([45, 90, 3]) second = np.ones([45, 90, 3]) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], first), 'second': (['lat', 'lon', 'time'], second), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90)}) actual = reg_op(ds=dataset, expr="6 * first - 3 * second", var="third") expected = xr.Dataset({ 'third': (['lat', 'lon', 'time'], 6 * first - 3 * second), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90)}) assert_dataset_equal(expected, actual) actual = reg_op(ds=dataset, expr="6 * first - 3 * second", var="third", copy=True) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], first), 'second': (['lat', 'lon', 'time'], second), 'third': (['lat', 'lon', 'time'], 6 * first - 3 * second), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90)}) assert_dataset_equal(expected, actual)
def test_registered_compute_with_context(self): reg_op = OP_REGISTRY.get_op(object_to_qualified_name(arithmetics.compute)) first = np.ones([45, 90, 3]) second = np.ones([45, 90, 3]) lon = np.linspace(-178, 178, 90) lat = np.linspace(-88, 88, 45) res_1 = xr.Dataset({ 'first': (['lat', 'lon', 'time'], first), 'lat': lat, 'lon': lon }) res_2 = xr.Dataset({ 'second': (['lat', 'lon', 'time'], second), 'lat': lat, 'lon': lon }) # Note, if executed from a workflow, _ctx will be set by the framework _ctx = dict(value_cache=dict(res_1=res_1, res_2=res_2)) actual = reg_op(ds=None, expr="6 * res_1.first - 3 * res_2.second", var="third", _ctx=_ctx) expected = xr.Dataset({ 'third': (['lat', 'lon', 'time'], 6 * first - 3 * second), 'lat': lat, 'lon': lon}) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test nominal execution through the operations registry. """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(anomaly.anomaly_internal)) ds = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [datetime(2000, x, 1) for x in range(1, 13)] }) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [datetime(2000, x, 1) for x in range(1, 13)] }) actual = reg_op(ds=ds, time_range='2000-01-01, 2000-04-01', region='-50, -50, 50, 50') assert_dataset_equal(expected, actual)
def test_registered(self): """ Test registered operation execution """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(temporal_aggregation)) ds = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 366])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 366])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': pd.date_range('2000-01-01', '2000-12-31') }) ds = adjust_temporal_attrs(ds) ex = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': pd.date_range('2000-01-01', freq='MS', periods=12) }) ex.first.attrs['cell_methods'] = 'time: mean within years' ex.second.attrs['cell_methods'] = 'time: mean within years' actual = reg_op(ds=ds) self.assertTrue(actual.broadcast_equals(ex))
def test_registered(self): """ Test the operation when invoked through the OP_REGISTRY """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(arithmetics.ds_arithmetics)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90) }) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90) }) actual = reg_op(ds=dataset, op='+2, -2, *3, /3, *4') assert_dataset_equal(expected * 4, actual)
def test_registered(self): """ Test if it runs as an operation registered in the op registry. """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(subset.subset_temporal_index)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': [ '2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01', '2000-05-01', '2000-06-01' ] }) actual = reg_op(ds=dataset, time_ind_min=2, time_ind_max=4) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': ['2000-03-01', '2000-04-01', '2000-05-01'] }) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test if it runs as an operation registered in the op registry. """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(subset.subset_temporal)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': [datetime(2000, x, 1) for x in range(1, 7)] }) actual = reg_op(ds=dataset, time_range='2000-01-10, 2000-04-01') expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': [datetime(2000, x, 1) for x in range(2, 5)] }) assert_dataset_equal(expected, actual)
def test_registered(self): reg_op = OP_REGISTRY.get_op(object_to_qualified_name(index.enso_nino34)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': ([datetime(2001, x, 1) for x in range(1, 13)] + [datetime(2002, x, 1) for x in range(1, 13)])}) lta = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [x for x in range(1, 13)]}) lta = 2 * lta expected_time = ([datetime(2001, x, 1) for x in range(3, 13)] + [datetime(2002, x, 1) for x in range(1, 11)]) expected = pd.DataFrame(data=(np.ones([20]) * -1), columns=['ENSO N3.4 Index'], index=expected_time) with create_tmp_file() as tmp_file: lta.to_netcdf(tmp_file) actual = reg_op(ds=dataset, var='first', file=tmp_file) self.assertTrue(expected.equals(actual))
def test_existing_method(self): op = OP_REGISTRY.get_op('cate.ops.timeseries.tseries_point', True) op_args, op_kwargs = main._parse_op_args(['ds=@ds', 'point=12.2,54.3', 'var=temperature', 'method=bfill'], input_props=op.op_meta_info.inputs) self.assertEqual(op_args, []) self.assertEqual(op_kwargs, OrderedDict([('ds', dict(source='ds')), ('point', dict(value=(12.2, 54.3))), ('var', dict(value='temperature')), ('method', dict(value='bfill'))]))
def test_registered(self): """ Test nominal execution as a registered operation """ op = OP_REGISTRY.get_op(object_to_qualified_name(identity)) self.assertEqual(op(value=True), True) self.assertEqual(op(value=42), 42) self.assertEqual(op(value=3.14), 3.14) self.assertEqual(op(value='ha'), 'ha') self.assertEqual(op(value=[3, 4, 5]), [3, 4, 5])
def test_registered(self): """ Test nominal execution as a registered operation """ op = OP_REGISTRY.get_op(object_to_qualified_name(literal)) self.assertEqual(op(value='True'), True) self.assertEqual(op(value='42'), 42) self.assertEqual(op(value='3.14'), 3.14) self.assertEqual(op(value='"ha"'), 'ha') self.assertEqual(op(value='[3,4,5]'), [3, 4, 5])
def test_registered(self): """ Test as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(harmonize)) dataset = xr.Dataset( {'first': (['latitude', 'longitude'], [[1, 2, 3], [2, 3, 4]])}) expected = xr.Dataset( {'first': (['lat', 'lon'], [[1, 2, 3], [2, 3, 4]])}) actual = reg_op(ds=dataset) assertDatasetEqual(actual, expected)
def test_registered(self): """ Test nominal execution of the function as a registered operation. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot)) # Test plot dataset = xr.Dataset({'first': np.random.rand(10)}) with create_tmp_file('remove_me', 'jpg') as tmp_file: reg_op(ds=dataset, var='first', file=tmp_file) self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self): """ Test nominal execution of the function as a registered operation. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot)) # Test plot dataset = xr.Dataset({ 'first': np.random.rand(10)}) with create_tmp_file('remove_me', 'jpg') as tmp_file: reg_op(ds=dataset, var='first', file=tmp_file) self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self): """ Test as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(normalize)) dataset = xr.Dataset({'first': (['latitude', 'longitude'], [[1, 2, 3], [2, 3, 4]])}) expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3], [2, 3, 4]])}) actual = reg_op(ds=dataset) assertDatasetEqual(actual, expected)
def test_registered(self): """ Test the method when run as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_data_frame)) data = {'A': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'B': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} df = pd.DataFrame(data=data, index=pd.date_range('2000-01-01', periods=10)) with create_tmp_file('remove_me', 'png') as tmp_file: reg_op(df=df, file=tmp_file) self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self): """ Test nominal execution of the function as a registered operation. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_line)) # Test plot dataset = xr.Dataset({ 'first': (['time'], np.random.rand(10)), 'second': (['time'], np.random.rand(10)), 'time': pd.date_range('2000-01-01', periods=10)}) with create_tmp_file('remove_me', 'jpg') as tmp_file: reg_op(ds=dataset, var_names=['first', 'second'], file=tmp_file) self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self): """ Test nominal execution of the function as a registered operation. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_map)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.random.rand(5, 10, 2)), 'second': (['lat', 'lon', 'time'], np.random.rand(5, 10, 2)), 'lat': np.linspace(-89.5, 89.5, 5), 'lon': np.linspace(-179.5, 179.5, 10), 'time': pd.date_range('2000-01-01', periods=2)}) with create_tmp_file('remove_me', 'png') as tmp_file: reg_op(ds=dataset, file=tmp_file) self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self): """ Test registered operation execution """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(long_term_average)) ds = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': pd.date_range('2000-01-01', freq='MS', periods=24)}) ds = adjust_temporal_attrs(ds) reg_op(ds=ds)
def test_registered(self): """ Test nominal execution of the function as a registered operation. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(plot_line)) # Test plot dataset = xr.Dataset({ 'first': (['time'], np.random.rand(10)), 'second': (['time'], np.random.rand(10)), 'time': pd.date_range('2000-01-01', periods=10) }) with create_tmp_file('remove_me', 'jpg') as tmp_file: reg_op(ds=dataset, var_names=['first', 'second'], file=tmp_file) self.assertTrue(os.path.isfile(tmp_file))
def test_registered(self): """ Test operation when run as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(pandas_fillna)) # Test na filling using a given method data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6], 'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]} expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6], 'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]} time = pd.date_range('2000-01-01', freq='MS', periods=12, tz=timezone.utc) expected = pd.DataFrame(data=expected, index=time, dtype=float) df = pd.DataFrame(data=data, index=time, dtype=float) actual = reg_op(df=df, method='ffill') self.assertTrue(actual.equals(expected))
def test_registered(self): """ Test if it runs as an operation registered in the op registry. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(subset.subset_spatial)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360)}) actual = reg_op(ds=dataset, region="-20, -10, 20, 10") expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([22, 42, 6])), 'second': (['lat', 'lon', 'time'], np.ones([22, 42, 6])), 'lat': np.linspace(-10.5, 10.5, 22), 'lon': np.linspace(-20.5, 20.5, 42)}) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test operation when run as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(pandas_fillna)) # Test na filling using a given method data = {'A': [1, 2, 3, np.nan, 4, 9, np.nan, np.nan, 1, 0, 4, 6], 'B': [5, 6, 8, 7, 5, np.nan, np.nan, np.nan, 1, 2, 7, 6]} expected = {'A': [1, 2, 3, 3, 4, 9, 9, 9, 1, 0, 4, 6], 'B': [5, 6, 8, 7, 5, 5, 5, 5, 1, 2, 7, 6]} time = pd.date_range('2000-01-01', freq='MS', periods=12) expected = pd.DataFrame(data=expected, index=time, dtype=float) df = pd.DataFrame(data=data, index=time, dtype=float) actual = reg_op(df=df, method='ffill') self.assertTrue(actual.equals(expected))
def run_op(self, op_name: str, op_args: List[str], validate_args=False, monitor=Monitor.NONE): assert op_name assert op_args op = OP_REGISTRY.get_op(op_name) if not op: raise WorkspaceError('Unknown operation "%s"' % op_name) with monitor.starting("Running operation '%s'" % op_name, 2): self.workflow.invoke(self.resource_cache, monitor=monitor.child(work=1)) op_kwargs = self._parse_op_args(op, op_args, self.resource_cache, validate_args) op(monitor=monitor.child(work=1), **op_kwargs)
def test_registered(self): """ Test the operation when invoked from the OP_REGISTRY """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(arithmetics.diff)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90)}) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 3])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90)}) actual = reg_op(ds=dataset, ds2=dataset * 2) assert_dataset_equal(expected * -1, actual)
def test_registered(self): """ Test execution as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(sel)) ds = new_ds() sel_ds = reg_op(ds=ds, time='2014-09-06') self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'}) self.assertEqual(sel_ds.dims['lon'], 4) self.assertEqual(sel_ds.dims['lat'], 2) self.assertNotIn('time', sel_ds.dims) sel_ds = reg_op(ds=ds, point=(34.51, 10.25)) self.assertEqual(set(sel_ds.coords.keys()), {'lon', 'lat', 'time', 'reference_time'}) self.assertNotIn('lon', sel_ds.dims) self.assertNotIn('lat', sel_ds.dims) self.assertEqual(sel_ds.dims['time'], 10)
def test_registered(self): """ Test if it runs as an operation registered in the op registry. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(subset.subset_temporal)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': [datetime(2000, x, 1) for x in range(1, 7)]}) actual = reg_op(ds=dataset, time_range='2000-01-10, 2000-04-01') expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': [datetime(2000, x, 1) for x in range(2, 5)]}) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test nominal execution as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(from_dataframe)) time = pd.date_range('2000-01-01', periods=10) data = {'A': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'B': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'time': time} df = pd.DataFrame(data) df = df.set_index('time') expected = xr.Dataset({ 'A': (['time'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), 'B': (['time'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), 'time': time}) actual = reg_op(df=df) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test nominal execution as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(from_dataframe)) time = pd.date_range('2000-01-01', periods=10, tz=timezone.utc) data = {'A': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'B': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'time': time} df = pd.DataFrame(data) df = df.set_index('time') expected = xr.Dataset({ 'A': (['time'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), 'B': (['time'], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), 'time': time}) actual = reg_op(df=df) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test the operation when it is invoked through the operation registry """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(anomaly.anomaly_external)) ref = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90) }) ds = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 24])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [datetime(2000, x, 1) for x in range(1, 13)] + [datetime(2001, x, 1) for x in range(1, 13)] }) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.zeros([45, 90, 24])), 'second': (['lat', 'lon', 'time'], np.zeros([45, 90, 24])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': [datetime(2000, x, 1) for x in range(1, 13)] + [datetime(2001, x, 1) for x in range(1, 13)] }) with create_tmp_file() as tmp_file: ref.to_netcdf(tmp_file, 'w') actual = reg_op(ds=ds, file=tmp_file) assert_dataset_equal(actual, expected)
def registered(self): """ Test tseries_point as a registered operation """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(tseries_mean)) dataset = xr.Dataset({ 'abs': (['lat', 'lon', 'time'], np.ones([4, 8, 6])), 'bbs': (['lat', 'lon', 'time'], np.ones([4, 8, 6])), 'lat': np.linspace(-67.5, 67.5, 4), 'lon': np.linspace(-157.5, 157.5, 8), 'time': ['2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01', '2000-05-01', '2000-06-01']}) actual = reg_op(ds=dataset, var='*bs') expected = xr.Dataset({ 'abs': (['time'], np.ones([6])), 'bbs': (['time'], np.ones([6])), 'abs_std': (['time'], np.zeros([6])), 'bbs_std': (['time'], np.zeros([6])), 'lat': np.linspace(-67.5, 67.5, 4), 'lon': np.linspace(-157.5, 157.5, 8), 'time': ['2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01', '2000-05-01', '2000-06-01']}) assertDatasetEqual(expected, actual)
def test_registered(self): """ Test if it runs as an operation registered in the op registry. """ reg_op = OP_REGISTRY.get_op( object_to_qualified_name(subset.subset_spatial)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360) }) actual = reg_op(ds=dataset, region="-20, -10, 20, 10") expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([20, 40, 6])), 'second': (['lat', 'lon', 'time'], np.ones([20, 40, 6])), 'lat': np.linspace(-9.5, 9.5, 20), 'lon': np.linspace(-19.5, 19.5, 40) }) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test if it runs as an operation registered in the op registry. """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(subset.subset_temporal_index)) dataset = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 6])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': ['2000-01-01', '2000-02-01', '2000-03-01', '2000-04-01', '2000-05-01', '2000-06-01']}) actual = reg_op(ds=dataset, time_ind_min=2, time_ind_max=4) expected = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'second': (['lat', 'lon', 'time'], np.ones([180, 360, 3])), 'lat': np.linspace(-89.5, 89.5, 180), 'lon': np.linspace(-179.5, 179.5, 360), 'time': ['2000-03-01', '2000-04-01', '2000-05-01']}) assert_dataset_equal(expected, actual)
def test_registered(self): """ Test registered operation execution """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(temporal_aggregation)) ds = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 366])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 366])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': pd.date_range('2000-01-01', '2000-12-31')}) ds = adjust_temporal_attrs(ds) ex = xr.Dataset({ 'first': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'second': (['lat', 'lon', 'time'], np.ones([45, 90, 12])), 'lat': np.linspace(-88, 88, 45), 'lon': np.linspace(-178, 178, 90), 'time': pd.date_range('2000-01-01', freq='MS', periods=12)}) ex.first.attrs['cell_methods'] = 'time: mean within years' ex.second.attrs['cell_methods'] = 'time: mean within years' actual = reg_op(ds=ds) self.assertTrue(actual.broadcast_equals(ex))
def write_op(self): return OP_REGISTRY.get_op('write_text')
def read_op(self): return OP_REGISTRY.get_op('read_json')
def write_op(self): return OP_REGISTRY.get_op('write_json')
def read_op(self): return OP_REGISTRY.get_op('read_netcdf')
def write_op(self): return OP_REGISTRY.get_op('write_netcdf4')
def test_registered(self): """ Test registered operation execution execution """ reg_op = OP_REGISTRY.get_op(object_to_qualified_name(coregister)) ds_fine = xr.Dataset({ 'first': (['time', 'lat', 'lon'], np.array([np.eye(4, 8), np.eye(4, 8)])), 'second': (['time', 'lat', 'lon'], np.array([np.eye(4, 8), np.eye(4, 8)])), 'lat': np.linspace(-67.5, 67.5, 4), 'lon': np.linspace(-157.5, 157.5, 8), 'time': np.array([1, 2])}) ds_coarse = xr.Dataset({ 'first': (['time', 'lat', 'lon'], np.array([np.eye(3, 6), np.eye(3, 6)])), 'second': (['time', 'lat', 'lon'], np.array([np.eye(3, 6), np.eye(3, 6)])), 'lat': np.linspace(-60, 60, 3), 'lon': np.linspace(-150, 150, 6), 'time': np.array([1, 2])}) # Test that the coarse dataset has been resampled onto the grid # of the finer dataset. ds_coarse_resampled = reg_op(ds_master=ds_fine, ds_replica=ds_coarse) expected = xr.Dataset({ 'first': (['time', 'lat', 'lon'], np.array([[[1., 0.28571429, 0., 0., 0., 0., 0., 0.], [0.33333333, 0.57142857, 0.38095238, 0., 0., 0., 0., 0.], [0., 0.47619048, 0.52380952, 0.28571429, 0.04761905, 0., 0., 0.], [0., 0., 0.42857143, 0.85714286, 0.14285714, 0., 0., 0.]], [[1., 0.28571429, 0., 0., 0., 0., 0., 0.], [0.33333333, 0.57142857, 0.38095238, 0., 0., 0., 0., 0.], [0., 0.47619048, 0.52380952, 0.28571429, 0.04761905, 0., 0., 0.], [0., 0., 0.42857143, 0.85714286, 0.14285714, 0., 0., 0.]]])), 'second': (['time', 'lat', 'lon'], np.array([[[1., 0.28571429, 0., 0., 0., 0., 0., 0.], [0.33333333, 0.57142857, 0.38095238, 0., 0., 0., 0., 0.], [0., 0.47619048, 0.52380952, 0.28571429, 0.04761905, 0., 0., 0.], [0., 0., 0.42857143, 0.85714286, 0.14285714, 0., 0., 0.]], [[1., 0.28571429, 0., 0., 0., 0., 0., 0.], [0.33333333, 0.57142857, 0.38095238, 0., 0., 0., 0., 0.], [0., 0.47619048, 0.52380952, 0.28571429, 0.04761905, 0., 0., 0.], [0., 0., 0.42857143, 0.85714286, 0.14285714, 0., 0., 0.]]])), 'lat': np.linspace(-67.5, 67.5, 4), 'lon': np.linspace(-157.5, 157.5, 8), 'time': np.array([1, 2])}) assert_almost_equal(ds_coarse_resampled['first'].values, expected['first'].values) # Test that the fine dataset has been resampled (aggregated) # onto the grid of the coarse dataset. ds_fine_resampled = reg_op(ds_master=ds_coarse, ds_replica=ds_fine) expected = xr.Dataset({ 'first': (['time', 'lat', 'lon'], np.array([[[0.625, 0.125, 0., 0., 0., 0.], [0.125, 0.5, 0.125, 0., 0., 0.], [0., 0.125, 0.625, 0., 0., 0.]], [[0.625, 0.125, 0., 0., 0., 0.], [0.125, 0.5, 0.125, 0., 0., 0.], [0., 0.125, 0.625, 0., 0., 0.]]])), 'second': (['time', 'lat', 'lon'], np.array([[[0.625, 0.125, 0., 0., 0., 0.], [0.125, 0.5, 0.125, 0., 0., 0.], [0., 0.125, 0.625, 0., 0., 0.]], [[0.625, 0.125, 0., 0., 0., 0.], [0.125, 0.5, 0.125, 0., 0., 0.], [0., 0.125, 0.625, 0., 0., 0.]]])), 'lat': np.linspace(-60, 60, 3), 'lon': np.linspace(-150, 150, 6), 'time': np.array([1, 2])}) assert_almost_equal(ds_fine_resampled['first'].values, expected['first'].values)