def test_adapters_with_ascat(): ascat_data_folder = os.path.join( os.path.dirname(__file__), "..", "test-data", "sat", "ascat", "netcdf", "55R22", ) ascat_grid_folder = os.path.join( os.path.dirname(__file__), "..", "test-data", "sat", "ascat", "netcdf", "grid", ) grid_fname = os.path.join(ascat_grid_folder, "TUW_WARP5_grid_info_2_1.nc") ascat_reader = AscatGriddedNcTs( ascat_data_folder, "TUW_METOP_ASCAT_WARP55R22_{:04d}", grid_filename=grid_fname, ) ascat_anom = AnomalyAdapter(ascat_reader, window_size=35, columns=["sm"]) data = ascat_anom.read_ts(12.891455, 45.923004) assert data is not None assert np.any(data["sm"].values != 0) data = ascat_anom.read(12.891455, 45.923004) assert data is not None assert np.any(data["sm"].values != 0) ascat_self = SelfMaskingAdapter(ascat_reader, ">", 0, "sm") data2 = ascat_self.read_ts(12.891455, 45.923004) assert data2 is not None assert np.all(data2["sm"].values > 0) data2 = ascat_self.read(12.891455, 45.923004) assert data2 is not None assert np.all(data2["sm"].values > 0) ascat_mask = MaskingAdapter(ascat_reader, ">", 0, "sm") data3 = ascat_mask.read_ts(12.891455, 45.923004) assert data3 is not None assert np.any(data3["sm"].values) data3 = ascat_mask.read(12.891455, 45.923004) assert data3 is not None assert np.any(data3["sm"].values) ascat_clim = AnomalyClimAdapter(ascat_reader, columns=["sm"]) data4 = ascat_clim.read_ts(12.891455, 45.923004) assert data4 is not None assert np.any(data["sm"].values != 0) data4 = ascat_clim.read(12.891455, 45.923004) assert data4 is not None assert np.any(data["sm"].values != 0)
def test_self_masking_adapter(): ref_x = np.arange(10) ref_y = np.arange(10) * 0.5 ds = TestDataset('', n=20) ds_mask = SelfMaskingAdapter(ds, '<', 10, 'x') data_masked = ds_mask.read_ts() data_masked2 = ds_mask.read() nptest.assert_almost_equal(data_masked['x'].values, ref_x) nptest.assert_almost_equal(data_masked2['x'].values, ref_x) nptest.assert_almost_equal(data_masked['y'].values, ref_y) nptest.assert_almost_equal(data_masked2['y'].values, ref_y)
def test_self_masking_adapter(): ref_x = np.arange(10) ref_y = np.arange(10) * 0.5 ds = TestDataset("", n=20) ds_mask = SelfMaskingAdapter(ds, "<", 10, "x") data_masked = ds_mask.read_ts() data_masked2 = ds_mask.read() nptest.assert_almost_equal(data_masked["x"].values, ref_x) nptest.assert_almost_equal(data_masked2["x"].values, ref_x) nptest.assert_almost_equal(data_masked["y"].values, ref_y) nptest.assert_almost_equal(data_masked2["y"].values, ref_y)
def test_self_masking_adapter(): ref_x = np.arange(10) ref_y = np.arange(10) * 0.5 ds = TestDataset('', n=20) ds_mask = SelfMaskingAdapter(ds, '<', 10, 'x') data_masked = ds_mask.read_ts() data_masked2 = ds_mask.read() nptest.assert_almost_equal(data_masked['x'].values,ref_x) nptest.assert_almost_equal(data_masked2['x'].values,ref_x) nptest.assert_almost_equal(data_masked['y'].values,ref_y) nptest.assert_almost_equal(data_masked2['y'].values,ref_y)
def test_timezone_removal(): tz_reader = TestTimezoneReader() reader_anom = AnomalyAdapter(tz_reader, window_size=35, columns=["data"]) assert reader_anom.read_ts(0) is not None reader_self = SelfMaskingAdapter(tz_reader, ">", 0, "data") assert reader_self.read_ts(0) is not None reader_mask = MaskingAdapter(tz_reader, ">", 0, "data") assert reader_mask.read_ts(0) is not None reader_clim = AnomalyClimAdapter(tz_reader, columns=["data"]) assert reader_clim.read_ts(0) is not None
def test_adapters_with_ascat(): ascat_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data', 'sat', 'ascat', 'netcdf', '55R22') ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data', 'sat', 'ascat', 'netcdf', 'grid') ascat_reader = AscatSsmCdr(ascat_data_folder, ascat_grid_folder, grid_filename='TUW_WARP5_grid_info_2_1.nc') ascat_anom = AnomalyAdapter(ascat_reader, window_size=35, columns=['sm']) data = ascat_anom.read_ts(12.891455, 45.923004) assert data is not None assert np.any(data['sm'].values != 0) data = ascat_anom.read(12.891455, 45.923004) assert data is not None assert np.any(data['sm'].values != 0) ascat_self = SelfMaskingAdapter(ascat_reader, '>', 0, 'sm') data2 = ascat_self.read_ts(12.891455, 45.923004) assert data2 is not None assert np.all(data2['sm'].values > 0) data2 = ascat_self.read(12.891455, 45.923004) assert data2 is not None assert np.all(data2['sm'].values > 0) ascat_mask = MaskingAdapter(ascat_reader, '>', 0, 'sm') data3 = ascat_mask.read_ts(12.891455, 45.923004) assert data3 is not None assert np.any(data3['sm'].values) data3 = ascat_mask.read(12.891455, 45.923004) assert data3 is not None assert np.any(data3['sm'].values) ascat_clim = AnomalyClimAdapter(ascat_reader, columns=['sm']) data4 = ascat_clim.read_ts(12.891455, 45.923004) assert data4 is not None assert np.any(data['sm'].values != 0) data4 = ascat_clim.read(12.891455, 45.923004) assert data4 is not None assert np.any(data['sm'].values != 0)
def test_adapters_custom_fct_name(): def assert_all_read_fcts(reader): assert (np.all(reader.read() == reader.read_ts())) assert (np.all(reader.read() == reader.alias_read())) base = TestDataset("", n=20) assert_all_read_fcts(base) sma = SelfMaskingAdapter(base, '>=', 5, column_name='y', read_name='alias_read') assert_all_read_fcts(sma) smanom = AnomalyAdapter(sma, read_name='alias_read') assert_all_read_fcts(smanom)
# # Masking datasets are datasets that return a pandas DataFrame with boolean values. `True` means that the observation # should be masked, `False` means it should be kept. All masking datasets are temporally matched in pairs to the # temporal reference dataset. Only observations for which all masking datasets have a value of `False` are kept for # further validation. # # The masking datasets have the same format as the dataset dictionary and can be specified in the Validation class # with the `masking_datasets` keyword. # # ### Masking adapter # # To easily transform an existing dataset into a masking dataset `pytesmo` offers a adapter class that calls the # `read_ts` method of an existing dataset and creates a masking dataset based on an operator, a given threshold, and (optionally) a column name. # In[12]: from pytesmo.validation_framework.adapters import MaskingAdapter ds_mask = MaskingAdapter(ismn_reader, '<', 0.2, 'soil moisture') print(ds_mask.read_ts(ids[0]).head()) # ### Self-masking adapter # `pytesmo` also has a class that masks a dataset "on-the-fly", based on one of the columns it contains and an operator and a threshold. In contrast to the masking adapter mentioned above, the output of the self-masking adapter is the masked data, not the the mask. The self-masking adapter wraps a data reader, which must have a `read_ts` or `read` method. Calling its `read_ts`/`read` method will return the masked data - more precisely a DataFrame with only rows where the masking condition is true. # In[13]: from pytesmo.validation_framework.adapters import SelfMaskingAdapter ds_mask = SelfMaskingAdapter(ismn_reader, '<', 0.2, 'soil moisture') print(ds_mask.read_ts(ids[0]).head())
icdr_dataset, exact_index=exact_index, **kwargs) self.grid = self.tcdr_reader.grid def read(self, *args, **kwargs): return pd.concat([self.tcdr_reader.read(*args, **kwargs), self.icdr_reader.read(*args, **kwargs)], axis=0) # check if implementation match with paths for ds in GeoC3Sv201706Ts._ds_implemented: assert ds in path_settings.keys() # check if FullCDR implementation match with paths for ds in GeoC3Sv201706FullCDRTs._ds_implemented: assert tuple(list(ds) + ['TCDR']) in path_settings.keys() assert tuple(list(ds) + ['ICDR']) in path_settings.keys() # ============================================================================== if __name__ == '__main__': for record in ['TCDR', 'ICDR']: for dataset in ['COMBINED', 'ACTIVE', 'PASSIVE']: reader = GeoC3Sv201706Ts( dataset_or_path=('C3S', 'v201706', dataset, 'DAILY', record), grid=None, exact_index=True, ioclass_kws={'read_bulk': True}, parameters=['sm', 'sm_uncertainty', 'flag'], scale_factors={'sm': 100.}) reader = SelfMaskingAdapter(reader, '==', 0, 'flag') ts = reader.read(-108,40) print(ts)