def test_alternative_region(self, tmp_path): # make the dataset (tmp_path / "raw/esa_cci_landcover").mkdir(parents=True) data_path = tmp_path / "raw/esa_cci_landcover/1992-v2.0.7b_testy_test.nc" dataset = self._make_ESA_CCI_dataset(size=(100, 100)) dataset.attrs["time_coverage_start"] = "20190101" dataset.to_netcdf(path=data_path) ethiopia = get_ethiopia() legend_path = tmp_path / "raw/esa_cci_landcover/legend.csv" self._make_ESA_CCI_legend().to_csv(legend_path) # regrid the datasets regrid_dataset, _, _ = _make_dataset( size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax, ) regrid_path = tmp_path / "regridder.nc" regrid_dataset.to_netcdf(regrid_path) # build the Preprocessor object and subset with a different subset_str processor = ESACCIPreprocessor(tmp_path) processor.preprocess(subset_str="ethiopia", regrid=regrid_path) expected_out_path = (tmp_path / "interim/static/esa_cci_landcover_preprocessed" "/esa_cci_landcover_ethiopia_one_hot.nc") assert (expected_out_path.exists( )), f"Expected processed file to be saved to {expected_out_path}"
def test_alternative_region(self, tmp_path): # make the dataset (tmp_path / "raw/chirps/global").mkdir(parents=True) data_path = tmp_path / "raw/chirps/global/testy_test.nc" dataset = self._make_chirps_dataset(size=(100, 100)) dataset.to_netcdf(path=data_path) ethiopia = get_ethiopia() # regrid the datasets regrid_dataset, _, _ = _make_dataset( size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax, ) regrid_path = tmp_path / "regridder.nc" regrid_dataset.to_netcdf(regrid_path) # build the Preprocessor object and subset with a different subset_str processor = CHIRPSPreprocessor(tmp_path) processor.preprocess(subset_str="ethiopia", regrid=regrid_path, parallel=False) expected_out_path = tmp_path / "interim/chirps_preprocessed/data_ethiopia.nc" assert (expected_out_path.exists( )), f"Expected processed file to be saved to {expected_out_path}"
def test_alternative_region(self, tmp_path): # make the dataset (tmp_path / "raw/boku_ndvi_1000").mkdir(parents=True) data_path = ( tmp_path / "raw/boku_ndvi_1000/MCD13A2.t200915.006.EAv1.1_km_10_days_NDVI.O1.nc" ) dataset = self._make_boku_ndvi_dataset(size=(100, 100)) dataset.to_netcdf(path=data_path) ethiopia = get_ethiopia() # regrid the datasets regrid_dataset, _, _ = _make_dataset( size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax, ) regrid_path = tmp_path / "regridder.nc" regrid_dataset.to_netcdf(regrid_path) # build the Preprocessor object and subset with a different subset_str processor = BokuNDVIPreprocessor(tmp_path) processor.preprocess(subset_str="ethiopia", regrid=regrid_path, parallel=False) expected_out_path = tmp_path / "interim/boku_ndvi_preprocessed/data_ethiopia.nc" assert (expected_out_path.exists( )), f"Expected processed file to be saved to {expected_out_path}"
def test_alternative_region(self, tmp_path): # make the dataset (tmp_path / 'raw/era5POS/global').mkdir(parents=True) data_path = tmp_path / 'raw/era5POS/global/testy_test.nc' dataset = self._make_era5POS_dataset(size=(100, 100)) dataset.to_netcdf(path=data_path) ethiopia = get_ethiopia() regrid_dataset, _, _ = _make_dataset(size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax) regrid_path = tmp_path / 'regridder.nc' regrid_dataset.to_netcdf(regrid_path) processor = PlanetOSPreprocessor(tmp_path) processor.preprocess(subset_str='ethiopia', regrid=regrid_path, parallel=False) expected_out_path = tmp_path / 'interim/era5POS_preprocessed/data_ethiopia.nc' assert expected_out_path.exists(), \ f'Expected processed file to be saved to {expected_out_path}'
def test_alternative_region(self, tmp_path): # make the dataset (tmp_path / "raw/era5POS/global").mkdir(parents=True) data_path = tmp_path / "raw/era5POS/global/testy_test.nc" dataset = self._make_era5POS_dataset(size=(100, 100)) dataset.to_netcdf(path=data_path) ethiopia = get_ethiopia() regrid_dataset, _, _ = _make_dataset( size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax, ) regrid_path = tmp_path / "regridder.nc" regrid_dataset.to_netcdf(regrid_path) processor = PlanetOSPreprocessor(tmp_path) processor.preprocess(subset_str="ethiopia", regrid=regrid_path, n_processes=1) expected_out_path = tmp_path / "interim/era5POS_preprocessed/data_ethiopia.nc" assert ( expected_out_path.exists() ), f"Expected processed file to be saved to {expected_out_path}"
def test_VCI(self, tmp_path): v = VHIPreprocessor(tmp_path, var="VCI") # get filename demo_raw_folder = v.raw_folder / "vhi" / "1981" demo_raw_folder.mkdir(parents=True, exist_ok=True) netcdf_filepath = demo_raw_folder / "VHP.G04.C07.NC.P1981035.VH.nc" raw_height, raw_width = 360, 100 v.raw_height = raw_height v.raw_width = raw_width # build dummy .nc object raw_ds = self._make_vhi_dataset(raw_height, raw_width) raw_ds.to_netcdf(netcdf_filepath) # get regridder ethiopia = get_ethiopia() regrid_dataset, _, _ = _make_dataset( size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax, ) regrid_path = tmp_path / "regridder.nc" regrid_dataset.to_netcdf(regrid_path) # run the preprocessing steps out = v._preprocess( netcdf_filepath=netcdf_filepath.as_posix(), output_dir=v.interim.as_posix(), subset_str="ethiopia", regrid=regrid_dataset, ) expected_out_path = (tmp_path / "interim/VCI_interim/\ STAR_VHP.G04.C07.NC_1981_8_31_ethiopia_VH.nc".replace(" ", "")) assert (expected_out_path.exists( )), f"Expected processed file to be saved to {expected_out_path}" assert (out == expected_out_path), f"Expected: {expected_out_path}, \ Got: {out}" output = xr.open_dataset(expected_out_path) assert "VCI" in list(output.data_vars) assert "VHI" not in list(output.data_vars)
def test_alternative_region_interim_creation(self, tmp_path): v = VHIPreprocessor(tmp_path) # get filename demo_raw_folder = (v.raw_folder / 'vhi' / '1981') demo_raw_folder.mkdir(parents=True, exist_ok=True) netcdf_filepath = demo_raw_folder / 'VHP.G04.C07.NC.P1981035.VH.nc' # build dummy .nc object raw_height, raw_width = 360, 100 v.raw_height = raw_height v.raw_width = raw_width raw_ds = self._make_vhi_dataset(raw_height, raw_width) raw_ds.to_netcdf(netcdf_filepath) # get regridder ethiopia = get_ethiopia() regrid_dataset, _, _ = _make_dataset(size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax) regrid_path = tmp_path / 'regridder.nc' regrid_dataset.to_netcdf(regrid_path) # run the preprocessing steps out = v._preprocess(netcdf_filepath=netcdf_filepath.as_posix(), output_dir=v.interim.as_posix(), subset_str='ethiopia', regrid=regrid_dataset) expected_out_path = tmp_path / 'interim/VHI_interim/\ STAR_VHP.G04.C07.NC_1981_8_31_ethiopia_VH.nc'.replace(' ', '') assert expected_out_path.exists(), \ f'Expected processed file to be saved to {expected_out_path}' assert out == expected_out_path, f"Expected: {expected_out_path}, \
def test_alternative_region(self, tmp_path): # make the dataset (tmp_path / 'raw/gleam/monthly').mkdir(parents=True) data_path = tmp_path / 'raw/gleam/monthly/testy_test.nc' dataset = self._make_gleam_dataset(size=(100, 100)) dataset.to_netcdf(path=data_path) ethiopia = get_ethiopia() # regrid the datasets regrid_dataset, _, _ = _make_dataset(size=(20, 20), latmin=ethiopia.latmin, latmax=ethiopia.latmax, lonmin=ethiopia.lonmin, lonmax=ethiopia.lonmax) regrid_path = tmp_path / 'regridder.nc' regrid_dataset.to_netcdf(regrid_path) # build the Preprocessor object and subset with a different subset_str processor = GLEAMPreprocessor(tmp_path) processor.preprocess(subset_str='ethiopia', regrid=regrid_path) expected_out_path = tmp_path / 'interim/gleam_preprocessed/data_ethiopia.nc' assert expected_out_path.exists(), \ f'Expected processed file to be saved to {expected_out_path}'