def test_alternative_region(self, tmp_path):
# make the dataset
(tmp_path / "raw/esa_cci_landcover").mkdir(parents=True)
data_path = tmp_path / "raw/esa_cci_landcover/1992-v2.0.7b_testy_test.nc"
dataset = self._make_ESA_CCI_dataset(size=(100, 100))
dataset.attrs["time_coverage_start"] = "20190101"
dataset.to_netcdf(path=data_path)
ethiopia = get_ethiopia()
legend_path = tmp_path / "raw/esa_cci_landcover/legend.csv"
self._make_ESA_CCI_legend().to_csv(legend_path)
# regrid the datasets
regrid_dataset, _, _ = _make_dataset(
size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax,
)
regrid_path = tmp_path / "regridder.nc"
regrid_dataset.to_netcdf(regrid_path)
# build the Preprocessor object and subset with a different subset_str
processor = ESACCIPreprocessor(tmp_path)
processor.preprocess(subset_str="ethiopia", regrid=regrid_path)
expected_out_path = (tmp_path /
"interim/static/esa_cci_landcover_preprocessed"
"/esa_cci_landcover_ethiopia_one_hot.nc")
assert (expected_out_path.exists(
)), f"Expected processed file to be saved to {expected_out_path}"
def test_alternative_region(self, tmp_path):
# make the dataset
(tmp_path / "raw/chirps/global").mkdir(parents=True)
data_path = tmp_path / "raw/chirps/global/testy_test.nc"
dataset = self._make_chirps_dataset(size=(100, 100))
dataset.to_netcdf(path=data_path)
ethiopia = get_ethiopia()
# regrid the datasets
regrid_dataset, _, _ = _make_dataset(
size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax,
)
regrid_path = tmp_path / "regridder.nc"
regrid_dataset.to_netcdf(regrid_path)
# build the Preprocessor object and subset with a different subset_str
processor = CHIRPSPreprocessor(tmp_path)
processor.preprocess(subset_str="ethiopia",
regrid=regrid_path,
parallel=False)
expected_out_path = tmp_path / "interim/chirps_preprocessed/data_ethiopia.nc"
assert (expected_out_path.exists(
)), f"Expected processed file to be saved to {expected_out_path}"
def test_alternative_region(self, tmp_path):
# make the dataset
(tmp_path / "raw/boku_ndvi_1000").mkdir(parents=True)
data_path = (
tmp_path /
"raw/boku_ndvi_1000/MCD13A2.t200915.006.EAv1.1_km_10_days_NDVI.O1.nc"
)
dataset = self._make_boku_ndvi_dataset(size=(100, 100))
dataset.to_netcdf(path=data_path)
ethiopia = get_ethiopia()
# regrid the datasets
regrid_dataset, _, _ = _make_dataset(
size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax,
)
regrid_path = tmp_path / "regridder.nc"
regrid_dataset.to_netcdf(regrid_path)
# build the Preprocessor object and subset with a different subset_str
processor = BokuNDVIPreprocessor(tmp_path)
processor.preprocess(subset_str="ethiopia",
regrid=regrid_path,
parallel=False)
expected_out_path = tmp_path / "interim/boku_ndvi_preprocessed/data_ethiopia.nc"
assert (expected_out_path.exists(
)), f"Expected processed file to be saved to {expected_out_path}"
def test_alternative_region(self, tmp_path):
# make the dataset
(tmp_path / 'raw/era5POS/global').mkdir(parents=True)
data_path = tmp_path / 'raw/era5POS/global/testy_test.nc'
dataset = self._make_era5POS_dataset(size=(100, 100))
dataset.to_netcdf(path=data_path)
ethiopia = get_ethiopia()
regrid_dataset, _, _ = _make_dataset(size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax)
regrid_path = tmp_path / 'regridder.nc'
regrid_dataset.to_netcdf(regrid_path)
processor = PlanetOSPreprocessor(tmp_path)
processor.preprocess(subset_str='ethiopia',
regrid=regrid_path,
parallel=False)
expected_out_path = tmp_path / 'interim/era5POS_preprocessed/data_ethiopia.nc'
assert expected_out_path.exists(), \
f'Expected processed file to be saved to {expected_out_path}'
def test_alternative_region(self, tmp_path):
# make the dataset
(tmp_path / "raw/era5POS/global").mkdir(parents=True)
data_path = tmp_path / "raw/era5POS/global/testy_test.nc"
dataset = self._make_era5POS_dataset(size=(100, 100))
dataset.to_netcdf(path=data_path)
ethiopia = get_ethiopia()
regrid_dataset, _, _ = _make_dataset(
size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax,
)
regrid_path = tmp_path / "regridder.nc"
regrid_dataset.to_netcdf(regrid_path)
processor = PlanetOSPreprocessor(tmp_path)
processor.preprocess(subset_str="ethiopia", regrid=regrid_path, n_processes=1)
expected_out_path = tmp_path / "interim/era5POS_preprocessed/data_ethiopia.nc"
assert (
expected_out_path.exists()
), f"Expected processed file to be saved to {expected_out_path}"
def test_VCI(self, tmp_path):
v = VHIPreprocessor(tmp_path, var="VCI")
# get filename
demo_raw_folder = v.raw_folder / "vhi" / "1981"
demo_raw_folder.mkdir(parents=True, exist_ok=True)
netcdf_filepath = demo_raw_folder / "VHP.G04.C07.NC.P1981035.VH.nc"
raw_height, raw_width = 360, 100
v.raw_height = raw_height
v.raw_width = raw_width
# build dummy .nc object
raw_ds = self._make_vhi_dataset(raw_height, raw_width)
raw_ds.to_netcdf(netcdf_filepath)
# get regridder
ethiopia = get_ethiopia()
regrid_dataset, _, _ = _make_dataset(
size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax,
)
regrid_path = tmp_path / "regridder.nc"
regrid_dataset.to_netcdf(regrid_path)
# run the preprocessing steps
out = v._preprocess(
netcdf_filepath=netcdf_filepath.as_posix(),
output_dir=v.interim.as_posix(),
subset_str="ethiopia",
regrid=regrid_dataset,
)
expected_out_path = (tmp_path / "interim/VCI_interim/\
STAR_VHP.G04.C07.NC_1981_8_31_ethiopia_VH.nc".replace(" ", ""))
assert (expected_out_path.exists(
)), f"Expected processed file to be saved to {expected_out_path}"
assert (out == expected_out_path), f"Expected: {expected_out_path}, \
Got: {out}"
output = xr.open_dataset(expected_out_path)
assert "VCI" in list(output.data_vars)
assert "VHI" not in list(output.data_vars)
def test_alternative_region_interim_creation(self, tmp_path):
v = VHIPreprocessor(tmp_path)
# get filename
demo_raw_folder = (v.raw_folder / 'vhi' / '1981')
demo_raw_folder.mkdir(parents=True, exist_ok=True)
netcdf_filepath = demo_raw_folder / 'VHP.G04.C07.NC.P1981035.VH.nc'
# build dummy .nc object
raw_height, raw_width = 360, 100
v.raw_height = raw_height
v.raw_width = raw_width
raw_ds = self._make_vhi_dataset(raw_height, raw_width)
raw_ds.to_netcdf(netcdf_filepath)
# get regridder
ethiopia = get_ethiopia()
regrid_dataset, _, _ = _make_dataset(size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax)
regrid_path = tmp_path / 'regridder.nc'
regrid_dataset.to_netcdf(regrid_path)
# run the preprocessing steps
out = v._preprocess(netcdf_filepath=netcdf_filepath.as_posix(),
output_dir=v.interim.as_posix(),
subset_str='ethiopia',
regrid=regrid_dataset)
expected_out_path = tmp_path / 'interim/VHI_interim/\
STAR_VHP.G04.C07.NC_1981_8_31_ethiopia_VH.nc'.replace(' ', '')
assert expected_out_path.exists(), \
f'Expected processed file to be saved to {expected_out_path}'
assert out == expected_out_path, f"Expected: {expected_out_path}, \
def test_alternative_region(self, tmp_path):
# make the dataset
(tmp_path / 'raw/gleam/monthly').mkdir(parents=True)
data_path = tmp_path / 'raw/gleam/monthly/testy_test.nc'
dataset = self._make_gleam_dataset(size=(100, 100))
dataset.to_netcdf(path=data_path)
ethiopia = get_ethiopia()
# regrid the datasets
regrid_dataset, _, _ = _make_dataset(size=(20, 20),
latmin=ethiopia.latmin,
latmax=ethiopia.latmax,
lonmin=ethiopia.lonmin,
lonmax=ethiopia.lonmax)
regrid_path = tmp_path / 'regridder.nc'
regrid_dataset.to_netcdf(regrid_path)
# build the Preprocessor object and subset with a different subset_str
processor = GLEAMPreprocessor(tmp_path)
processor.preprocess(subset_str='ethiopia', regrid=regrid_path)
expected_out_path = tmp_path / 'interim/gleam_preprocessed/data_ethiopia.nc'
assert expected_out_path.exists(), \
f'Expected processed file to be saved to {expected_out_path}'
