def test_regridder_save(self, tmp_path):
size_reference = (10, 10)
size_target = (20, 20)
reference_ds, _, _ = _make_dataset(size_reference)
target_ds, _, _ = _make_dataset(size_target)
processor = BasePreProcessor(tmp_path)
processor.regrid(target_ds, reference_ds)
weight_filename = 'nearest_s2d_100x100_10x10.nc'
assert (processor.preprocessed_folder / weight_filename).exists() is False, \
f'Regridder weight file not deleted!'
def test_incorrect_method(self, tmp_path):
size_reference = (10, 10)
size_target = (100, 100)
reference_ds, _, _ = _make_dataset(size_reference)
target_ds, _, _ = _make_dataset(size_target)
processor = BasePreProcessor(tmp_path)
with pytest.raises(AssertionError) as e:
processor.regrid(target_ds, reference_ds, method='woops!')
expected_message_contains = 'not an acceptable regridding method. Must be one of'
assert expected_message_contains in str(e), \
f'Expected {e} to contain {expected_message_contains}'
def test_regridding(self, tmp_path):
size_reference = (10, 10)
size_target = (20, 20)
reference_ds, _, _ = _make_dataset(size_reference)
target_ds, _, _ = _make_dataset(size_target)
processor = BasePreProcessor(tmp_path)
regridded_ds = processor.regrid(target_ds, reference_ds)
# add the time dimension
assert regridded_ds.VHI.values.shape[1:] == size_reference, \
f'Expected regridded Dataset to have shape {size_reference}, ' \
f'got {regridded_ds.VHI.values.shape}'
# ------------------------------------------------------------------------------
# Test PREPROCESSING
# ------------------------------------------------------------------------------
from src.preprocess.base import BasePreProcessor
b = BasePreProcessor()
ds1_kenya = b.chop_roi(ds1, inverse_lat=True)
ds2_kenya = b.chop_roi(ds2, inverse_lat=True)
# concat across initialisation dates
ds_kenya = xr.concat([ds1_kenya, ds2_kenya], dim="initialisation_date")
stacked = ds_kenya.stack(time=("initialisation_date", "forecast_horizon"))
# stack each individually
k1 = ds1_kenya.stack(time=("initialisation_date", "forecast_horizon"))
k2 = ds2_kenya.stack(time=("initialisation_date", "forecast_horizon"))
# test selectors
stacked.sel(forecast_horizon=np.timedelta64(28, "D"))
stacked.sel(initialisation_date="1997-01-01")
stacked.swap_dims({"time": "valid_time"}).sel(valid_time="1997-04")
# test regridding
ref_ds = b.load_reference_grid(path_to_grid=Path("data/interim/chirps_preprocessed.nc"))
ds1_kenya_regrid = b.regrid(
ds1_kenya.rename({"latitude": "lat", "longitude": "lon"}), ref_ds
)
# test resampling time
b.resample_time()
# wrong shapes!
datasets = engineer._get_preprocessed_files()
ds_list = [xr.open_dataset(ds) for ds in datasets]
dims_list = [[dim for dim in ds.dims] for ds in ds_list]
variable_list = [[var for var in ds.variables if var not in dims_list[i]][0]
for i, ds in enumerate(ds_list)]
da_list = [ds[variable_list[i]] for i, ds in enumerate(ds_list)]
pp = BasePreProcessor(data_path)
c_ds = ds_list[0]
e_ds = ds_list[1]
v_ds = ds_list[2]
v_ds = pp.resample_time(v_ds)
c_ds = pp.regrid(c_ds, v_ds)
c_ds = pp.resample_time(c_ds)
v_ds.to_netcdf(vhi_path.home() / vhi_path.parent / "vhi_kenya_regrid.nc")
v_ds.to_netcdf(chirps_path.home() / chirps_path.parent /
"chirps_kenya_regrid.nc")
# engineer process
engineer._get_preprocessed_files
engineer._make_dataset
engineer.stratify_xy
engineer.stratify_xy
engineer._train_test_split
engineer.stratify_xy
engineer._stratify_training_data
engineer._save
import xarray as xr
import sys
sys.path.append("../..")
from scripts.utils import get_data_path
from src.preprocess.base import BasePreProcessor
if __name__ == "__main__":
data_dir = get_data_path()
vci = xr.open_dataset(data_dir / "interim/VCI_preprocessed/data_india.nc")
regrid_ds = xr.open_dataset(
data_dir / "interim/reanalysis-era5-land_preprocessed/data_india.nc"
)
print("** Begin Regridding **")
processor = BasePreProcessor(data_dir)
vci = processor.regrid(ds=vci, reference_ds=regrid_ds)
print("** Saving file **")
vci.to_netcdf(data_dir / "interim/VCI_preprocessed/regrid_data_india.nc")