def normalise(data_refs):
print(f'[INFO] Working on data refs: {data_refs}')
norm_dsets = collections.OrderedDict()
for data_ref, file_paths in data_refs.items():
xr_dset = open_dataset(data_ref, file_paths)
norm_dsets[data_ref] = xr_dset
return norm_dsets
def normalise(collection):
LOGGER.info(f"Working on datasets: {collection}")
norm_collection = collections.OrderedDict()
for dset, file_paths in collection.items():
ds = open_dataset(dset, file_paths)
norm_collection[dset] = ds
return norm_collection
def normalise(collection, apply_fixes=True):
"""
Takes file paths and opens and fixes the dataset they make up.
:param collection: Ordered dictionary of ds ids and their related file paths.
:param apply_fixes: Boolean. If True fixes will be applied to datasets if needed. Default is True.
:return: An ordered dictionary of ds ids and their fixed xarray Dataset.
"""
LOGGER.info(f"Working on datasets: {collection}")
norm_collection = collections.OrderedDict()
for dset, file_paths in collection.items():
ds = open_dataset(dset, file_paths, apply_fixes)
norm_collection[dset] = ds
return norm_collection
def test_open_dataset_without_fix(load_esgf_test_data):
ds = xr.open_mfdataset(fpath, use_cftime=True, combine="by_coords")
not_fixed_ds = open_dataset(ds_id, fpath, apply_fixes=False)
assert ds.dims == not_fixed_ds.dims
assert "lev" in ds.dims
assert "lev" in not_fixed_ds.dims
def test_open_dataset_with_fix(load_esgf_test_data):
unfixed_ds = xr.open_mfdataset(fpath, use_cftime=True, combine="by_coords")
fixed_ds = open_dataset(ds_id, fpath)
assert unfixed_ds.dims != fixed_ds.dims
assert "lev" in unfixed_ds.dims
assert "lev" not in fixed_ds.dims