def concat_cubes(dataset, historical_key=None):
"""Concatenate cubes into a dataset spanning the full time frame"""
if historical_key is None:
historical_key = default_config['data']['attributes'][
'historical_experiment']
concatenated = pd.DataFrame(columns=dataset.columns)
for model, group in dataset.groupby('model'):
future_sel = group['experiment'] != historical_key
for _, row in group.loc[future_sel, :].iterrows():
cube = row['cube']
matchid = row['index_match_run']
histcube = dataset.loc[matchid, 'cube']
time = cube.coord('time')
start = time.units.num2date(time.points)[0]
time = histcube.coord('time')
end = time.units.num2date(time.points)[-1]
if end > start:
logger.warning(
"Historical experiment ends past the start of future experiment: "
"trimming historical dataset to match %s - %s r%di%dp%d",
model, row['experiment'], row['realization'],
row['initialization'], row['physics'])
logger.debug("Historical end: %s. Future start: %s", end,
start)
logger.info("Constraining historical run to end before %s",
start)
# pylint: disable=cell-var-from-loop
constraint = iris.Constraint(
time=lambda cell: cell.point < start)
histcube = constraint.extract(histcube)
time = histcube.coord('time')
end = time.units.num2date(time.points)[-1]
# Since we may have matched on different realizations, set them to
# equal, otherwise the concatenation procedure will fail
histcube.replace_coord(cube.coord('realization'))
cubes = iris.cube.CubeList([histcube, cube])
equalise_attributes(cubes)
unify_time_units(cubes)
try:
row['cube'] = cubes.concatenate_cube()
except iris.exceptions.ConcatenateError as exc:
logger.warning(
"DATA SKIPPED: Error concatenating cubes: %s - %s r%di%dp%d: %s",
model, row['experiment'], row['realization'],
row['initialization'], row['physics'], exc)
continue
logger.info("Concatenated %s - %s r%di%dp%d", model,
row['experiment'], row['realization'],
row['initialization'], row['physics'])
# By adding the rows with concatenated cubes,
# we can construct a new DataFrame with only concatenated cubes,
# but with all the essential attributes from the input dataset.
concatenated = concatenated.append(row)
concatenated.reset_index(inplace=True)
logger.info("concatenated a total of %d realizations", len(concatenated))
return concatenated
def plot_means(cubes, type):
# compute and plot means
equalise_attributes(cubes["mean_pr"])
equalise_attributes(cubes["rx1day"])
# also clear up attributes on season number coord in rx1day
for c in cubes["rx1day"]:
c.coord("season_number").attributes = None
means = {}
means["mean"] = cubes["mean_pr"].merge_cube().collapsed(
'multi_model', iris.analysis.MEAN)
means["rx1day"] = cubes["rx1day"].merge_cube().collapsed(
'multi_model', iris.analysis.MEAN)
plot_3panel(means["mean"], means["rx1day"], f"{type} mean change")
def normalize_average_dataset(cubes,
season=None,
average_years=True,
relative=False,
reference_period=None):
"""Normalize and average a given iterable of cubes
The dataset is normalized by, and averaged across, its individual
ensemble runs. Thus, this works best (only), if the dataset
belongs to the same model and experiment, and has no other
ensemble variations other than its realization.
The dataset should already be concatenated across the historical
and future experiment, if necessary.
Each Iris cube inside the dataset should have a scalar realization
coordinate, with its value given the realization number. If not,
these are added on the fly, equal to the iteration index of the
cubes.
"""
if reference_period is None:
reference_period = default_config['data']['extra']['control_period']
if season:
cubes = extract_season(cubes, season)
if average_years:
cubes = average_year(cubes, season=season)
refvalues = calc_reference_values(cubes,
reference_period=reference_period,
normby='run')
cubes = normalize(cubes, refvalues, relative=relative)
for i, cube in enumerate(cubes):
if not cube.coords('realization'):
coord = iris.coords.AuxCoord(i,
standard_name='realization',
long_name='realization',
var_name='realization')
cube.add_aux_coord(coord)
cubes = iris.cube.CubeList(cubes)
equalise_attributes(cubes)
cube2d = cubes.merge_cube()
mean = cube2d.collapsed('realization', iris.analysis.MEAN)
return mean
def load_input_cube(src, varname):
"""Load input file(s) into one cube."""
with warnings.catch_warnings():
# Suppress psu warning
warnings.filterwarnings(
action='ignore',
message="Ignoring netCDF variable",
category=UserWarning,
)
month_cubes = iris.load(src, varname)
if len(month_cubes) == 1:
month_cube = remove_unique_attributes(month_cubes[0])
return month_cube
equalise_attributes(
month_cubes) # 'timeStamp' and 'uuid' would cause ConcatenateError
leg_cube = month_cubes.concatenate_cube()
return leg_cube
def _test(self, cubes, expect_attributes):
"""Test."""
working_cubes = [cube.copy() for cube in cubes]
original_working_list = [cube for cube in working_cubes]
# Exercise basic operation
equalise_attributes(working_cubes)
# Check they are the same cubes
self.assertEqual(working_cubes, original_working_list)
# Check resulting attributes all match the expected set
for cube in working_cubes:
self.assertEqual(cube.attributes, expect_attributes)
# Check everything else remains the same
for new_cube, old_cube in zip(working_cubes, cubes):
cube_before_noatts = old_cube.copy()
cube_before_noatts.attributes.clear()
cube_after_noatts = new_cube.copy()
cube_after_noatts.attributes.clear()
self.assertEqual(cube_after_noatts, cube_before_noatts)
def test_hybrid_pressure(self):
filepath = self.get_testdata_path('faked_sample_hp_grib_data.grib2')
# Load and save temperature cube and reference (air_pressure at
# surface) cube separately because this is the only way to save the
# hybrid pressure coordinate.
cube, ref_cube = load_cubes(filepath,
('air_temperature', 'air_pressure'))
with self.temp_filename() as tmp_save_path:
save([cube, ref_cube], tmp_save_path, saver='grib2')
# Only need to reload temperature cube to compare with unsaved
# temperature cube.
saved_cube = load_cube(tmp_save_path, 'air_temperature')
# Currently all attributes are lost when saving to grib, so we must
# equalise them in order to successfully compare all other aspects.
equalise_attributes([saved_cube, cube])
self.assertTrue(saved_cube == cube)
def load_cube(paths, variable_name=None):
"""Read datasets from paths into Iris cubes.
Combines cubes if there are more than one dataset in the same file.
Returns a list of lists. Inner lists corresponds to the areas (in
order), outer lists corresponds to the paths
"""
if isinstance(paths, (str, pathlib.Path)):
if variable_name:
cubes = iris.load_cubes(str(paths), constraints=variable_name)
else:
cubes = iris.load_cubes(str(paths))
else:
if variable_name:
cubes = iris.load([str(path) for path in paths],
constraints=variable_name)
else:
cubes = iris.load([str(path) for path in paths])
# Select only the cubes with 3/4D data (time, lat, long, height)
cubes = iris.cube.CubeList(
[cube for cube in cubes if len(cube.coords()) >= 3])
if len(cubes) == 0:
return None
equalise_attributes(cubes)
unify_time_units(cubes)
try:
cube = cubes.concatenate_cube()
except iris.exceptions.ConcatenateError as exc:
logger.warning("%s for %s", exc, str(paths))
logger.warning("Using only the first cube of [%s]", cubes)
cube = cubes[
0] # iris.load always returns a cubelist, so just take the first element
return cube