def tsplot_save_data(cfg, data_info, temp, salt, depth_model):
"""Save data for TS plots."""
ofiles = {}
data_info['variable'] = 'thetao'
ofiles['ofilename_t'] = genfilename(**data_info, data_type='tsplot')
np.save(ofiles['ofilename_t'], temp)
provenance_record = get_provenance_record(data_info, 'tsplot', 'npy')
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(ofiles['ofilename_t'] + '.npy',
provenance_record)
data_info['variable'] = 'so'
ofiles['ofilename_s'] = genfilename(**data_info, data_type='tsplot')
np.save(ofiles['ofilename_s'], salt)
provenance_record = get_provenance_record(data_info, 'tsplot', 'npy')
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(ofiles['ofilename_s'] + '.npy',
provenance_record)
data_info['variable'] = 'depth'
ofiles['ofilename_depth'] = genfilename(**data_info, data_type='tsplot')
np.save(ofiles['ofilename_depth'], depth_model)
provenance_record = get_provenance_record(data_info, 'tsplot', 'npy')
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(ofiles['ofilename_depth'] + '.npy',
provenance_record)
def hofm_save_data(cfg, data_info, oce_hofm):
"""Save data for Hovmoeller diagrams."""
ofiles = {}
ofiles['ofilename'] = genfilename(**data_info, data_type='hofm')
ofiles['ofilename_levels'] = genfilename(**data_info, data_type='levels')
ofiles['ofilename_time'] = genfilename(**data_info, data_type='time')
np.save(ofiles['ofilename'], oce_hofm)
provenance_record = get_provenance_record(data_info, 'hofm', 'npy')
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(ofiles['ofilename'] + '.npy', provenance_record)
if isinstance(data_info['levels'], np.ma.core.MaskedArray):
np.save(ofiles['ofilename_levels'],
data_info['levels'][0:data_info['lev_limit']].filled())
else:
np.save(ofiles['ofilename_levels'],
data_info['levels'][0:data_info['lev_limit']])
provenance_record = get_provenance_record(data_info, 'lev', 'npy')
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(ofiles['ofilename_levels'] + '.npy',
provenance_record)
np.save(ofiles['ofilename_time'], data_info['time'])
provenance_record = get_provenance_record(data_info, 'time', 'npy')
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(ofiles['ofilename_time'] + '.npy',
provenance_record)
def main(cfg):
"""Run the diagnostic."""
input_data = (
select_metadata(cfg['input_data'].values(), short_name='tas') +
select_metadata(cfg['input_data'].values(), short_name='tasa'))
if not input_data:
raise ValueError("This diagnostics needs 'tas' or 'tasa' variable")
# Calculate psi for every dataset
psis = {}
psi_attrs = {
'short_name': 'psi',
'long_name': 'Temperature variability metric',
'units': 'K',
}
grouped_data = group_metadata(input_data, 'dataset')
for (dataset, [data]) in grouped_data.items():
logger.info("Processing %s", dataset)
cube = iris.load_cube(data['filename'])
iris.coord_categorisation.add_year(cube, 'time')
cube = cube.aggregated_by('year', iris.analysis.MEAN)
psi_cube = calculate_psi(cube, cfg)
data.update(psi_attrs)
data.pop('standard_name', '')
# Provenance
caption = ("Temporal evolution of temperature variability metric psi "
"between {start_year} and {end_year} for {dataset}.".format(
**data))
provenance_record = get_provenance_record(caption, [data['filename']])
out_path = get_diagnostic_filename('psi_' + dataset, cfg)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(out_path, provenance_record)
# Save psi for every dataset
data['filename'] = out_path
io.metadata_to_netcdf(psi_cube, data)
# Save averaged psi
psis[dataset] = np.mean(psi_cube.data)
# Save averaged psis for every dataset in one file
out_path = get_diagnostic_filename('psi', cfg)
io.save_scalar_data(psis,
out_path,
psi_attrs,
attributes=psi_cube.attributes)
# Provenance
caption = "{long_name} for mutliple climate models.".format(**psi_attrs)
ancestor_files = [d['filename'] for d in input_data]
provenance_record = get_provenance_record(caption, ancestor_files)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(out_path, provenance_record)
def main(diag_config):
"""
Diagnostic function to compare the zonal turnover time.
Argument:
--------
diag_config - nested dictionary of metadata
"""
model_data_dict = group_metadata(diag_config['input_data'].values(),
'dataset')
fig_config = _get_fig_config(diag_config)
zonal_tau_mod = {}
for model_name, model_dataset in model_data_dict.items():
zonal_tau_mod[model_name] = {}
ctotal = _load_variable(model_dataset, 'ctotal')
gpp = _load_variable(model_dataset, 'gpp')
zonal_tau_mod[model_name] = _calc_zonal_tau(gpp, ctotal, fig_config)
zonal_tau_obs = _get_obs_data_zonal(diag_config)
obs_var = diag_config.get('obs_variable')[0]
tau_obs = zonal_tau_obs[obs_var]
base_name = '{title}_{source_label}_{grid_label}z'.format(
title=tau_obs.long_name,
source_label=diag_config['obs_info']['source_label'],
grid_label=diag_config['obs_info']['grid_label'])
provenance_record = _get_provenance_record(
"Comparison of latitudinal (zonal) variations of observation-based and"
" modelled ecosystem carbon turnover time. The zonal turnover time is"
" calculated as the ratio of zonal `ctotal` and `gpp`. Reproduces "
" figure 2a and 2b in Carvalhais et al. (2014).", ['mean', 'perc'],
['zonal'], _get_ancestor_files(diag_config, obs_var))
if diag_config['write_netcdf']:
model_cubes = [
c for c in zonal_tau_mod.values() if isinstance(c, iris.cube.Cube)
]
obs_cubes = [
c for c in zonal_tau_obs.values() if isinstance(c, iris.cube.Cube)
]
netcdf_path = get_diagnostic_filename(base_name, diag_config)
save_cubes = iris.cube.CubeList(model_cubes + obs_cubes)
iris.save(save_cubes, netcdf_path)
with ProvenanceLogger(diag_config) as provenance_logger:
provenance_logger.log(netcdf_path, provenance_record)
if diag_config['write_plots']:
plot_path = get_plot_filename(base_name, diag_config)
_plot_zonal_tau(plot_path, zonal_tau_mod, zonal_tau_obs, diag_config)
with ProvenanceLogger(diag_config) as provenance_logger:
provenance_logger.log(plot_path, provenance_record)
def _provenance_time_series_spei(cfg, data_dict):
"""Provenance for time series plots."""
caption = 'Time series of ' + \
data_dict['var'] + \
' at' + data_dict['area'] + '.'
if cfg['indexname'].lower == "spei":
set_refs = ['vicente10jclim', ]
elif cfg['indexname'].lower == "spi":
set_refs = ['mckee93proc', ]
else:
set_refs = ['martin18grl', ]
provenance_record = get_provenance_record([data_dict['filename']],
caption,
['reg'], set_refs,
plot_type='times')
diagnostic_file = get_diagnostic_filename(cfg['indexname'] +
'_time_series_' +
data_dict['area'] +
'_' +
data_dict['dataset_name'], cfg)
logger.info("Saving analysis results to %s", diagnostic_file)
cubesave = cube_to_save_ploted_ts(data_dict)
iris.save(cubesave, target=diagnostic_file)
logger.info("Recording provenance of %s:\n%s", diagnostic_file,
pformat(provenance_record))
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(diagnostic_file, provenance_record)
def main(cfg):
"""Process data for use as input to the HYPE hydrological model."""
input_data = cfg['input_data'].values()
grouped_input_data = group_metadata(input_data,
'long_name',
sort='dataset')
for long_name in grouped_input_data:
logger.info("Processing variable %s", long_name)
for attributes in grouped_input_data[long_name]:
logger.info("Processing dataset %s", attributes['dataset'])
output_file = get_diagnostic_filename(get_output_stem(attributes),
cfg, 'txt')
Path(output_file).parent.mkdir(exist_ok=True)
data, times, ids = get_data_times_and_ids(attributes)
frame = pandas.DataFrame(data, index=times, columns=ids)
frame.to_csv(output_file,
sep=' ',
index_label="DATE",
float_format='%.3f')
# Store provenance
provenance_record = get_provenance_record(attributes)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(output_file, provenance_record)
def select_final_subset(cfg, subsets, prov=None):
"""Select sample with minimal reuse of ensemble segments.
Final set of eight samples should have with minimal reuse
of the same ensemble member for the same period.
From 10.000 randomly selected sets of 8 samples, count
and penalize re-used segments (1 for 3*reuse, 5 for 4*reuse).
Choose the set with the lowest penalty.
"""
n_samples = cfg['n_samples']
all_scenarios = {}
for scenario, dataframes in subsets.items():
# Make a table with the final indices
LOGGER.info("Selecting %s final samples for scenario %s", n_samples,
scenario)
control = _best_subset(dataframes['control'].combination, n_samples)
future = _best_subset(dataframes['future'].combination, n_samples)
table = pd.concat([control, future],
axis=1,
keys=['control', 'future'])
all_scenarios[scenario] = table
# Store the output
filename = get_diagnostic_filename(f'indices_{scenario}',
cfg,
extension='csv')
table.to_csv(filename)
LOGGER.info("Selected recombinations for scenario %s: \n %s", scenario,
table)
LOGGER.info('Output stored as %s', filename)
# Write provenance information
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(filename, prov)
return all_scenarios
def log_provenance(caption: str, filename: str, cfg: dict, ancestors: list):
"""Log provenance info."""
provenance_record = get_provenance_record(caption, ancestors=ancestors)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(filename, provenance_record)
logger.info('Output stored as %s', filename)
def _write_albedochanges_to_disk(alb_lc, template_cube, datadict, cfg):
transition_cube = template_cube
# Remove attributes that are not applicable to derived data
for att in ['comment', 'modeling_realm', 'table_id']:
if att in transition_cube.attributes:
transition_cube.attributes.pop(att)
# Set correct unit
transition_cube.units = '1'
result_dict = {
'lc1': alb_lc[0, :, :],
'lc2': alb_lc[1, :, :],
'lc3': alb_lc[2, :, :]
}
names = {
'lc1': '-'.join(cfg['params']['lc1_class']),
'lc2': '-'.join(cfg['params']['lc2_class']),
'lc3': '-'.join(cfg['params']['lc3_class'])
}
for ikey, jkey in it.product(result_dict.keys(), result_dict.keys()):
if not ikey == jkey:
# Take out Frac for readability
transition_cube.data = result_dict[jkey] - result_dict[ikey]
transition_cube.rename("albedo_change_from_{0}_to_{1}".format(
names[ikey], names[jkey]).replace('Frac', ''))
logger.info("Calculated: %s", transition_cube.name())
# Get some usefull info for constructing the filenames
month_string = template_cube.coord('time').units.num2date(
template_cube.coord('time').points)[0].strftime('%b')
basename = '{0}-{1}-{2}'.format(month_string,
datadict['alb']['dataset'],
transition_cube.name())
transition_cube.attributes['plottitle'] = month_string + '-'\
+ datadict['alb']['dataset']
transition_cube.attributes['plotsuptitle'] = transition_cube.name()
savename_nc = os.path.join(cfg['work_dir'],
'{0}.nc'.format(basename))
logger.info("Saving file as: %s", savename_nc)
iris.save(transition_cube, savename_nc)
# Create provenance record
# Create caption
prov_rec = {
'caption':
'{0} {1}'.format(transition_cube.attributes['plottitle'],
transition_cube.attributes['plotsuptitle']),
'statistics': ['other'],
'domains': ['global'],
'plot_type':
'other',
'authors': [
'lejeune_quentin',
'crezee_bas',
],
'project': [
'crescendo',
],
}
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(os.path.join(cfg['work_dir'], basename),
prov_rec)
def calculate_tcr(cfg):
"""Calculate transient climate response (TCR)."""
tcr = {}
# Get anomaly cubes
(anomaly_cubes, ancestors) = _get_anomaly_cubes(cfg)
# Iterate over cubes and calculate TCR
for (dataset_name, anomaly_cube) in anomaly_cubes.items():
tas_2x = anomaly_cube[START_YEAR_IDX:END_YEAR_IDX].collapsed(
'time', iris.analysis.MEAN).data
new_tcr = tas_2x
tcr[dataset_name] = new_tcr
logger.info("TCR (%s) = %.2f %s", dataset_name, new_tcr,
anomaly_cube.units)
# Plot
(path, provenance_record) = _plot(cfg, anomaly_cube, dataset_name,
new_tcr)
if path is not None:
provenance_record['ancestors'] = ancestors[dataset_name]
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(path, provenance_record)
return tcr
def set_provenance(cfg):
"""Add provenance to all image files that the cvdp package creates."""
def _get_provenance_record(filename, ancestors):
return {
'caption': _get_caption(filename),
'statistics': [_get_stat(filename)],
'domain': 'global',
'plot_type': _get_plot_type(filename),
'plot_file': filename,
'authors': [
'phillips_adam',
],
'references': [
'acknow_project',
'phillips14eos',
],
'ancestors': ancestors,
}
ancestors = _get_global_ancestors(cfg)
logger.info("Path to work_dir: %s", cfg['work_dir'])
with ProvenanceLogger(cfg) as provenance_logger:
for root, _, files in os.walk(cfg['work_dir']):
for datei in files:
path = os.path.join(root, datei)
if _is_png(path):
logger.info("Name of file: %s", path)
provenance_record = _get_provenance_record(path, ancestors)
logger.info("Recording provenance of %s:\n%s", path,
provenance_record)
provenance_logger.log(path, provenance_record)
def main(cfg):
"""Run the diagnostic."""
sns.set(**cfg.get('seaborn_settings', {}))
ecs_file = io.get_ancestor_file(cfg, 'ecs.nc')
tcr_file = io.get_ancestor_file(cfg, 'tcr.nc')
ecs_cube = iris.load_cube(ecs_file)
tcr_cube = iris.load_cube(tcr_file)
# Project
if (ecs_cube.attributes.get('project', 'a') != tcr_cube.attributes.get(
'project', 'b')):
raise ValueError(
"ECS and TCR input files have either no 'project' attribute or "
"differ in it")
project = ecs_cube.attributes['project']
# Remove missing data and use equal coordinate
[ecs_cube, tcr_cube
] = iris_helpers.intersect_dataset_coordinates([ecs_cube, tcr_cube])
# Create plot
plot_path = plot_data(cfg, ecs_cube, tcr_cube)
# Write netcdf file
netcdf_path = write_data(cfg, ecs_cube, tcr_cube)
# Provenance
ancestor_files = [ecs_file, tcr_file]
provenance_record = get_provenance_record(project, ancestor_files)
provenance_record.update({
'plot_file': plot_path,
'plot_types': ['scatter'],
})
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, provenance_record)
def _provenance_map_spei(cfg, name_dict, spei, dataset_name):
"""Set provenance for plot_map_spei."""
caption = 'Global map of ' + \
name_dict['drought_char'] + \
' [' + name_dict['unit'] + '] ' + \
'based on ' + cfg['indexname'] + '.'
if cfg['indexname'].lower == "spei":
set_refs = ['martin18grl', 'vicente10jclim', ]
elif cfg['indexname'].lower == "spi":
set_refs = ['martin18grl', 'mckee93proc', ]
else:
set_refs = ['martin18grl', ]
provenance_record = get_provenance_record([name_dict['input_filenames']],
caption,
['global'],
set_refs)
diagnostic_file = get_diagnostic_filename(cfg['indexname'] + '_map' +
name_dict['add_to_filename'] +
'_' +
dataset_name, cfg)
logger.info("Saving analysis results to %s", diagnostic_file)
cubesave = cube_to_save_ploted(spei, name_dict)
iris.save(cubesave, target=diagnostic_file)
logger.info("Recording provenance of %s:\n%s", diagnostic_file,
pformat(provenance_record))
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(diagnostic_file, provenance_record)
def main(cfg):
"""Run the diagnostic."""
sns.set(**cfg.get('seaborn_settings', {}))
patterns = cfg.get('patterns')
if patterns is None:
input_files = io.get_all_ancestor_files(cfg)
else:
input_files = []
for pattern in patterns:
input_files.extend(io.get_all_ancestor_files(cfg, pattern=pattern))
if not input_files:
raise ValueError("No input files found")
logger.info("Found input files:\n%s", pformat(input_files))
# Iterate over all files and extract data
(all_data, all_files, metadata) = get_all_data(cfg, input_files)
# Create plot and netcdf file
plot_path = plot_data(cfg, all_data, metadata)
netcdf_path = write_data(cfg, all_data, metadata)
# Provenance
caption = f"{metadata['long_name']} for multiple datasets."
provenance_record = get_provenance_record(caption, all_files)
if plot_path is not None:
provenance_record.update({
'plot_file': plot_path,
'plot_types': ['bar'],
})
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, provenance_record)
def make_daily_var(cfg, input_data, short_name, getweights, metadata,
scale=1, offset=0):
"""Wrapper for dry_spell_rwr.utc_to_lt.make_daily_var() to derive some
args from the ESMValTool config.
"""
var_meta = select_metadata(input_data, short_name=short_name)[0]
logger.info(var_meta)
files_var = [var_meta["filename"], ]
var_name = var_meta["short_name"]
local_time = var_meta["local_time"]
model_grid, file_sftlf = _get_model_grid(input_data)
ut_var, ts_pad = _get_time_axis(files_var[0])
logger.info("ts_pad = %s", ts_pad)
file_out = _get_filename(var_meta, cfg)
utc.make_daily_var(files_var, var_name, local_time, getweights,
model_grid, metadata, ut_var, tsPad=ts_pad,
scale=scale, offset=offset,
file_out=file_out)
record_var = _get_provenance_record({}, files_var + [file_sftlf, ])
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(file_out, record_var)
return file_out
def main(cfg):
"""Run the diagnostic."""
cfg = get_default_settings(cfg)
diag = check_cfg(cfg)
sns.set(**cfg.get('seaborn_settings', {}))
# Get input data
input_data = list(cfg['input_data'].values())
input_data.extend(io.netcdf_to_metadata(cfg, pattern=cfg.get('pattern')))
input_data = deepcopy(input_data)
check_input_data(input_data)
grouped_data = group_metadata(input_data, 'dataset')
# Calculate X-axis of emergent constraint
diag_func = globals()[diag]
(diag_data, var_attrs, attrs) = diag_func(grouped_data, cfg)
attrs.update(get_global_attributes(input_data, cfg))
# Save data
netcdf_path = get_diagnostic_filename(diag, cfg)
io.save_scalar_data(diag_data, netcdf_path, var_attrs, attributes=attrs)
logger.info("Found data:\n%s", pformat(diag_data))
# Provenance
provenance_record = ec.get_provenance_record(
{diag: attrs}, [diag],
caption=attrs['plot_xlabel'],
ancestors=[d['filename'] for d in input_data])
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, provenance_record)
def write_data(cfg, cubes, var, prov_rec):
"""Write intermediate datafield for one variable.
Parameters
----------
cfg : dict
Configuration dictionary of the recipe.
cubes : dict
collection of iris data cubes.
var : str
variable short name
prov_rec : dict
contains information for provenance tracking.
"""
# Compile output path
filepath = os.path.join(cfg[diag.names.WORK_DIR],
'_'.join(['postproc', var]) + '.nc')
# Join cubes in one list with ref being the last entry
outcubes = cubes['exp'][var] + cubes['ref'][var]
if cfg[diag.names.WRITE_NETCDF]:
iris.save(outcubes, filepath)
logger.info("Writing %s", filepath)
# provenance tracking
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(filepath, prov_rec[var])
def write_cube(cfg, cube, data):
"""Write cube (check for MLR attributes and existing files first)."""
if not mlr.datasets_have_mlr_attributes([data], log_level='error'):
raise ValueError(
f"Cannot write cube {cube.summary(shorten=True)} using metadata "
f"{data}")
# Get new path
new_path = data['filename']
if os.path.exists(new_path):
now = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S%f")
data['filename'] = new_path.replace('.nc', f'_{now}.nc')
# Provenance
ancestors = [data.pop('original_filename')]
opts = [opt for opt in cfg if opt in globals()]
caption = (f"{cube.long_name} for {mlr.get_alias(data)} preprocessed with "
f"operations {opts}.")
record = {
'ancestors': ancestors,
'authors': ['schlund_manuel'],
'caption': caption,
'references': ['schlund20jgr'],
}
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(data['filename'], record)
# Write file
io.metadata_to_netcdf(cube, data)
def main(cfg):
"""Process data for use as input to the PCR-GLOBWB hydrological model."""
input_data = cfg['input_data'].values()
grouped_input_data = group_metadata(input_data,
'standard_name',
sort='dataset')
for standard_name in grouped_input_data:
logger.info("Processing variable %s", standard_name)
for attributes in grouped_input_data[standard_name]:
logger.info("Processing dataset %s", attributes['dataset'])
input_file = attributes['filename']
cube = iris.load_cube(input_file)
# Round times to integer number of days
time_coord = cube.coord('time')
time_coord.points = da.floor(time_coord.core_points())
time_coord.bounds = None
# Set lat from highest to lowest value
cube = cube[:, ::-1, ...]
# Save data
output_file = get_diagnostic_filename(
Path(input_file).stem + '_pcrglobwb', cfg)
iris.save(cube, output_file, fill_value=1.e20)
# Store provenance
provenance_record = get_provenance_record(input_file)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(output_file, provenance_record)
def save(self, egr, alias, data):
"""Save results and write provenance."""
script = self.cfg[names.SCRIPT]
info = data[alias][0]
keys = [
str(info[key]) for key in ('project', 'dataset', 'exp', 'ensemble',
'diagnostic', 'start_year', 'end_year')
if key in info
]
output_name = '_'.join(keys) + '.nc'
output_file = os.path.join(self.cfg[names.WORK_DIR], output_name)
iris.save(egr, output_file)
script_name = script.replace(" ", '_')
caption = (f"{script_name} between {info['start_year']} "
f"and {info['end_year']} according to {info['dataset']}")
ancestors = []
for i in range(len(data[alias])):
ancestors.append(data[alias][i]['filename'])
record = {
'caption': caption,
'domains': ['global'],
'authors': ['sanchez-gomez_emilia', 'moreno-chamarro_eduardo'],
'references': ['acknow_project'],
'ancestors': ancestors
}
with ProvenanceLogger(self.cfg) as provenance_logger:
provenance_logger.log(output_file, record)
def _write_xy_provenance(cfg, cubes, plot_path, title, *attrs):
"""Write provenance information for X-Y plots."""
cubes = cubes.copy()
if isinstance(cubes, iris.cube.Cube):
cubes = iris.cube.CubeList([cubes])
ancestors = []
for attr in attrs:
ancestors.extend(attr['filename'].split('|'))
netcdf_path = mlr.get_new_path(cfg, plot_path)
io.iris_save(cubes, netcdf_path)
long_name = ' and '.join([cube.long_name for cube in cubes])
caption = f"Line plot of {long_name}"
if title:
caption += f" for {title}."
else:
caption += '.'
record = {
'ancestors': ancestors,
'authors': ['schlund_manuel'],
'caption': caption,
'plot_file': plot_path,
'plot_types': ['line'],
'references': ['schlund20jgr'],
}
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, record)
def plot_cdf(cfg, psi_cube, ecs_cube, obs_cube):
"""Plot cumulative distribution function of ECS."""
confidence_level = cfg.get('confidence_level', 0.66)
(ecs_lin, ecs_pdf) = ec.gaussian_pdf(psi_cube.data, ecs_cube.data,
np.mean(obs_cube.data),
np.std(obs_cube.data))
ecs_cdf = ec.cdf(ecs_lin, ecs_pdf)
# Provenance
filename = 'cdf_{}'.format(obs_cube.attributes['dataset'])
netcdf_path = get_diagnostic_filename(filename, cfg)
cube = iris.cube.Cube(ecs_cdf,
var_name='cdf',
long_name='Cumulative distribution function',
units='1')
cube.add_aux_coord(
iris.coords.AuxCoord(ecs_lin, **ih.convert_to_iris(ECS_ATTRS)), 0)
io.iris_save(cube, netcdf_path)
project = _get_project(cfg)
provenance_record = get_provenance_record(
"The CDF for ECS. The horizontal dot-dashed lines show the {}% "
"confidence limits. The orange histograms show the prior "
"distributions that arise from equal weighting of the {} models in "
"0.5 K bins.".format(int(confidence_level * 100), project), ['mean'],
['other'], _get_ancestor_files(cfg, obs_cube.attributes['dataset']))
# Plot
if cfg['write_plots']:
AXES.plot(ecs_lin,
ecs_cdf,
color='black',
linewidth=2.0,
label='Emergent constraint')
AXES.hist(ecs_cube.data,
bins=6,
range=(2.0, 5.0),
cumulative=True,
density=True,
color='orange',
label='{} models'.format(project))
AXES.axhline((1.0 - confidence_level) / 2.0,
color='black',
linestyle='dashdot')
AXES.axhline((1.0 + confidence_level) / 2.0,
color='black',
linestyle='dashdot')
# Plot appearance
AXES.set_title('CDF of emergent constraint')
AXES.set_xlabel('ECS / K')
AXES.set_ylabel('CDF')
legend = AXES.legend(loc='upper left')
# Save plot
provenance_record['plot_file'] = _save_fig(cfg, filename, legend)
# Write provenance
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, provenance_record)
def main(cfg):
"""
Run the zonal-mean NAM diagnostic.
Calling in order:
- preprocessing
- index calculation
- regression and plot
"""
logger.setLevel(cfg['log_level'].upper())
input_files = cfg['input_data']
plot_dir = cfg['plot_dir']
out_dir = cfg['work_dir']
write_plots = cfg['write_plots']
fig_fmt = cfg['output_file_type']
filenames_cat = []
fileprops_cat = []
# Loop over input cfg
for key, value in input_files.items():
# Collect file names
filenames_cat.append(key)
# Collect relevant information for outputs naming
fileprops_cat.append([
value['project'], value['dataset'], value['exp'],
value['ensemble'],
str(value['start_year']) + '-' + str(value['end_year'])
])
# Go to work_dir for running
os.chdir(out_dir)
# Process list of input files
for indfile, ifile in enumerate(filenames_cat):
ifile_props = fileprops_cat[indfile]
# Call diagnostics functions
print("prepro")
(file_da_an_zm, file_mo_an) = zmnam_preproc(ifile)
print("calc")
outfiles = zmnam_calc(file_da_an_zm, out_dir + '/', ifile_props)
provenance_record = get_provenance_record(list(
input_files.values())[0],
ancestor_files=ifile)
if write_plots:
print("plot_files")
plot_files = zmnam_plot(file_mo_an, out_dir + '/', plot_dir + '/',
ifile_props, fig_fmt, write_plots)
else:
plot_files = []
for file in outfiles + plot_files:
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(file, provenance_record)
def su(grouped_data, cfg):
"""Su et al. (2014) constraint."""
metric = cfg['metric']
logger.info("Found metric '%s' for Su et al. (2014) constraint", metric)
# Extract cubes
(var_name, reference_datasets) = _get_su_variable(grouped_data)
cube_dict = _get_su_cube_dict(grouped_data, var_name, reference_datasets)
diag_data = {}
ref_cube = cube_dict[reference_datasets]
# Variable attributes
var_attrs = {
'short_name': 'alpha' if metric == 'regression_slope' else 'rho',
'long_name': f"Error in vertically-resolved tropospheric "
f"zonal-average {ref_cube.long_name} between 40°N and "
f"45°S expressed as {metric.replace('_', ' ')} between "
f"model data and observations",
'units': '1',
}
attrs = {
'plot_xlabel': f'Model performance in {ref_cube.long_name} [1]',
'plot_title': 'Su et al. (2014) constraint',
'provenance_authors': ['schlund_manuel'],
'provenance_domains': ['trop', 'midlat'],
'provenance_realms': ['atmos'],
'provenance_references': ['su14jgr'],
'provenance_statistics': ['corr'],
'provenance_themes': ['EC'],
}
# Calculate constraint
for (dataset_name, cube) in cube_dict.items():
logger.info("Processing dataset '%s'", dataset_name)
# Plot cube
if cube.ndim == 2:
iris.quickplot.contourf(cube)
filename = f"su_{dataset_name.replace('|', '_')}"
plot_path = get_plot_filename(filename, cfg)
plt.savefig(plot_path, **cfg['savefig_kwargs'])
logger.info("Wrote %s", plot_path)
plt.close()
# Provenance
netcdf_path = get_diagnostic_filename(filename, cfg)
io.iris_save(cube, netcdf_path)
ancestors = cube.attributes['ancestors'].split('|')
provenance_record = ec.get_provenance_record(
{'su': attrs}, ['su'],
caption=f'{cube.long_name} for {dataset_name}.',
plot_type='zonal', plot_file=plot_path, ancestors=ancestors)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, provenance_record)
# Similarity metric
diag_data[dataset_name] = _similarity_metric(cube, ref_cube, metric)
return (diag_data, var_attrs, attrs)
def plot_bar_deangelis(cfg, data_var_sum, available_exp, available_vars):
"""Plot linear regression used to calculate ECS."""
if not cfg[n.WRITE_PLOTS]:
return
# Plot data
fig, axx = plt.subplots()
set_colors = [
'cornflowerblue', 'orange', 'silver', 'limegreen', 'rosybrown',
'orchid'
]
bar_width = 1.0 / float(len(available_vars))
for iii, iexp in enumerate(available_exp):
axx.bar(np.arange(len(available_vars)) + bar_width * float(iii),
data_var_sum[iexp],
bar_width,
color=set_colors[iii],
label=iexp)
axx.set_xlabel(' ')
axx.set_ylabel(r'Model mean (W m$^{-2}$)')
axx.set_title(' ')
axx.set_xticks(np.arange(len(available_vars)) + bar_width)
axx.set_xticklabels(available_vars)
axx.legend(loc=1)
fig.tight_layout()
fig.savefig(get_plot_filename('bar_all', cfg), dpi=300)
plt.close()
caption = 'Global average multi-model mean comparing different ' + \
'model experiments and flux variables.'
provenance_record = get_provenance_record(
_get_sel_files_var(cfg, available_vars), caption, ['mean'], ['global'])
diagnostic_file = get_diagnostic_filename('bar_all', cfg)
logger.info("Saving analysis results to %s", diagnostic_file)
list_dict = {}
list_dict["data"] = []
list_dict["name"] = []
for iexp in available_exp:
list_dict["data"].append(data_var_sum[iexp])
list_dict["name"].append({
'var_name': iexp + '_all',
'long_name': 'Fluxes for ' + iexp + ' experiment',
'units': 'W m-2'
})
iris.save(cube_to_save_vars(list_dict), target=diagnostic_file)
logger.info("Recording provenance of %s:\n%s", diagnostic_file,
pformat(provenance_record))
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(diagnostic_file, provenance_record)
def _plot_single_map(plot_path, _dat, _datglobal, _name, provenance_record,
diag_config):
"""
Plot a map for a given variable.
Argument:
--------
diag_config - nested dictionary of metadata
cube - the cube to plot
dataset - name of the dataset to plot
"""
# figure configuration
fig_config = _get_fig_config(diag_config)
# colormap configuration
cb_info = _get_diagonal_colorbar_info()
# define the figure and axis
plt.figure(figsize=(5, 3))
_ax = plt.axes([0.1, 0.1, 0.9, 0.9],
projection=ccrs.Robinson(central_longitude=0),
frameon=False)
# plot data over the map
plt.imshow(_get_data_to_plot(_dat.data),
norm=mpl.colors.BoundaryNorm(cb_info['tickBounds'],
len(cb_info['tickBounds'])),
cmap=cb_info['colMap'],
origin='upper',
vmin=cb_info['tickBounds'][0],
vmax=cb_info['tickBounds'][-1],
transform=ccrs.PlateCarree())
_fix_map(_ax)
# get the data and set the title of the map
_dat_median = np.nanmedian(
_remove_invalid(_dat.data, fill_value=fig_config['fill_value']))
title_str = (f'{_dat.long_name} ({_dat.units}), {_name},\n'
f'global = {_datglobal:.2f}, median = {_dat_median:.2f}')
plt.title(title_str, fontsize=0.98 * fig_config['ax_fs'])
# draw the colorbar
_axcol_dia = [0.254, fig_config['y_colo_single'], 0.6, 0.035]
plut.mk_colo_tau(_axcol_dia,
cb_info['tickBounds'],
cb_info['colMap'],
tick_locs=cb_info['ticksLoc'],
cbfs=0.86 * fig_config['ax_fs'],
cbtitle='',
cbrt=90)
# save and close figure
t_x = plt.figtext(0.5, 0.5, ' ', transform=plt.gca().transAxes)
plut.save_figure(plot_path, _extr_art=[t_x])
plt.close()
with ProvenanceLogger(diag_config) as provenance_logger:
provenance_logger.log(plot_path, provenance_record)
def make_plot(metadata, scenarios, cfg, provenance):
"""Make figure 3, left graph.
Multimodel values as line, reference value in black square,
steering variables in dark dots.
"""
fig, axes = plt.subplots()
for member in select_metadata(metadata, variable_group='tas_cmip'):
filename = member['filename']
dataset = xr.open_dataset(filename)
if 'MultiModel' not in filename:
axes.plot(dataset.time.dt.year,
dataset.tas.values,
c='grey',
alpha=0.3,
lw=.5,
label='CMIP members')
else:
# Only display stats for the future period:
dataset = dataset.sel(time=slice('2010', None, None))
axes.plot(dataset.time.dt.year,
dataset.tas.values,
color='k',
linewidth=2,
label='CMIP ' + Path(filename).stem.split('_')[0][10:])
for member in select_metadata(metadata, variable_group='tas_target'):
filename = member['filename']
dataset = xr.open_dataset(filename)
if 'MultiModel' not in filename:
axes.plot(dataset.time.dt.year,
dataset.tas.values,
color='blue',
linewidth=1,
label=member['dataset'])
# Add the scenario's with dots at the cmip dt and bars for the periods
for i, scenario in enumerate(scenarios):
axes.scatter(scenario['year'],
scenario['cmip_dt'],
s=50,
zorder=10,
color='r',
label=r"Scenarios' steering $\Delta T_{CMIP}$")
_timeline(axes, i, scenario['period_bounds'])
handles, labels = plt.gca().get_legend_handles_labels()
by_label = dict(zip(labels, handles)) # dict removes dupes
axes.legend(by_label.values(), by_label.keys())
axes.set_xlabel('Year')
axes.set_ylabel(r'Global mean $\Delta T$ (K) w.r.t. reference period')
# Save figure
filename = get_plot_filename('global_matching', cfg)
fig.savefig(filename, bbox_inches='tight', dpi=300)
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(filename, provenance)
def save(output, cfg, provenance):
"""Save the output as csv file."""
scenarios = pd.DataFrame(output)
filename = get_diagnostic_filename('scenarios', cfg, extension='csv')
scenarios.to_csv(filename)
print(scenarios.round(2))
print(f"Output written to {filename}")
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(filename, provenance)
def main(cfg):
"""Run the diagnostic."""
sns.set(**cfg.get('seaborn_settings', {}))
# Read external file if desired
if cfg.get('read_external_file'):
(ecs, feedback_parameter, external_file) = read_external_file(cfg)
else:
check_input_data(cfg)
ecs = {}
feedback_parameter = {}
external_file = None
# Read and preprocess data
all_ancestors = []
(tas_data, rtnt_data) = preprocess_data(cfg)
# Iterate over all datasets and save ECS and feedback parameter
for dataset_name in tas_data:
logger.info("Processing '%s'", dataset_name)
if dataset_name not in rtnt_data:
raise ValueError(f"No 'rtnt' data for '{dataset_name}' available")
tas_cube = tas_data[dataset_name][0]['cube']
rtnt_cube = rtnt_data[dataset_name][0]['cube']
ancestor_files = (tas_data[dataset_name][0]['ancestors'] +
rtnt_data[dataset_name][0]['ancestors'])
# Perform linear regression
reg = stats.linregress(tas_cube.data, rtnt_cube.data)
# Plot ECS regression if desired
(path,
provenance_record) = plot_ecs_regression(cfg, dataset_name, tas_cube,
rtnt_cube, reg)
# Provenance
if path is not None:
provenance_record['ancestors'] = ancestor_files
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(path, provenance_record)
# Save data
if cfg.get('read_external_file') and dataset_name in ecs:
logger.warning(
"Overwriting externally given ECS and climate feedback "
"parameter from file '%s' for '%s'", external_file,
dataset_name)
ecs[dataset_name] = -reg.intercept / (2 * reg.slope)
feedback_parameter[dataset_name] = -reg.slope
all_ancestors.extend(ancestor_files)
# Write data
if external_file is not None:
all_ancestors.append(external_file)
all_ancestors = list(set(all_ancestors))
write_data(ecs, feedback_parameter, all_ancestors, cfg)
def write_provenance(cfg, netcdf_path, ancestors, caption):
"""Write provenance information."""
record = {
'ancestors': ancestors,
'authors': ['schlund_manuel'],
'caption': caption,
'references': ['schlund20jgr'],
}
with ProvenanceLogger(cfg) as provenance_logger:
provenance_logger.log(netcdf_path, record)
