def test_colocator__find_var_matches_model_add_vars(col):
r = ReadGridded('TM5-met2010_CTRL-TEST')
model_var = 'abs550aer'
obs_var = 'od550aer'
col.model_add_vars = {obs_var: model_var}
var_matches = col._find_var_matches(obs_var, r)
assert var_matches == {model_var: obs_var, obs_var: obs_var}
def test_colocator__check_add_model_read_aux():
coloc = Colocator(raise_exceptions=True)
r = ReadGridded('TM5-met2010_CTRL-TEST')
assert not coloc._check_add_model_read_aux('od550aer', r)
coloc.model_read_aux = {
'od550aer': dict(vars_required=['od550aer', 'od550aer'], fun=add_cubes)
}
assert coloc._check_add_model_read_aux('od550aer', r)
def test_model_add_vars(col_tm5_aero):
model_var = 'abs550aer'
obs_var = 'od550aer'
col_tm5_aero.model_add_vars = {obs_var: model_var}
model_reader = ReadGridded(col_tm5_aero.model_id)
var_matches = col_tm5_aero._check_model_add_var(obs_var, model_reader, {})
assert var_matches == {model_var: obs_var}
var_matches = col_tm5_aero._find_var_matches([obs_var], model_reader)
assert len(var_matches) == 2
assert (model_var in var_matches)
assert (obs_var in var_matches)
def test_colocator__find_var_matches(col):
r = ReadGridded('TM5-met2010_CTRL-TEST')
with pytest.raises(DataCoverageError):
col._find_var_matches('invalid', r)
var_matches = col._find_var_matches('od550aer', r)
assert var_matches == {'od550aer': 'od550aer'}
obs_vars = 'conco3'
col.obs_vars = [obs_vars]
col.model_use_vars = {obs_vars: 'od550aer'}
var_matches = col._find_var_matches('conco3', r)
assert var_matches == {'od550aer': 'conco3'}
def create_varinfo_table(model_ids,
vars_or_var_patterns,
read_data=False,
sort_by_cols=['Var', 'Model']):
"""Create an info table for model list based on variables
The method iterates over all models in :attr:`model_list` and creates an
instance of :class:`ReadGridded`. Variable matches are searched based on
input list :attr:`vars_or_var_patterns` (you may also use wildcards to
specify a family of variables) and for each match the information below
is collected. The search also includes variables that are not directly
available in the model data but can be computed from other available
variables. That is, all variables that are defined in
:attr:`ReadGridded.AUX_REQUIRES`.
The output table (DataFrame) then consists of the following columns:
- Var: variable name
- Model: model name
- Years: available years
- Freq: frequency
- Vertical: information about vertical dimension (inferred from \
Aerocom file name)
- At stations: data is at stations (inferred from filename)
- AUX vars: Auxiliary variable required to compute Var (col 1). Only \
relevant for variables that are computed by the interface
- Dim: number of dimensions (only retrieved if *read_data* is True)
- Dim names: names of dimension coordinates (only retrieved if \
*read_data* is True)
- Shape: Shape of data (only retrieved if *read_data* is True)
- Read ok: reading was successful (only retrieved if *read_data* \
is True)
Parameters
----------
model_ids : list
list of model ids to be analysed (can also be string -> single model)
vars_or_var_patterns : list
list of variables or variable patterns to be analysed (can also be
string -> single variable or variable family)
read_data : bool
if True, more information about the imported data will be available
in the table (e.g. no. of dimensions, names of dimension coords)
but the routine will run longer since the data is imported
sort_by_cols : list
column sort order (use header names in listing above). Defaults
to `['Var', 'Model']`
Returns
-------
pandas.DataFrame
dataframe including result table (ready to be saved as csv or other
tabular format or to be displayed in a jupyter notebook)
Example
-------
>>> from pyaerocom import create_varinfo_table
>>> models = ['INCA-BCext_CTRL2016-PD',
'GEOS5-freegcm_CTRL2016-PD']
>>> vars = ['ang4487aer', 'od550aer', 'ec*']
>>> df = create_varinfo_table(models, vars)
>>> print(df)
"""
if isinstance(model_ids, str):
model_ids = [model_ids]
if isinstance(vars_or_var_patterns, str):
vars_or_var_patterns = [vars_or_var_patterns]
failed = []
header = [
'Var', 'Model', 'Years', 'Freq', 'Vertical', 'At stations', 'AUX vars',
'Dim', 'Dim names', 'Shape', 'Read ok'
]
result = []
table_cols = ['year', 'ts_type', 'vert_code', 'is_at_stations', 'aux_vars']
for i, model in enumerate(model_ids):
print('At model: {} ({} of {})'.format(model, i, len(model_ids)))
try:
reader = ReadGridded(model)
var_info = reader.get_var_info_from_files()
for var_avail, info in var_info.items():
for var in vars_or_var_patterns:
if fnmatch.fnmatch(var_avail, var):
for freq in info['ts_type']:
sub_res = [var_avail, model]
for key in table_cols:
if key in info:
sub_res.append(info[key])
else:
sub_res.append(None)
try:
if not read_data:
raise Exception
data = reader.read_var(var_avail,
ts_type=freq,
flex_ts_type=False)
dim_names = [
d.name() for d in data.grid.dim_coords
]
sub_res.extend(
[data.ndim, dim_names, data.shape, True])
except Exception:
sub_res.extend([None, None, None, False])
result.append(sub_res)
except IOError as e:
dummy = [None] * len(header)
dummy[1] = model
result.append(dummy)
print(repr(e))
failed.append(model)
df = pd.DataFrame(result, columns=header)
#df.set_index(['Var'], inplace=True)
if sort_by_cols:
df.sort_values(sort_by_cols, inplace=True)
return df
def _run_gridded_gridded(self, var_name=None):
start, stop = start_stop(self.start, self.stop)
model_reader = ReadGridded(self.model_id)
obs_reader = ReadGridded(self.obs_id)
if 'obs_filters' in self:
remaining_filters = self._eval_obs_filters()
if bool(remaining_filters):
raise NotImplementedError(
'Cannot apply filters {} to gridded '
'observation data.'.format(remaining_filters))
obs_vars = self.obs_vars
obs_vars_avail = obs_reader.vars_provided
for obs_var in obs_vars:
if not obs_var in obs_vars_avail:
raise DataCoverageError(
'Variable {} is not supported by {}'.format(
obs_var, self.obs_id))
var_matches = self._find_var_matches(obs_vars, model_reader, var_name)
if self.remove_outliers:
self._update_var_outlier_ranges(var_matches)
all_ts_types = const.GRID_IO.TS_TYPES
ts_type = self.ts_type
data_objs = {}
for model_var, obs_var in var_matches.items():
print_log.info('Running {} / {} ({}, {})'.format(
self.model_id, self.obs_id, model_var, obs_var))
try:
model_data = self._read_gridded(reader=model_reader,
var_name=model_var,
start=start,
stop=stop,
is_model=True)
except Exception as e:
msg = (
'Failed to load gridded data: {} / {}. Reason {}'.format(
self.model_id, model_var, repr(e)))
const.print_log.warning(msg)
self._write_log(msg + '\n')
if self.raise_exceptions:
self._close_log()
raise Exception(msg)
else:
continue
if not model_data.ts_type in all_ts_types:
raise TemporalResolutionError('Invalid temporal resolution {} '
'in model {}'.format(
model_data.ts_type,
self.model_id))
try:
obs_data = self._read_gridded(reader=obs_reader,
var_name=obs_var,
start=start,
stop=stop,
is_model=False)
except Exception as e:
msg = (
'Failed to load gridded data: {} / {}. Reason {}'.format(
self.model_id, model_var, repr(e)))
const.print_log.warning(msg)
self._write_log(msg + '\n')
if self.raise_exceptions:
self._close_log()
raise Exception(msg)
else:
continue
if not obs_data.ts_type in all_ts_types:
raise TemporalResolutionError('Invalid temporal resolution {} '
'in obs {}'.format(
obs_data.ts_type,
self.model_id))
# update colocation ts_type, based on the available resolution in
# model and obs.
lowest = self.get_lowest_resolution(ts_type, model_data.ts_type,
obs_data.ts_type)
if lowest != ts_type:
print_log.info('Updating ts_type from {} to {} (highest '
'available in {} / {} combination)'.format(
ts_type, lowest, self.model_id,
self.obs_id))
ts_type = lowest
if self.save_coldata:
out_dir = chk_make_subdir(self.basedir_coldata, self.model_id)
savename = self._coldata_savename(model_data,
start,
stop,
ts_type,
var_name=model_var)
file_exists = self._check_coldata_exists(
self.model_id, savename)
if file_exists:
if not self.reanalyse_existing:
if self._log:
self._write_log('SKIP: {}\n'.format(savename))
print_log.info('Skip {} (file already '
'exists)'.format(savename))
continue
else:
os.remove(os.path.join(out_dir, savename))
try:
by = None
if self.model_use_climatology:
by = to_pandas_timestamp(start).year
coldata = colocate_gridded_gridded(
gridded_data=model_data,
gridded_data_ref=obs_data,
ts_type=ts_type,
start=start, stop=stop,
filter_name=self.filter_name,
regrid_res_deg=self.regrid_res_deg,
remove_outliers=self.remove_outliers,
vert_scheme=self.vert_scheme,
harmonise_units=self.harmonise_units,
var_outlier_ranges=self.var_outlier_ranges,
var_ref_outlier_ranges=self.var_ref_outlier_ranges,
update_baseyear_gridded=by,
apply_time_resampling_constraints=\
self.apply_time_resampling_constraints,
min_num_obs=self.min_num_obs,
colocate_time=self.colocate_time,
var_keep_outliers=self.model_keep_outliers,
var_ref_keep_outliers=self.obs_keep_outliers)
if self.save_coldata:
self._save_coldata(coldata, savename, out_dir, model_var,
model_data, obs_var)
#coldata.to_netcdf(out_dir, savename=savename)
if self._log:
self._write_log('WRITE: {}\n'.format(savename))
print_log.info('Writing file {}'.format(savename))
data_objs[model_var] = coldata
except Exception as e:
msg = ('Colocation between model {} / {} and obs {} / {} '
'failed: Reason {}'.format(self.model_id, model_var,
self.obs_id, obs_var,
repr(e)))
const.print_log.warning(msg)
self._write_log(msg)
if self.raise_exceptions:
self._close_log()
raise Exception(msg)
return data_objs
def _run_gridded_ungridded(self, var_name=None):
"""Analysis method for gridded vs. ungridded data"""
model_reader = ReadGridded(self.model_id)
obs_reader = ReadUngridded(self.obs_id)
obs_vars_supported = obs_reader.get_reader(
self.obs_id).PROVIDES_VARIABLES
obs_vars = list(np.intersect1d(self.obs_vars, obs_vars_supported))
if len(obs_vars) == 0:
raise DataCoverageError(
'No observation variable matches found for '
'{}'.format(self.obs_id))
var_matches = self._find_var_matches(obs_vars, model_reader, var_name)
if self.read_opts_ungridded is not None:
ropts = self.read_opts_ungridded
else:
ropts = {}
obs_data = obs_reader.read(datasets_to_read=self.obs_id,
vars_to_retrieve=obs_vars,
**ropts)
if 'obs_filters' in self:
remaining_filters = self._eval_obs_filters()
obs_data = obs_data.apply_filters(**remaining_filters)
if self.remove_outliers:
self._update_var_outlier_ranges(var_matches)
#all_ts_types = const.GRID_IO.TS_TYPES
data_objs = {}
for model_var, obs_var in var_matches.items():
ts_type = self.ts_type
start, stop = start_stop(self.start, self.stop)
print_log.info('Running {} / {} ({}, {})'.format(
self.model_id, self.obs_id, model_var, obs_var))
try:
model_data = self._read_gridded(reader=model_reader,
var_name=model_var,
start=start,
stop=stop,
is_model=True)
except Exception as e:
msg = (
'Failed to load gridded data: {} / {}. Reason {}'.format(
self.model_id, model_var, repr(e)))
const.print_log.warning(msg)
self._write_log(msg + '\n')
if self.raise_exceptions:
self._close_log()
raise Exception(msg)
else:
continue
ts_type_src = model_data.ts_type
# =============================================================================
# if not model_data.ts_type in all_ts_types:
# raise TemporalResolutionError('Invalid temporal resolution {} '
# 'in model {}'.format(model_data.ts_type,
# self.model_id))
# =============================================================================
ignore_stats = None
if self.ignore_station_names is not None:
ignore_stats = self.ignore_station_names
if isinstance(ignore_stats, dict):
if obs_var in ignore_stats:
ignore_stats = ignore_stats[obs_var]
else:
ignore_stats = None
#ts_type_src = model_data.ts_type
if TsType(ts_type_src) < TsType(
ts_type): # < all_ts_types.index(ts_type_src):
print_log.info('Updating ts_type from {} to {} (highest '
'available in model {})'.format(
ts_type, ts_type_src, self.model_id))
ts_type = ts_type_src
if self.save_coldata:
savename = self._coldata_savename(model_data,
start,
stop,
ts_type,
var_name=model_var)
file_exists = self._check_coldata_exists(
model_data.data_id, savename)
out_dir = chk_make_subdir(self.basedir_coldata, self.model_id)
if file_exists:
if not self.reanalyse_existing:
if self._log:
self._write_log('SKIP: {}\n'.format(savename))
print_log.info('Skip {} (file already '
'exists)'.format(savename))
self.file_status[savename] = 'skipped'
continue
else:
print_log.info(
'Deleting and recomputing existing '
'colocated data file {}'.format(savename))
print_log.info('REMOVE: {}\n'.format(savename))
os.remove(os.path.join(out_dir, savename))
try:
by = None
if self.model_use_climatology:
by = start.year
coldata = colocate_gridded_ungridded(
gridded_data=model_data,
ungridded_data=obs_data,
ts_type=ts_type,
start=start,
stop=stop,
var_ref=obs_var,
filter_name=self.filter_name,
regrid_res_deg=self.regrid_res_deg,
remove_outliers=self.remove_outliers,
vert_scheme=self.vert_scheme,
harmonise_units=self.harmonise_units,
var_outlier_ranges=self.var_outlier_ranges,
var_ref_outlier_ranges=self.var_ref_outlier_ranges,
update_baseyear_gridded=by,
ignore_station_names=ignore_stats,
apply_time_resampling_constraints=self.
apply_time_resampling_constraints,
min_num_obs=self.min_num_obs,
colocate_time=self.colocate_time,
var_keep_outliers=self.model_keep_outliers,
var_ref_keep_outliers=self.obs_keep_outliers)
if self.save_coldata:
self._save_coldata(coldata, savename, out_dir, model_var,
model_data, obs_var)
data_objs[model_var] = coldata
except Exception as e:
msg = ('Colocation between model {} / {} and obs {} / {} '
'failed: Reason {}'.format(self.model_id, model_var,
self.obs_id, obs_var,
repr(e)))
const.print_log.warning(msg)
self._write_log(msg + '\n')
if self.raise_exceptions:
self._close_log()
raise Exception(msg)
return data_objs
def _load_cams_rean():
from pyaerocom.io import ReadGridded
r = ReadGridded(data_id="ECMWF_CAMS_REAN")
return r.read_var('od550aer', ts_type='daily', start=2010, stop=2013)
def _run_gridded_gridded(self):
start, stop = self.start, self.stop
model_reader = ReadGridded(self.model_id, start, stop)
obs_reader = ReadGridded(self.obs_id, start, stop)
vars_to_analyse = self.vars_to_analyse
if vars_to_analyse is None:
vars_to_analyse = model_reader.vars_provided
var_matches = {}
for var in vars_to_analyse:
if var in model_reader.vars_provided: #candidate
# first check if the variable pair was defined explicitely
if var in self.alt_vars:
if self.alt_vars[var] in obs_reader.vars_provided:
var_matches[var] = self.alt_vars[var]
else:
if var in obs_reader.vars_provided:
var_matches[var] = var
if len(var_matches) == 0:
raise DataCoverageError('No variable matches between {} and {} for '
'input vars: {}'.format(self.model_id,
self.obs_id,
self.vars_to_analyse))
all_ts_types = const.GRID_IO.TS_TYPES
ts_types_ana = self.ts_types_ana
if ts_types_ana is None:
ts_types_ana = self._setup.TS_TYPES_ANA_DEFAULT['gridded']
ts_types_read = self.ts_types_read
if ts_types_read is None:
ts_types_read = model_reader.ts_types
vars_model = list(var_matches.keys())
vars_obs = list(var_matches.values())
flex_obs = self._setup.options.TS_TYPE_OBS_FLEX
for ts_type_read in ts_types_read:
# reads only year if starttime is provided but not stop time
model_data_vars = model_reader.read(vars_model,
start=start,
stop=stop,
ts_type=ts_type_read,
flex_ts_type=False)
if len(model_data_vars) == 0:
if self._log:
self._log.write('No model data available ({}-{}, {})\n'
.format(start, stop, ts_type_read))
continue
obs_data_vars = obs_reader.read(vars_obs,
start=start,
stop=stop,
ts_type=ts_type_read,
flex_ts_type=flex_obs)
if len(obs_data_vars) == 0:
if self._log:
self._log.write('No obs data available for variables {} '
'({}-{}, {})\n'
.format(vars_obs, start, stop,
ts_type_read))
continue
for model_data in model_data_vars:
var = model_data.var_name
obs_data = None
for _obs in obs_data_vars:
if _obs.var_name == var_matches[var]:
obs_data = _obs
break
if obs_data is None:
if self._log:
self._log.write('No obs data available for model var {} '
'({}-{}, {})\n'
.format(var, start, stop,
ts_type_read))
continue
for ts_type_ana in ts_types_ana:
# model resolution (ts_type) must be equal or higher
# than the current analysis setting (since )
if all_ts_types.index(ts_type_ana) >= all_ts_types.index(ts_type_read):
out_dir = chk_make_subdir(self.output_dir('colocate'),
self.model_id)
savename = self._coldata_save_name(model_data,
ts_type_ana,
start,
stop)
file_exists = self._check_coldata_exists(self.model_id,
savename)
if file_exists:
if not self.options.REANALYSE_EXISTING:
if self._log:
self._log.write('SKIP: {}\n'.format(savename))
print_log.info('Skip {} (file already '
'exists)'.format(savename))
continue
else:
os.remove(os.path.join(out_dir, savename))
data_coll = colocate_gridded_gridded(
model_data, obs_data,
ts_type=ts_type_ana,
start=start, stop=stop,
filter_name=self.filter_name)
self._last_coldata = data_coll
if data_coll.save_name_aerocom + '.nc' != savename:
raise Exception
data_coll.to_netcdf(out_dir)
if self._log:
self._log.write('WRITE: {}\n'.format(savename))
print_log.info('Writing {}'.format(savename))
def _run_gridded_ungridded(self):
"""Analysis method for gridded vs. ungridded data"""
start, stop = self.start, self.stop
model_reader = ReadGridded(self.model_id, start, stop)
obs_reader = ReadUngridded(self.obs_id)
obs_vars = obs_reader.get_reader(self.obs_id).PROVIDES_VARIABLES
vars_to_analyse = self.vars_to_analyse
if vars_to_analyse is None:
vars_to_analyse = model_reader.vars_provided
var_matches = {}
for var in vars_to_analyse:
if var in model_reader.vars_provided: #candidate
if var in self.alt_vars:
if self.alt_vars[var] in obs_vars:
var_matches[var] = self.alt_vars[var]
else:
if var in obs_vars:
var_matches[var] = var
if len(var_matches) == 0:
raise DataCoverageError('No variable matches between '
'{} and {} for input vars: {}'
.format(self.model_id,
self.obs_id,
self.vars_to_analyse))
all_ts_types = const.GRID_IO.TS_TYPES
ts_types_ana = self.ts_types_ana
if ts_types_ana is None:
ts_types_ana = self._setup.TS_TYPES_ANA_DEFAULT['ungridded']
ts_types_read = self.ts_types_read
if ts_types_read is None:
ts_types_read = model_reader.ts_types
vars_model = list(var_matches.keys())
vars_obs = list(var_matches.values())
obs_data = obs_reader.read(datasets_to_read=self.obs_id,
vars_to_retrieve=vars_obs)
for ts_type_read in ts_types_read:
model_data_vars = model_reader.read(vars_model,
start=start,
stop=stop,
ts_type=ts_type_read,
flex_ts_type=False)
if len(model_data_vars)==0:
if self._log:
self._log.write('No model data available ({}-{}, {})\n'
.format(start, stop, ts_type_read))
continue
for model_data in model_data_vars:
var = model_data.var_info.var_name
obs_var = var_matches[var]
if not obs_var in obs_reader.data:
if self._log:
self._log.write('No obs data available for variable {} '
'({}-{}, {})\n'
.format(obs_var, start, stop,
ts_type_read))
continue
for ts_type_ana in ts_types_ana:
if all_ts_types.index(ts_type_ana) >= all_ts_types.index(ts_type_read):
out_dir = chk_make_subdir(self.output_dir('colocate'),
self.model_id)
savename = self._coldata_save_name(model_data,
ts_type_ana,
start,
stop)
file_exists = self._check_coldata_exists(
self.model_id,
savename)
if file_exists:
if not self.options.REANALYSE_EXISTING:
if self._log:
self._log.write('SKIP: {}\n'
.format(savename))
print_log.info('Skip {} (file already '
'exists)'.format(savename))
continue
else:
os.remove(os.path.join(out_dir, savename))
data_coll = colocate_gridded_ungridded_2D(
model_data, obs_data,
ts_type=ts_type_ana,
start=start, stop=stop,
var_ref=obs_var,
filter_name=self.filter_name)
self._last_coldata = data_coll
data_coll.to_netcdf(out_dir)
if self._log:
self._log.write('WRITE: {}\n'.format(savename))
print_log.info('Writing {}'.format(savename))
plt.close('all')
def test_colocator__infer_start_stop():
col = Colocator()
reader = ReadGridded('TM5-met2010_CTRL-TEST')
col._infer_start_stop(reader)
assert col.start == 2010
assert col.stop == None