def _run_gridded_gridded(self): start, stop = self.start, self.stop model_reader = ReadGridded(self.model_id, start, stop) obs_reader = ReadGridded(self.obs_id, start, stop) vars_to_analyse = self.vars_to_analyse if vars_to_analyse is None: vars_to_analyse = model_reader.vars_provided var_matches = {} for var in vars_to_analyse: if var in model_reader.vars_provided: #candidate # first check if the variable pair was defined explicitely if var in self.alt_vars: if self.alt_vars[var] in obs_reader.vars_provided: var_matches[var] = self.alt_vars[var] else: if var in obs_reader.vars_provided: var_matches[var] = var if len(var_matches) == 0: raise DataCoverageError('No variable matches between {} and {} for ' 'input vars: {}'.format(self.model_id, self.obs_id, self.vars_to_analyse)) all_ts_types = const.GRID_IO.TS_TYPES ts_types_ana = self.ts_types_ana if ts_types_ana is None: ts_types_ana = self._setup.TS_TYPES_ANA_DEFAULT['gridded'] ts_types_read = self.ts_types_read if ts_types_read is None: ts_types_read = model_reader.ts_types vars_model = list(var_matches.keys()) vars_obs = list(var_matches.values()) flex_obs = self._setup.options.TS_TYPE_OBS_FLEX for ts_type_read in ts_types_read: # reads only year if starttime is provided but not stop time model_data_vars = model_reader.read(vars_model, start=start, stop=stop, ts_type=ts_type_read, flex_ts_type=False) if len(model_data_vars) == 0: if self._log: self._log.write('No model data available ({}-{}, {})\n' .format(start, stop, ts_type_read)) continue obs_data_vars = obs_reader.read(vars_obs, start=start, stop=stop, ts_type=ts_type_read, flex_ts_type=flex_obs) if len(obs_data_vars) == 0: if self._log: self._log.write('No obs data available for variables {} ' '({}-{}, {})\n' .format(vars_obs, start, stop, ts_type_read)) continue for model_data in model_data_vars: var = model_data.var_name obs_data = None for _obs in obs_data_vars: if _obs.var_name == var_matches[var]: obs_data = _obs break if obs_data is None: if self._log: self._log.write('No obs data available for model var {} ' '({}-{}, {})\n' .format(var, start, stop, ts_type_read)) continue for ts_type_ana in ts_types_ana: # model resolution (ts_type) must be equal or higher # than the current analysis setting (since ) if all_ts_types.index(ts_type_ana) >= all_ts_types.index(ts_type_read): out_dir = chk_make_subdir(self.output_dir('colocate'), self.model_id) savename = self._coldata_save_name(model_data, ts_type_ana, start, stop) file_exists = self._check_coldata_exists(self.model_id, savename) if file_exists: if not self.options.REANALYSE_EXISTING: if self._log: self._log.write('SKIP: {}\n'.format(savename)) print_log.info('Skip {} (file already ' 'exists)'.format(savename)) continue else: os.remove(os.path.join(out_dir, savename)) data_coll = colocate_gridded_gridded( model_data, obs_data, ts_type=ts_type_ana, start=start, stop=stop, filter_name=self.filter_name) self._last_coldata = data_coll if data_coll.save_name_aerocom + '.nc' != savename: raise Exception data_coll.to_netcdf(out_dir) if self._log: self._log.write('WRITE: {}\n'.format(savename)) print_log.info('Writing {}'.format(savename))
def _run_gridded_ungridded(self): """Analysis method for gridded vs. ungridded data""" start, stop = self.start, self.stop model_reader = ReadGridded(self.model_id, start, stop) obs_reader = ReadUngridded(self.obs_id) obs_vars = obs_reader.get_reader(self.obs_id).PROVIDES_VARIABLES vars_to_analyse = self.vars_to_analyse if vars_to_analyse is None: vars_to_analyse = model_reader.vars_provided var_matches = {} for var in vars_to_analyse: if var in model_reader.vars_provided: #candidate if var in self.alt_vars: if self.alt_vars[var] in obs_vars: var_matches[var] = self.alt_vars[var] else: if var in obs_vars: var_matches[var] = var if len(var_matches) == 0: raise DataCoverageError('No variable matches between ' '{} and {} for input vars: {}' .format(self.model_id, self.obs_id, self.vars_to_analyse)) all_ts_types = const.GRID_IO.TS_TYPES ts_types_ana = self.ts_types_ana if ts_types_ana is None: ts_types_ana = self._setup.TS_TYPES_ANA_DEFAULT['ungridded'] ts_types_read = self.ts_types_read if ts_types_read is None: ts_types_read = model_reader.ts_types vars_model = list(var_matches.keys()) vars_obs = list(var_matches.values()) obs_data = obs_reader.read(datasets_to_read=self.obs_id, vars_to_retrieve=vars_obs) for ts_type_read in ts_types_read: model_data_vars = model_reader.read(vars_model, start=start, stop=stop, ts_type=ts_type_read, flex_ts_type=False) if len(model_data_vars)==0: if self._log: self._log.write('No model data available ({}-{}, {})\n' .format(start, stop, ts_type_read)) continue for model_data in model_data_vars: var = model_data.var_info.var_name obs_var = var_matches[var] if not obs_var in obs_reader.data: if self._log: self._log.write('No obs data available for variable {} ' '({}-{}, {})\n' .format(obs_var, start, stop, ts_type_read)) continue for ts_type_ana in ts_types_ana: if all_ts_types.index(ts_type_ana) >= all_ts_types.index(ts_type_read): out_dir = chk_make_subdir(self.output_dir('colocate'), self.model_id) savename = self._coldata_save_name(model_data, ts_type_ana, start, stop) file_exists = self._check_coldata_exists( self.model_id, savename) if file_exists: if not self.options.REANALYSE_EXISTING: if self._log: self._log.write('SKIP: {}\n' .format(savename)) print_log.info('Skip {} (file already ' 'exists)'.format(savename)) continue else: os.remove(os.path.join(out_dir, savename)) data_coll = colocate_gridded_ungridded_2D( model_data, obs_data, ts_type=ts_type_ana, start=start, stop=stop, var_ref=obs_var, filter_name=self.filter_name) self._last_coldata = data_coll data_coll.to_netcdf(out_dir) if self._log: self._log.write('WRITE: {}\n'.format(savename)) print_log.info('Writing {}'.format(savename)) plt.close('all')