def _apply_operator_cubes(cube1, cube2, operator_name, allow_coord_merge=True): if not operator_name in CUBE_MATHS: raise NotImplementedError('No such arithmetic cube operator ' 'implemented: {}'.format(operator_name)) fun = CUBE_MATHS[operator_name] try: return fun(cube1, cube2) except ValueError as e: print_log.warning('Could not {} cubes straight out of the box. Trying ' 'to correct for dimension definition errors' .format(operator_name)) if 'differing coordinates (time)' in repr(e): iris.util.unify_time_units([cube1, cube2]) attrs = {} attrs.update(cube1.coord('time').attributes) attrs.update(cube2.coord('time').attributes) cube2.coord('time').attributes = attrs cube1.coord('time').attributes = attrs elif allow_coord_merge: copy_coords_cube(to_cube=cube2, from_cube=cube1, inplace=True) return fun(cube1, cube2)
def check_and_regrid_lons_cube(cube): """Checks and corrects for if longitudes of :attr:`grid` are 0 -> 360 Note ---- This method checks if the maximum of the current longitudes array exceeds 180. Thus, it is not recommended to use this function after subsetting a cube, rather, it should be checked directly when the file is loaded (cf. :func:`load_input`) Parameters ---------- cube : iris.cube.Cube gridded data loaded as iris.Cube Returns ------- bool True, if longitudes were on 0 -> 360 and have been rolled, else False """ from pyaerocom import print_log try: if cube.coord("longitude").points.max() > 180: logger.info("Rolling longitudes to -180 -> 180 definition") cube = cube.intersection(longitude=(-180, 180)) except Exception as e: print_log.warning('Failed to roll longitudes: {}'.format(repr(e))) return cube
def _apply_gridded(self, data_obj): """Apply filter to instance of class :class:`GriddedData` """ print_log.warning( 'Applying regional cropping in GriddedData using Filter ' 'class. Note that this does not yet include potential ' 'cropping in the vertical dimension. Coming soon...') return data_obj.crop(region=self._region)
def merge_meta_cubes(cube1, cube2): try: return merge_dicts(cube1.attributes, cube2.attributes) except Exception as e: print_log.warning('Failed to merge Cube metadata. Error: {}'.format( repr(e))) return { 'NOTE': 'MERGE_FAILED', 'meta1': cube1.attributes, 'meta2': cube2.attributes }
def get_default_vert_code(self): """Get default vertical code for variable name""" if self.default_vert_code is not None: return self.default_vert_code try: return VarNameInfo(self.var_name_aerocom).get_default_vert_code() except ValueError: print_log.warning('default_vert_code not set for {} and ' 'could also not be inferred'.format( self.var_name_aerocom)) return None
def datasets_to_read(self, datasets): if isinstance(datasets, str): datasets = [datasets] elif not isinstance(datasets, (tuple, list)): raise IOError('Invalid input for parameter datasets_to_read') avail = [] for ds in datasets: try: self.find_read_class(ds) avail.append(ds) except NetworkNotSupported: print_log.warning( 'Removing {} from list of datasets to read ' 'in ReadUngridded class. Reason: network ' 'not supported or data is not available'.format(ds)) self._datasets_to_read = avail
def merge_meta_cubes(cube1, cube2): try: return merge_dicts(cube1.attributes, cube2.attributes) except Exception: print_log.warning('WARNING: Failed to merge Cube metadata. Reason:\n{}' .format(format_exc())) ts_type = None try: if cube1.attributes['ts_type']==cube2.attributes['ts_type']: ts_type = cube1.attributes['ts_type'] except: pass return {'NOTE' : 'METADATA_MERGE_FAILED', 'meta1' : cube1.attributes, 'meta2' : cube2.attributes, 'ts_type' : ts_type}
def plot_map(data, *args, **kwargs): """Map plot of grid data Note ---- Deprecated name of method. Please use :func:`plot_griddeddata_on_map` in the future. Parameters ---------- data data (2D numpy array or instance of GriddedData class. The latter is deprecated, but will continue to work) *args, **kwargs See :func:`plot_griddeddata_on_map` Returns ------- See :func:`plot_griddeddata_on_map` """ from pyaerocom import print_log, GriddedData print_log.warning( DeprecationWarning('Method name plot_map is deprecated. ' 'Please use plot_griddeddata_on_map')) if isinstance(data, GriddedData): if 'time' in data and len(data['time'].points) > 1: logger.warning( "Input data contains more than one time stamp, using " "first time stamp") data = data[0] if not all([x in data for x in ('longitude', 'latitude')]): raise AttributeError( 'GriddedData does not contain either longitude ' 'or latitude coordinates') return plot_griddeddata_on_map(data.grid.data, data.longitude.points, data.latitude.points, *args, **kwargs) return plot_griddeddata_on_map(data, *args, **kwargs)
def get_topo_data(lat0, lon0, lat1=None, lon1=None, topo_dataset='srtm', topodata_loc=None, try_etopo1=False): """Retrieve topographic altitude for a certain location Currently works only if :mod:`geonum` is installed. Supports topography datasets supported by geonum. These are currently (20 Feb. 19) srtm (SRTM dataset, default, automatic access if online) and etopo1 (ETOPO1 dataset, lower resolution, must be available on local machine or server). Parameters ---------- lat0 : float start longitude for data extraction lon0 : float start latitude for data extraction lat1 : float stop longitude for data extraction (default: None). If None only data around lon0, lat0 will be extracted. lon1 : float stop latitude for data extraction (default: None). If None only data around lon0, lat0 will be extracted topo_dataset : str name of topography dataset topodata_loc : str filepath or directory containing supported topographic datasets try_etopo1 : bool if True and if access fails via input arg `topo_dataset`, then try to access altitude using ETOPO1 dataset. Returns ------- geonum.TopoData data object containing topography data in specified range Raises ------ ValueError if altitude data cannot be accessed """ if not GEONUM_AVAILABLE: raise ModuleNotFoundError('Feature disabled: geonum library is not ' 'installed') import geonum if topodata_loc is None: from pyaerocom import const if topo_dataset in const.SUPPLDIRS and os.path.exists(const.SUPPLDIRS[topo_dataset]): topodata_loc = const.SUPPLDIRS[topo_dataset] print_log.info('Found default location for {} topodata at\n{}' .format(topo_dataset, topodata_loc)) try: access = geonum.TopoDataAccess(topo_dataset, local_path=topodata_loc) return access.get_data(lat0, lon0, lat1, lon1) except Exception as e: if try_etopo1 and not topo_dataset=='etopo1': print_log.warning('Failed to access topography data for {}. ' 'Trying ETOPO1.\nError: {}'.format(topo_dataset, repr(e))) return get_topo_data(lat0, lon0, lat1, lon1, topo_dataset='etopo1', topodata_loc=topodata_loc, try_etopo1=False) raise
def read_dataset(self, dataset_to_read, vars_to_retrieve=None, only_cached=False, **kwargs): """Read dataset into an instance of :class:`ReadUngridded` Parameters ---------- dataset_to_read : str name of dataset vars_to_retrieve : str or list variable or list of variables to be imported only_cached : bool if True, then nothing is reloaded but only data is loaded that is available as cached objects (not recommended to use but may be used if working offline without connection to database) **kwargs additional reading constraints. If any are provided, caching is deactivated and the data will be read from disk. Returns -------- UngriddedData data object """ _caching = None if len(kwargs) > 0: _caching = const.CACHING const.CACHING = False print_log.info('Received additional reading constraints, ' 'ignoring caching') reader = self.get_reader(dataset_to_read) if vars_to_retrieve is not None: # Note: self.vars_to_retrieve may be None as well, then # default variables of each network are read self.vars_to_retrieve = vars_to_retrieve if self.vars_to_retrieve is None: self.vars_to_retrieve = reader.PROVIDES_VARIABLES vars_to_retrieve = varlist_aerocom(self.vars_to_retrieve) # data_dir will be None in most cases, but can be specified when # creating the instance, by default, data_dir is inferred automatically # in the reading class, using database location data_dir = self._get_data_dir(dataset_to_read) if data_dir is not None: if not os.path.exists(data_dir): raise FileNotFoundError( 'Trying to read {} from specified data_dir {} failed. ' 'Directory does not exist'.format(dataset_to_read, data_dir)) reader._dataset_path = data_dir const.print_log.info( 'Reading {} from specified data loaction: {}'.format( dataset_to_read, data_dir)) # Since this interface enables to load multiple datasets, each of # which support a number of variables, here, only the variables are # considered that are supported by the dataset vars_available = [ var for var in vars_to_retrieve if reader.var_supported(var) ] if len(vars_available) == 0: raise DataRetrievalError('None of the input variables ({}) is ' 'supported by {} interface'.format( vars_to_retrieve, dataset_to_read)) cache = CacheHandlerUngridded(reader) if not self.ignore_cache: # initate cache handler for var in vars_available: try: cache.check_and_load(var, force_use_outdated=only_cached) except Exception: self.logger.exception( 'Fatal: compatibility error between ' 'old cache file {} and current version ' 'of code ') if not only_cached: vars_to_read = [ v for v in vars_available if not v in cache.loaded_data ] else: vars_to_read = [] data_read = None if len(vars_to_read) > 0: _loglevel = print_log.level print_log.setLevel(logging.INFO) data_read = reader.read(vars_to_read, **kwargs) print_log.setLevel(_loglevel) for var in vars_to_read: # write the cache file if not self.ignore_cache: try: cache.write(data_read, var) except Exception as e: _caching = False print_log.warning( 'Failed to write to cache directory. ' 'Error: {}. Deactivating caching in ' 'pyaerocom'.format(repr(e))) if len(vars_to_read) == len(vars_available): data_out = data_read else: data_out = UngriddedData() for var in vars_available: if var in cache.loaded_data: data_out.append(cache.loaded_data[var]) if data_read is not None: data_out.append(data_read) if _caching is not None: const.CACHING = _caching return data_out
def _apply_colocated(self, data_obj): print_log.warning( 'Applying regional cropping in ColocatedData using Filter ' 'class. Note that this does not yet include potential ' 'cropping in the vertical dimension. Coming soon...') return data_obj.apply_latlon_filter(region_id=self.region_name)
def read_datasetOLD(self, dataset_to_read, vars_to_retrieve=None, **kwargs): """Read single dataset into instance of :class:`ReadUngridded` Note ---- This method does not write class attribute :attr:`data` (only :func:`read` does) Parameters ---------- dataset_to_read : str name of dataset vars_to_retrieve : list list of variables to be retrieved. If None (default), the default variables of each reading routine are imported Returns -------- UngriddedData data object """ _caching = None if len(kwargs) > 0: _caching = const.CACHING const.CACHING = False print_log.info('Received additional reading constraints, ' 'ignoring caching') if vars_to_retrieve is None: # Note: self.vars_to_retrieve may be None as well, then # default variables of each network are read vars_to_retrieve = self.vars_to_retrieve reader = self.get_reader(dataset_to_read) if vars_to_retrieve is None: vars_to_retrieve = reader.PROVIDES_VARIABLES elif isinstance(vars_to_retrieve, str): vars_to_retrieve = [vars_to_retrieve] # Since this interface enables to load multiple datasets, each of # which support a number of variables, here, only the variables are # considered that are supported by the dataset vars_available = [ var for var in vars_to_retrieve if var in reader.PROVIDES_VARIABLES ] # read the data sets cache_hit_flag = False if not self.ignore_cache: # initate cache handler try: cache = CacheHandlerUngridded(reader, vars_available, **kwargs) if cache.check_and_load(): all_avail = True for var in vars_available: if not var in cache.loaded_data: all_avail = False break if all_avail: print_log.info( 'Found Cache match for {}'.format(dataset_to_read)) cache_hit_flag = True data = cache.loaded_data except: self.logger.exception( 'Fatal: compatibility error between old ' 'cache file and current version of code ') cache_hit_flag = False if not cache_hit_flag: print_log.info('No Cache match found for {} in {}. ' 'Reading from files (this ' 'may take a while)'.format(dataset_to_read, const.CACHEDIR)) _loglevel = print_log.level print_log.setLevel(logging.INFO) data = reader.read(vars_available, **kwargs) print_log.setLevel(_loglevel) self.revision[dataset_to_read] = reader.data_revision self.data_version[dataset_to_read] = reader.__version__ # write the cache file if not cache_hit_flag and not self.ignore_cache: try: cache.write(data) except Exception as e: _caching = False print_log.warning('Failed to write to cache directory:\n{}.\n' 'Deactivating caching in pyaerocom'.format( repr(e))) if _caching is not None: const.CACHING = _caching return data
def read_dataset(self, dataset_to_read, vars_to_retrieve=None, **kwargs): """Read dataset into an instance of :class:`ReadUngridded` Note ---- This method does not assign loaded data obj to class attribute :attr:`data` (only :func:`read` does) Parameters ---------- dataset_to_read : str name of dataset vars_to_retrieve : list list of variables to be retrieved. If None (default), the default variables of each reading routine are imported Returns -------- UngriddedData data object """ _caching = None if len(kwargs) > 0: _caching = const.CACHING const.CACHING = False print_log.info('Received additional reading constraints, ' 'ignoring caching') if vars_to_retrieve is None: # Note: self.vars_to_retrieve may be None as well, then # default variables of each network are read vars_to_retrieve = self.vars_to_retrieve reader = self.get_reader(dataset_to_read) if vars_to_retrieve is None: vars_to_retrieve = reader.PROVIDES_VARIABLES elif isinstance(vars_to_retrieve, str): vars_to_retrieve = [vars_to_retrieve] # Since this interface enables to load multiple datasets, each of # which support a number of variables, here, only the variables are # considered that are supported by the dataset vars_available = [ var for var in vars_to_retrieve if var in reader.PROVIDES_VARIABLES ] cache = CacheHandlerUngridded(reader) if not self.ignore_cache: # initate cache handler for var in vars_available: try: cache.check_and_load(var_name=var) except: self.logger.exception( 'Fatal: compatibility error between ' 'old cache file {} and current version ' 'of code ') vars_to_read = [ v for v in vars_available if not v in cache.loaded_data ] data_read = None if len(vars_to_read) > 0: _loglevel = print_log.level print_log.setLevel(logging.INFO) data_read = reader.read(vars_to_read, **kwargs) print_log.setLevel(_loglevel) for var in vars_to_read: # write the cache file if not self.ignore_cache: try: cache.write(data_read, var) except Exception as e: _caching = False print_log.warning( 'Failed to write to cache directory. ' 'Error: {}. Deactivating caching in ' 'pyaerocom'.format(repr(e))) if len(vars_to_read) == len(vars_available): data_out = data_read else: data_out = UngriddedData() for var in vars_available: if var in cache.loaded_data: data_out.append(cache.loaded_data[var]) if data_read is not None: data_out.append(data_read) if _caching is not None: const.CACHING = _caching return data_out