def _apply_operator_cubes(cube1, cube2, operator_name,
allow_coord_merge=True):
if not operator_name in CUBE_MATHS:
raise NotImplementedError('No such arithmetic cube operator '
'implemented: {}'.format(operator_name))
fun = CUBE_MATHS[operator_name]
try:
return fun(cube1, cube2)
except ValueError as e:
print_log.warning('Could not {} cubes straight out of the box. Trying '
'to correct for dimension definition errors'
.format(operator_name))
if 'differing coordinates (time)' in repr(e):
iris.util.unify_time_units([cube1, cube2])
attrs = {}
attrs.update(cube1.coord('time').attributes)
attrs.update(cube2.coord('time').attributes)
cube2.coord('time').attributes = attrs
cube1.coord('time').attributes = attrs
elif allow_coord_merge:
copy_coords_cube(to_cube=cube2,
from_cube=cube1,
inplace=True)
return fun(cube1, cube2)
def check_and_regrid_lons_cube(cube):
"""Checks and corrects for if longitudes of :attr:`grid` are 0 -> 360
Note
----
This method checks if the maximum of the current longitudes array
exceeds 180. Thus, it is not recommended to use this function after
subsetting a cube, rather, it should be checked directly when the
file is loaded (cf. :func:`load_input`)
Parameters
----------
cube : iris.cube.Cube
gridded data loaded as iris.Cube
Returns
-------
bool
True, if longitudes were on 0 -> 360 and have been rolled, else
False
"""
from pyaerocom import print_log
try:
if cube.coord("longitude").points.max() > 180:
logger.info("Rolling longitudes to -180 -> 180 definition")
cube = cube.intersection(longitude=(-180, 180))
except Exception as e:
print_log.warning('Failed to roll longitudes: {}'.format(repr(e)))
return cube
def _apply_gridded(self, data_obj):
"""Apply filter to instance of class :class:`GriddedData`
"""
print_log.warning(
'Applying regional cropping in GriddedData using Filter '
'class. Note that this does not yet include potential '
'cropping in the vertical dimension. Coming soon...')
return data_obj.crop(region=self._region)
def merge_meta_cubes(cube1, cube2):
try:
return merge_dicts(cube1.attributes, cube2.attributes)
except Exception as e:
print_log.warning('Failed to merge Cube metadata. Error: {}'.format(
repr(e)))
return {
'NOTE': 'MERGE_FAILED',
'meta1': cube1.attributes,
'meta2': cube2.attributes
}
def get_default_vert_code(self):
"""Get default vertical code for variable name"""
if self.default_vert_code is not None:
return self.default_vert_code
try:
return VarNameInfo(self.var_name_aerocom).get_default_vert_code()
except ValueError:
print_log.warning('default_vert_code not set for {} and '
'could also not be inferred'.format(
self.var_name_aerocom))
return None
def datasets_to_read(self, datasets):
if isinstance(datasets, str):
datasets = [datasets]
elif not isinstance(datasets, (tuple, list)):
raise IOError('Invalid input for parameter datasets_to_read')
avail = []
for ds in datasets:
try:
self.find_read_class(ds)
avail.append(ds)
except NetworkNotSupported:
print_log.warning(
'Removing {} from list of datasets to read '
'in ReadUngridded class. Reason: network '
'not supported or data is not available'.format(ds))
self._datasets_to_read = avail
def merge_meta_cubes(cube1, cube2):
try:
return merge_dicts(cube1.attributes,
cube2.attributes)
except Exception:
print_log.warning('WARNING: Failed to merge Cube metadata. Reason:\n{}'
.format(format_exc()))
ts_type = None
try:
if cube1.attributes['ts_type']==cube2.attributes['ts_type']:
ts_type = cube1.attributes['ts_type']
except:
pass
return {'NOTE' : 'METADATA_MERGE_FAILED',
'meta1' : cube1.attributes,
'meta2' : cube2.attributes,
'ts_type' : ts_type}
def plot_map(data, *args, **kwargs):
"""Map plot of grid data
Note
----
Deprecated name of method. Please use :func:`plot_griddeddata_on_map` in
the future.
Parameters
----------
data
data (2D numpy array or instance of GriddedData class. The latter is
deprecated, but will continue to work)
*args, **kwargs
See :func:`plot_griddeddata_on_map`
Returns
-------
See :func:`plot_griddeddata_on_map`
"""
from pyaerocom import print_log, GriddedData
print_log.warning(
DeprecationWarning('Method name plot_map is deprecated. '
'Please use plot_griddeddata_on_map'))
if isinstance(data, GriddedData):
if 'time' in data and len(data['time'].points) > 1:
logger.warning(
"Input data contains more than one time stamp, using "
"first time stamp")
data = data[0]
if not all([x in data for x in ('longitude', 'latitude')]):
raise AttributeError(
'GriddedData does not contain either longitude '
'or latitude coordinates')
return plot_griddeddata_on_map(data.grid.data, data.longitude.points,
data.latitude.points, *args, **kwargs)
return plot_griddeddata_on_map(data, *args, **kwargs)
def get_topo_data(lat0, lon0, lat1=None, lon1=None, topo_dataset='srtm',
topodata_loc=None, try_etopo1=False):
"""Retrieve topographic altitude for a certain location
Currently works only if :mod:`geonum` is installed. Supports topography
datasets supported by geonum. These are currently (20 Feb. 19) srtm
(SRTM dataset, default, automatic access if online) and etopo1
(ETOPO1 dataset, lower resolution, must be available on local machine or
server).
Parameters
----------
lat0 : float
start longitude for data extraction
lon0 : float
start latitude for data extraction
lat1 : float
stop longitude for data extraction (default: None). If None only
data around lon0, lat0 will be extracted.
lon1 : float
stop latitude for data extraction (default: None).
If None only data around lon0, lat0 will be extracted
topo_dataset : str
name of topography dataset
topodata_loc : str
filepath or directory containing supported topographic datasets
try_etopo1 : bool
if True and if access fails via input arg `topo_dataset`, then try
to access altitude using ETOPO1 dataset.
Returns
-------
geonum.TopoData
data object containing topography data in specified range
Raises
------
ValueError
if altitude data cannot be accessed
"""
if not GEONUM_AVAILABLE:
raise ModuleNotFoundError('Feature disabled: geonum library is not '
'installed')
import geonum
if topodata_loc is None:
from pyaerocom import const
if topo_dataset in const.SUPPLDIRS and os.path.exists(const.SUPPLDIRS[topo_dataset]):
topodata_loc = const.SUPPLDIRS[topo_dataset]
print_log.info('Found default location for {} topodata at\n{}'
.format(topo_dataset, topodata_loc))
try:
access = geonum.TopoDataAccess(topo_dataset, local_path=topodata_loc)
return access.get_data(lat0, lon0, lat1, lon1)
except Exception as e:
if try_etopo1 and not topo_dataset=='etopo1':
print_log.warning('Failed to access topography data for {}. '
'Trying ETOPO1.\nError: {}'.format(topo_dataset, repr(e)))
return get_topo_data(lat0, lon0, lat1, lon1,
topo_dataset='etopo1',
topodata_loc=topodata_loc,
try_etopo1=False)
raise
def read_dataset(self,
dataset_to_read,
vars_to_retrieve=None,
only_cached=False,
**kwargs):
"""Read dataset into an instance of :class:`ReadUngridded`
Parameters
----------
dataset_to_read : str
name of dataset
vars_to_retrieve : str or list
variable or list of variables to be imported
only_cached : bool
if True, then nothing is reloaded but only data is loaded that is
available as cached objects (not recommended to use but may be
used if working offline without connection to database)
**kwargs
additional reading constraints. If any are provided, caching is
deactivated and the data will be read from disk.
Returns
--------
UngriddedData
data object
"""
_caching = None
if len(kwargs) > 0:
_caching = const.CACHING
const.CACHING = False
print_log.info('Received additional reading constraints, '
'ignoring caching')
reader = self.get_reader(dataset_to_read)
if vars_to_retrieve is not None:
# Note: self.vars_to_retrieve may be None as well, then
# default variables of each network are read
self.vars_to_retrieve = vars_to_retrieve
if self.vars_to_retrieve is None:
self.vars_to_retrieve = reader.PROVIDES_VARIABLES
vars_to_retrieve = varlist_aerocom(self.vars_to_retrieve)
# data_dir will be None in most cases, but can be specified when
# creating the instance, by default, data_dir is inferred automatically
# in the reading class, using database location
data_dir = self._get_data_dir(dataset_to_read)
if data_dir is not None:
if not os.path.exists(data_dir):
raise FileNotFoundError(
'Trying to read {} from specified data_dir {} failed. '
'Directory does not exist'.format(dataset_to_read,
data_dir))
reader._dataset_path = data_dir
const.print_log.info(
'Reading {} from specified data loaction: {}'.format(
dataset_to_read, data_dir))
# Since this interface enables to load multiple datasets, each of
# which support a number of variables, here, only the variables are
# considered that are supported by the dataset
vars_available = [
var for var in vars_to_retrieve if reader.var_supported(var)
]
if len(vars_available) == 0:
raise DataRetrievalError('None of the input variables ({}) is '
'supported by {} interface'.format(
vars_to_retrieve, dataset_to_read))
cache = CacheHandlerUngridded(reader)
if not self.ignore_cache:
# initate cache handler
for var in vars_available:
try:
cache.check_and_load(var, force_use_outdated=only_cached)
except Exception:
self.logger.exception(
'Fatal: compatibility error between '
'old cache file {} and current version '
'of code ')
if not only_cached:
vars_to_read = [
v for v in vars_available if not v in cache.loaded_data
]
else:
vars_to_read = []
data_read = None
if len(vars_to_read) > 0:
_loglevel = print_log.level
print_log.setLevel(logging.INFO)
data_read = reader.read(vars_to_read, **kwargs)
print_log.setLevel(_loglevel)
for var in vars_to_read:
# write the cache file
if not self.ignore_cache:
try:
cache.write(data_read, var)
except Exception as e:
_caching = False
print_log.warning(
'Failed to write to cache directory. '
'Error: {}. Deactivating caching in '
'pyaerocom'.format(repr(e)))
if len(vars_to_read) == len(vars_available):
data_out = data_read
else:
data_out = UngriddedData()
for var in vars_available:
if var in cache.loaded_data:
data_out.append(cache.loaded_data[var])
if data_read is not None:
data_out.append(data_read)
if _caching is not None:
const.CACHING = _caching
return data_out
def _apply_colocated(self, data_obj):
print_log.warning(
'Applying regional cropping in ColocatedData using Filter '
'class. Note that this does not yet include potential '
'cropping in the vertical dimension. Coming soon...')
return data_obj.apply_latlon_filter(region_id=self.region_name)
def read_datasetOLD(self,
dataset_to_read,
vars_to_retrieve=None,
**kwargs):
"""Read single dataset into instance of :class:`ReadUngridded`
Note
----
This method does not write class attribute :attr:`data` (only
:func:`read` does)
Parameters
----------
dataset_to_read : str
name of dataset
vars_to_retrieve : list
list of variables to be retrieved. If None (default), the default
variables of each reading routine are imported
Returns
--------
UngriddedData
data object
"""
_caching = None
if len(kwargs) > 0:
_caching = const.CACHING
const.CACHING = False
print_log.info('Received additional reading constraints, '
'ignoring caching')
if vars_to_retrieve is None:
# Note: self.vars_to_retrieve may be None as well, then
# default variables of each network are read
vars_to_retrieve = self.vars_to_retrieve
reader = self.get_reader(dataset_to_read)
if vars_to_retrieve is None:
vars_to_retrieve = reader.PROVIDES_VARIABLES
elif isinstance(vars_to_retrieve, str):
vars_to_retrieve = [vars_to_retrieve]
# Since this interface enables to load multiple datasets, each of
# which support a number of variables, here, only the variables are
# considered that are supported by the dataset
vars_available = [
var for var in vars_to_retrieve if var in reader.PROVIDES_VARIABLES
]
# read the data sets
cache_hit_flag = False
if not self.ignore_cache:
# initate cache handler
try:
cache = CacheHandlerUngridded(reader, vars_available, **kwargs)
if cache.check_and_load():
all_avail = True
for var in vars_available:
if not var in cache.loaded_data:
all_avail = False
break
if all_avail:
print_log.info(
'Found Cache match for {}'.format(dataset_to_read))
cache_hit_flag = True
data = cache.loaded_data
except:
self.logger.exception(
'Fatal: compatibility error between old '
'cache file and current version of code ')
cache_hit_flag = False
if not cache_hit_flag:
print_log.info('No Cache match found for {} in {}. '
'Reading from files (this '
'may take a while)'.format(dataset_to_read,
const.CACHEDIR))
_loglevel = print_log.level
print_log.setLevel(logging.INFO)
data = reader.read(vars_available, **kwargs)
print_log.setLevel(_loglevel)
self.revision[dataset_to_read] = reader.data_revision
self.data_version[dataset_to_read] = reader.__version__
# write the cache file
if not cache_hit_flag and not self.ignore_cache:
try:
cache.write(data)
except Exception as e:
_caching = False
print_log.warning('Failed to write to cache directory:\n{}.\n'
'Deactivating caching in pyaerocom'.format(
repr(e)))
if _caching is not None:
const.CACHING = _caching
return data
def read_dataset(self, dataset_to_read, vars_to_retrieve=None, **kwargs):
"""Read dataset into an instance of :class:`ReadUngridded`
Note
----
This method does not assign loaded data obj to class attribute
:attr:`data` (only :func:`read` does)
Parameters
----------
dataset_to_read : str
name of dataset
vars_to_retrieve : list
list of variables to be retrieved. If None (default), the default
variables of each reading routine are imported
Returns
--------
UngriddedData
data object
"""
_caching = None
if len(kwargs) > 0:
_caching = const.CACHING
const.CACHING = False
print_log.info('Received additional reading constraints, '
'ignoring caching')
if vars_to_retrieve is None:
# Note: self.vars_to_retrieve may be None as well, then
# default variables of each network are read
vars_to_retrieve = self.vars_to_retrieve
reader = self.get_reader(dataset_to_read)
if vars_to_retrieve is None:
vars_to_retrieve = reader.PROVIDES_VARIABLES
elif isinstance(vars_to_retrieve, str):
vars_to_retrieve = [vars_to_retrieve]
# Since this interface enables to load multiple datasets, each of
# which support a number of variables, here, only the variables are
# considered that are supported by the dataset
vars_available = [
var for var in vars_to_retrieve if var in reader.PROVIDES_VARIABLES
]
cache = CacheHandlerUngridded(reader)
if not self.ignore_cache:
# initate cache handler
for var in vars_available:
try:
cache.check_and_load(var_name=var)
except:
self.logger.exception(
'Fatal: compatibility error between '
'old cache file {} and current version '
'of code ')
vars_to_read = [
v for v in vars_available if not v in cache.loaded_data
]
data_read = None
if len(vars_to_read) > 0:
_loglevel = print_log.level
print_log.setLevel(logging.INFO)
data_read = reader.read(vars_to_read, **kwargs)
print_log.setLevel(_loglevel)
for var in vars_to_read:
# write the cache file
if not self.ignore_cache:
try:
cache.write(data_read, var)
except Exception as e:
_caching = False
print_log.warning(
'Failed to write to cache directory. '
'Error: {}. Deactivating caching in '
'pyaerocom'.format(repr(e)))
if len(vars_to_read) == len(vars_available):
data_out = data_read
else:
data_out = UngriddedData()
for var in vars_available:
if var in cache.loaded_data:
data_out.append(cache.loaded_data[var])
if data_read is not None:
data_out.append(data_read)
if _caching is not None:
const.CACHING = _caching
return data_out