def shape(self):
shape_realization = get_default_or_apply(self.realization,len,1)
shape_temporal = get_default_or_apply(self.temporal,len,1)
shape_level = get_default_or_apply(self.level,len,1)
shape_spatial = get_default_or_apply(self.spatial,lambda x: x.shape,(1,1))
ret = (shape_realization,shape_temporal,shape_level,shape_spatial[0],shape_spatial[1])
return(ret)
def shape(self):
"""
:returns: The shape of the field as a five-element tuple: (realization, time, level, row, column)
:rtype: tuple
"""
shape_realization = get_default_or_apply(self.realization, len, 1)
shape_temporal = get_default_or_apply(self.temporal, len, 1)
shape_level = get_default_or_apply(self.level, len, 1)
shape_spatial = get_default_or_apply(self.spatial, lambda x: x.shape, (1, 1))
ret = (shape_realization, shape_temporal, shape_level, shape_spatial[0], shape_spatial[1])
return ret
def __init__(self,
alias=None,
dtype=None,
field=None,
file_only=False,
vc=None,
parms=None,
tgd=None,
calc_sample_size=False,
fill_value=None,
meta_attrs=None,
tag=TagName.DATA_VARIABLES,
spatial_aggregation=False):
self._curr_variable = None
self._current_conformed_array = None
self._dtype = dtype
self.alias = alias or self.key
self.fill_value = fill_value
self.vc = vc or VariableCollection()
self.field = field
self.file_only = file_only
self.parms = get_default_or_apply(parms,
self._format_parms_,
default={})
self.tgd = tgd
self.calc_sample_size = calc_sample_size
self.meta_attrs = deepcopy(meta_attrs)
self.tag = tag
self.spatial_aggregation = spatial_aggregation
def __init__(
self,
alias=None,
dtype=None,
field=None,
file_only=False,
vc=None,
parms=None,
tgd=None,
use_raw_values=False,
calc_sample_size=False,
fill_value=None,
meta_attrs=None,
add_parents=False,
):
self._dtype = self.get_dtype(overload=dtype)
self.alias = alias or self.key
self.fill_value = fill_value
self.vc = vc or VariableCollection()
self.field = field
self.file_only = file_only
self.parms = get_default_or_apply(parms, self._format_parms_, default={})
self.tgd = tgd
self.use_raw_values = use_raw_values
self.calc_sample_size = calc_sample_size
self.meta_attrs = deepcopy(meta_attrs)
self.add_parents = add_parents
def __init__(self,alias=None,dtype=None,field=None,file_only=False,vc=None,
parms=None,tgd=None,use_raw_values=False,calc_sample_size=False,
fill_value=None):
self.alias = alias or self.key
self.dtype = dtype or self.dtype
self.fill_value = fill_value
self.vc = vc or VariableCollection()
self.field = field
self.file_only = file_only
self.parms = get_default_or_apply(parms,self._format_parms_,default={})
self.tgd = tgd
self.use_raw_values = use_raw_values
self.calc_sample_size = calc_sample_size
def __init__(self, alias=None, dtype=None, field=None, file_only=False, vc=None, parms=None, tgd=None,
calc_sample_size=False, fill_value=None, meta_attrs=None, tag=TagName.DATA_VARIABLES,
spatial_aggregation=False):
self._curr_variable = None
self._current_conformed_array = None
self._dtype = dtype
self.alias = alias or self.key
self.fill_value = fill_value
self.vc = vc or VariableCollection()
self.field = field
self.file_only = file_only
self.parms = get_default_or_apply(parms, self._format_parms_, default={})
self.tgd = tgd
self.calc_sample_size = calc_sample_size
self.meta_attrs = deepcopy(meta_attrs)
self.tag = tag
self.spatial_aggregation = spatial_aggregation
def _iter_collections_(self):
'''
:yields: :class:`~ocgis.SpatialCollection`
'''
## multivariate calculations require datasets come in as a list with all
## variable inputs part of the same sequence.
if self._has_multivariate_calculations:
itr_rd = [[r for r in self.ops.dataset.itervalues()]]
## otherwise, process geometries expects a single element sequence
else:
itr_rd = [[rd] for rd in self.ops.dataset.itervalues()]
## configure the progress object
self._progress.n_subsettables = len(itr_rd)
self._progress.n_geometries = get_default_or_apply(self.ops.geom,len,default=1)
self._progress.n_calculations = get_default_or_apply(self.ops.calc,len,default=0)
## send some messages
msg = '{0} dataset collection(s) to process.'.format(self._progress.n_subsettables)
ocgis_lh(msg=msg,logger=self._subset_log)
if self.ops.geom is None:
msg = 'Entire spatial domain returned. No selection geometries requested.'
else:
msg = 'Each data collection will be subsetted by {0} selection geometries.'.format(self._progress.n_geometries)
ocgis_lh(msg=msg,logger=self._subset_log)
if self._progress.n_calculations == 0:
msg = 'No calculations requested.'
else:
msg = 'The following calculations will be applied to each data collection: {0}.'.\
format(', '.join([_['func'] for _ in self.ops.calc]))
ocgis_lh(msg=msg,logger=self._subset_log)
## process the data collections
for rds in itr_rd:
msg = 'Processing URI(s): {0}'.format([rd.uri for rd in rds])
ocgis_lh(msg=msg,logger=self._subset_log)
for coll in self._process_subsettables_(rds):
## if there are calculations, do those now and return a new type of collection
if self.cengine is not None:
ocgis_lh('Starting calculations.',
self._subset_log,
alias=coll.items()[0][1].keys()[0],
ugid=coll.keys()[0])
## look for any optimizations for temporal grouping.
if self.ops.optimizations is None:
tgds = None
else:
tgds = self.ops.optimizations.get('tgds')
## execute the calculations
coll = self.cengine.execute(coll,file_only=self.ops.file_only,
tgds=tgds)
else:
## if there are no calculations, mark progress to indicate
## a geometry has been completed.
self._progress.mark()
## conversion of groups.
if self.ops.output_grouping is not None:
raise(NotImplementedError)
else:
ocgis_lh('subset yielding',self._subset_log,level=logging.DEBUG)
yield(coll)
def _iter_collections_(self):
""":rtype: :class:`ocgis.collection.base.AbstractCollection`"""
# Multivariate calculations require datasets come in as a list with all variable inputs part of the same
# sequence.
if self._has_multivariate_calculations:
itr_rd = [[rd for rd in self.ops.dataset]]
# Otherwise, process geometries expects a single element sequence.
else:
itr_rd = [[rd] for rd in self.ops.dataset]
# Configure the progress object.
self._progress.n_subsettables = len(itr_rd)
self._progress.n_geometries = get_default_or_apply(self.ops.geom,
len,
default=1)
self._progress.n_calculations = get_default_or_apply(self.ops.calc,
len,
default=0)
# Some introductory logging.
msg = '{0} dataset collection(s) to process.'.format(
self._progress.n_subsettables)
ocgis_lh(msg=msg, logger=self._subset_log)
if self.ops.geom is None:
msg = 'Entire spatial domain returned. No selection geometries requested.'
else:
msg = 'Each data collection will be subsetted by {0} selection geometries.'.format(
self._progress.n_geometries)
ocgis_lh(msg=msg, logger=self._subset_log)
if self._progress.n_calculations == 0:
msg = 'No calculations requested.'
else:
msg = 'The following calculations will be applied to each data collection: {0}.'. \
format(', '.join([_['func'] for _ in self.ops.calc]))
ocgis_lh(msg=msg, logger=self._subset_log)
# Process the incoming datasets. Convert from request datasets to fields as needed.
for rds in itr_rd:
try:
msg = 'Processing URI(s): {0}'.format([rd.uri for rd in rds])
except AttributeError:
# Field objects have no URIs. Multivariate calculations change how the request dataset iterator is
# configured as well.
msg = []
for rd in rds:
try:
msg.append(rd.uri)
except AttributeError:
# Likely a field object which does have a name.
msg.append(rd.name)
msg = 'Processing URI(s) / field names: {0}'.format(msg)
ocgis_lh(msg=msg, logger=self._subset_log)
for coll in self._process_subsettables_(rds):
# If there are calculations, do those now and return a collection.
if not vm.is_null and self.cengine is not None:
ocgis_lh('Starting calculations.', self._subset_log)
raise_if_empty(coll)
# Look for any temporal grouping optimizations.
if self.ops.optimizations is None:
tgds = None
else:
tgds = self.ops.optimizations.get('tgds')
# Execute the calculations.
coll = self.cengine.execute(coll,
file_only=self.ops.file_only,
tgds=tgds)
# If we need to spatially aggregate and calculations used raw values, update the collection
# fields and subset geometries.
if self.ops.aggregate and self.ops.calc_raw:
coll_to_itr = coll.copy()
for sfield, container in coll_to_itr.iter_fields(
yield_container=True):
sfield = _update_aggregation_wrapping_crs_(
self, None, sfield, container, None)
coll.add_field(sfield, container, force=True)
else:
# If there are no calculations, mark progress to indicate a geometry has been completed.
self._progress.mark()
# Conversion of groups.
if self.ops.output_grouping is not None:
raise NotImplementedError
else:
ocgis_lh('_iter_collections_ yielding',
self._subset_log,
level=logging.DEBUG)
yield coll
def _iter_collections_(self):
""":rtype: :class:`ocgis.collection.base.AbstractCollection`"""
# Multivariate calculations require datasets come in as a list with all variable inputs part of the same
# sequence.
if self._has_multivariate_calculations:
itr_rd = [[rd for rd in self.ops.dataset]]
# Otherwise, process geometries expects a single element sequence.
else:
itr_rd = [[rd] for rd in self.ops.dataset]
# Configure the progress object.
self._progress.n_subsettables = len(itr_rd)
self._progress.n_geometries = get_default_or_apply(self.ops.geom, len, default=1)
self._progress.n_calculations = get_default_or_apply(self.ops.calc, len, default=0)
# Some introductory logging.
msg = '{0} dataset collection(s) to process.'.format(self._progress.n_subsettables)
ocgis_lh(msg=msg, logger=self._subset_log)
if self.ops.geom is None:
msg = 'Entire spatial domain returned. No selection geometries requested.'
else:
msg = 'Each data collection will be subsetted by {0} selection geometries.'.format(
self._progress.n_geometries)
ocgis_lh(msg=msg, logger=self._subset_log)
if self._progress.n_calculations == 0:
msg = 'No calculations requested.'
else:
msg = 'The following calculations will be applied to each data collection: {0}.'. \
format(', '.join([_['func'] for _ in self.ops.calc]))
ocgis_lh(msg=msg, logger=self._subset_log)
# Process the incoming datasets. Convert from request datasets to fields as needed.
for rds in itr_rd:
try:
msg = 'Processing URI(s): {0}'.format([rd.uri for rd in rds])
except AttributeError:
# Field objects have no URIs. Multivariate calculations change how the request dataset iterator is
# configured as well.
msg = []
for rd in rds:
try:
msg.append(rd.uri)
except AttributeError:
# Likely a field object which does have a name.
msg.append(rd.name)
msg = 'Processing URI(s) / field names: {0}'.format(msg)
ocgis_lh(msg=msg, logger=self._subset_log)
for coll in self._process_subsettables_(rds):
# If there are calculations, do those now and return a collection.
if not vm.is_null and self.cengine is not None:
ocgis_lh('Starting calculations.', self._subset_log)
raise_if_empty(coll)
# Look for any temporal grouping optimizations.
if self.ops.optimizations is None:
tgds = None
else:
tgds = self.ops.optimizations.get('tgds')
# Execute the calculations.
coll = self.cengine.execute(coll, file_only=self.ops.file_only, tgds=tgds)
# If we need to spatially aggregate and calculations used raw values, update the collection
# fields and subset geometries.
if self.ops.aggregate and self.ops.calc_raw:
coll_to_itr = coll.copy()
for sfield, container in coll_to_itr.iter_fields(yield_container=True):
sfield = _update_aggregation_wrapping_crs_(self, None, sfield, container, None)
coll.add_field(sfield, container, force=True)
else:
# If there are no calculations, mark progress to indicate a geometry has been completed.
self._progress.mark()
# Conversion of groups.
if self.ops.output_grouping is not None:
raise NotImplementedError
else:
ocgis_lh('_iter_collections_ yielding', self._subset_log, level=logging.DEBUG)
yield coll