def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'Siemens_per_meter'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'Pascal'
pdict.add_context(pres_ctxt)
if parameter_dict_name == 'input_param_for_L0':
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
else:
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_kelvin'
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = 'g/m'
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = 'PSU'
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == 'input_param_for_L0':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
elif pc[1].name == 'temp':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _create_input_param_dict_for_test(self, parameter_dict_name=""):
pdict = ParameterDictionary()
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=numpy.dtype("float64")))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = "seconds since 01-01-1900"
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext("conductivity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
cond_ctxt.uom = ""
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext("pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
pres_ctxt.uom = ""
pdict.add_context(pres_ctxt)
if parameter_dict_name == "input_param_dict":
temp_ctxt = ParameterContext("temperature", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
else:
temp_ctxt = ParameterContext("temp", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
temp_ctxt.uom = ""
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext("density", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
dens_ctxt.uom = ""
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext("salinity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
sal_ctxt.uom = ""
pdict.add_context(sal_ctxt)
# create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == "input_param_dict":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
elif parameter_dict_name == "output_param_dict" and pc[1].name == "temp":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
class SimplexCoverage(AbstractCoverage):
"""
A concrete implementation of AbstractCoverage consisting of 2 domains (temporal and spatial)
and a collection of parameters associated with one or both of the domains. Each parameter is defined by a
ParameterContext object (provided via the ParameterDictionary) and has content represented by a concrete implementation
of the AbstractParameterValue class.
"""
def __init__(self, root_dir, persistence_guid, name=None, parameter_dictionary=None, temporal_domain=None, spatial_domain=None, mode=None, in_memory_storage=False, bricking_scheme=None, inline_data_writes=True, auto_flush_values=True):
"""
Constructor for SimplexCoverage
@param root_dir The root directory for storage of this coverage
@param persistence_guid The persistence uuid for this coverage
@param name The name of the coverage
@param parameter_dictionary a ParameterDictionary object expected to contain one or more valid ParameterContext objects
@param spatial_domain a concrete instance of AbstractDomain for the spatial domain component
@param temporal_domain a concrete instance of AbstractDomain for the temporal domain component
@param mode the file mode for the coverage; one of 'r', 'a', 'r+', or 'w'; defaults to 'r'
@param in_memory_storage if False (default), HDF5 persistence is used; otherwise, nothing is written to disk and all data is held in memory only
@param bricking_scheme the bricking scheme for the coverage; a dict of the form {'brick_size': #, 'chunk_size': #}
@param inline_data_writes if True (default), brick data is written as it is set; otherwise it is written out-of-band by worker processes or threads
@param auto_flush_values if True (default), brick data is flushed immediately; otherwise it is buffered until SimplexCoverage.flush_values() is called
"""
AbstractCoverage.__init__(self, mode=mode)
try:
# Make sure root_dir and persistence_guid are both not None and are strings
if not isinstance(root_dir, str) or not isinstance(persistence_guid, str):
raise TypeError('\'root_dir\' and \'persistence_guid\' must be instances of str')
root_dir = root_dir if not root_dir.endswith(persistence_guid) else os.path.split(root_dir)[0]
pth=os.path.join(root_dir, persistence_guid)
def _doload(self):
# Make sure the coverage directory exists
if not os.path.exists(pth):
raise SystemError('Cannot find specified coverage: {0}'.format(pth))
# All appears well - load it up!
self._persistence_layer = PersistenceLayer(root_dir, persistence_guid, mode=self.mode)
self.name = self._persistence_layer.name
self.spatial_domain = self._persistence_layer.sdom
self.temporal_domain = self._persistence_layer.tdom
self._range_dictionary = ParameterDictionary()
self._range_value = RangeValues()
self._bricking_scheme = self._persistence_layer.global_bricking_scheme
self._in_memory_storage = False
auto_flush_values = self._persistence_layer.auto_flush_values
inline_data_writes = self._persistence_layer.inline_data_writes
from coverage_model.persistence import PersistedStorage
for parameter_name in self._persistence_layer.parameter_metadata.keys():
md = self._persistence_layer.parameter_metadata[parameter_name]
pc = md.parameter_context
self._range_dictionary.add_context(pc)
s = PersistedStorage(md, self._persistence_layer.brick_dispatcher, dtype=pc.param_type.storage_encoding, fill_value=pc.param_type.fill_value, mode=self.mode, inline_data_writes=inline_data_writes, auto_flush=auto_flush_values)
self._range_value[parameter_name] = get_value_class(param_type=pc.param_type, domain_set=pc.dom, storage=s)
if name is None or parameter_dictionary is None:
# This appears to be a load
_doload(self)
else:
# This appears to be a new coverage
# Make sure name and parameter_dictionary are not None
if name is None or parameter_dictionary is None:
raise SystemError('\'name\' and \'parameter_dictionary\' cannot be None')
# Make sure the specified root_dir exists
if not in_memory_storage and not os.path.exists(root_dir):
raise SystemError('Cannot find specified \'root_dir\': {0}'.format(root_dir))
# If the coverage directory exists, load it instead!!
if os.path.exists(pth):
log.warn('The specified coverage already exists - performing load of \'{0}\''.format(pth))
_doload(self)
return
# We've checked everything we can - this is a new coverage!!!
# Check the mode - must be in 'a' for a new coverage
if self.mode != 'a':
self.mode = 'a'
self.name = name
if temporal_domain is None:
self.temporal_domain = GridDomain(GridShape('temporal',[0]), CRS.standard_temporal(), MutabilityEnum.EXTENSIBLE)
elif isinstance(temporal_domain, AbstractDomain):
self.temporal_domain = deepcopy(temporal_domain)
else:
raise TypeError('\'temporal_domain\' must be an instance of AbstractDomain')
if spatial_domain is None or isinstance(spatial_domain, AbstractDomain):
self.spatial_domain = deepcopy(spatial_domain)
else:
raise TypeError('\'spatial_domain\' must be an instance of AbstractDomain')
if not isinstance(parameter_dictionary, ParameterDictionary):
raise TypeError('\'parameter_dictionary\' must be of type ParameterDictionary')
self._range_dictionary = ParameterDictionary()
self._range_value = RangeValues()
self._bricking_scheme = bricking_scheme or {'brick_size':10000,'chunk_size':500}
self._in_memory_storage = in_memory_storage
if self._in_memory_storage:
self._persistence_layer = InMemoryPersistenceLayer()
else:
self._persistence_layer = PersistenceLayer(root_dir, persistence_guid, name=name, tdom=temporal_domain, sdom=spatial_domain, mode=self.mode, bricking_scheme=self._bricking_scheme, inline_data_writes=inline_data_writes, auto_flush_values=auto_flush_values)
for o, pc in parameter_dictionary.itervalues():
self._append_parameter(pc)
except:
self._closed = True
raise
@classmethod
def _fromdict(cls, cmdict, arg_masks=None):
return super(SimplexCoverage, cls)._fromdict(cmdict, {'parameter_dictionary':'_range_dictionary'})
@property
def temporal_parameter_name(self):
return self._range_dictionary.temporal_parameter_name
@property
def parameter_dictionary(self):
return deepcopy(self._range_dictionary)
@property
def persistence_guid(self):
if isinstance(self._persistence_layer, InMemoryPersistenceLayer):
return None
else:
return self._persistence_layer.guid
@property
def persistence_dir(self):
if isinstance(self._persistence_layer, InMemoryPersistenceLayer):
return None
else:
return self._persistence_layer.master_manager.root_dir
def append_parameter(self, parameter_context):
"""
Append a ParameterContext to the coverage
@deprecated use a ParameterDictionary during construction of the coverage
"""
log.warn('SimplexCoverage.append_parameter() is deprecated: use a ParameterDictionary during construction of the coverage')
self._append_parameter(parameter_context)
def _append_parameter(self, parameter_context):
"""
Appends a ParameterContext object to the internal set for this coverage.
A <b>deep copy</b> of the supplied ParameterContext is added to self._range_dictionary. An AbstractParameterValue of the type
indicated by ParameterContext.param_type is added to self._range_value. If the ParameterContext indicates that
the parameter is a coordinate parameter, it is associated with the indicated axis of the appropriate CRS.
@param parameter_context The ParameterContext to append to the coverage <b>as a copy</b>
@throws StandardError If the ParameterContext.axis indicates that it is temporal and a temporal parameter
already exists in the coverage
"""
if self.closed:
raise IOError('I/O operation on closed file')
if self.mode == 'r':
raise IOError('Coverage not open for writing: mode == \'{0}\''.format(self.mode))
if not isinstance(parameter_context, ParameterContext):
raise TypeError('\'parameter_context\' must be an instance of ParameterContext')
# Create a deep copy of the ParameterContext
pcontext = deepcopy(parameter_context)
pname = pcontext.name
no_sdom = self.spatial_domain is None
## Determine the correct array shape
# Get the parameter variability; assign to VariabilityEnum.NONE if None
pv=pcontext.variability or VariabilityEnum.NONE
if no_sdom and pv in (VariabilityEnum.SPATIAL, VariabilityEnum.BOTH):
log.warn('Provided \'parameter_context\' indicates Spatial variability, but coverage has no Spatial Domain')
if pv == VariabilityEnum.TEMPORAL: # Only varies in the Temporal Domain
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, None)
elif pv == VariabilityEnum.SPATIAL: # Only varies in the Spatial Domain
pcontext.dom = DomainSet(None, self.spatial_domain.shape.extents)
elif pv == VariabilityEnum.BOTH: # Varies in both domains
# If the Spatial Domain is only a single point on a 0d Topology, the parameter's shape is that of the Temporal Domain only
if no_sdom or (len(self.spatial_domain.shape.extents) == 1 and self.spatial_domain.shape.extents[0] == 0):
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, None)
else:
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, self.spatial_domain.shape.extents)
elif pv == VariabilityEnum.NONE: # No variance; constant
# CBM TODO: Not sure we can have this constraint - precludes situations like a TextType with Variablity==None...
# # This is a constant - if the ParameterContext is not a ConstantType, make it one with the default 'x' expr
# if not isinstance(pcontext.param_type, ConstantType):
# pcontext.param_type = ConstantType(pcontext.param_type)
# The domain is the total domain - same value everywhere!!
# If the Spatial Domain is only a single point on a 0d Topology, the parameter's shape is that of the Temporal Domain only
if no_sdom or (len(self.spatial_domain.shape.extents) == 1 and self.spatial_domain.shape.extents[0] == 0):
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, None)
else:
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, self.spatial_domain.shape.extents)
else:
# Should never get here...but...
raise SystemError('Must define the variability of the ParameterContext: a member of VariabilityEnum')
# Assign the pname to the CRS (if applicable) and select the appropriate domain (default is the spatial_domain)
dom = self.spatial_domain
if not pcontext.axis is None and AxisTypeEnum.is_member(pcontext.axis, AxisTypeEnum.TIME):
dom = self.temporal_domain
dom.crs.axes[pcontext.axis] = pcontext.name
elif not no_sdom and (pcontext.axis in self.spatial_domain.crs.axes):
dom.crs.axes[pcontext.axis] = pcontext.name
self._range_dictionary.add_context(pcontext)
s = self._persistence_layer.init_parameter(pcontext, self._bricking_scheme)
self._range_value[pname] = get_value_class(param_type=pcontext.param_type, domain_set=pcontext.dom, storage=s)
def get_parameter(self, param_name):
"""
Get a Parameter object by name
The Parameter object contains the ParameterContext and AbstractParameterValue associated with the param_name
@param param_name The local name of the parameter to return
@returns A Parameter object containing the context and value for the specified parameter
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if self.closed:
raise ValueError('I/O operation on closed file')
if param_name in self._range_dictionary:
p = Parameter(deepcopy(self._range_dictionary.get_context(param_name)), self._range_value[param_name].shape, self._range_value[param_name])
return p
else:
raise KeyError('Coverage does not contain parameter \'{0}\''.format(param_name))
def list_parameters(self, coords_only=False, data_only=False):
"""
List the names of the parameters contained in the coverage
@param coords_only List only the coordinate parameters
@param data_only List only the data parameters (non-coordinate) - superseded by coords_only
@returns A list of parameter names
"""
if coords_only:
lst=[x for x, v in self._range_dictionary.iteritems() if v[1].is_coordinate]
elif data_only:
lst=[x for x, v in self._range_dictionary.iteritems() if not v[1].is_coordinate]
else:
lst=[x for x in self._range_dictionary]
lst.sort()
return lst
def insert_timesteps(self, count, origin=None, oob=True):
"""
Insert count # of timesteps beginning at the origin
The specified # of timesteps are inserted into the temporal value array at the indicated origin. This also
expands the temporal dimension of the AbstractParameterValue for each parameters
@param count The number of timesteps to insert
@param origin The starting location, from which to begin the insertion
@param oob Out of band operations, True will use greenlets, False will be in-band.
"""
if self.closed:
raise IOError('I/O operation on closed file')
if self.mode == 'r':
raise IOError('Coverage not open for writing: mode == \'{0}\''.format(self.mode))
# Get the current shape of the temporal_dimension
shp = self.temporal_domain.shape
# If not provided, set the origin to the end of the array
if origin is None or not isinstance(origin, int):
origin = shp.extents[0]
# Expand the shape of the temporal_domain - following works if extents is a list or tuple
shp.extents = (shp.extents[0]+count,)+tuple(shp.extents[1:])
# Expand the temporal dimension of each of the parameters - the parameter determines how to apply the change
for n in self._range_dictionary:
pc = self._range_dictionary.get_context(n)
# Update the dom of the parameter_context
if pc.dom.tdom is not None:
pc.dom.tdom = self.temporal_domain.shape.extents
self._persistence_layer.expand_domain(pc)
self._range_value[n].expand_content(VariabilityEnum.TEMPORAL, origin, count)
# Update the temporal_domain in the master_manager, do NOT flush!!
self._persistence_layer.update_domain(tdom=self.temporal_domain, do_flush=False)
# Flush the master_manager & parameter_managers in a separate greenlet
if oob:
spawn(self._persistence_layer.flush)
else:
self._persistence_layer.flush()
def set_time_values(self, value, tdoa=None):
"""
Convenience method for setting time values
@param value The value to set
@param tdoa The temporal DomainOfApplication; default to full Domain
"""
return self.set_parameter_values(self.temporal_parameter_name, value, tdoa, None)
def get_time_values(self, tdoa=None, return_value=None):
"""
Convenience method for retrieving time values
Delegates to get_parameter_values, supplying the temporal parameter name and sdoa == None
@param tdoa The temporal DomainOfApplication; default to full Domain
@param return_value If supplied, filled with response value
"""
return self.get_parameter_values(self.temporal_parameter_name, tdoa, None, return_value)
@property
def num_timesteps(self):
"""
The current number of timesteps
"""
return self.temporal_domain.shape.extents[0]
def set_parameter_values(self, param_name, value, tdoa=None, sdoa=None):
"""
Assign value to the specified parameter
Assigns the value to param_name within the coverage. Temporal and spatial DomainOfApplication objects can be
applied to constrain the assignment. See DomainOfApplication for details
@param param_name The name of the parameter
@param value The value to set
@param tdoa The temporal DomainOfApplication
@param sdoa The spatial DomainOfApplication
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if self.closed:
raise IOError('I/O operation on closed file')
if self.mode == 'r':
raise IOError('Coverage not open for writing: mode == \'{0}\''.format(self.mode))
if not param_name in self._range_value:
raise KeyError('Parameter \'{0}\' not found in coverage_model'.format(param_name))
slice_ = []
tdoa = get_valid_DomainOfApplication(tdoa, self.temporal_domain.shape.extents)
log.debug('Temporal doa: %s', tdoa.slices)
slice_.extend(tdoa.slices)
if self.spatial_domain is not None:
sdoa = get_valid_DomainOfApplication(sdoa, self.spatial_domain.shape.extents)
log.debug('Spatial doa: %s', sdoa.slices)
slice_.extend(sdoa.slices)
log.debug('Setting slice: %s', slice_)
self._range_value[param_name][slice_] = value
def get_parameter_values(self, param_name, tdoa=None, sdoa=None, return_value=None):
"""
Retrieve the value for a parameter
Returns the value from param_name. Temporal and spatial DomainOfApplication objects can be used to
constrain the response. See DomainOfApplication for details.
@param param_name The name of the parameter
@param tdoa The temporal DomainOfApplication
@param sdoa The spatial DomainOfApplication
@param return_value If supplied, filled with response value - currently via OVERWRITE
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if self.closed:
raise ValueError('I/O operation on closed file')
if not param_name in self._range_value:
raise KeyError('Parameter \'{0}\' not found in coverage'.format(param_name))
if return_value is not None:
log.warn('Provided \'return_value\' will be OVERWRITTEN')
slice_ = []
tdoa = get_valid_DomainOfApplication(tdoa, self.temporal_domain.shape.extents)
log.debug('Temporal doa: %s', tdoa.slices)
slice_.extend(tdoa.slices)
if self.spatial_domain is not None:
sdoa = get_valid_DomainOfApplication(sdoa, self.spatial_domain.shape.extents)
log.debug('Spatial doa: %s', sdoa.slices)
slice_.extend(sdoa.slices)
log.debug('Getting slice: %s', slice_)
return_value = self._range_value[param_name][slice_]
return return_value
def get_parameter_context(self, param_name):
"""
Retrieve a deepcopy of the ParameterContext object for the specified parameter
@param param_name The name of the parameter for which to retrieve context
@returns A deepcopy of the specified ParameterContext object
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if not param_name in self._range_dictionary:
raise KeyError('Parameter \'{0}\' not found in coverage'.format(param_name))
return deepcopy(self._range_dictionary.get_context(param_name))
def __axis_arg_to_params(self, axis=None):
"""
Helper function to compose a list of parameter names based on the <i>axis</i> argument
If <i>axis</i> is None, all coordinate parameters are included
@param axis A member of AxisTypeEnum; may be an iterable of such members
"""
params = []
if axis is None:
params.extend(pn for pk, pn in self.temporal_domain.crs.axes.iteritems())
params.extend(pn for pk, pn in self.spatial_domain.crs.axes.iteritems())
elif hasattr(axis, '__iter__'):
for a in axis:
if a in self.temporal_domain.crs.axes:
params.append(self.temporal_domain.crs.axes[a])
elif a in self.spatial_domain.crs.axes:
params.append(self.spatial_domain.crs.axes[a])
else:
raise ValueError('Specified axis ({0}) not found in coverage'.format(a))
elif axis in self.temporal_domain.crs.axes:
params.append(self.temporal_domain.crs.axes[axis])
elif axis in self.spatial_domain.crs.axes:
params.append(self.spatial_domain.crs.axes[axis])
else:
raise ValueError('Specified axis ({0}) not found in coverage'.format(axis))
return params
def __parameter_name_arg_to_params(self, parameter_name=None):
"""
Helper function to compose a list of parameter names based on the <i>parameter_name</i> argument
If <i>parameter_name</i> is None, all parameters in the coverage are included
@param parameter_name A string parameter name; may be an iterable of such members
"""
params = []
if parameter_name is None:
params.extend(self._range_dictionary.keys())
elif hasattr(parameter_name, '__iter__'):
params.extend(pn for pn in parameter_name if pn in self._range_dictionary.keys())
else:
params.append(parameter_name)
return params
def get_data_bounds(self, parameter_name=None):
"""
Returns the bounds (min, max) for the parameter(s) indicated by <i>parameter_name</i>
If <i>parameter_name</i> is None, all parameters in the coverage are included
If more than one parameter is indicated by <i>parameter_name</i>, a dict of {key:(min,max)} is returned;
otherwise, only the (min, max) tuple is returned
@param parameter_name A string parameter name; may be an iterable of such members
"""
from coverage_model import QuantityType, ConstantType
ret = {}
for pn in self.__parameter_name_arg_to_params(parameter_name):
ctxt = self._range_dictionary.get_context(pn)
fv = ctxt.fill_value
if isinstance(ctxt.param_type, QuantityType) or isinstance(ctxt.param_type, ConstantType):
varr = np.ma.masked_equal(self._range_value[pn][:], fv, copy=False)
r = (varr.min(), varr.max())
ret[pn] = tuple([fv if isinstance(x, np.ma.core.MaskedConstant) else x for x in r])
else:
# CBM TODO: Sort out if this is an appropriate way to deal with non-numeric types
ret[pn] = (fv, fv)
if len(ret) == 1:
ret = ret.values()[0]
return ret
def get_data_bounds_by_axis(self, axis=None):
"""
Returns the bounds (min, max) for the coordinate parameter(s) indicated by <i>axis</i>
If <i>axis</i> is None, all coordinate parameters are included
If more than one parameter is indicated by <i>axis</i>, a dict of {key:(min,max)} is returned;
otherwise, only the (min, max) tuple is returned
@param axis A member of AxisTypeEnum; may be an iterable of such members
"""
return self.get_data_bounds(self.__axis_arg_to_params(axis))
def get_data_extents(self, parameter_name=None):
"""
Returns the extents (dim_0,dim_1,...,dim_n) for the parameter(s) indicated by <i>parameter_name</i>
If <i>parameter_name</i> is None, all parameters in the coverage are included
If more than one parameter is indicated by <i>parameter_name</i>, a dict of {key:(dim_0,dim_1,...,dim_n)} is returned;
otherwise, only the (dim_0,dim_1,...,dim_n) tuple is returned
@param parameter_name A string parameter name; may be an iterable of such members
"""
ret = {}
for pn in self.__parameter_name_arg_to_params(parameter_name):
p = self._range_dictionary.get_context(pn)
ret[pn] = p.dom.total_extents
if len(ret) == 1:
ret = ret.values()[0]
return ret
def get_data_extents_by_axis(self, axis=None):
"""
Returns the extents (dim_0,dim_1,...,dim_n) for the coordinate parameter(s) indicated by <i>axis</i>
If <i>axis</i> is None, all coordinate parameters are included
If more than one parameter is indicated by <i>axis</i>, a dict of {key:(dim_0,dim_1,...,dim_n)} is returned;
otherwise, only the (dim_0,dim_1,...,dim_n) tuple is returned
@param axis A member of AxisTypeEnum; may be an iterable of such members
"""
return self.get_data_extents(self.__axis_arg_to_params(axis))
def get_data_size(self, parameter_name=None, slice_=None, in_bytes=False):
"""
Returns the size of the <b>data values</b> for the parameter(s) indicated by <i>parameter_name</i>.
ParameterContext and Coverage metadata is <b>NOT</b> included in the returned size.
If <i>parameter_name</i> is None, all parameters in the coverage are included
If more than one parameter is indicated by <i>parameter_name</i>, the sum of the indicated parameters is returned
If <i>slice_</i> is not None, it is applied to each parameter (after being run through utils.fix_slice) before
calculation of size
Sizes are calculated as:
size = itemsize * total_extent_size
where:
itemsize == the per-item size based on the data type of the parameter
total_extent_size == the total number of elements after slicing is applied (if applicable)
Sizes are in MB unless <i>in_bytes</i> == True
@param parameter_name A string parameter name; may be an iterable of such members
@param slice_ If not None, applied to each parameter before calculation of size
@param in_bytes If True, returns the size in bytes; otherwise, returns the size in MB (default)
"""
size = 0
if parameter_name is None:
for pn in self._range_dictionary.keys():
size += self.get_data_size(pn, in_bytes=in_bytes)
for pn in self.__parameter_name_arg_to_params(parameter_name):
p = self._range_dictionary.get_context(pn)
te=p.dom.total_extents
dt = np.dtype(p.param_type.value_encoding)
if slice_ is not None:
slice_ = utils.fix_slice(slice_, te)
a=np.empty(te, dtype=dt)[slice_]
size += a.nbytes
else:
size += dt.itemsize * utils.prod(te)
if not in_bytes:
size *= 9.53674e-7
return size
@property
def info(self):
"""
Returns a detailed string representation of the coverage contents
@returns string of coverage contents
"""
lst = []
indent = ' '
lst.append('ID: {0}'.format(self._id))
lst.append('Name: {0}'.format(self.name))
lst.append('Temporal Domain:\n{0}'.format(self.temporal_domain.__str__(indent*2)))
lst.append('Spatial Domain:\n{0}'.format(self.spatial_domain.__str__(indent*2)))
lst.append('Parameters:')
for x in self._range_value:
lst.append('{0}{1} {2}\n{3}'.format(indent*2,x,self._range_value[x].shape,self._range_dictionary.get_context(x).__str__(indent*4)))
return '\n'.join(lst)
def __str__(self):
lst = []
indent = ' '
lst.append('ID: {0}'.format(self._id))
lst.append('Name: {0}'.format(self.name))
lst.append('TemporalDomain: Shape=>{0} Axes=>{1}'.format(self.temporal_domain.shape.extents, self.temporal_domain.crs.axes))
lst.append('SpatialDomain: Shape=>{0} Axes=>{1}'.format(self.spatial_domain.shape.extents, self.spatial_domain.crs.axes))
lst.append('Coordinate Parameters: {0}'.format(self.list_parameters(coords_only=True)))
lst.append('Data Parameters: {0}'.format(self.list_parameters(coords_only=False, data_only=True)))
return '\n'.join(lst)
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(name='example data agent',
handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance(
name='example dataset agent instance')
eda_inst_id = dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(name='example data provider',
institution=Institution(),
contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(name='example datasource',
protocol_type='FILE',
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters[
'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters[
'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = []
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
#dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(
external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'data',
param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
pc_list.append(dms_cli.create_parameter_context(
pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict',
pc_list)
streamdef_id = pubsub_cli.create_stream_definition(
name="ruv",
description="stream def for ruv testing",
parameter_dictionary_id=pdict_id)
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product')
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(
data_product=dprod, stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({
'ExternalDataset': ds_id,
'ExternalDataProvider': ext_dprov_id,
'DataSource': ext_dsrc_id,
'DataSourceModel': ext_dsrc_model_id,
'DataProducer': dproducer_id,
'DataProduct': dproduct_id,
'Stream': stream_id
}))
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='ruv',
stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'external_dataset_res': dset,
'param_dictionary': pdict.dump(),
'data_producer_id':
dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(name='example data agent', handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance(name='example dataset agent instance')
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(name='example datasource', protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters['list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters['date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
#dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict', pc_list)
streamdef_id = pubsub_cli.create_stream_definition(name="ruv", description="stream def for ruv testing", parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product',
temporal_domain=tdom,
spatial_domain=sdom)
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id}))
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='ruv', stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'external_dataset_res': dset,
'param_dictionary': pdict.dump(),
'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
class SimplexCoverage(AbstractCoverage):
"""
A concrete implementation of AbstractCoverage consisting of 2 domains (temporal and spatial)
and a collection of parameters associated with one or both of the domains. Each parameter is defined by a
ParameterContext object (provided via the ParameterDictionary) and has content represented by a concrete implementation
of the AbstractParameterValue class.
"""
def __init__(self, root_dir, persistence_guid, name=None, parameter_dictionary=None, temporal_domain=None, spatial_domain=None, in_memory_storage=False, bricking_scheme=None):
"""
Constructor for SimplexCoverage
@param root_dir The root directory for storage of this coverage
@param persistence_guid The persistence uuid for this coverage
@param name The name of the coverage
@param parameter_dictionary a ParameterDictionary object expected to contain one or more valid ParameterContext objects
@param spatial_domain a concrete instance of AbstractDomain for the spatial domain component
@param temporal_domain a concrete instance of AbstractDomain for the temporal domain component
"""
# Make sure root_dir and persistence_guid are both not None and are strings
if not isinstance(root_dir, str) or not isinstance(persistence_guid, str):
raise SystemError('\'root_dir\' and \'persistence_guid\' must be instances of str')
pth=os.path.join(root_dir, persistence_guid)
def _doload(self):
# Make sure the coverage directory exists
if not os.path.exists(pth):
raise SystemError('Cannot find specified coverage: {0}'.format(pth))
# All appears well - load it up!
self._persistence_layer = PersistenceLayer(root_dir, persistence_guid)
self.name = self._persistence_layer.name
self.spatial_domain = self._persistence_layer.sdom
self.temporal_domain = self._persistence_layer.tdom
self._range_dictionary = ParameterDictionary()
self._range_value = RangeValues()
self._bricking_scheme = self._persistence_layer.global_bricking_scheme
self._temporal_param_name = self._persistence_layer.temporal_param_name
self._in_memory_storage = False
from coverage_model.persistence import PersistedStorage
for parameter_name in self._persistence_layer.parameter_metadata.keys():
md = self._persistence_layer.parameter_metadata[parameter_name]
pc = md.parameter_context
self._range_dictionary.add_context(pc)
s = PersistedStorage(md, self._persistence_layer.brick_dispatcher, dtype=pc.param_type.value_encoding, fill_value=pc.param_type.fill_value)
self._range_value[parameter_name] = get_value_class(param_type=pc.param_type, domain_set=pc.dom, storage=s)
AbstractCoverage.__init__(self)
if name is None or parameter_dictionary is None:
# This appears to be a load
_doload(self)
else:
# This appears to be a new coverage
# Make sure name and parameter_dictionary are not None
if name is None or parameter_dictionary is None:
raise SystemError('\'name\' and \'parameter_dictionary\' cannot be None')
# Make sure the specified root_dir exists
if not in_memory_storage and not os.path.exists(root_dir):
raise SystemError('Cannot find specified \'root_dir\': {0}'.format(root_dir))
# If the coverage directory exists, load it instead!!
if os.path.exists(pth):
log.warn('The specified coverage already exists - performing load of \'{0}\''.format(pth))
_doload(self)
return
self.name = name
self.spatial_domain = deepcopy(spatial_domain)
self.temporal_domain = deepcopy(temporal_domain) or GridDomain(GridShape('temporal',[0]), CRS.standard_temporal(), MutabilityEnum.EXTENSIBLE)
if not isinstance(parameter_dictionary, ParameterDictionary):
raise TypeError('\'parameter_dictionary\' must be of type ParameterDictionary')
self._range_dictionary = ParameterDictionary()
self._range_value = RangeValues()
self._bricking_scheme = bricking_scheme or {'brick_size':10,'chunk_size':5}
self._temporal_param_name = None
self._in_memory_storage = in_memory_storage
if self._in_memory_storage:
self._persistence_layer = InMemoryPersistenceLayer()
else:
self._persistence_layer = PersistenceLayer(root_dir, persistence_guid, name=name, tdom=temporal_domain, sdom=spatial_domain, bricking_scheme=self._bricking_scheme)
for o, pc in parameter_dictionary.itervalues():
self._append_parameter(pc)
@classmethod
def _fromdict(cls, cmdict, arg_masks=None):
return super(SimplexCoverage, cls)._fromdict(cmdict, {'parameter_dictionary':'_range_dictionary'})
@property
def parameter_dictionary(self):
return deepcopy(self._range_dictionary)
@property
def persistence_guid(self):
if isinstance(self._persistence_layer, InMemoryPersistenceLayer):
return None
else:
return self._persistence_layer.guid
def append_parameter(self, parameter_context):
"""
Append a ParameterContext to the coverage
@deprecated use a ParameterDictionary during construction of the coverage
"""
log.warn('SimplexCoverage.append_parameter() is deprecated: use a ParameterDictionary during construction of the coverage')
self._append_parameter(parameter_context)
def _append_parameter(self, parameter_context):
"""
Appends a ParameterContext object to the internal set for this coverage.
A <b>deep copy</b> of the supplied ParameterContext is added to self._range_dictionary. An AbstractParameterValue of the type
indicated by ParameterContext.param_type is added to self._range_value. If the ParameterContext indicates that
the parameter is a coordinate parameter, it is associated with the indicated axis of the appropriate CRS.
@param parameter_context The ParameterContext to append to the coverage <b>as a copy</b>
@throws StandardError If the ParameterContext.axis indicates that it is temporal and a temporal parameter
already exists in the coverage
"""
if not isinstance(parameter_context, ParameterContext):
raise TypeError('\'parameter_context\' must be an instance of ParameterContext')
# Create a deep copy of the ParameterContext
pcontext = deepcopy(parameter_context)
pname = pcontext.name
no_sdom = self.spatial_domain is None
## Determine the correct array shape
# Get the parameter variability; assign to VariabilityEnum.NONE if None
pv=pcontext.variability or VariabilityEnum.NONE
if no_sdom and pv in (VariabilityEnum.SPATIAL, VariabilityEnum.BOTH):
log.warn('Provided \'parameter_context\' indicates Spatial variability, but coverage has no Spatial Domain')
if pv == VariabilityEnum.TEMPORAL: # Only varies in the Temporal Domain
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, None)
elif pv == VariabilityEnum.SPATIAL: # Only varies in the Spatial Domain
pcontext.dom = DomainSet(None, self.spatial_domain.shape.extents)
elif pv == VariabilityEnum.BOTH: # Varies in both domains
# If the Spatial Domain is only a single point on a 0d Topology, the parameter's shape is that of the Temporal Domain only
if no_sdom or (len(self.spatial_domain.shape.extents) == 1 and self.spatial_domain.shape.extents[0] == 0):
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, None)
else:
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, self.spatial_domain.shape.extents)
elif pv == VariabilityEnum.NONE: # No variance; constant
# CBM TODO: Not sure we can have this constraint - precludes situations like a TextType with Variablity==None...
# # This is a constant - if the ParameterContext is not a ConstantType, make it one with the default 'x' expr
# if not isinstance(pcontext.param_type, ConstantType):
# pcontext.param_type = ConstantType(pcontext.param_type)
# The domain is the total domain - same value everywhere!!
# If the Spatial Domain is only a single point on a 0d Topology, the parameter's shape is that of the Temporal Domain only
if no_sdom or (len(self.spatial_domain.shape.extents) == 1 and self.spatial_domain.shape.extents[0] == 0):
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, None)
else:
pcontext.dom = DomainSet(self.temporal_domain.shape.extents, self.spatial_domain.shape.extents)
else:
# Should never get here...but...
raise SystemError('Must define the variability of the ParameterContext: a member of VariabilityEnum')
# Assign the pname to the CRS (if applicable) and select the appropriate domain (default is the spatial_domain)
dom = self.spatial_domain
is_tparam = False
if not pcontext.reference_frame is None and AxisTypeEnum.is_member(pcontext.reference_frame, AxisTypeEnum.TIME):
if self._temporal_param_name is None:
self._temporal_param_name = pname
is_tparam = True
else:
raise StandardError("temporal_parameter already defined.")
dom = self.temporal_domain
dom.crs.axes[pcontext.reference_frame] = pcontext.name
elif not no_sdom and (pcontext.reference_frame in self.spatial_domain.crs.axes):
dom.crs.axes[pcontext.reference_frame] = pcontext.name
self._range_dictionary.add_context(pcontext)
s = self._persistence_layer.init_parameter(pcontext, self._bricking_scheme, is_temporal_param=is_tparam)
self._range_value[pname] = get_value_class(param_type=pcontext.param_type, domain_set=pcontext.dom, storage=s)
def get_parameter(self, param_name):
"""
Get a Parameter object by name
The Parameter object contains the ParameterContext and AbstractParameterValue associated with the param_name
@param param_name The local name of the parameter to return
@returns A Parameter object containing the context and value for the specified parameter
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if param_name in self._range_dictionary:
p = Parameter(self._range_dictionary.get_context(param_name), self._range_value[param_name].shape, self._range_value[param_name])
return p
else:
raise KeyError('Coverage does not contain parameter \'{0}\''.format(param_name))
def list_parameters(self, coords_only=False, data_only=False):
"""
List the names of the parameters contained in the coverage
@param coords_only List only the coordinate parameters
@param data_only List only the data parameters (non-coordinate) - superseded by coords_only
@returns A list of parameter names
"""
if coords_only:
lst=[x for x, v in self._range_dictionary.iteritems() if v[1].is_coordinate]
elif data_only:
lst=[x for x, v in self._range_dictionary.iteritems() if not v[1].is_coordinate]
else:
lst=[x for x in self._range_dictionary]
lst.sort()
return lst
def insert_timesteps(self, count, origin=None):
"""
Insert count # of timesteps beginning at the origin
The specified # of timesteps are inserted into the temporal value array at the indicated origin. This also
expands the temporal dimension of the AbstractParameterValue for each parameters
@param count The number of timesteps to insert
@param origin The starting location, from which to begin the insertion
"""
# Get the current shape of the temporal_dimension
shp = self.temporal_domain.shape
# If not provided, set the origin to the end of the array
if origin is None or not isinstance(origin, int):
origin = shp.extents[0]
# Expand the shape of the temporal_domain - following works if extents is a list or tuple
shp.extents = (shp.extents[0]+count,)+tuple(shp.extents[1:])
# Expand the temporal dimension of each of the parameters - the parameter determines how to apply the change
for n in self._range_dictionary:
pc = self._range_dictionary.get_context(n)
# Update the dom of the parameter_context
if pc.dom.tdom is not None:
pc.dom.tdom = self.temporal_domain.shape.extents
self._persistence_layer.expand_domain(pc, tdom=self.temporal_domain)
self._range_value[n].expand_content(VariabilityEnum.TEMPORAL, origin, count)
def set_time_values(self, value, tdoa):
"""
Convenience method for setting time values
@param value The value to set
@param tdoa The temporal DomainOfApplication; default to full Domain
"""
return self.set_parameter_values(self._temporal_param_name, value, tdoa, None)
def get_time_values(self, tdoa=None, return_value=None):
"""
Convenience method for retrieving time values
Delegates to get_parameter_values, supplying the temporal parameter name and sdoa == None
@param tdoa The temporal DomainOfApplication; default to full Domain
@param return_value If supplied, filled with response value
"""
return self.get_parameter_values(self._temporal_param_name, tdoa, None, return_value)
@property
def num_timesteps(self):
"""
The current number of timesteps
"""
return self.temporal_domain.shape.extents[0]
def set_parameter_values(self, param_name, value, tdoa=None, sdoa=None):
"""
Assign value to the specified parameter
Assigns the value to param_name within the coverage. Temporal and spatial DomainOfApplication objects can be
applied to constrain the assignment. See DomainOfApplication for details
@param param_name The name of the parameter
@param value The value to set
@param tdoa The temporal DomainOfApplication
@param sdoa The spatial DomainOfApplication
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if not param_name in self._range_value:
raise KeyError('Parameter \'{0}\' not found in coverage_model'.format(param_name))
slice_ = []
tdoa = get_valid_DomainOfApplication(tdoa, self.temporal_domain.shape.extents)
log.debug('Temporal doa: %s', tdoa.slices)
slice_.extend(tdoa.slices)
if self.spatial_domain is not None:
sdoa = get_valid_DomainOfApplication(sdoa, self.spatial_domain.shape.extents)
log.debug('Spatial doa: %s', sdoa.slices)
slice_.extend(sdoa.slices)
log.debug('Setting slice: %s', slice_)
self._range_value[param_name][slice_] = value
def get_parameter_values(self, param_name, tdoa=None, sdoa=None, return_value=None):
"""
Retrieve the value for a parameter
Returns the value from param_name. Temporal and spatial DomainOfApplication objects can be used to
constrain the response. See DomainOfApplication for details.
@param param_name The name of the parameter
@param tdoa The temporal DomainOfApplication
@param sdoa The spatial DomainOfApplication
@param return_value If supplied, filled with response value - currently via OVERWRITE
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if not param_name in self._range_value:
raise KeyError('Parameter \'{0}\' not found in coverage'.format(param_name))
if return_value is not None:
log.warn('Provided \'return_value\' will be OVERWRITTEN')
slice_ = []
tdoa = get_valid_DomainOfApplication(tdoa, self.temporal_domain.shape.extents)
log.debug('Temporal doa: %s', tdoa.slices)
slice_.extend(tdoa.slices)
if self.spatial_domain is not None:
sdoa = get_valid_DomainOfApplication(sdoa, self.spatial_domain.shape.extents)
log.debug('Spatial doa: %s', sdoa.slices)
slice_.extend(sdoa.slices)
log.debug('Getting slice: %s', slice_)
return_value = self._range_value[param_name][slice_]
return return_value
def get_parameter_context(self, param_name):
"""
Retrieve the ParameterContext object for the specified parameter
@param param_name The name of the parameter for which to retrieve context
@returns A ParameterContext object
@throws KeyError The coverage does not contain a parameter with name 'param_name'
"""
if not param_name in self._range_dictionary:
raise KeyError('Parameter \'{0}\' not found in coverage'.format(param_name))
return self._range_dictionary.get_context(param_name)
@property
def info(self):
"""
Returns a detailed string representation of the coverage contents
@returns string of coverage contents
"""
lst = []
indent = ' '
lst.append('ID: {0}'.format(self._id))
lst.append('Name: {0}'.format(self.name))
lst.append('Temporal Domain:\n{0}'.format(self.temporal_domain.__str__(indent*2)))
lst.append('Spatial Domain:\n{0}'.format(self.spatial_domain.__str__(indent*2)))
lst.append('Parameters:')
for x in self._range_value:
lst.append('{0}{1} {2}\n{3}'.format(indent*2,x,self._range_value[x].shape,self._range_dictionary.get_context(x).__str__(indent*4)))
return '\n'.join(lst)
def __str__(self):
lst = []
indent = ' '
lst.append('ID: {0}'.format(self._id))
lst.append('Name: {0}'.format(self.name))
lst.append('TemporalDomain: Shape=>{0} Axes=>{1}'.format(self.temporal_domain.shape.extents, self.temporal_domain.crs.axes))
lst.append('SpatialDomain: Shape=>{0} Axes=>{1}'.format(self.spatial_domain.shape.extents, self.spatial_domain.crs.axes))
lst.append('Coordinate Parameters: {0}'.format(self.list_parameters(coords_only=True)))
lst.append('Data Parameters: {0}'.format(self.list_parameters(coords_only=False, data_only=True)))
return '\n'.join(lst)