def _compare_pc(cls, pc1, pc2):
if pc1:
pc1 = ParameterContext.load(pc1) or {}
if pc2:
pc2 = ParameterContext.load(pc2) or {}
if hasattr(pc1,'lookup_value') or hasattr(pc2,'lookup_value'):
if hasattr(pc1,'lookup_value') and hasattr(pc2,'lookup_value'):
return bool(pc1 == pc2) and pc1.document_key == pc2.document_key
return False
return bool(pc1 == pc2)
def _compare_pc(cls, pc1, pc2):
if pc1:
pc1 = ParameterContext.load(pc1) or {}
if pc2:
pc2 = ParameterContext.load(pc2) or {}
if hasattr(pc1, 'lookup_value') or hasattr(pc2, 'lookup_value'):
if hasattr(pc1, 'lookup_value') and hasattr(pc2, 'lookup_value'):
return bool(
pc1 == pc2) and pc1.document_key == pc2.document_key
return False
return bool(pc1 == pc2)
def _create_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management_client.create_parameter_context(name='ingestion_timestamp', parameter_context=t_ctxt.dump())
context_ids.append(t_ctxt_id)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.uom = ''
context_ids.append(self.dataset_management_client.create_parameter_context(name='raw', parameter_context=raw_ctxt.dump()))
return context_ids, t_ctxt_id
def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''):
cov = self._get_simplex_coverage(dataset_id, mode='r+')
parameter_ctx_res = self.read_parameter_context(parameter_context_id)
pc = ParameterContext.load(parameter_ctx_res.parameter_context)
cov.append_parameter(pc)
cov.close()
return True
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''):
validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.')
validate_is_not_none(spatial_domain, 'A spatial domain must be supplied to register a new dataset.')
validate_is_not_none(temporal_domain, 'A temporal domain must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
parameter_dict = parameter_dict.dump()
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.temporal_domain = temporal_domain
dataset.spatial_domain = spatial_domain
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id,stream_id)
log.debug('creating dataset: %s', dataset_id)
cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain)
self._save_coverage(cov)
cov.close()
return dataset_id
def get_parameter_context_by_name(cls, name=''):
dms_cli = DatasetManagementServiceClient()
pc_res = dms_cli.read_parameter_context_by_name(name=name,
id_only=False)
pc = ParameterContext.load(pc_res.parameter_context)
pc._identifier = pc_res._id
return pc
def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''):
cov = self._get_simplex_coverage(dataset_id, mode='r+')
parameter_ctx_res = self.read_parameter_context(parameter_context_id)
pc = ParameterContext.load(parameter_ctx_res.parameter_context)
cov.append_parameter(pc)
cov.close()
return True
def _coverage_parameter_dictionary(self, parameter_dictionary_id):
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id,
id_only=False)
parameter_dict = self._merge_contexts(
[ParameterContext.load(i.parameter_context) for i in pcs],
pd.temporal_context)
return parameter_dict
def build_parameter_dictionary(cls, parameter_dictionary_obj,
parameter_contexts):
pdict = cls._merge_contexts([
ParameterContext.load(i.parameter_context)
for i in parameter_contexts
], parameter_dictionary_obj.temporal_context)
pdict._identifier = parameter_dictionary_obj._id
return pdict
def get_parameter_context(cls, parameter_context_id=''):
'''
Preferred client-side class method for constructing a parameter context
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pc_res = dms_cli.read_parameter_context(parameter_context_id=parameter_context_id)
pc = ParameterContext.load(pc_res.parameter_context)
pc._identifier = pc_res._id
return pc
def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''):
cov = self._get_simplex_coverage(dataset_id, mode='r+')
parameter_ctx_res = self.read_parameter_context(parameter_context_id)
pc = ParameterContext.load(parameter_ctx_res.parameter_context)
cov.append_parameter(pc)
cov.close()
dataset = self.read_dataset(dataset_id)
pdict = cov.parameter_dictionary
dataset.parameter_dictionary = pdict.dump()
self.update_dataset(dataset)
return True
def create_dataset(self,
name='',
datastore_name='',
view_name='',
stream_id='',
parameter_dict=None,
spatial_domain=None,
temporal_domain=None,
parameter_dictionary_id='',
description=''):
validate_true(
parameter_dict or parameter_dictionary_id,
'A parameter dictionary must be supplied to register a new dataset.'
)
validate_is_not_none(
spatial_domain,
'A spatial domain must be supplied to register a new dataset.')
validate_is_not_none(
temporal_domain,
'A temporal domain must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id,
id_only=False)
parameter_dict = self._merge_contexts(
[ParameterContext.load(i.parameter_context) for i in pcs],
pd.temporal_context)
parameter_dict = parameter_dict.dump()
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.temporal_domain = temporal_domain
dataset.spatial_domain = spatial_domain
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id, stream_id)
log.debug('creating dataset: %s', dataset_id)
cov = self._create_coverage(dataset_id, description or dataset_id,
parameter_dict, spatial_domain,
temporal_domain)
self._save_coverage(cov)
cov.close()
return dataset_id
def get_parameter_context(cls, parameter_context_id=''):
'''
Preferred client-side class method for constructing a parameter context
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pc_res = dms_cli.read_parameter_context(
parameter_context_id=parameter_context_id)
pc = ParameterContext.load(pc_res.parameter_context)
pc._identifier = pc_res._id
return pc
def _create_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'ingestion_timestamp',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management_client.create_parameter_context(
name='ingestion_timestamp', parameter_context=t_ctxt.dump())
context_ids.append(t_ctxt_id)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.uom = ''
context_ids.append(
self.dataset_management_client.create_parameter_context(
name='raw', parameter_context=raw_ctxt.dump()))
return context_ids, t_ctxt_id
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
pdict._identifier = parameter_dictionary_id
return pdict
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(
parameter_dictionary_id=parameter_dictionary_id, id_only=False)
pdict = cls._merge_contexts(
[ParameterContext.load(i.parameter_context) for i in pcs],
pd.temporal_context)
pdict._identifier = parameter_dictionary_id
return pdict
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, parameter_dictionary_id='', description='', parent_dataset_id=''):
validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
parameter_dict = parameter_dict.dump()
parameter_dict = self.numpy_walk(parameter_dict)
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id, stream_id)
log.debug('creating dataset: %s', dataset_id)
if parent_dataset_id:
vcov = self._create_view_coverage(dataset_id, description or dataset_id, parent_dataset_id)
vcov.close()
return dataset_id
cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict)
self._save_coverage(cov)
cov.close()
#table loader create resource
if self._get_eoi_service_available():
log.debug('DM:create dataset: %s -- dataset_id: %s', name, dataset_id)
self._create_single_resource(dataset_id, parameter_dict)
return dataset_id
def _add_location_time_ctxt(self, pdict):
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
return pdict
def _create_parameter(self):
pdict = ParameterDictionary()
pdict = self._add_location_time_ctxt(pdict)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
raw_fixed_ctxt = ParameterContext('raw_fixed', param_type=QuantityType(value_encoding=numpy.float32))
raw_fixed_ctxt.uom = 'unknown'
raw_fixed_ctxt.fill_value = 0e0
pdict.add_context(raw_fixed_ctxt)
raw_blob_ctxt = ParameterContext('raw_blob', param_type=QuantityType(value_encoding=numpy.float32))
raw_blob_ctxt.uom = 'unknown'
raw_blob_ctxt.fill_value = 0e0
pdict.add_context(raw_blob_ctxt)
return pdict
def _compare_pc(cls, pc1, pc2):
if pc1:
pc1 = ParameterContext.load(pc1) or {}
if pc2:
pc2 = ParameterContext.load(pc2) or {}
return bool(pc1 == pc2)
def _coverage_parameter_dictionary(self, parameter_dictionary_id):
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
return parameter_dict
def _compare_pc(cls, pc1, pc2):
if pc1:
pc1 = ParameterContext.load(pc1) or {}
if pc2:
pc2 = ParameterContext.load(pc2) or {}
return bool(pc1 == pc2)
def get_coverage_parameter(cls, parameter_context):
"""
Creates a Coverage Model based Parameter Context given the
ParameterContext IonObject.
Note: If the parameter is a parameter function and depends on dynamically
created calibrations, this will fail.
"""
# Only CF and netCDF compliant variable names
parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_',
parameter_context.name)
from ion.services.dm.utility.types import TypesManager
# The TypesManager does all the parsing and converting to the coverage model instances
tm = TypesManager(None, {}, {})
# First thing to do is create the parameter type
param_type = tm.get_parameter_type(
parameter_context.parameter_type, parameter_context.value_encoding,
parameter_context.code_report,
parameter_context.parameter_function_id,
parameter_context.parameter_function_map)
# Ugh, I hate it but I did copy this section from
# ion/processes/bootstrap/ion_loader.py
context = ParameterContext(name=parameter_context.name,
param_type=param_type)
# Now copy over all the attrs
context.uom = parameter_context.units
try:
if isinstance(context.uom, basestring):
tm.get_unit(context.uom)
except UdunitsError:
log.warning('Parameter %s has invalid units: %s',
parameter_context.name, context.uom)
# Fill values can be a bit tricky...
context.fill_value = tm.get_fill_value(
parameter_context.fill_value, parameter_context.value_encoding,
param_type)
context.reference_urls = parameter_context.reference_urls
context.internal_name = parameter_context.name
context.display_name = parameter_context.display_name
context.standard_name = parameter_context.standard_name
context.ooi_short_name = parameter_context.ooi_short_name
context.description = parameter_context.description
context.precision = parameter_context.precision
context.visible = parameter_context.visible
return context
def build_parameter_dictionary(cls, parameter_dictionary_obj, parameter_contexts):
pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in parameter_contexts],
parameter_dictionary_obj.temporal_context)
pdict._identifier = parameter_dictionary_obj._id
return pdict
def get_parameter_context_by_name(cls, name=''):
dms_cli = DatasetManagementServiceClient()
pc_res = dms_cli.read_parameter_context_by_name(name=name, id_only=False)
pc = ParameterContext.load(pc_res.parameter_context)
pc._identifier = pc_res._id
return pc
def get_coverage_parameter(cls, parameter_context):
"""
Creates a Coverage Model based Parameter Context given the
ParameterContext IonObject.
Note: If the parameter is a parameter function and depends on dynamically
created calibrations, this will fail.
"""
# Only CF and netCDF compliant variable names
parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name)
from ion.services.dm.utility.types import TypesManager
# The TypesManager does all the parsing and converting to the coverage model instances
tm = TypesManager(None, {}, {})
# First thing to do is create the parameter type
param_type = tm.get_parameter_type(
parameter_context.parameter_type,
parameter_context.value_encoding,
parameter_context.code_report,
parameter_context.parameter_function_id,
parameter_context.parameter_function_map)
# Ugh, I hate it but I did copy this section from
# ion/processes/bootstrap/ion_loader.py
context = ParameterContext(name=parameter_context.name, param_type=param_type)
# Now copy over all the attrs
context.uom = parameter_context.units
try:
if isinstance(context.uom, basestring):
tm.get_unit(context.uom)
except UdunitsError:
log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom)
# Fill values can be a bit tricky...
context.fill_value = tm.get_fill_value(parameter_context.fill_value,
parameter_context.value_encoding,
param_type)
context.reference_urls = parameter_context.reference_urls
context.internal_name = parameter_context.name
context.display_name = parameter_context.display_name
context.standard_name = parameter_context.standard_name
context.ooi_short_name = parameter_context.ooi_short_name
context.description = parameter_context.description
context.precision = parameter_context.precision
context.visible = parameter_context.visible
return context
