def _compare_pc(cls, pc1, pc2): if pc1: pc1 = ParameterContext.load(pc1) or {} if pc2: pc2 = ParameterContext.load(pc2) or {} if hasattr(pc1,'lookup_value') or hasattr(pc2,'lookup_value'): if hasattr(pc1,'lookup_value') and hasattr(pc2,'lookup_value'): return bool(pc1 == pc2) and pc1.document_key == pc2.document_key return False return bool(pc1 == pc2)
def _compare_pc(cls, pc1, pc2): if pc1: pc1 = ParameterContext.load(pc1) or {} if pc2: pc2 = ParameterContext.load(pc2) or {} if hasattr(pc1, 'lookup_value') or hasattr(pc2, 'lookup_value'): if hasattr(pc1, 'lookup_value') and hasattr(pc2, 'lookup_value'): return bool( pc1 == pc2) and pc1.document_key == pc2.document_key return False return bool(pc1 == pc2)
def _create_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt.fill_value = -9999 t_ctxt_id = self.dataset_management_client.create_parameter_context(name='ingestion_timestamp', parameter_context=t_ctxt.dump()) context_ids.append(t_ctxt_id) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.uom = '' context_ids.append(self.dataset_management_client.create_parameter_context(name='raw', parameter_context=raw_ctxt.dump())) return context_ids, t_ctxt_id
def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''): cov = self._get_simplex_coverage(dataset_id, mode='r+') parameter_ctx_res = self.read_parameter_context(parameter_context_id) pc = ParameterContext.load(parameter_ctx_res.parameter_context) cov.append_parameter(pc) cov.close() return True
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''): validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.') validate_is_not_none(spatial_domain, 'A spatial domain must be supplied to register a new dataset.') validate_is_not_none(temporal_domain, 'A temporal domain must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.temporal_domain = temporal_domain dataset.spatial_domain = spatial_domain dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id,stream_id) log.debug('creating dataset: %s', dataset_id) cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain) self._save_coverage(cov) cov.close() return dataset_id
def get_parameter_context_by_name(cls, name=''): dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context_by_name(name=name, id_only=False) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc
def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''): cov = self._get_simplex_coverage(dataset_id, mode='r+') parameter_ctx_res = self.read_parameter_context(parameter_context_id) pc = ParameterContext.load(parameter_ctx_res.parameter_context) cov.append_parameter(pc) cov.close() return True
def _coverage_parameter_dictionary(self, parameter_dictionary_id): pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts( [ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) return parameter_dict
def build_parameter_dictionary(cls, parameter_dictionary_obj, parameter_contexts): pdict = cls._merge_contexts([ ParameterContext.load(i.parameter_context) for i in parameter_contexts ], parameter_dictionary_obj.temporal_context) pdict._identifier = parameter_dictionary_obj._id return pdict
def get_parameter_context(cls, parameter_context_id=''): ''' Preferred client-side class method for constructing a parameter context from a service call. ''' dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context(parameter_context_id=parameter_context_id) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc
def add_parameter_to_dataset(self, parameter_context_id='', dataset_id=''): cov = self._get_simplex_coverage(dataset_id, mode='r+') parameter_ctx_res = self.read_parameter_context(parameter_context_id) pc = ParameterContext.load(parameter_ctx_res.parameter_context) cov.append_parameter(pc) cov.close() dataset = self.read_dataset(dataset_id) pdict = cov.parameter_dictionary dataset.parameter_dictionary = pdict.dump() self.update_dataset(dataset) return True
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''): validate_true( parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.' ) validate_is_not_none( spatial_domain, 'A spatial domain must be supplied to register a new dataset.') validate_is_not_none( temporal_domain, 'A temporal domain must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts( [ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.temporal_domain = temporal_domain dataset.spatial_domain = spatial_domain dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id, stream_id) log.debug('creating dataset: %s', dataset_id) cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain) self._save_coverage(cov) cov.close() return dataset_id
def get_parameter_context(cls, parameter_context_id=''): ''' Preferred client-side class method for constructing a parameter context from a service call. ''' dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context( parameter_context_id=parameter_context_id) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc
def _create_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt.fill_value = -9999 t_ctxt_id = self.dataset_management_client.create_parameter_context( name='ingestion_timestamp', parameter_context=t_ctxt.dump()) context_ids.append(t_ctxt_id) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.uom = '' context_ids.append( self.dataset_management_client.create_parameter_context( name='raw', parameter_context=raw_ctxt.dump())) return context_ids, t_ctxt_id
def get_parameter_dictionary(cls, parameter_dictionary_id=''): ''' Preferred client-side class method for constructing a parameter dictionary from a service call. ''' dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False) pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) pdict._identifier = parameter_dictionary_id return pdict
def get_parameter_dictionary(cls, parameter_dictionary_id=''): ''' Preferred client-side class method for constructing a parameter dictionary from a service call. ''' dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts( parameter_dictionary_id=parameter_dictionary_id, id_only=False) pdict = cls._merge_contexts( [ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) pdict._identifier = parameter_dictionary_id return pdict
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, parameter_dictionary_id='', description='', parent_dataset_id=''): validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.') if parameter_dictionary_id: pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) parameter_dict = parameter_dict.dump() parameter_dict = self.numpy_walk(parameter_dict) dataset = Dataset() dataset.description = description dataset.name = name dataset.primary_view_key = stream_id or None dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE dataset.view_name = view_name or self.DEFAULT_VIEW dataset.parameter_dictionary = parameter_dict dataset.registered = False dataset_id, _ = self.clients.resource_registry.create(dataset) if stream_id: self.add_stream(dataset_id, stream_id) log.debug('creating dataset: %s', dataset_id) if parent_dataset_id: vcov = self._create_view_coverage(dataset_id, description or dataset_id, parent_dataset_id) vcov.close() return dataset_id cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict) self._save_coverage(cov) cov.close() #table loader create resource if self._get_eoi_service_available(): log.debug('DM:create dataset: %s -- dataset_id: %s', name, dataset_id) self._create_single_resource(dataset_id, parameter_dict) return dataset_id
def _add_location_time_ctxt(self, pdict): t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) return pdict
def _create_parameter(self): pdict = ParameterDictionary() pdict = self._add_location_time_ctxt(pdict) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) raw_fixed_ctxt = ParameterContext('raw_fixed', param_type=QuantityType(value_encoding=numpy.float32)) raw_fixed_ctxt.uom = 'unknown' raw_fixed_ctxt.fill_value = 0e0 pdict.add_context(raw_fixed_ctxt) raw_blob_ctxt = ParameterContext('raw_blob', param_type=QuantityType(value_encoding=numpy.float32)) raw_blob_ctxt.uom = 'unknown' raw_blob_ctxt.fill_value = 0e0 pdict.add_context(raw_blob_ctxt) return pdict
def _compare_pc(cls, pc1, pc2): if pc1: pc1 = ParameterContext.load(pc1) or {} if pc2: pc2 = ParameterContext.load(pc2) or {} return bool(pc1 == pc2)
def _coverage_parameter_dictionary(self, parameter_dictionary_id): pd = self.read_parameter_dictionary(parameter_dictionary_id) pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False) parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) return parameter_dict
def _compare_pc(cls, pc1, pc2): if pc1: pc1 = ParameterContext.load(pc1) or {} if pc2: pc2 = ParameterContext.load(pc2) or {} return bool(pc1 == pc2)
def get_coverage_parameter(cls, parameter_context): """ Creates a Coverage Model based Parameter Context given the ParameterContext IonObject. Note: If the parameter is a parameter function and depends on dynamically created calibrations, this will fail. """ # Only CF and netCDF compliant variable names parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name) from ion.services.dm.utility.types import TypesManager # The TypesManager does all the parsing and converting to the coverage model instances tm = TypesManager(None, {}, {}) # First thing to do is create the parameter type param_type = tm.get_parameter_type( parameter_context.parameter_type, parameter_context.value_encoding, parameter_context.code_report, parameter_context.parameter_function_id, parameter_context.parameter_function_map) # Ugh, I hate it but I did copy this section from # ion/processes/bootstrap/ion_loader.py context = ParameterContext(name=parameter_context.name, param_type=param_type) # Now copy over all the attrs context.uom = parameter_context.units try: if isinstance(context.uom, basestring): tm.get_unit(context.uom) except UdunitsError: log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom) # Fill values can be a bit tricky... context.fill_value = tm.get_fill_value( parameter_context.fill_value, parameter_context.value_encoding, param_type) context.reference_urls = parameter_context.reference_urls context.internal_name = parameter_context.name context.display_name = parameter_context.display_name context.standard_name = parameter_context.standard_name context.ooi_short_name = parameter_context.ooi_short_name context.description = parameter_context.description context.precision = parameter_context.precision context.visible = parameter_context.visible return context
def build_parameter_dictionary(cls, parameter_dictionary_obj, parameter_contexts): pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in parameter_contexts], parameter_dictionary_obj.temporal_context) pdict._identifier = parameter_dictionary_obj._id return pdict
def get_parameter_context_by_name(cls, name=''): dms_cli = DatasetManagementServiceClient() pc_res = dms_cli.read_parameter_context_by_name(name=name, id_only=False) pc = ParameterContext.load(pc_res.parameter_context) pc._identifier = pc_res._id return pc
def get_coverage_parameter(cls, parameter_context): """ Creates a Coverage Model based Parameter Context given the ParameterContext IonObject. Note: If the parameter is a parameter function and depends on dynamically created calibrations, this will fail. """ # Only CF and netCDF compliant variable names parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name) from ion.services.dm.utility.types import TypesManager # The TypesManager does all the parsing and converting to the coverage model instances tm = TypesManager(None, {}, {}) # First thing to do is create the parameter type param_type = tm.get_parameter_type( parameter_context.parameter_type, parameter_context.value_encoding, parameter_context.code_report, parameter_context.parameter_function_id, parameter_context.parameter_function_map) # Ugh, I hate it but I did copy this section from # ion/processes/bootstrap/ion_loader.py context = ParameterContext(name=parameter_context.name, param_type=param_type) # Now copy over all the attrs context.uom = parameter_context.units try: if isinstance(context.uom, basestring): tm.get_unit(context.uom) except UdunitsError: log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom) # Fill values can be a bit tricky... context.fill_value = tm.get_fill_value(parameter_context.fill_value, parameter_context.value_encoding, param_type) context.reference_urls = parameter_context.reference_urls context.internal_name = parameter_context.name context.display_name = parameter_context.display_name context.standard_name = parameter_context.standard_name context.ooi_short_name = parameter_context.ooi_short_name context.description = parameter_context.description context.precision = parameter_context.precision context.visible = parameter_context.visible return context