def test_replay_with_parameters(self): # -------------------------------------------------------------------------------- # Create the configurations and the dataset # -------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext("binary", param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext("records", param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( "replay_pdict", parameter_context_ids=context_ids, temporal_context="time" ) stream_def_id = self.pubsub_management.create_stream_definition( "replay_stream", parameter_dictionary_id=pdict_id ) stream_id, route = self.pubsub_management.create_stream( "replay_with_params", exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id ) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Coerce the datastore into existence (beats race condition) # -------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 40) query = { "start_time": 0 - 2208988800, "end_time": 20 - 2208988800, "stride_time": 2, "parameters": ["time", "temp"], } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0, 20, 2) == rdt["time"] self.assertTrue(comp.all(), "%s" % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=["time", "temp"]) self.assertTrue(extents["time"] >= 20) self.assertTrue(extents["temp"] >= 20) self.streams.append(stream_id) self.stop_ingestion(stream_id)
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64"))) t_ctxt.uom = "seconds since 01-01-1970" context_ids.append( self.dataset_management.create_parameter_context(name="time", parameter_context=t_ctxt.dump()) ) cnd_ctxt = ParameterContext("conductivity", param_type=ArrayType()) cnd_ctxt.uom = "mmho/cm" context_ids.append( self.dataset_management.create_parameter_context(name="conductivity", parameter_context=cnd_ctxt.dump()) ) temp_ctxt = ParameterContext("temperature", param_type=ArrayType()) temp_ctxt.uom = "degC" context_ids.append( self.dataset_management.create_parameter_context(name="temperature", parameter_context=temp_ctxt.dump()) ) press_ctxt = ParameterContext("pressure", param_type=ArrayType()) press_ctxt.uom = "decibars" context_ids.append( self.dataset_management.create_parameter_context(name="pressure", parameter_context=press_ctxt.dump()) ) oxy_ctxt = ParameterContext("oxygen", param_type=ArrayType()) oxy_ctxt.uom = "Hz" context_ids.append( self.dataset_management.create_parameter_context(name="oxygen", parameter_context=oxy_ctxt.dump()) ) return context_ids
def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id) es.start() self.addCleanup(es.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_modified.wait(30)) query = { 'start_time': 0 - 2208988800, 'end_time': 20 - 2208988800, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0,20,2) == rdt['time'] self.assertTrue(comp.all(),'%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20) self.streams.append(stream_id) self.stop_ingestion(stream_id)
def build_context(self,record): context = ParameterContext(name=record['Name'], param_type=self.param_type(record['Parameter Type'])) context.uom = record['Unit of Measure'] if record['Fill Value']: context.fill_value = self.fill_value(record['Fill Value'], record['Parameter Type']) if record['Axis']: context.axis = self.ref_frame(record['Axis']) for key in self.additional_attrs.iterkeys(): if key in record and record[key]: setattr(context,self.additional_attrs[key],record[key]) return context
def build_context(self, record): context = ParameterContext(name=record['Name'], param_type=self.param_type( record['Parameter Type'])) context.uom = record['Unit of Measure'] if record['Fill Value']: context.fill_value = self.fill_value(record['Fill Value'], record['Parameter Type']) if record['Axis']: context.axis = self.ref_frame(record['Axis']) for key in self.additional_attrs.iterkeys(): if key in record and record[key]: setattr(context, self.additional_attrs[key], record[key]) return context
def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_monitor.wait()) query = { 'start_time': 0 - 2208988800, 'end_time': 19 - 2208988800, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2)) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20)
def get_param_dict(param_dict_name=None): raise NotImplementedError( 'This method has been replaced by DatasetManagementService, please use read_parameter_dictionary_by_name instead' ) # read the file just once, not every time needed global _PARAMETER_DICTIONARIES global _PARAMETER_CONTEXTS if not _PARAMETER_DICTIONARIES: param_dict_defs_file = "res/config/param_dict_defs.yml" with open(param_dict_defs_file, "r") as f_dict: dict_string = f_dict.read() _PARAMETER_DICTIONARIES = yaml.load(dict_string) param_context_defs_file = "res/config/param_context_defs.yml" with open(param_context_defs_file, "r") as f_ctxt: ctxt_string = f_ctxt.read() _PARAMETER_CONTEXTS = yaml.load(ctxt_string) # make sure we have the one requested context_names = _PARAMETER_DICTIONARIES[param_dict_name] for name in context_names: if not _PARAMETER_CONTEXTS.has_key(name): raise AssertionError( 'The parameter dict has a context that does not exist in the parameter context defs specified in yml: %s' % name) # package and ship pdict = ParameterDictionary() for ctxt_name in context_names: param_context = ParameterContext.load(_PARAMETER_CONTEXTS[ctxt_name]) pdict.add_context(param_context) return pdict
def get_param_dict(param_dict_name = None): raise NotImplementedError('This method has been replaced by DatasetManagementService, please use read_parameter_dictionary_by_name instead') # read the file just once, not every time needed global _PARAMETER_DICTIONARIES global _PARAMETER_CONTEXTS if not _PARAMETER_DICTIONARIES: param_dict_defs_file = "res/config/param_dict_defs.yml" with open(param_dict_defs_file, "r") as f_dict: dict_string = f_dict.read() _PARAMETER_DICTIONARIES = yaml.load(dict_string) param_context_defs_file = "res/config/param_context_defs.yml" with open(param_context_defs_file, "r") as f_ctxt: ctxt_string = f_ctxt.read() _PARAMETER_CONTEXTS = yaml.load(ctxt_string) # make sure we have the one requested context_names = _PARAMETER_DICTIONARIES[param_dict_name] for name in context_names: if not _PARAMETER_CONTEXTS.has_key(name): raise AssertionError('The parameter dict has a context that does not exist in the parameter context defs specified in yml: %s' % name) # package and ship pdict = ParameterDictionary() for ctxt_name in context_names: param_context = ParameterContext.load(_PARAMETER_CONTEXTS[ctxt_name]) pdict.add_context(param_context) return pdict
def _setup_resources(self): pdict = ParameterDictionary() t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) stream_id, stream_route, stream_def = self.create_stream_and_logger(name='fibonacci_stream', pdict=pdict) # tx = TaxyTool() # tx.add_taxonomy_set('data', 'external_data') self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'stream_def': stream_def, 'data_producer_id': 'fibonacci_data_producer_id', 'max_records': 4, }
def test_lookup_value_check(self): func = NumexprFunction('f', 'coeff_a * x', ['coeff_a', 'x'], param_map={ 'x': 'x', 'coeff_a': 'coeff_a' }) func.lookup_values = ['abc123'] test_context = ParameterContext('test', param_type=ParameterFunctionType(func)) tm = TypesManager(None, None, None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123'])
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())) cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType()) cnd_ctxt.uom = 'mmho/cm' context_ids.append(self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cnd_ctxt.dump())) temp_ctxt = ParameterContext('temperature', param_type=ArrayType()) temp_ctxt.uom = 'degC' context_ids.append(self.dataset_management.create_parameter_context(name='temperature', parameter_context=temp_ctxt.dump())) press_ctxt = ParameterContext('pressure', param_type=ArrayType()) press_ctxt.uom = 'decibars' context_ids.append(self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump())) oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType()) oxy_ctxt.uom = 'Hz' context_ids.append(self.dataset_management.create_parameter_context(name='oxygen', parameter_context=oxy_ctxt.dump())) return context_ids
def _create_parameter(self): pdict = ParameterDictionary() pdict = self._add_location_time_ctxt(pdict) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) return pdict
def _create_parameter_dictionary(self): pdict = ParameterDictionary() lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) return pdict
def rdt_to_granule(self, context, value_array, comp_val=None): time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict = ParameterDictionary() pdict.add_context(time, is_temporal=True) pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['time'] = np.arange(len(value_array)) rdt['test'] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2['test'] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual)
def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) self.addCleanup(shutil.rmtree, cov.persistence_dir) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0, len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())) ut_ctxt = ParameterContext('upload_time', param_type=QuantityType(value_encoding=np.dtype('int64'))) ut_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='upload_time', parameter_context=ut_ctxt.dump())) vela_ctxt = ParameterContext('VelA', param_type=ArrayType()) vela_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelA', parameter_context=vela_ctxt.dump())) velb_ctxt = ParameterContext('VelB', param_type=ArrayType()) velb_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelB', parameter_context=velb_ctxt.dump())) velc_ctxt = ParameterContext('VelC', param_type=ArrayType()) velc_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelC', parameter_context=velc_ctxt.dump())) veld_ctxt = ParameterContext('VelD', param_type=ArrayType()) veld_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelD', parameter_context=veld_ctxt.dump())) mx_ctxt = ParameterContext('Mx', param_type=ArrayType()) mx_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Mx', parameter_context=mx_ctxt.dump())) my_ctxt = ParameterContext('My', param_type=ArrayType()) my_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='My', parameter_context=my_ctxt.dump())) mz_ctxt = ParameterContext('Mz', param_type=ArrayType()) mz_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Mz', parameter_context=mz_ctxt.dump())) pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType()) pitch_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Pitch', parameter_context=pitch_ctxt.dump())) roll_ctxt = ParameterContext('Roll', param_type=ArrayType()) roll_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Roll', parameter_context=roll_ctxt.dump())) return context_ids
def test_build_granule_and_load_from_granule(self): pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) rdt = RecordDictionaryTool(param_dictionary=pdict) #Create some arrays and fill them with random values temp_array = np.random.standard_normal(100) cond_array = np.random.standard_normal(100) pres_array = np.random.standard_normal(100) time_array = np.random.standard_normal(100) lat_array = np.random.standard_normal(100) lon_array = np.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['conductivity'] = cond_array rdt['pres'] = pres_array rdt['time'] = time_array rdt['lat'] = lat_array rdt['lon'] = lon_array g = build_granule(data_producer_id='john', record_dictionary=rdt, param_dictionary=pdict) l_pd = ParameterDictionary.load(g.param_dictionary) #l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object #self.assertEquals(l_pd, pdict) self.assertEquals(l_pd.ord_from_key('temp'), pdict.ord_from_key('temp')) self.assertEquals(l_pd.ord_from_key('conductivity'), pdict.ord_from_key('conductivity')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) #self.assertEquals(l_rd._param_dict, rdt._param_dict) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, np.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_simplex_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop()
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='time', parameter_context=t_ctxt.dump())) ut_ctxt = ParameterContext( 'upload_time', param_type=QuantityType(value_encoding=np.dtype('int64'))) ut_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='upload_time', parameter_context=ut_ctxt.dump())) vela_ctxt = ParameterContext('VelA', param_type=ArrayType()) vela_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelA', parameter_context=vela_ctxt.dump())) velb_ctxt = ParameterContext('VelB', param_type=ArrayType()) velb_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelB', parameter_context=velb_ctxt.dump())) velc_ctxt = ParameterContext('VelC', param_type=ArrayType()) velc_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelC', parameter_context=velc_ctxt.dump())) veld_ctxt = ParameterContext('VelD', param_type=ArrayType()) veld_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelD', parameter_context=veld_ctxt.dump())) mx_ctxt = ParameterContext('Mx', param_type=ArrayType()) mx_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Mx', parameter_context=mx_ctxt.dump())) my_ctxt = ParameterContext('My', param_type=ArrayType()) my_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='My', parameter_context=my_ctxt.dump())) mz_ctxt = ParameterContext('Mz', param_type=ArrayType()) mz_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Mz', parameter_context=mz_ctxt.dump())) pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType()) pitch_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Pitch', parameter_context=pitch_ctxt.dump())) roll_ctxt = ParameterContext('Roll', param_type=ArrayType()) roll_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Roll', parameter_context=roll_ctxt.dump())) return context_ids
def get_param_dict(self): pdict = ParameterDictionary() cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float64)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float64)) pres_ctxt.uom = 'unknown' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=np.float64)) temp_ctxt.uom = 'unknown' temp_ctxt.fill_value = 0x0 pdict.add_context(temp_ctxt) oxy_ctxt = ParameterContext('oxygen', param_type=QuantityType(value_encoding=np.float64)) oxy_ctxt.uom = 'unknown' oxy_ctxt.fill_value = 0x0 pdict.add_context(oxy_ctxt) internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 pdict.add_context(internal_ts_ctxt, is_temporal=True) driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 pdict.add_context(driver_ts_ctxt) return pdict
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'usgs_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['vertical_dimension'] = 'z' dset.dataset_description.parameters['variables'] = [ 'water_temperature', 'streamflow', 'water_temperature_bottom', 'water_temperature_middle', 'specific_conductance', 'data_qualifier', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp") craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dprod = IonObject(RT.DataProduct, name='usgs_parsed_product', description='parsed usgs product', temporal_domain = tdom, spatial_domain = sdom) # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names # ttool = TaxyTool() # ttool.add_taxonomy_set('time','time') # ttool.add_taxonomy_set('lon','longitude') # ttool.add_taxonomy_set('lat','latitude') # ttool.add_taxonomy_set('z','water depth') # ttool.add_taxonomy_set('water_temperature', 'average water temperature') # ttool.add_taxonomy_set('water_temperature_bottom','water temperature at bottom of water column') # ttool.add_taxonomy_set('water_temperature_middle', 'water temperature at middle of water column') # ttool.add_taxonomy_set('streamflow', 'flow velocity of stream') # ttool.add_taxonomy_set('specific_conductance', 'specific conductance of water') # ttool.add_taxonomy_set('data_qualifier','data qualifier flag') # # ttool.add_taxonomy_set('coords','This group contains coordinate parameters') # ttool.add_taxonomy_set('data','This group contains data parameters') # Create the logger for receiving publications self.create_stream_and_logger(name='usgs',stream_id=stream_id) pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding = numpy.dtype('float32'))) temp_ctxt.uom = 'meters' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, #'taxonomy':ttool.dump(), 'param_dictionary':pdict.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':4, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example data agent', handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance( name='example dataset agent instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters[ 'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters[ 'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') #dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) pdict = ParameterDictionary() t_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): pc_list.append(dms_cli.create_parameter_context( pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition( name="ruv", description="stream def for ruv testing", parameter_dictionary_id=pdict_id) dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product') # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='ruv', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'external_dataset_res': dset, 'param_dictionary': pdict.dump(), 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
def setUp(self): self._tx = TaxyTool() self._tx.add_taxonomy_set('temp', 'long_temp_name') self._tx.add_taxonomy_set('cond', 'long_cond_name') self._tx.add_taxonomy_set('pres', 'long_pres_name') self._tx.add_taxonomy_set('rdt') self._tx.add_taxonomy_set('rdt2') # map is {<local name>: <granule name or path>} self._rdt = RecordDictionaryTool(taxonomy=self._tx) self._pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' self._pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' self._pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' self._pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' self._pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' self._pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' self._pdict.add_context(pres_ctxt) self._rdt_pdict = RecordDictionaryTool(param_dictionary=self._pdict)
def build_contexts(): ''' Builds the relevant parameter context objects ''' contexts = [] cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 contexts.append(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 contexts.append(pres_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 contexts.append(sal_ctxt) den_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=np.float32)) den_ctxt.uom = 'kg/m3' den_ctxt.fill_value = 0x0 contexts.append(den_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 contexts.append(temp_ctxt) t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 contexts.append(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 contexts.append(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 contexts.append(lon_ctxt) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.description = 'raw binary string values' raw_ctxt.uom = 'utf-8 byte string' raw_ctxt.fill_value = 0x0 contexts.append(raw_ctxt) port_ts_ctxt = ParameterContext( name='port_timestamp', param_type=QuantityType(value_encoding=np.float64)) port_ts_ctxt._derived_from_name = 'time' port_ts_ctxt.uom = 'seconds' port_ts_ctxt.fill_value = -1 contexts.append(port_ts_ctxt) driver_ts_ctxt = ParameterContext( name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 contexts.append(driver_ts_ctxt) internal_ts_ctxt = ParameterContext( name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 contexts.append(internal_ts_ctxt) timer_num_ctxt = ParameterContext( name='timer', param_type=QuantityType(value_encoding=np.float64)) timer_num_ctxt.fill_value = -1 contexts.append(timer_num_ctxt) serial_num_ctxt = ParameterContext( name='serial_num', param_type=QuantityType(value_encoding=np.int32)) serial_num_ctxt.fill_value = -1 contexts.append(serial_num_ctxt) count_ctxt = ParameterContext( name='counts', param_type=QuantityType(value_encoding=np.uint64)) count_ctxt.fill_value = -1 contexts.append(count_ctxt) checksum_ctxt = ParameterContext( name='checksum', param_type=QuantityType(value_encoding=np.int32)) checksum_ctxt.fill_value = -1 contexts.append(checksum_ctxt) pref_ts_ctxt = ParameterContext( name='preferred_timestamp', param_type=QuantityType(value_encoding=np.uint64)) pref_ts_ctxt.description = 'name of preferred timestamp' pref_ts_ctxt.fill_value = 0x0 contexts.append(pref_ts_ctxt) # TODO: This should probably be of type CategoryType when implemented qual_flag_ctxt = ParameterContext(name='quality_flag', param_type=ArrayType()) qual_flag_ctxt.description = 'flag indicating quality' qual_flag_ctxt.fill_value = None contexts.append(qual_flag_ctxt) viz_ts_ctxt = ParameterContext( name='viz_timestamp', param_type=QuantityType(value_encoding=np.float64)) viz_ts_ctxt._derived_from_name = 'time' viz_ts_ctxt.uom = 'seconds' viz_ts_ctxt.fill_value = -1 contexts.append(viz_ts_ctxt) viz_prod_type_ctxt = ParameterContext(name='viz_product_type', param_type=ArrayType()) viz_prod_type_ctxt.fill_value = None contexts.append(viz_prod_type_ctxt) image_obj_ctxt = ParameterContext(name='image_obj', param_type=ArrayType()) image_obj_ctxt.fill_value = None contexts.append(image_obj_ctxt) image_name_ctxt = ParameterContext(name='image_name', param_type=ArrayType()) contexts.append(image_name_ctxt) content_type_ctxt = ParameterContext(name='content_type', param_type=ArrayType()) contexts.append(content_type_ctxt) gdt_ctxt = ParameterContext(name='google_dt_components', param_type=RecordType()) contexts.append(gdt_ctxt) mpl_ctxt = ParameterContext(name='mpl_graph', param_type=RecordType()) contexts.append(mpl_ctxt) dummy_ctxt = ParameterContext( name='dummy', param_type=QuantityType(value_encoding=np.int64)) contexts.append(dummy_ctxt) return contexts
def create(self, path): mkdir_silent(path) tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT]) tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) dens_ctxt = ParameterContext('data_quantity', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) serial_ctxt = ParameterContext('data_array', param_type=ArrayType()) serial_ctxt.uom = 'unknown' serial_ctxt.fill_value = 0x0 pdict.add_context(serial_ctxt) guid = str(uuid.uuid4()).upper() self.path = path self.cov = SimplexCoverage(path, guid, name='test_cov', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream( 'replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage( dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber( self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop()
def create_parameters(cls): ''' WARNING: This method is a wrapper intended only for tests, it should not be used in production code. It probably will not align to most datasets. ''' pdict = ParameterDictionary() t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) data_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=np.int8)) data_ctxt.uom = 'byte' data_ctxt.fill_value = 0x0 pdict.add_context(data_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 pdict.add_context(sal_ctxt) dens_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) return pdict
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='Time_Time', parameter_context=t_ctxt.dump())) core_current_ctxt = ParameterContext('Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_current_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Core_Current', parameter_context=core_current_ctxt.dump())) core_voltage_ctxt = ParameterContext('Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_voltage_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Core_Voltage', parameter_context=core_voltage_ctxt.dump())) core_pressure_ctxt = ParameterContext('Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_pressure_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Core_Pressure', parameter_context=core_pressure_ctxt.dump())) fluorometer_value_ctxt = ParameterContext('Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) fluorometer_value_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump())) fluorometer_gain_ctxt = ParameterContext('Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) fluorometer_gain_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump())) turbidity_value_ctxt = ParameterContext('Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) turbidity_value_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump())) turbidity_gain_ctxt = ParameterContext('Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) turbidity_gain_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump())) optode_oxygen_ctxt = ParameterContext('Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_oxygen_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump())) optode_temp_ctxt = ParameterContext('Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_temp_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Temp', parameter_context=optode_temp_ctxt.dump())) par_value_ctxt = ParameterContext('Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) par_value_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Par_Value', parameter_context=par_value_ctxt.dump())) puck_scatter_ctxt = ParameterContext('Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_scatter_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump())) puck_chla_ctxt = ParameterContext('Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_chla_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Chla', parameter_context=puck_chla_ctxt.dump())) puck_cdom_ctxt = ParameterContext('Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_cdom_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump())) biosuite_scatter_ctxt = ParameterContext('BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_scatter_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump())) biosuite_chla_ctxt = ParameterContext('BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_chla_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump())) biosuite_cdom_ctxt = ParameterContext('BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_cdom_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump())) biosuite_temp_ctxt = ParameterContext('BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_temp_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump())) biosuite_par_ctxt = ParameterContext('BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_par_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump())) flbb_chla_ctxt = ParameterContext('FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_chla_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump())) flbb_turb_ctxt = ParameterContext('FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_turb_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump())) flbb_temp_ctxt = ParameterContext('FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_temp_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump())) return context_ids
def load(): from coverage_model.parameter import ParameterContext from coverage_model.parameter_types import QuantityType, ArrayType, RecordType from coverage_model.basic_types import AxisTypeEnum import numpy as np contexts = [] cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 contexts.append(cond_ctxt) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 contexts.append(pres_ctxt) sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 contexts.append(sal_ctxt) den_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=np.float32)) den_ctxt.uom = 'kg/m3' den_ctxt.fill_value = 0x0 contexts.append(den_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 contexts.append(temp_ctxt) t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 contexts.append(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 contexts.append(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 contexts.append(lon_ctxt) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.description = 'raw binary string values' raw_ctxt.uom = 'utf-8 byte string' contexts.append(raw_ctxt) port_ts_ctxt = ParameterContext(name='port_timestamp', param_type=QuantityType(value_encoding=np.float64)) port_ts_ctxt._derived_from_name = 'time' port_ts_ctxt.uom = 'seconds' port_ts_ctxt.fill_value = -1 contexts.append(port_ts_ctxt) driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 contexts.append(driver_ts_ctxt) internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 contexts.append(internal_ts_ctxt) timer_num_ctxt = ParameterContext(name='timer', param_type=QuantityType(value_encoding=np.float64)) timer_num_ctxt.fill_value = -1 contexts.append(timer_num_ctxt) serial_num_ctxt = ParameterContext(name='serial_num', param_type=QuantityType(value_encoding=np.int32)) serial_num_ctxt.fill_value = -1 contexts.append(serial_num_ctxt) count_ctxt = ParameterContext(name='counts', param_type=QuantityType(value_encoding=np.uint64)) count_ctxt.fill_value = -1 contexts.append(count_ctxt) checksum_ctxt = ParameterContext(name='checksum', param_type=QuantityType(value_encoding=np.int32)) checksum_ctxt.fill_value = -1 contexts.append(checksum_ctxt) pref_ts_ctxt = ParameterContext(name='preferred_timestamp', param_type=ArrayType()) pref_ts_ctxt.description = 'name of preferred timestamp' contexts.append(pref_ts_ctxt) # TODO: This should probably be of type CategoryType when implemented qual_flag_ctxt = ParameterContext(name='quality_flag', param_type=ArrayType()) qual_flag_ctxt.description = 'flag indicating quality' contexts.append(qual_flag_ctxt) viz_ts_ctxt = ParameterContext(name='viz_timestamp', param_type=QuantityType(value_encoding=np.float64)) viz_ts_ctxt._derived_from_name = 'time' viz_ts_ctxt.uom = 'seconds' viz_ts_ctxt.fill_value = -1 contexts.append(viz_ts_ctxt) viz_prod_type_ctxt = ParameterContext(name='viz_product_type', param_type=ArrayType()) contexts.append(viz_prod_type_ctxt) image_obj_ctxt = ParameterContext(name='image_obj', param_type=ArrayType()) contexts.append(image_obj_ctxt) image_name_ctxt = ParameterContext(name='image_name', param_type=ArrayType()) contexts.append(image_name_ctxt) content_type_ctxt = ParameterContext(name='content_type', param_type=ArrayType()) contexts.append(content_type_ctxt) gdt_ctxt = ParameterContext(name='google_dt_components', param_type=RecordType()) contexts.append(gdt_ctxt) mpl_ctxt = ParameterContext(name='mpl_graph', param_type=RecordType()) contexts.append(mpl_ctxt) dummy_ctxt = ParameterContext(name='dummy', param_type=QuantityType(value_encoding=np.int64)) contexts.append(dummy_ctxt) return contexts
def create_contexts(self): context_ids = [] cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump())) pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump())) sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump())) temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump())) t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump())) return context_ids
def create_parameters(cls): ''' WARNING: This method is a wrapper intended only for tests, it should not be used in production code. It probably will not align to most datasets. ''' pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) data_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=np.int8)) data_ctxt.uom = 'byte' data_ctxt.fill_value = 0x0 pdict.add_context(data_ctxt) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 pdict.add_context(sal_ctxt) dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) return pdict
def _make_coverage(path): tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT]) tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) cat = {0:'lemon',1:'apple',2:'banana',99:'None'} cat_ctxt = ParameterContext('category', param_type=CategoryType(categories=cat)) cat_ctxt.long_name = "example of category" pdict.add_context(cat_ctxt) dens_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) const_ctxt = ParameterContext('constant', param_type=ConstantType()) const_ctxt.long_name = 'example of a parameter of type ConstantType' pdict.add_context(const_ctxt) rec_ctxt = ParameterContext('boolean', param_type=BooleanType()) rec_ctxt.long_name = 'example of a parameter of type BooleanType' pdict.add_context(rec_ctxt) rec_ctxt = ParameterContext('range', param_type=ConstantRangeType()) rec_ctxt.long_name = 'Range example' rec_ctxt.fill_value = 0x0 pdict.add_context(rec_ctxt) rec_ctxt = ParameterContext('record', param_type=RecordType()) rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries' rec_ctxt.fill_value = 0x0 pdict.add_context(rec_ctxt) serial_ctxt = ParameterContext('array', param_type=ArrayType()) serial_ctxt.uom = 'unknown' serial_ctxt.fill_value = 0x0 pdict.add_context(serial_ctxt) guid = create_guid() guid = guid.replace("-", "") cov = SimplexCoverage(path, guid, name="sample_cov", parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) return (cov,path+os.sep+guid)
def defining_parameter_dictionary(self): # Define the parameter context objects t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 height_ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=np.float32)) height_ctxt.reference_frame = AxisTypeEnum.HEIGHT height_ctxt.uom = 'meters' height_ctxt.fill_value = 0e0 pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'degree_Celsius' pres_ctxt.fill_value = 0e0 temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 cond_ctxt = ParameterContext('cond', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 data_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=np.int8)) data_ctxt.uom = 'byte' data_ctxt.fill_value = 0x0 # Define the parameter dictionary objects self.pres = ParameterDictionary() self.pres.add_context(t_ctxt) self.pres.add_context(lat_ctxt) self.pres.add_context(lon_ctxt) self.pres.add_context(height_ctxt) self.pres.add_context(pres_ctxt) self.pres.add_context(data_ctxt) self.temp = ParameterDictionary() self.temp.add_context(t_ctxt) self.temp.add_context(lat_ctxt) self.temp.add_context(lon_ctxt) self.temp.add_context(height_ctxt) self.temp.add_context(temp_ctxt) self.temp.add_context(data_ctxt) self.cond = ParameterDictionary() self.cond.add_context(t_ctxt) self.cond.add_context(lat_ctxt) self.cond.add_context(lon_ctxt) self.cond.add_context(height_ctxt) self.cond.add_context(cond_ctxt) self.cond.add_context(data_ctxt)
def get_param_dict(self): pdict = ParameterDictionary() cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float64)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float64)) pres_ctxt.uom = 'unknown' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temperature', param_type=QuantityType(value_encoding=np.float64)) temp_ctxt.uom = 'unknown' temp_ctxt.fill_value = 0x0 pdict.add_context(temp_ctxt) oxy_ctxt = ParameterContext( 'oxygen', param_type=QuantityType(value_encoding=np.float64)) oxy_ctxt.uom = 'unknown' oxy_ctxt.fill_value = 0x0 pdict.add_context(oxy_ctxt) internal_ts_ctxt = ParameterContext( name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 pdict.add_context(internal_ts_ctxt, is_temporal=True) driver_ts_ctxt = ParameterContext( name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 pdict.add_context(driver_ts_ctxt) return pdict
def _create_parameter_dictionary(self): pdict = ParameterDictionary() lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'meters' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) return pdict
def _create_parameter(self): pdict = ParameterDictionary() pdict = self._add_location_time_ctxt(pdict) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=numpy.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=numpy.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=numpy.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) raw_fixed_ctxt = ParameterContext( 'raw_fixed', param_type=QuantityType(value_encoding=numpy.float32)) raw_fixed_ctxt.uom = 'unknown' raw_fixed_ctxt.fill_value = 0e0 pdict.add_context(raw_fixed_ctxt) raw_blob_ctxt = ParameterContext( 'raw_blob', param_type=QuantityType(value_encoding=numpy.float32)) raw_blob_ctxt.uom = 'unknown' raw_blob_ctxt.fill_value = 0e0 pdict.add_context(raw_blob_ctxt) return pdict
def _create_parameter_dictionary(self): pdict = ParameterDictionary() t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) return pdict
def _add_location_time_ctxt(self, pdict): t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) return pdict
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='time', parameter_context=t_ctxt.dump())) cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType( )) # param_type=QuantityType(value_encoding=np.dtype('float32'))) cnd_ctxt.uom = 'mmho/cm' context_ids.append( self.dataset_management.create_parameter_context( name='conductivity', parameter_context=cnd_ctxt.dump())) temp_ctxt = ParameterContext('temperature', param_type=ArrayType( )) # param_type=QuantityType(value_encoding=np.dtype('float32'))) temp_ctxt.uom = 'degC' context_ids.append( self.dataset_management.create_parameter_context( name='temperature', parameter_context=temp_ctxt.dump())) press_ctxt = ParameterContext('pressure', param_type=ArrayType( )) # param_type=QuantityType(value_encoding=np.dtype('float32'))) press_ctxt.uom = 'decibars' context_ids.append( self.dataset_management.create_parameter_context( name='pressure', parameter_context=press_ctxt.dump())) oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType( )) # param_type=QuantityType(value_encoding=np.dtype('float32'))) oxy_ctxt.uom = 'Hz' context_ids.append( self.dataset_management.create_parameter_context( name='oxygen', parameter_context=oxy_ctxt.dump())) return context_ids
def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,40) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) subscriber.start() self.addCleanup(subscriber.stop) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b,bool) else b)
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='Time_Time', parameter_context=t_ctxt.dump())) core_current_ctxt = ParameterContext( 'Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_current_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Current', parameter_context=core_current_ctxt.dump())) core_voltage_ctxt = ParameterContext( 'Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_voltage_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Voltage', parameter_context=core_voltage_ctxt.dump())) core_pressure_ctxt = ParameterContext( 'Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_pressure_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Pressure', parameter_context=core_pressure_ctxt.dump())) fluorometer_value_ctxt = ParameterContext( 'Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) fluorometer_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump())) fluorometer_gain_ctxt = ParameterContext( 'Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) fluorometer_gain_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump())) turbidity_value_ctxt = ParameterContext( 'Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) turbidity_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump())) turbidity_gain_ctxt = ParameterContext( 'Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) turbidity_gain_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump())) optode_oxygen_ctxt = ParameterContext( 'Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_oxygen_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump())) optode_temp_ctxt = ParameterContext( 'Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Optode_Temp', parameter_context=optode_temp_ctxt.dump())) par_value_ctxt = ParameterContext( 'Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) par_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Par_Value', parameter_context=par_value_ctxt.dump())) puck_scatter_ctxt = ParameterContext( 'Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_scatter_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump())) puck_chla_ctxt = ParameterContext( 'Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_Chla', parameter_context=puck_chla_ctxt.dump())) puck_cdom_ctxt = ParameterContext( 'Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_cdom_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump())) biosuite_scatter_ctxt = ParameterContext( 'BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_scatter_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump())) biosuite_chla_ctxt = ParameterContext( 'BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump())) biosuite_cdom_ctxt = ParameterContext( 'BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_cdom_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump())) biosuite_temp_ctxt = ParameterContext( 'BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump())) biosuite_par_ctxt = ParameterContext( 'BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_par_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump())) flbb_chla_ctxt = ParameterContext( 'FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump())) flbb_turb_ctxt = ParameterContext( 'FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_turb_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump())) flbb_temp_ctxt = ParameterContext( 'FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_wpt_y_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_water_cond", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_y_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext( "u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_hd_fin_ap_inflection_holdoff", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_m_present_time", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext( "m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="m_leakdetect_voltage_forward", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_bb3slo_b660_scaled", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_science_send_all", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_gps_status", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_water_vx", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_water_vy", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_heading", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_fl3slo_chlor_units", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_hd_fin_ap_gain", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_vacuum", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_min_water_depth", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_gps_lat", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_veh_temp", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="f_fin_offset", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext( "u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_hd_fin_ap_hardover_holdoff", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_alt_time", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_present_time", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_heading", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_bb3slo_b532_scaled", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_fl3slo_cdom_units", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_fin", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="x_cycle_overrun_in_ms", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_water_pressure", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_hd_fin_ap_igain", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_fl3slo_phyco_units", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_battpos", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_bb3slo_b470_scaled", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_lat", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_gps_lon", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_ctd41cp_timestamp", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_pressure", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_wpt_x_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_ballast_pumped", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="x_lmc_xy_source", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_lon", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_avg_speed", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_water_temp", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_pitch_ap_gain", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_roll", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_tot_num_inflections", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="m_tot_num_inflections", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_x_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_pitch_ap_deadband", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_final_water_vy", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_final_water_vx", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_water_depth", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_leakdetect_voltage", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="m_leakdetect_voltage", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext( "u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_pitch_max_delta_battpos", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_coulomb_amphr", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_pitch", parameter_context=t_ctxt.dump()) ) return context_ids
def _add_location_time_ctxt(self, pdict): t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=numpy.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=numpy.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=numpy.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) return pdict
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters['list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters['date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ ] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product', temporal_domain = tdom, spatial_domain = sdom) streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp") # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names #ttool = TaxyTool() # #ttool.add_taxonomy_set('data','test data') pdict = ParameterDictionary() t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name='ruv',stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, 'external_dataset_res':dset, 'param_dictionary':pdict.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':20, }
def get_context(self, ptype, encoding, fill_value, codeset=None): ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset) context = ParameterContext(name='test', param_type=ptype) context.fill_value = self.types_manager.get_fill_value(fill_value, encoding, ptype) return context
def get_context(self, ptype, encoding, fill_value, codeset=None): ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset) context = ParameterContext(name='test', param_type=ptype) context.fill_value = self.types_manager.get_fill_value( fill_value, encoding, ptype) return context
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = "Christopher Mueller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="FILE", institution=Institution(), contact=ContactInformation()) dsrc.connection_params["base_data_url"] = "" dsrc.contact.name = "Tim Giguere" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset ds_name = "slocum_test_dataset" dset = ExternalDataset( name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation(), ) dset.dataset_description.parameters["base_url"] = "test_data/slocum/" dset.dataset_description.parameters["list_pattern"] = "ru05-2012-021-0-0-sbd.dat" dset.dataset_description.parameters["date_pattern"] = "%Y %j" dset.dataset_description.parameters["date_extraction_pattern"] = "ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat" dset.dataset_description.parameters["temporal_dimension"] = None dset.dataset_description.parameters["zonal_dimension"] = None dset.dataset_description.parameters["meridional_dimension"] = None dset.dataset_description.parameters["vertical_dimension"] = None dset.dataset_description.parameters["variables"] = [ "c_wpt_y_lmc", "sci_water_cond", "m_y_lmc", "u_hd_fin_ap_inflection_holdoff", "sci_m_present_time", "m_leakdetect_voltage_forward", "sci_bb3slo_b660_scaled", "c_science_send_all", "m_gps_status", "m_water_vx", "m_water_vy", "c_heading", "sci_fl3slo_chlor_units", "u_hd_fin_ap_gain", "m_vacuum", "u_min_water_depth", "m_gps_lat", "m_veh_temp", "f_fin_offset", "u_hd_fin_ap_hardover_holdoff", "c_alt_time", "m_present_time", "m_heading", "sci_bb3slo_b532_scaled", "sci_fl3slo_cdom_units", "m_fin", "x_cycle_overrun_in_ms", "sci_water_pressure", "u_hd_fin_ap_igain", "sci_fl3slo_phyco_units", "m_battpos", "sci_bb3slo_b470_scaled", "m_lat", "m_gps_lon", "sci_ctd41cp_timestamp", "m_pressure", "c_wpt_x_lmc", "c_ballast_pumped", "x_lmc_xy_source", "m_lon", "m_avg_speed", "sci_water_temp", "u_pitch_ap_gain", "m_roll", "m_tot_num_inflections", "m_x_lmc", "u_pitch_ap_deadband", "m_final_water_vy", "m_final_water_vx", "m_water_depth", "m_leakdetect_voltage", "u_pitch_max_delta_battpos", "m_coulomb_amphr", "m_pitch", ] # Create DataSourceModel dsrc_model = DataSourceModel(name="slocum_model") dsrc_model.model = "SLOCUM" dsrc_model.data_handler_module = "N/A" dsrc_model.data_handler_class = "N/A" ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id ) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # create temp streamdef so the data product can create the stream streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp") # Generate the data product and associate it to the ExternalDataset craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dprod = IonObject( RT.DataProduct, name="slocum_parsed_product", description="parsed slocum product", temporal_domain=tdom, spatial_domain=sdom, ) dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary ) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects( subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True ) stream_id = stream_id[0] log.info( "Created resources: {0}".format( { "ExternalDataset": ds_id, "ExternalDataProvider": ext_dprov_id, "DataSource": ext_dsrc_id, "DataSourceModel": ext_dsrc_model_id, "DataProducer": dproducer_id, "DataProduct": dproduct_id, "Stream": stream_id, } ) ) # CBM: Use CF standard_names # ttool = TaxyTool() # # ttool.add_taxonomy_set('c_wpt_y_lmc'), # ttool.add_taxonomy_set('sci_water_cond'), # ttool.add_taxonomy_set('m_y_lmc'), # ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'), # ttool.add_taxonomy_set('sci_m_present_time'), # ttool.add_taxonomy_set('m_leakdetect_voltage_forward'), # ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'), # ttool.add_taxonomy_set('c_science_send_all'), # ttool.add_taxonomy_set('m_gps_status'), # ttool.add_taxonomy_set('m_water_vx'), # ttool.add_taxonomy_set('m_water_vy'), # ttool.add_taxonomy_set('c_heading'), # ttool.add_taxonomy_set('sci_fl3slo_chlor_units'), # ttool.add_taxonomy_set('u_hd_fin_ap_gain'), # ttool.add_taxonomy_set('m_vacuum'), # ttool.add_taxonomy_set('u_min_water_depth'), # ttool.add_taxonomy_set('m_gps_lat'), # ttool.add_taxonomy_set('m_veh_temp'), # ttool.add_taxonomy_set('f_fin_offset'), # ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'), # ttool.add_taxonomy_set('c_alt_time'), # ttool.add_taxonomy_set('m_present_time'), # ttool.add_taxonomy_set('m_heading'), # ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'), # ttool.add_taxonomy_set('sci_fl3slo_cdom_units'), # ttool.add_taxonomy_set('m_fin'), # ttool.add_taxonomy_set('x_cycle_overrun_in_ms'), # ttool.add_taxonomy_set('sci_water_pressure'), # ttool.add_taxonomy_set('u_hd_fin_ap_igain'), # ttool.add_taxonomy_set('sci_fl3slo_phyco_units'), # ttool.add_taxonomy_set('m_battpos'), # ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'), # ttool.add_taxonomy_set('m_lat'), # ttool.add_taxonomy_set('m_gps_lon'), # ttool.add_taxonomy_set('sci_ctd41cp_timestamp'), # ttool.add_taxonomy_set('m_pressure'), # ttool.add_taxonomy_set('c_wpt_x_lmc'), # ttool.add_taxonomy_set('c_ballast_pumped'), # ttool.add_taxonomy_set('x_lmc_xy_source'), # ttool.add_taxonomy_set('m_lon'), # ttool.add_taxonomy_set('m_avg_speed'), # ttool.add_taxonomy_set('sci_water_temp'), # ttool.add_taxonomy_set('u_pitch_ap_gain'), # ttool.add_taxonomy_set('m_roll'), # ttool.add_taxonomy_set('m_tot_num_inflections'), # ttool.add_taxonomy_set('m_x_lmc'), # ttool.add_taxonomy_set('u_pitch_ap_deadband'), # ttool.add_taxonomy_set('m_final_water_vy'), # ttool.add_taxonomy_set('m_final_water_vx'), # ttool.add_taxonomy_set('m_water_depth'), # ttool.add_taxonomy_set('m_leakdetect_voltage'), # ttool.add_taxonomy_set('u_pitch_max_delta_battpos'), # ttool.add_taxonomy_set('m_coulomb_amphr'), # ttool.add_taxonomy_set('m_pitch'), pdict = ParameterDictionary() t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "m_tot_num_inflections", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "m_leakdetect_voltage", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) # CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name="slocum", stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG["dh_cfg"] = { "TESTING": True, "stream_id": stream_id, "external_dataset_res": dset, "param_dictionary": pdict.dump(), "data_producer_id": dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? "max_records": 20, }
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_y_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_cond', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_y_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_inflection_holdoff', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_m_present_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage_forward', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b660_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_science_send_all', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_status', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vx', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vy', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_heading', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_chlor_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_gain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_vacuum', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_min_water_depth', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lat', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_veh_temp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='f_fin_offset', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_hardover_holdoff', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_alt_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_present_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_heading', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b532_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_cdom_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_fin', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='x_cycle_overrun_in_ms', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_pressure', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_igain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_phyco_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_battpos', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b470_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_lat', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lon', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_ctd41cp_timestamp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_pressure', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_x_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_ballast_pumped', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='x_lmc_xy_source', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_lon', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_avg_speed', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_temp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_gain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_roll', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_tot_num_inflections', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_x_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_deadband', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vy', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vx', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_depth', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_max_delta_battpos', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_coulomb_amphr', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_pitch', parameter_context=t_ctxt.dump())) return context_ids
def adhoc_get_parameter_dictionary(stream_name): """ @param stream_name IGNORED in this adhoc function; it returns the same ParameterDictionary definition always. @retval corresponding ParameterDictionary. """ #@TODO Luke - Maybe we can make this a bit more versatile, we could make this a standard pdict... pdict = ParameterDictionary() # ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.float32)) ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) ctxt.uom = 'unknown' ctxt.fill_value = 0e0 pdict.add_context(ctxt) ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) ctxt.axis = AxisTypeEnum.TIME ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(ctxt) ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.LON ctxt.uom = 'degree_east' pdict.add_context(ctxt) ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.LAT ctxt.uom = 'degree_north' pdict.add_context(ctxt) ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.HEIGHT ctxt.uom = 'unknown' pdict.add_context(ctxt) return pdict