def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='time', parameter_context=t_ctxt.dump())) cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType()) cnd_ctxt.uom = 'mmho/cm' context_ids.append( self.dataset_management.create_parameter_context( name='conductivity', parameter_context=cnd_ctxt.dump())) temp_ctxt = ParameterContext('temperature', param_type=ArrayType()) temp_ctxt.uom = 'degC' context_ids.append( self.dataset_management.create_parameter_context( name='temperature', parameter_context=temp_ctxt.dump())) press_ctxt = ParameterContext('pressure', param_type=ArrayType()) press_ctxt.uom = 'decibars' context_ids.append( self.dataset_management.create_parameter_context( name='pressure', parameter_context=press_ctxt.dump())) oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType()) oxy_ctxt.uom = 'Hz' context_ids.append( self.dataset_management.create_parameter_context( name='oxygen', parameter_context=oxy_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_y_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_cond', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_y_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_inflection_holdoff', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_m_present_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage_forward', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b660_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_science_send_all', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_status', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vx', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vy', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_heading', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_chlor_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_gain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_vacuum', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_min_water_depth', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lat', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_veh_temp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='f_fin_offset', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_hardover_holdoff', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_alt_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_present_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_heading', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b532_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_cdom_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_fin', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='x_cycle_overrun_in_ms', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_pressure', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_igain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_phyco_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_battpos', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b470_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_lat', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lon', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_ctd41cp_timestamp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_pressure', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_x_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_ballast_pumped', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='x_lmc_xy_source', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_lon', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_avg_speed', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_temp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_gain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_roll', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_tot_num_inflections', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_x_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_deadband', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vy', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vx', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_depth', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_max_delta_battpos', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_coulomb_amphr', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_pitch', parameter_context=t_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())) cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType()) cnd_ctxt.uom = 'mmho/cm' context_ids.append(self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cnd_ctxt.dump())) temp_ctxt = ParameterContext('temperature', param_type=ArrayType()) temp_ctxt.uom = 'degC' context_ids.append(self.dataset_management.create_parameter_context(name='temperature', parameter_context=temp_ctxt.dump())) press_ctxt = ParameterContext('pressure', param_type=ArrayType()) press_ctxt.uom = 'decibars' context_ids.append(self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump())) oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType()) oxy_ctxt.uom = 'Hz' context_ids.append(self.dataset_management.create_parameter_context(name='oxygen', parameter_context=oxy_ctxt.dump())) return context_ids
def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_monitor.wait()) query = { 'start_time': 0 - 2208988800, 'end_time': 19 - 2208988800, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2)) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20)
def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,40) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) subscriber.start() self.addCleanup(subscriber.stop) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b,bool) else b)
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='Time_Time', parameter_context=t_ctxt.dump())) core_current_ctxt = ParameterContext( 'Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_current_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Current', parameter_context=core_current_ctxt.dump())) core_voltage_ctxt = ParameterContext( 'Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_voltage_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Voltage', parameter_context=core_voltage_ctxt.dump())) core_pressure_ctxt = ParameterContext( 'Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_pressure_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Pressure', parameter_context=core_pressure_ctxt.dump())) fluorometer_value_ctxt = ParameterContext( 'Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) fluorometer_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump())) fluorometer_gain_ctxt = ParameterContext( 'Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) fluorometer_gain_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump())) turbidity_value_ctxt = ParameterContext( 'Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) turbidity_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump())) turbidity_gain_ctxt = ParameterContext( 'Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) turbidity_gain_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump())) optode_oxygen_ctxt = ParameterContext( 'Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_oxygen_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump())) optode_temp_ctxt = ParameterContext( 'Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Optode_Temp', parameter_context=optode_temp_ctxt.dump())) par_value_ctxt = ParameterContext( 'Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) par_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Par_Value', parameter_context=par_value_ctxt.dump())) puck_scatter_ctxt = ParameterContext( 'Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_scatter_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump())) puck_chla_ctxt = ParameterContext( 'Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_Chla', parameter_context=puck_chla_ctxt.dump())) puck_cdom_ctxt = ParameterContext( 'Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_cdom_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump())) biosuite_scatter_ctxt = ParameterContext( 'BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_scatter_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump())) biosuite_chla_ctxt = ParameterContext( 'BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump())) biosuite_cdom_ctxt = ParameterContext( 'BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_cdom_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump())) biosuite_temp_ctxt = ParameterContext( 'BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump())) biosuite_par_ctxt = ParameterContext( 'BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_par_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump())) flbb_chla_ctxt = ParameterContext( 'FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump())) flbb_turb_ctxt = ParameterContext( 'FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_turb_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump())) flbb_temp_ctxt = ParameterContext( 'FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='time', parameter_context=t_ctxt.dump())) ut_ctxt = ParameterContext( 'upload_time', param_type=QuantityType(value_encoding=np.dtype('int64'))) ut_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='upload_time', parameter_context=ut_ctxt.dump())) vela_ctxt = ParameterContext('VelA', param_type=ArrayType()) vela_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelA', parameter_context=vela_ctxt.dump())) velb_ctxt = ParameterContext('VelB', param_type=ArrayType()) velb_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelB', parameter_context=velb_ctxt.dump())) velc_ctxt = ParameterContext('VelC', param_type=ArrayType()) velc_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelC', parameter_context=velc_ctxt.dump())) veld_ctxt = ParameterContext('VelD', param_type=ArrayType()) veld_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelD', parameter_context=veld_ctxt.dump())) mx_ctxt = ParameterContext('Mx', param_type=ArrayType()) mx_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Mx', parameter_context=mx_ctxt.dump())) my_ctxt = ParameterContext('My', param_type=ArrayType()) my_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='My', parameter_context=my_ctxt.dump())) mz_ctxt = ParameterContext('Mz', param_type=ArrayType()) mz_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Mz', parameter_context=mz_ctxt.dump())) pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType()) pitch_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Pitch', parameter_context=pitch_ctxt.dump())) roll_ctxt = ParameterContext('Roll', param_type=ArrayType()) roll_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Roll', parameter_context=roll_ctxt.dump())) return context_ids
def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_simplex_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop()
def test_replay_with_parameters(self): # -------------------------------------------------------------------------------- # Create the configurations and the dataset # -------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext("binary", param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext("records", param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( "replay_pdict", parameter_context_ids=context_ids, temporal_context="time" ) stream_def_id = self.pubsub_management.create_stream_definition( "replay_stream", parameter_dictionary_id=pdict_id ) stream_id, route = self.pubsub_management.create_stream( "replay_with_params", exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id ) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Coerce the datastore into existence (beats race condition) # -------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 40) query = { "start_time": 0 - 2208988800, "end_time": 20 - 2208988800, "stride_time": 2, "parameters": ["time", "temp"], } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0, 20, 2) == rdt["time"] self.assertTrue(comp.all(), "%s" % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=["time", "temp"]) self.assertTrue(extents["time"] >= 20) self.assertTrue(extents["temp"] >= 20) self.streams.append(stream_id) self.stop_ingestion(stream_id)
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_y_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_cond', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_y_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_inflection_holdoff', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_m_present_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage_forward', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b660_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_science_send_all', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_status', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vx', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vy', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_heading', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_chlor_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_gain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_vacuum', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_min_water_depth', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lat', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_veh_temp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='f_fin_offset', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_hardover_holdoff', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_alt_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_present_time', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_heading', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b532_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_cdom_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_fin', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='x_cycle_overrun_in_ms', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_pressure', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_igain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_phyco_units', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_battpos', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b470_scaled', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_lat', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lon', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_ctd41cp_timestamp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_pressure', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_x_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='c_ballast_pumped', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='x_lmc_xy_source', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_lon', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_avg_speed', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_temp', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_gain', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_roll', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_tot_num_inflections', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_x_lmc', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_deadband', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vy', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vx', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_water_depth', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_max_delta_battpos', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_coulomb_amphr', parameter_context=t_ctxt.dump())) t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=np.dtype('float32'))) t_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='m_pitch', parameter_context=t_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='Time_Time', parameter_context=t_ctxt.dump())) core_current_ctxt = ParameterContext('Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_current_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Core_Current', parameter_context=core_current_ctxt.dump())) core_voltage_ctxt = ParameterContext('Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_voltage_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Core_Voltage', parameter_context=core_voltage_ctxt.dump())) core_pressure_ctxt = ParameterContext('Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_pressure_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Core_Pressure', parameter_context=core_pressure_ctxt.dump())) fluorometer_value_ctxt = ParameterContext('Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) fluorometer_value_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump())) fluorometer_gain_ctxt = ParameterContext('Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) fluorometer_gain_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump())) turbidity_value_ctxt = ParameterContext('Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) turbidity_value_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump())) turbidity_gain_ctxt = ParameterContext('Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) turbidity_gain_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump())) optode_oxygen_ctxt = ParameterContext('Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_oxygen_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump())) optode_temp_ctxt = ParameterContext('Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_temp_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Temp', parameter_context=optode_temp_ctxt.dump())) par_value_ctxt = ParameterContext('Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) par_value_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Par_Value', parameter_context=par_value_ctxt.dump())) puck_scatter_ctxt = ParameterContext('Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_scatter_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump())) puck_chla_ctxt = ParameterContext('Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_chla_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Chla', parameter_context=puck_chla_ctxt.dump())) puck_cdom_ctxt = ParameterContext('Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_cdom_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump())) biosuite_scatter_ctxt = ParameterContext('BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_scatter_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump())) biosuite_chla_ctxt = ParameterContext('BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_chla_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump())) biosuite_cdom_ctxt = ParameterContext('BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_cdom_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump())) biosuite_temp_ctxt = ParameterContext('BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_temp_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump())) biosuite_par_ctxt = ParameterContext('BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_par_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump())) flbb_chla_ctxt = ParameterContext('FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_chla_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump())) flbb_turb_ctxt = ParameterContext('FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_turb_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump())) flbb_temp_ctxt = ParameterContext('FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_temp_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())) ut_ctxt = ParameterContext('upload_time', param_type=QuantityType(value_encoding=np.dtype('int64'))) ut_ctxt.uom = 'seconds since 01-01-1970' context_ids.append(self.dataset_management.create_parameter_context(name='upload_time', parameter_context=ut_ctxt.dump())) vela_ctxt = ParameterContext('VelA', param_type=ArrayType()) vela_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelA', parameter_context=vela_ctxt.dump())) velb_ctxt = ParameterContext('VelB', param_type=ArrayType()) velb_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelB', parameter_context=velb_ctxt.dump())) velc_ctxt = ParameterContext('VelC', param_type=ArrayType()) velc_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelC', parameter_context=velc_ctxt.dump())) veld_ctxt = ParameterContext('VelD', param_type=ArrayType()) veld_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='VelD', parameter_context=veld_ctxt.dump())) mx_ctxt = ParameterContext('Mx', param_type=ArrayType()) mx_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Mx', parameter_context=mx_ctxt.dump())) my_ctxt = ParameterContext('My', param_type=ArrayType()) my_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='My', parameter_context=my_ctxt.dump())) mz_ctxt = ParameterContext('Mz', param_type=ArrayType()) mz_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Mz', parameter_context=mz_ctxt.dump())) pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType()) pitch_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Pitch', parameter_context=pitch_ctxt.dump())) roll_ctxt = ParameterContext('Roll', param_type=ArrayType()) roll_ctxt.uom = 'unknown' context_ids.append(self.dataset_management.create_parameter_context(name='Roll', parameter_context=roll_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_wpt_y_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_water_cond", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_y_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext( "u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_hd_fin_ap_inflection_holdoff", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_m_present_time", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext( "m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="m_leakdetect_voltage_forward", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_bb3slo_b660_scaled", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_science_send_all", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_gps_status", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_water_vx", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_water_vy", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_heading", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_fl3slo_chlor_units", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_hd_fin_ap_gain", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_vacuum", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_min_water_depth", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_gps_lat", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_veh_temp", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="f_fin_offset", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext( "u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_hd_fin_ap_hardover_holdoff", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_alt_time", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_present_time", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_heading", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_bb3slo_b532_scaled", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_fl3slo_cdom_units", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_fin", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="x_cycle_overrun_in_ms", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_water_pressure", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_hd_fin_ap_igain", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_fl3slo_phyco_units", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_battpos", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_bb3slo_b470_scaled", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_lat", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_gps_lon", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="sci_ctd41cp_timestamp", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_pressure", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_wpt_x_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="c_ballast_pumped", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="x_lmc_xy_source", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_lon", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_avg_speed", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="sci_water_temp", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="u_pitch_ap_gain", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_roll", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_tot_num_inflections", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="m_tot_num_inflections", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_x_lmc", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_pitch_ap_deadband", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_final_water_vy", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_final_water_vx", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_water_depth", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_leakdetect_voltage", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="m_leakdetect_voltage", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext( "u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=np.dtype("float32")) ) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context( name="u_pitch_max_delta_battpos", parameter_context=t_ctxt.dump() ) ) t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_coulomb_amphr", parameter_context=t_ctxt.dump()) ) t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=np.dtype("float32"))) t_ctxt.uom = "unknown" context_ids.append( self.dataset_management.create_parameter_context(name="m_pitch", parameter_context=t_ctxt.dump()) ) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64"))) t_ctxt.uom = "seconds since 01-01-1970" context_ids.append( self.dataset_management.create_parameter_context(name="time", parameter_context=t_ctxt.dump()) ) cnd_ctxt = ParameterContext("conductivity", param_type=ArrayType()) cnd_ctxt.uom = "mmho/cm" context_ids.append( self.dataset_management.create_parameter_context(name="conductivity", parameter_context=cnd_ctxt.dump()) ) temp_ctxt = ParameterContext("temperature", param_type=ArrayType()) temp_ctxt.uom = "degC" context_ids.append( self.dataset_management.create_parameter_context(name="temperature", parameter_context=temp_ctxt.dump()) ) press_ctxt = ParameterContext("pressure", param_type=ArrayType()) press_ctxt.uom = "decibars" context_ids.append( self.dataset_management.create_parameter_context(name="pressure", parameter_context=press_ctxt.dump()) ) oxy_ctxt = ParameterContext("oxygen", param_type=ArrayType()) oxy_ctxt.uom = "Hz" context_ids.append( self.dataset_management.create_parameter_context(name="oxygen", parameter_context=oxy_ctxt.dump()) ) return context_ids
def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream( 'replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage( dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber( self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop()
def create_contexts(self): context_ids = [] cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump())) pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump())) sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump())) temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump())) t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump())) return context_ids