def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='time', parameter_context=t_ctxt.dump()))
cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType())
cnd_ctxt.uom = 'mmho/cm'
context_ids.append(
self.dataset_management.create_parameter_context(
name='conductivity', parameter_context=cnd_ctxt.dump()))
temp_ctxt = ParameterContext('temperature', param_type=ArrayType())
temp_ctxt.uom = 'degC'
context_ids.append(
self.dataset_management.create_parameter_context(
name='temperature', parameter_context=temp_ctxt.dump()))
press_ctxt = ParameterContext('pressure', param_type=ArrayType())
press_ctxt.uom = 'decibars'
context_ids.append(
self.dataset_management.create_parameter_context(
name='pressure', parameter_context=press_ctxt.dump()))
oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType())
oxy_ctxt.uom = 'Hz'
context_ids.append(
self.dataset_management.create_parameter_context(
name='oxygen', parameter_context=oxy_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_y_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_cond', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_y_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_inflection_holdoff', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_m_present_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage_forward', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b660_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_science_send_all', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_status', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vx', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vy', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_heading', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_chlor_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_gain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_vacuum', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_min_water_depth', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lat', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_veh_temp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='f_fin_offset', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_hardover_holdoff', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_alt_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_present_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_heading', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b532_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_cdom_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_fin', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='x_cycle_overrun_in_ms', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_pressure', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_igain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_phyco_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_battpos', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b470_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_lat', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lon', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_ctd41cp_timestamp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_pressure', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_x_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_ballast_pumped', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='x_lmc_xy_source', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_lon', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_avg_speed', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_temp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_gain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_roll', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_tot_num_inflections', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_x_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_deadband', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vy', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vx', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_depth', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_max_delta_battpos', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_coulomb_amphr', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_pitch', parameter_context=t_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()))
cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType())
cnd_ctxt.uom = 'mmho/cm'
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cnd_ctxt.dump()))
temp_ctxt = ParameterContext('temperature', param_type=ArrayType())
temp_ctxt.uom = 'degC'
context_ids.append(self.dataset_management.create_parameter_context(name='temperature', parameter_context=temp_ctxt.dump()))
press_ctxt = ParameterContext('pressure', param_type=ArrayType())
press_ctxt.uom = 'decibars'
context_ids.append(self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump()))
oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType())
oxy_ctxt.uom = 'Hz'
context_ids.append(self.dataset_management.create_parameter_context(name='oxygen', parameter_context=oxy_ctxt.dump()))
return context_ids
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_monitor.wait())
query = {
'start_time': 0 - 2208988800,
'end_time': 19 - 2208988800,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2))
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
subscriber.start()
self.addCleanup(subscriber.stop)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'Time_Time',
param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Time_Time', parameter_context=t_ctxt.dump()))
core_current_ctxt = ParameterContext(
'Core_Current',
param_type=QuantityType(value_encoding=np.dtype('float32')))
core_current_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Core_Current',
parameter_context=core_current_ctxt.dump()))
core_voltage_ctxt = ParameterContext(
'Core_Voltage',
param_type=QuantityType(value_encoding=np.dtype('float32')))
core_voltage_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Core_Voltage',
parameter_context=core_voltage_ctxt.dump()))
core_pressure_ctxt = ParameterContext(
'Core_Pressure',
param_type=QuantityType(value_encoding=np.dtype('float32')))
core_pressure_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Core_Pressure',
parameter_context=core_pressure_ctxt.dump()))
fluorometer_value_ctxt = ParameterContext(
'Fluorometer_Value',
param_type=QuantityType(value_encoding=np.dtype('float32')))
fluorometer_value_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Fluorometer_Value',
parameter_context=fluorometer_value_ctxt.dump()))
fluorometer_gain_ctxt = ParameterContext(
'Fluorometer_Gain',
param_type=QuantityType(value_encoding=np.dtype('int32')))
fluorometer_gain_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Fluorometer_Gain',
parameter_context=fluorometer_gain_ctxt.dump()))
turbidity_value_ctxt = ParameterContext(
'Turbidity_Value',
param_type=QuantityType(value_encoding=np.dtype('float32')))
turbidity_value_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Turbidity_Value',
parameter_context=turbidity_value_ctxt.dump()))
turbidity_gain_ctxt = ParameterContext(
'Turbidity_Gain',
param_type=QuantityType(value_encoding=np.dtype('int32')))
turbidity_gain_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Turbidity_Gain',
parameter_context=turbidity_gain_ctxt.dump()))
optode_oxygen_ctxt = ParameterContext(
'Optode_Oxygen',
param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_oxygen_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Optode_Oxygen',
parameter_context=optode_oxygen_ctxt.dump()))
optode_temp_ctxt = ParameterContext(
'Optode_Temp',
param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_temp_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Optode_Temp', parameter_context=optode_temp_ctxt.dump()))
par_value_ctxt = ParameterContext(
'Par_Value',
param_type=QuantityType(value_encoding=np.dtype('float32')))
par_value_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Par_Value', parameter_context=par_value_ctxt.dump()))
puck_scatter_ctxt = ParameterContext(
'Puck_Scatter',
param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_scatter_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Puck_Scatter',
parameter_context=puck_scatter_ctxt.dump()))
puck_chla_ctxt = ParameterContext(
'Puck_Chla',
param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_chla_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Puck_Chla', parameter_context=puck_chla_ctxt.dump()))
puck_cdom_ctxt = ParameterContext(
'Puck_CDOM',
param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_cdom_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump()))
biosuite_scatter_ctxt = ParameterContext(
'BioSuite_Scatter',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_scatter_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Scatter',
parameter_context=biosuite_scatter_ctxt.dump()))
biosuite_chla_ctxt = ParameterContext(
'BioSuite_Chla',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_chla_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Chla',
parameter_context=biosuite_chla_ctxt.dump()))
biosuite_cdom_ctxt = ParameterContext(
'BioSuite_CDOM',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_cdom_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_CDOM',
parameter_context=biosuite_cdom_ctxt.dump()))
biosuite_temp_ctxt = ParameterContext(
'BioSuite_Temp',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_temp_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Temp',
parameter_context=biosuite_temp_ctxt.dump()))
biosuite_par_ctxt = ParameterContext(
'BioSuite_Par',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_par_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Par',
parameter_context=biosuite_par_ctxt.dump()))
flbb_chla_ctxt = ParameterContext(
'FLBB_Chla',
param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_chla_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump()))
flbb_turb_ctxt = ParameterContext(
'FLBB_Turb',
param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_turb_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump()))
flbb_temp_ctxt = ParameterContext(
'FLBB_Temp',
param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_temp_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='time', parameter_context=t_ctxt.dump()))
ut_ctxt = ParameterContext(
'upload_time',
param_type=QuantityType(value_encoding=np.dtype('int64')))
ut_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='upload_time', parameter_context=ut_ctxt.dump()))
vela_ctxt = ParameterContext('VelA', param_type=ArrayType())
vela_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelA', parameter_context=vela_ctxt.dump()))
velb_ctxt = ParameterContext('VelB', param_type=ArrayType())
velb_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelB', parameter_context=velb_ctxt.dump()))
velc_ctxt = ParameterContext('VelC', param_type=ArrayType())
velc_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelC', parameter_context=velc_ctxt.dump()))
veld_ctxt = ParameterContext('VelD', param_type=ArrayType())
veld_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelD', parameter_context=veld_ctxt.dump()))
mx_ctxt = ParameterContext('Mx', param_type=ArrayType())
mx_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Mx', parameter_context=mx_ctxt.dump()))
my_ctxt = ParameterContext('My', param_type=ArrayType())
my_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='My', parameter_context=my_ctxt.dump()))
mz_ctxt = ParameterContext('Mz', param_type=ArrayType())
mz_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Mz', parameter_context=mz_ctxt.dump()))
pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType())
pitch_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Pitch', parameter_context=pitch_ctxt.dump()))
roll_ctxt = ParameterContext('Roll', param_type=ArrayType())
roll_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Roll', parameter_context=roll_ctxt.dump()))
return context_ids
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_simplex_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def test_replay_with_parameters(self):
# --------------------------------------------------------------------------------
# Create the configurations and the dataset
# --------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext("binary", param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext("records", param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
"replay_pdict", parameter_context_ids=context_ids, temporal_context="time"
)
stream_def_id = self.pubsub_management.create_stream_definition(
"replay_stream", parameter_dictionary_id=pdict_id
)
stream_id, route = self.pubsub_management.create_stream(
"replay_with_params", exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id
)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Coerce the datastore into existence (beats race condition)
# --------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id, 40)
query = {
"start_time": 0 - 2208988800,
"end_time": 20 - 2208988800,
"stride_time": 2,
"parameters": ["time", "temp"],
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(0, 20, 2) == rdt["time"]
self.assertTrue(comp.all(), "%s" % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"]))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=["time", "temp"])
self.assertTrue(extents["time"] >= 20)
self.assertTrue(extents["temp"] >= 20)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_y_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_cond', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_y_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_inflection_holdoff', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_m_present_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage_forward', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b660_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_science_send_all', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_status', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vx', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vy', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_heading', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_chlor_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_gain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_vacuum', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_min_water_depth', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lat', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_veh_temp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='f_fin_offset', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_hardover_holdoff', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_alt_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_present_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_heading', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b532_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_cdom_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_fin', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='x_cycle_overrun_in_ms', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_pressure', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_igain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_phyco_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_battpos', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b470_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_lat', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lon', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_ctd41cp_timestamp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_pressure', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_x_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_ballast_pumped', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='x_lmc_xy_source', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_lon', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_avg_speed', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_temp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_gain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_roll', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_tot_num_inflections', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_x_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_deadband', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vy', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vx', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_depth', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_max_delta_battpos', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_coulomb_amphr', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_pitch', parameter_context=t_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='Time_Time', parameter_context=t_ctxt.dump()))
core_current_ctxt = ParameterContext('Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32')))
core_current_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Core_Current', parameter_context=core_current_ctxt.dump()))
core_voltage_ctxt = ParameterContext('Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32')))
core_voltage_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Core_Voltage', parameter_context=core_voltage_ctxt.dump()))
core_pressure_ctxt = ParameterContext('Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
core_pressure_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Core_Pressure', parameter_context=core_pressure_ctxt.dump()))
fluorometer_value_ctxt = ParameterContext('Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32')))
fluorometer_value_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump()))
fluorometer_gain_ctxt = ParameterContext('Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32')))
fluorometer_gain_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump()))
turbidity_value_ctxt = ParameterContext('Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32')))
turbidity_value_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump()))
turbidity_gain_ctxt = ParameterContext('Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32')))
turbidity_gain_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump()))
optode_oxygen_ctxt = ParameterContext('Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_oxygen_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump()))
optode_temp_ctxt = ParameterContext('Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_temp_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Temp', parameter_context=optode_temp_ctxt.dump()))
par_value_ctxt = ParameterContext('Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32')))
par_value_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Par_Value', parameter_context=par_value_ctxt.dump()))
puck_scatter_ctxt = ParameterContext('Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_scatter_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump()))
puck_chla_ctxt = ParameterContext('Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_chla_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Chla', parameter_context=puck_chla_ctxt.dump()))
puck_cdom_ctxt = ParameterContext('Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_cdom_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump()))
biosuite_scatter_ctxt = ParameterContext('BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_scatter_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump()))
biosuite_chla_ctxt = ParameterContext('BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_chla_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump()))
biosuite_cdom_ctxt = ParameterContext('BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_cdom_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump()))
biosuite_temp_ctxt = ParameterContext('BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_temp_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump()))
biosuite_par_ctxt = ParameterContext('BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_par_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump()))
flbb_chla_ctxt = ParameterContext('FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_chla_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump()))
flbb_turb_ctxt = ParameterContext('FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_turb_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump()))
flbb_temp_ctxt = ParameterContext('FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_temp_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()))
ut_ctxt = ParameterContext('upload_time', param_type=QuantityType(value_encoding=np.dtype('int64')))
ut_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='upload_time', parameter_context=ut_ctxt.dump()))
vela_ctxt = ParameterContext('VelA', param_type=ArrayType())
vela_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelA', parameter_context=vela_ctxt.dump()))
velb_ctxt = ParameterContext('VelB', param_type=ArrayType())
velb_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelB', parameter_context=velb_ctxt.dump()))
velc_ctxt = ParameterContext('VelC', param_type=ArrayType())
velc_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelC', parameter_context=velc_ctxt.dump()))
veld_ctxt = ParameterContext('VelD', param_type=ArrayType())
veld_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelD', parameter_context=veld_ctxt.dump()))
mx_ctxt = ParameterContext('Mx', param_type=ArrayType())
mx_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Mx', parameter_context=mx_ctxt.dump()))
my_ctxt = ParameterContext('My', param_type=ArrayType())
my_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='My', parameter_context=my_ctxt.dump()))
mz_ctxt = ParameterContext('Mz', param_type=ArrayType())
mz_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Mz', parameter_context=mz_ctxt.dump()))
pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType())
pitch_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Pitch', parameter_context=pitch_ctxt.dump()))
roll_ctxt = ParameterContext('Roll', param_type=ArrayType())
roll_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Roll', parameter_context=roll_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_wpt_y_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_water_cond", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_y_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext(
"u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_hd_fin_ap_inflection_holdoff", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_m_present_time", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext(
"m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="m_leakdetect_voltage_forward", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_bb3slo_b660_scaled", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_science_send_all", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_gps_status", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_water_vx", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_water_vy", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_heading", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_fl3slo_chlor_units", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_hd_fin_ap_gain", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_vacuum", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_min_water_depth", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_gps_lat", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_veh_temp", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="f_fin_offset", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext(
"u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_hd_fin_ap_hardover_holdoff", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_alt_time", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_present_time", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_heading", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_bb3slo_b532_scaled", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_fl3slo_cdom_units", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_fin", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="x_cycle_overrun_in_ms", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_water_pressure", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_hd_fin_ap_igain", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_fl3slo_phyco_units", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_battpos", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_bb3slo_b470_scaled", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_lat", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_gps_lon", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_ctd41cp_timestamp", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_pressure", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_wpt_x_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_ballast_pumped", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="x_lmc_xy_source", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_lon", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_avg_speed", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_water_temp", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_pitch_ap_gain", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_roll", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_tot_num_inflections", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="m_tot_num_inflections", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_x_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_pitch_ap_deadband", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_final_water_vy", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_final_water_vx", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_water_depth", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_leakdetect_voltage", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="m_leakdetect_voltage", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext(
"u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_pitch_max_delta_battpos", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_coulomb_amphr", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_pitch", parameter_context=t_ctxt.dump())
)
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64")))
t_ctxt.uom = "seconds since 01-01-1970"
context_ids.append(
self.dataset_management.create_parameter_context(name="time", parameter_context=t_ctxt.dump())
)
cnd_ctxt = ParameterContext("conductivity", param_type=ArrayType())
cnd_ctxt.uom = "mmho/cm"
context_ids.append(
self.dataset_management.create_parameter_context(name="conductivity", parameter_context=cnd_ctxt.dump())
)
temp_ctxt = ParameterContext("temperature", param_type=ArrayType())
temp_ctxt.uom = "degC"
context_ids.append(
self.dataset_management.create_parameter_context(name="temperature", parameter_context=temp_ctxt.dump())
)
press_ctxt = ParameterContext("pressure", param_type=ArrayType())
press_ctxt.uom = "decibars"
context_ids.append(
self.dataset_management.create_parameter_context(name="pressure", parameter_context=press_ctxt.dump())
)
oxy_ctxt = ParameterContext("oxygen", param_type=ArrayType())
oxy_ctxt.uom = "Hz"
context_ids.append(
self.dataset_management.create_parameter_context(name="oxygen", parameter_context=oxy_ctxt.dump())
)
return context_ids
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(
self.dataset_management.create_parameter_context(
'binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(
self.dataset_management.create_parameter_context(
'records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'replay_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition(
'replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream(
'replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(
dataset_id,
bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(
self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(
dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
