def test_replay_with_parameters(self):
# --------------------------------------------------------------------------------
# Create the configurations and the dataset
# --------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext("binary", param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext("records", param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
"replay_pdict", parameter_context_ids=context_ids, temporal_context="time"
)
stream_def_id = self.pubsub_management.create_stream_definition(
"replay_stream", parameter_dictionary_id=pdict_id
)
stream_id, route = self.pubsub_management.create_stream(
"replay_with_params", exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id
)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Coerce the datastore into existence (beats race condition)
# --------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id, 40)
query = {
"start_time": 0 - 2208988800,
"end_time": 20 - 2208988800,
"stride_time": 2,
"parameters": ["time", "temp"],
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(0, 20, 2) == rdt["time"]
self.assertTrue(comp.all(), "%s" % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"]))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=["time", "temp"])
self.assertTrue(extents["time"] >= 20)
self.assertTrue(extents["temp"] >= 20)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64")))
t_ctxt.uom = "seconds since 01-01-1970"
context_ids.append(
self.dataset_management.create_parameter_context(name="time", parameter_context=t_ctxt.dump())
)
cnd_ctxt = ParameterContext("conductivity", param_type=ArrayType())
cnd_ctxt.uom = "mmho/cm"
context_ids.append(
self.dataset_management.create_parameter_context(name="conductivity", parameter_context=cnd_ctxt.dump())
)
temp_ctxt = ParameterContext("temperature", param_type=ArrayType())
temp_ctxt.uom = "degC"
context_ids.append(
self.dataset_management.create_parameter_context(name="temperature", parameter_context=temp_ctxt.dump())
)
press_ctxt = ParameterContext("pressure", param_type=ArrayType())
press_ctxt.uom = "decibars"
context_ids.append(
self.dataset_management.create_parameter_context(name="pressure", parameter_context=press_ctxt.dump())
)
oxy_ctxt = ParameterContext("oxygen", param_type=ArrayType())
oxy_ctxt.uom = "Hz"
context_ids.append(
self.dataset_management.create_parameter_context(name="oxygen", parameter_context=oxy_ctxt.dump())
)
return context_ids
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id)
es.start()
self.addCleanup(es.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_modified.wait(30))
query = {
'start_time': 0 - 2208988800,
'end_time': 20 - 2208988800,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(0,20,2) == rdt['time']
self.assertTrue(comp.all(),'%s' % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def build_context(self,record):
context = ParameterContext(name=record['Name'], param_type=self.param_type(record['Parameter Type']))
context.uom = record['Unit of Measure']
if record['Fill Value']:
context.fill_value = self.fill_value(record['Fill Value'], record['Parameter Type'])
if record['Axis']:
context.axis = self.ref_frame(record['Axis'])
for key in self.additional_attrs.iterkeys():
if key in record and record[key]:
setattr(context,self.additional_attrs[key],record[key])
return context
def build_context(self, record):
context = ParameterContext(name=record['Name'],
param_type=self.param_type(
record['Parameter Type']))
context.uom = record['Unit of Measure']
if record['Fill Value']:
context.fill_value = self.fill_value(record['Fill Value'],
record['Parameter Type'])
if record['Axis']:
context.axis = self.ref_frame(record['Axis'])
for key in self.additional_attrs.iterkeys():
if key in record and record[key]:
setattr(context, self.additional_attrs[key], record[key])
return context
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_monitor.wait())
query = {
'start_time': 0 - 2208988800,
'end_time': 19 - 2208988800,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2))
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
def get_param_dict(param_dict_name=None):
raise NotImplementedError(
'This method has been replaced by DatasetManagementService, please use read_parameter_dictionary_by_name instead'
)
# read the file just once, not every time needed
global _PARAMETER_DICTIONARIES
global _PARAMETER_CONTEXTS
if not _PARAMETER_DICTIONARIES:
param_dict_defs_file = "res/config/param_dict_defs.yml"
with open(param_dict_defs_file, "r") as f_dict:
dict_string = f_dict.read()
_PARAMETER_DICTIONARIES = yaml.load(dict_string)
param_context_defs_file = "res/config/param_context_defs.yml"
with open(param_context_defs_file, "r") as f_ctxt:
ctxt_string = f_ctxt.read()
_PARAMETER_CONTEXTS = yaml.load(ctxt_string)
# make sure we have the one requested
context_names = _PARAMETER_DICTIONARIES[param_dict_name]
for name in context_names:
if not _PARAMETER_CONTEXTS.has_key(name):
raise AssertionError(
'The parameter dict has a context that does not exist in the parameter context defs specified in yml: %s'
% name)
# package and ship
pdict = ParameterDictionary()
for ctxt_name in context_names:
param_context = ParameterContext.load(_PARAMETER_CONTEXTS[ctxt_name])
pdict.add_context(param_context)
return pdict
def get_param_dict(param_dict_name = None):
raise NotImplementedError('This method has been replaced by DatasetManagementService, please use read_parameter_dictionary_by_name instead')
# read the file just once, not every time needed
global _PARAMETER_DICTIONARIES
global _PARAMETER_CONTEXTS
if not _PARAMETER_DICTIONARIES:
param_dict_defs_file = "res/config/param_dict_defs.yml"
with open(param_dict_defs_file, "r") as f_dict:
dict_string = f_dict.read()
_PARAMETER_DICTIONARIES = yaml.load(dict_string)
param_context_defs_file = "res/config/param_context_defs.yml"
with open(param_context_defs_file, "r") as f_ctxt:
ctxt_string = f_ctxt.read()
_PARAMETER_CONTEXTS = yaml.load(ctxt_string)
# make sure we have the one requested
context_names = _PARAMETER_DICTIONARIES[param_dict_name]
for name in context_names:
if not _PARAMETER_CONTEXTS.has_key(name):
raise AssertionError('The parameter dict has a context that does not exist in the parameter context defs specified in yml: %s' % name)
# package and ship
pdict = ParameterDictionary()
for ctxt_name in context_names:
param_context = ParameterContext.load(_PARAMETER_CONTEXTS[ctxt_name])
pdict.add_context(param_context)
return pdict
def _setup_resources(self):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
stream_id, stream_route, stream_def = self.create_stream_and_logger(name='fibonacci_stream', pdict=pdict)
# tx = TaxyTool()
# tx.add_taxonomy_set('data', 'external_data')
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'stream_def': stream_def,
'data_producer_id': 'fibonacci_data_producer_id',
'max_records': 4,
}
def test_lookup_value_check(self):
func = NumexprFunction('f',
'coeff_a * x', ['coeff_a', 'x'],
param_map={
'x': 'x',
'coeff_a': 'coeff_a'
})
func.lookup_values = ['abc123']
test_context = ParameterContext('test',
param_type=ParameterFunctionType(func))
tm = TypesManager(None, None, None)
self.assertTrue(tm.has_lookup_value(test_context))
self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123'])
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()))
cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType())
cnd_ctxt.uom = 'mmho/cm'
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cnd_ctxt.dump()))
temp_ctxt = ParameterContext('temperature', param_type=ArrayType())
temp_ctxt.uom = 'degC'
context_ids.append(self.dataset_management.create_parameter_context(name='temperature', parameter_context=temp_ctxt.dump()))
press_ctxt = ParameterContext('pressure', param_type=ArrayType())
press_ctxt.uom = 'decibars'
context_ids.append(self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump()))
oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType())
oxy_ctxt.uom = 'Hz'
context_ids.append(self.dataset_management.create_parameter_context(name='oxygen', parameter_context=oxy_ctxt.dump()))
return context_ids
def _create_parameter(self):
pdict = ParameterDictionary()
pdict = self._add_location_time_ctxt(pdict)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
return pdict
def _create_parameter_dictionary(self):
pdict = ParameterDictionary()
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
return pdict
def rdt_to_granule(self, context, value_array, comp_val=None):
time = ParameterContext(
name='time', param_type=QuantityType(value_encoding=np.float64))
pdict = ParameterDictionary()
pdict.add_context(time, is_temporal=True)
pdict.add_context(context)
rdt = RecordDictionaryTool(param_dictionary=pdict)
rdt['time'] = np.arange(len(value_array))
rdt['test'] = value_array
granule = rdt.to_granule()
rdt2 = RecordDictionaryTool.load_from_granule(granule)
testval = comp_val if comp_val is not None else value_array
actual = rdt2['test']
if isinstance(testval, basestring):
self.assertEquals(testval, actual)
else:
np.testing.assert_array_equal(testval, actual)
def cov_io(self, context, value_array, comp_val=None):
pdict = ParameterDictionary()
time = ParameterContext(
name='time', param_type=QuantityType(value_encoding=np.float64))
pdict.add_context(context)
pdict.add_context(time, True)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs,
MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial',
[0]), scrs, MutabilityEnum.IMMUTABLE
) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
cov = SimplexCoverage('test_data',
create_guid(),
'sample coverage_model',
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom)
self.addCleanup(shutil.rmtree, cov.persistence_dir)
cov.insert_timesteps(len(value_array))
cov.set_parameter_values('test',
tdoa=slice(0, len(value_array)),
value=value_array)
comp_val = comp_val if comp_val is not None else value_array
testval = cov.get_parameter_values('test')
try:
np.testing.assert_array_equal(testval, comp_val)
except:
print repr(value_array)
raise
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()))
ut_ctxt = ParameterContext('upload_time', param_type=QuantityType(value_encoding=np.dtype('int64')))
ut_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='upload_time', parameter_context=ut_ctxt.dump()))
vela_ctxt = ParameterContext('VelA', param_type=ArrayType())
vela_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelA', parameter_context=vela_ctxt.dump()))
velb_ctxt = ParameterContext('VelB', param_type=ArrayType())
velb_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelB', parameter_context=velb_ctxt.dump()))
velc_ctxt = ParameterContext('VelC', param_type=ArrayType())
velc_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelC', parameter_context=velc_ctxt.dump()))
veld_ctxt = ParameterContext('VelD', param_type=ArrayType())
veld_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='VelD', parameter_context=veld_ctxt.dump()))
mx_ctxt = ParameterContext('Mx', param_type=ArrayType())
mx_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Mx', parameter_context=mx_ctxt.dump()))
my_ctxt = ParameterContext('My', param_type=ArrayType())
my_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='My', parameter_context=my_ctxt.dump()))
mz_ctxt = ParameterContext('Mz', param_type=ArrayType())
mz_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Mz', parameter_context=mz_ctxt.dump()))
pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType())
pitch_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Pitch', parameter_context=pitch_ctxt.dump()))
roll_ctxt = ParameterContext('Roll', param_type=ArrayType())
roll_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Roll', parameter_context=roll_ctxt.dump()))
return context_ids
def test_build_granule_and_load_from_granule(self):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
rdt = RecordDictionaryTool(param_dictionary=pdict)
#Create some arrays and fill them with random values
temp_array = np.random.standard_normal(100)
cond_array = np.random.standard_normal(100)
pres_array = np.random.standard_normal(100)
time_array = np.random.standard_normal(100)
lat_array = np.random.standard_normal(100)
lon_array = np.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['conductivity'] = cond_array
rdt['pres'] = pres_array
rdt['time'] = time_array
rdt['lat'] = lat_array
rdt['lon'] = lon_array
g = build_granule(data_producer_id='john', record_dictionary=rdt, param_dictionary=pdict)
l_pd = ParameterDictionary.load(g.param_dictionary)
#l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
#self.assertEquals(l_pd, pdict)
self.assertEquals(l_pd.ord_from_key('temp'), pdict.ord_from_key('temp'))
self.assertEquals(l_pd.ord_from_key('conductivity'), pdict.ord_from_key('conductivity'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
#self.assertEquals(l_rd._param_dict, rdt._param_dict)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, np.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_simplex_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='time', parameter_context=t_ctxt.dump()))
ut_ctxt = ParameterContext(
'upload_time',
param_type=QuantityType(value_encoding=np.dtype('int64')))
ut_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='upload_time', parameter_context=ut_ctxt.dump()))
vela_ctxt = ParameterContext('VelA', param_type=ArrayType())
vela_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelA', parameter_context=vela_ctxt.dump()))
velb_ctxt = ParameterContext('VelB', param_type=ArrayType())
velb_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelB', parameter_context=velb_ctxt.dump()))
velc_ctxt = ParameterContext('VelC', param_type=ArrayType())
velc_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelC', parameter_context=velc_ctxt.dump()))
veld_ctxt = ParameterContext('VelD', param_type=ArrayType())
veld_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='VelD', parameter_context=veld_ctxt.dump()))
mx_ctxt = ParameterContext('Mx', param_type=ArrayType())
mx_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Mx', parameter_context=mx_ctxt.dump()))
my_ctxt = ParameterContext('My', param_type=ArrayType())
my_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='My', parameter_context=my_ctxt.dump()))
mz_ctxt = ParameterContext('Mz', param_type=ArrayType())
mz_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Mz', parameter_context=mz_ctxt.dump()))
pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType())
pitch_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Pitch', parameter_context=pitch_ctxt.dump()))
roll_ctxt = ParameterContext('Roll', param_type=ArrayType())
roll_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Roll', parameter_context=roll_ctxt.dump()))
return context_ids
def get_param_dict(self):
pdict = ParameterDictionary()
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float64))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float64))
pres_ctxt.uom = 'unknown'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=np.float64))
temp_ctxt.uom = 'unknown'
temp_ctxt.fill_value = 0x0
pdict.add_context(temp_ctxt)
oxy_ctxt = ParameterContext('oxygen', param_type=QuantityType(value_encoding=np.float64))
oxy_ctxt.uom = 'unknown'
oxy_ctxt.fill_value = 0x0
pdict.add_context(oxy_ctxt)
internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
pdict.add_context(internal_ts_ctxt, is_temporal=True)
driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
pdict.add_context(driver_ts_ctxt)
return pdict
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'usgs_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
# The usgs.nc test dataset is a download of the R1 dataset found here:
# http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml
dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc'
dset.dataset_description.parameters['temporal_dimension'] = 'time'
dset.dataset_description.parameters['zonal_dimension'] = 'lon'
dset.dataset_description.parameters['meridional_dimension'] = 'lat'
dset.dataset_description.parameters['vertical_dimension'] = 'z'
dset.dataset_description.parameters['variables'] = [
'water_temperature',
'streamflow',
'water_temperature_bottom',
'water_temperature_middle',
'specific_conductance',
'data_qualifier',
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='dap_model')
dsrc_model.model = 'DAP'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dprod = IonObject(RT.DataProduct,
name='usgs_parsed_product',
description='parsed usgs product',
temporal_domain = tdom,
spatial_domain = sdom)
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id,
parameter_dictionary=parameter_dictionary)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
# ttool = TaxyTool()
# ttool.add_taxonomy_set('time','time')
# ttool.add_taxonomy_set('lon','longitude')
# ttool.add_taxonomy_set('lat','latitude')
# ttool.add_taxonomy_set('z','water depth')
# ttool.add_taxonomy_set('water_temperature', 'average water temperature')
# ttool.add_taxonomy_set('water_temperature_bottom','water temperature at bottom of water column')
# ttool.add_taxonomy_set('water_temperature_middle', 'water temperature at middle of water column')
# ttool.add_taxonomy_set('streamflow', 'flow velocity of stream')
# ttool.add_taxonomy_set('specific_conductance', 'specific conductance of water')
# ttool.add_taxonomy_set('data_qualifier','data qualifier flag')
#
# ttool.add_taxonomy_set('coords','This group contains coordinate parameters')
# ttool.add_taxonomy_set('data','This group contains data parameters')
# Create the logger for receiving publications
self.create_stream_and_logger(name='usgs',stream_id=stream_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding = numpy.dtype('float32')))
temp_ctxt.uom = 'meters'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
#'taxonomy':ttool.dump(),
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':4,
}
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(name='example data agent',
handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance(
name='example dataset agent instance')
eda_inst_id = dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(name='example data provider',
institution=Institution(),
contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(name='example datasource',
protocol_type='FILE',
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters[
'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters[
'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = []
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
#dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(
external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'data',
param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
pc_list.append(dms_cli.create_parameter_context(
pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict',
pc_list)
streamdef_id = pubsub_cli.create_stream_definition(
name="ruv",
description="stream def for ruv testing",
parameter_dictionary_id=pdict_id)
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product')
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(
data_product=dprod, stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({
'ExternalDataset': ds_id,
'ExternalDataProvider': ext_dprov_id,
'DataSource': ext_dsrc_id,
'DataSourceModel': ext_dsrc_model_id,
'DataProducer': dproducer_id,
'DataProduct': dproduct_id,
'Stream': stream_id
}))
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='ruv',
stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'external_dataset_res': dset,
'param_dictionary': pdict.dump(),
'data_producer_id':
dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
def setUp(self):
self._tx = TaxyTool()
self._tx.add_taxonomy_set('temp', 'long_temp_name')
self._tx.add_taxonomy_set('cond', 'long_cond_name')
self._tx.add_taxonomy_set('pres', 'long_pres_name')
self._tx.add_taxonomy_set('rdt')
self._tx.add_taxonomy_set('rdt2')
# map is {<local name>: <granule name or path>}
self._rdt = RecordDictionaryTool(taxonomy=self._tx)
self._pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
self._pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
self._pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
self._pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
self._pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
self._pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
self._pdict.add_context(pres_ctxt)
self._rdt_pdict = RecordDictionaryTool(param_dictionary=self._pdict)
def build_contexts():
'''
Builds the relevant parameter context objects
'''
contexts = []
cond_ctxt = ParameterContext(
'conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
contexts.append(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
contexts.append(pres_ctxt)
sal_ctxt = ParameterContext(
'salinity', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
contexts.append(sal_ctxt)
den_ctxt = ParameterContext(
'density', param_type=QuantityType(value_encoding=np.float32))
den_ctxt.uom = 'kg/m3'
den_ctxt.fill_value = 0x0
contexts.append(den_ctxt)
temp_ctxt = ParameterContext(
'temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
contexts.append(temp_ctxt)
t_ctxt = ParameterContext('time',
param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
contexts.append(t_ctxt)
lat_ctxt = ParameterContext(
'lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
contexts.append(lat_ctxt)
lon_ctxt = ParameterContext(
'lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
contexts.append(lon_ctxt)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.description = 'raw binary string values'
raw_ctxt.uom = 'utf-8 byte string'
raw_ctxt.fill_value = 0x0
contexts.append(raw_ctxt)
port_ts_ctxt = ParameterContext(
name='port_timestamp',
param_type=QuantityType(value_encoding=np.float64))
port_ts_ctxt._derived_from_name = 'time'
port_ts_ctxt.uom = 'seconds'
port_ts_ctxt.fill_value = -1
contexts.append(port_ts_ctxt)
driver_ts_ctxt = ParameterContext(
name='driver_timestamp',
param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
contexts.append(driver_ts_ctxt)
internal_ts_ctxt = ParameterContext(
name='internal_timestamp',
param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
contexts.append(internal_ts_ctxt)
timer_num_ctxt = ParameterContext(
name='timer', param_type=QuantityType(value_encoding=np.float64))
timer_num_ctxt.fill_value = -1
contexts.append(timer_num_ctxt)
serial_num_ctxt = ParameterContext(
name='serial_num', param_type=QuantityType(value_encoding=np.int32))
serial_num_ctxt.fill_value = -1
contexts.append(serial_num_ctxt)
count_ctxt = ParameterContext(
name='counts', param_type=QuantityType(value_encoding=np.uint64))
count_ctxt.fill_value = -1
contexts.append(count_ctxt)
checksum_ctxt = ParameterContext(
name='checksum', param_type=QuantityType(value_encoding=np.int32))
checksum_ctxt.fill_value = -1
contexts.append(checksum_ctxt)
pref_ts_ctxt = ParameterContext(
name='preferred_timestamp',
param_type=QuantityType(value_encoding=np.uint64))
pref_ts_ctxt.description = 'name of preferred timestamp'
pref_ts_ctxt.fill_value = 0x0
contexts.append(pref_ts_ctxt)
# TODO: This should probably be of type CategoryType when implemented
qual_flag_ctxt = ParameterContext(name='quality_flag',
param_type=ArrayType())
qual_flag_ctxt.description = 'flag indicating quality'
qual_flag_ctxt.fill_value = None
contexts.append(qual_flag_ctxt)
viz_ts_ctxt = ParameterContext(
name='viz_timestamp',
param_type=QuantityType(value_encoding=np.float64))
viz_ts_ctxt._derived_from_name = 'time'
viz_ts_ctxt.uom = 'seconds'
viz_ts_ctxt.fill_value = -1
contexts.append(viz_ts_ctxt)
viz_prod_type_ctxt = ParameterContext(name='viz_product_type',
param_type=ArrayType())
viz_prod_type_ctxt.fill_value = None
contexts.append(viz_prod_type_ctxt)
image_obj_ctxt = ParameterContext(name='image_obj', param_type=ArrayType())
image_obj_ctxt.fill_value = None
contexts.append(image_obj_ctxt)
image_name_ctxt = ParameterContext(name='image_name',
param_type=ArrayType())
contexts.append(image_name_ctxt)
content_type_ctxt = ParameterContext(name='content_type',
param_type=ArrayType())
contexts.append(content_type_ctxt)
gdt_ctxt = ParameterContext(name='google_dt_components',
param_type=RecordType())
contexts.append(gdt_ctxt)
mpl_ctxt = ParameterContext(name='mpl_graph', param_type=RecordType())
contexts.append(mpl_ctxt)
dummy_ctxt = ParameterContext(
name='dummy', param_type=QuantityType(value_encoding=np.int64))
contexts.append(dummy_ctxt)
return contexts
def create(self, path):
mkdir_silent(path)
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT])
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
dens_ctxt = ParameterContext('data_quantity', param_type=QuantityType(value_encoding=np.float32))
dens_ctxt.uom = 'unknown'
dens_ctxt.fill_value = 0x0
pdict.add_context(dens_ctxt)
serial_ctxt = ParameterContext('data_array', param_type=ArrayType())
serial_ctxt.uom = 'unknown'
serial_ctxt.fill_value = 0x0
pdict.add_context(serial_ctxt)
guid = str(uuid.uuid4()).upper()
self.path = path
self.cov = SimplexCoverage(path, guid, name='test_cov', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(
self.dataset_management.create_parameter_context(
'binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(
self.dataset_management.create_parameter_context(
'records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'replay_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition(
'replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream(
'replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(
dataset_id,
bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(
self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(
dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def create_parameters(cls):
'''
WARNING: This method is a wrapper intended only for tests, it should not be used in production code.
It probably will not align to most datasets.
'''
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext(
'lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext(
'lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext(
'temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext(
'conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
data_ctxt = ParameterContext(
'data', param_type=QuantityType(value_encoding=np.int8))
data_ctxt.uom = 'byte'
data_ctxt.fill_value = 0x0
pdict.add_context(data_ctxt)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
sal_ctxt = ParameterContext(
'salinity', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
pdict.add_context(sal_ctxt)
dens_ctxt = ParameterContext(
'density', param_type=QuantityType(value_encoding=np.float32))
dens_ctxt.uom = 'unknown'
dens_ctxt.fill_value = 0x0
pdict.add_context(dens_ctxt)
return pdict
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(self.dataset_management.create_parameter_context(name='Time_Time', parameter_context=t_ctxt.dump()))
core_current_ctxt = ParameterContext('Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32')))
core_current_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Core_Current', parameter_context=core_current_ctxt.dump()))
core_voltage_ctxt = ParameterContext('Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32')))
core_voltage_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Core_Voltage', parameter_context=core_voltage_ctxt.dump()))
core_pressure_ctxt = ParameterContext('Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
core_pressure_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Core_Pressure', parameter_context=core_pressure_ctxt.dump()))
fluorometer_value_ctxt = ParameterContext('Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32')))
fluorometer_value_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump()))
fluorometer_gain_ctxt = ParameterContext('Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32')))
fluorometer_gain_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump()))
turbidity_value_ctxt = ParameterContext('Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32')))
turbidity_value_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump()))
turbidity_gain_ctxt = ParameterContext('Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32')))
turbidity_gain_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump()))
optode_oxygen_ctxt = ParameterContext('Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_oxygen_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump()))
optode_temp_ctxt = ParameterContext('Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_temp_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Optode_Temp', parameter_context=optode_temp_ctxt.dump()))
par_value_ctxt = ParameterContext('Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32')))
par_value_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Par_Value', parameter_context=par_value_ctxt.dump()))
puck_scatter_ctxt = ParameterContext('Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_scatter_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump()))
puck_chla_ctxt = ParameterContext('Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_chla_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Puck_Chla', parameter_context=puck_chla_ctxt.dump()))
puck_cdom_ctxt = ParameterContext('Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_cdom_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump()))
biosuite_scatter_ctxt = ParameterContext('BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_scatter_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump()))
biosuite_chla_ctxt = ParameterContext('BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_chla_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump()))
biosuite_cdom_ctxt = ParameterContext('BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_cdom_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump()))
biosuite_temp_ctxt = ParameterContext('BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_temp_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump()))
biosuite_par_ctxt = ParameterContext('BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_par_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump()))
flbb_chla_ctxt = ParameterContext('FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_chla_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump()))
flbb_turb_ctxt = ParameterContext('FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_turb_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump()))
flbb_temp_ctxt = ParameterContext('FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_temp_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump()))
return context_ids
def load():
from coverage_model.parameter import ParameterContext
from coverage_model.parameter_types import QuantityType, ArrayType, RecordType
from coverage_model.basic_types import AxisTypeEnum
import numpy as np
contexts = []
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
contexts.append(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
contexts.append(pres_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
contexts.append(sal_ctxt)
den_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=np.float32))
den_ctxt.uom = 'kg/m3'
den_ctxt.fill_value = 0x0
contexts.append(den_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
contexts.append(temp_ctxt)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
contexts.append(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
contexts.append(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
contexts.append(lon_ctxt)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.description = 'raw binary string values'
raw_ctxt.uom = 'utf-8 byte string'
contexts.append(raw_ctxt)
port_ts_ctxt = ParameterContext(name='port_timestamp', param_type=QuantityType(value_encoding=np.float64))
port_ts_ctxt._derived_from_name = 'time'
port_ts_ctxt.uom = 'seconds'
port_ts_ctxt.fill_value = -1
contexts.append(port_ts_ctxt)
driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
contexts.append(driver_ts_ctxt)
internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
contexts.append(internal_ts_ctxt)
timer_num_ctxt = ParameterContext(name='timer', param_type=QuantityType(value_encoding=np.float64))
timer_num_ctxt.fill_value = -1
contexts.append(timer_num_ctxt)
serial_num_ctxt = ParameterContext(name='serial_num', param_type=QuantityType(value_encoding=np.int32))
serial_num_ctxt.fill_value = -1
contexts.append(serial_num_ctxt)
count_ctxt = ParameterContext(name='counts', param_type=QuantityType(value_encoding=np.uint64))
count_ctxt.fill_value = -1
contexts.append(count_ctxt)
checksum_ctxt = ParameterContext(name='checksum', param_type=QuantityType(value_encoding=np.int32))
checksum_ctxt.fill_value = -1
contexts.append(checksum_ctxt)
pref_ts_ctxt = ParameterContext(name='preferred_timestamp', param_type=ArrayType())
pref_ts_ctxt.description = 'name of preferred timestamp'
contexts.append(pref_ts_ctxt)
# TODO: This should probably be of type CategoryType when implemented
qual_flag_ctxt = ParameterContext(name='quality_flag', param_type=ArrayType())
qual_flag_ctxt.description = 'flag indicating quality'
contexts.append(qual_flag_ctxt)
viz_ts_ctxt = ParameterContext(name='viz_timestamp', param_type=QuantityType(value_encoding=np.float64))
viz_ts_ctxt._derived_from_name = 'time'
viz_ts_ctxt.uom = 'seconds'
viz_ts_ctxt.fill_value = -1
contexts.append(viz_ts_ctxt)
viz_prod_type_ctxt = ParameterContext(name='viz_product_type', param_type=ArrayType())
contexts.append(viz_prod_type_ctxt)
image_obj_ctxt = ParameterContext(name='image_obj', param_type=ArrayType())
contexts.append(image_obj_ctxt)
image_name_ctxt = ParameterContext(name='image_name', param_type=ArrayType())
contexts.append(image_name_ctxt)
content_type_ctxt = ParameterContext(name='content_type', param_type=ArrayType())
contexts.append(content_type_ctxt)
gdt_ctxt = ParameterContext(name='google_dt_components', param_type=RecordType())
contexts.append(gdt_ctxt)
mpl_ctxt = ParameterContext(name='mpl_graph', param_type=RecordType())
contexts.append(mpl_ctxt)
dummy_ctxt = ParameterContext(name='dummy', param_type=QuantityType(value_encoding=np.int64))
contexts.append(dummy_ctxt)
return contexts
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
def create_parameters(cls):
'''
WARNING: This method is a wrapper intended only for tests, it should not be used in production code.
It probably will not align to most datasets.
'''
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
data_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=np.int8))
data_ctxt.uom = 'byte'
data_ctxt.fill_value = 0x0
pdict.add_context(data_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
pdict.add_context(sal_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=np.float32))
dens_ctxt.uom = 'unknown'
dens_ctxt.fill_value = 0x0
pdict.add_context(dens_ctxt)
return pdict
def _make_coverage(path):
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT])
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # Dimensionality is excluded for now
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
cat = {0:'lemon',1:'apple',2:'banana',99:'None'}
cat_ctxt = ParameterContext('category', param_type=CategoryType(categories=cat))
cat_ctxt.long_name = "example of category"
pdict.add_context(cat_ctxt)
dens_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.float32))
dens_ctxt.uom = 'unknown'
dens_ctxt.fill_value = 0x0
pdict.add_context(dens_ctxt)
const_ctxt = ParameterContext('constant', param_type=ConstantType())
const_ctxt.long_name = 'example of a parameter of type ConstantType'
pdict.add_context(const_ctxt)
rec_ctxt = ParameterContext('boolean', param_type=BooleanType())
rec_ctxt.long_name = 'example of a parameter of type BooleanType'
pdict.add_context(rec_ctxt)
rec_ctxt = ParameterContext('range', param_type=ConstantRangeType())
rec_ctxt.long_name = 'Range example'
rec_ctxt.fill_value = 0x0
pdict.add_context(rec_ctxt)
rec_ctxt = ParameterContext('record', param_type=RecordType())
rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries'
rec_ctxt.fill_value = 0x0
pdict.add_context(rec_ctxt)
serial_ctxt = ParameterContext('array', param_type=ArrayType())
serial_ctxt.uom = 'unknown'
serial_ctxt.fill_value = 0x0
pdict.add_context(serial_ctxt)
guid = create_guid()
guid = guid.replace("-", "")
cov = SimplexCoverage(path, guid, name="sample_cov", parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
return (cov,path+os.sep+guid)
def defining_parameter_dictionary(self):
# Define the parameter context objects
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
height_ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=np.float32))
height_ctxt.reference_frame = AxisTypeEnum.HEIGHT
height_ctxt.uom = 'meters'
height_ctxt.fill_value = 0e0
pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'degree_Celsius'
pres_ctxt.fill_value = 0e0
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
cond_ctxt = ParameterContext('cond', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
data_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=np.int8))
data_ctxt.uom = 'byte'
data_ctxt.fill_value = 0x0
# Define the parameter dictionary objects
self.pres = ParameterDictionary()
self.pres.add_context(t_ctxt)
self.pres.add_context(lat_ctxt)
self.pres.add_context(lon_ctxt)
self.pres.add_context(height_ctxt)
self.pres.add_context(pres_ctxt)
self.pres.add_context(data_ctxt)
self.temp = ParameterDictionary()
self.temp.add_context(t_ctxt)
self.temp.add_context(lat_ctxt)
self.temp.add_context(lon_ctxt)
self.temp.add_context(height_ctxt)
self.temp.add_context(temp_ctxt)
self.temp.add_context(data_ctxt)
self.cond = ParameterDictionary()
self.cond.add_context(t_ctxt)
self.cond.add_context(lat_ctxt)
self.cond.add_context(lon_ctxt)
self.cond.add_context(height_ctxt)
self.cond.add_context(cond_ctxt)
self.cond.add_context(data_ctxt)
def get_param_dict(self):
pdict = ParameterDictionary()
cond_ctxt = ParameterContext(
'conductivity', param_type=QuantityType(value_encoding=np.float64))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=np.float64))
pres_ctxt.uom = 'unknown'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature', param_type=QuantityType(value_encoding=np.float64))
temp_ctxt.uom = 'unknown'
temp_ctxt.fill_value = 0x0
pdict.add_context(temp_ctxt)
oxy_ctxt = ParameterContext(
'oxygen', param_type=QuantityType(value_encoding=np.float64))
oxy_ctxt.uom = 'unknown'
oxy_ctxt.fill_value = 0x0
pdict.add_context(oxy_ctxt)
internal_ts_ctxt = ParameterContext(
name='internal_timestamp',
param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
pdict.add_context(internal_ts_ctxt, is_temporal=True)
driver_ts_ctxt = ParameterContext(
name='driver_timestamp',
param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
pdict.add_context(driver_ts_ctxt)
return pdict
def _create_parameter_dictionary(self):
pdict = ParameterDictionary()
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'meters'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
return pdict
def _create_parameter(self):
pdict = ParameterDictionary()
pdict = self._add_location_time_ctxt(pdict)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=numpy.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temp', param_type=QuantityType(value_encoding=numpy.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
raw_fixed_ctxt = ParameterContext(
'raw_fixed', param_type=QuantityType(value_encoding=numpy.float32))
raw_fixed_ctxt.uom = 'unknown'
raw_fixed_ctxt.fill_value = 0e0
pdict.add_context(raw_fixed_ctxt)
raw_blob_ctxt = ParameterContext(
'raw_blob', param_type=QuantityType(value_encoding=numpy.float32))
raw_blob_ctxt.uom = 'unknown'
raw_blob_ctxt.fill_value = 0e0
pdict.add_context(raw_blob_ctxt)
return pdict
def _create_parameter_dictionary(self):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
return pdict
def _add_location_time_ctxt(self, pdict):
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
return pdict
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='time', parameter_context=t_ctxt.dump()))
cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType(
)) # param_type=QuantityType(value_encoding=np.dtype('float32')))
cnd_ctxt.uom = 'mmho/cm'
context_ids.append(
self.dataset_management.create_parameter_context(
name='conductivity', parameter_context=cnd_ctxt.dump()))
temp_ctxt = ParameterContext('temperature', param_type=ArrayType(
)) # param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'degC'
context_ids.append(
self.dataset_management.create_parameter_context(
name='temperature', parameter_context=temp_ctxt.dump()))
press_ctxt = ParameterContext('pressure', param_type=ArrayType(
)) # param_type=QuantityType(value_encoding=np.dtype('float32')))
press_ctxt.uom = 'decibars'
context_ids.append(
self.dataset_management.create_parameter_context(
name='pressure', parameter_context=press_ctxt.dump()))
oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType(
)) # param_type=QuantityType(value_encoding=np.dtype('float32')))
oxy_ctxt.uom = 'Hz'
context_ids.append(
self.dataset_management.create_parameter_context(
name='oxygen', parameter_context=oxy_ctxt.dump()))
return context_ids
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
subscriber.start()
self.addCleanup(subscriber.stop)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'Time_Time',
param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Time_Time', parameter_context=t_ctxt.dump()))
core_current_ctxt = ParameterContext(
'Core_Current',
param_type=QuantityType(value_encoding=np.dtype('float32')))
core_current_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Core_Current',
parameter_context=core_current_ctxt.dump()))
core_voltage_ctxt = ParameterContext(
'Core_Voltage',
param_type=QuantityType(value_encoding=np.dtype('float32')))
core_voltage_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Core_Voltage',
parameter_context=core_voltage_ctxt.dump()))
core_pressure_ctxt = ParameterContext(
'Core_Pressure',
param_type=QuantityType(value_encoding=np.dtype('float32')))
core_pressure_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Core_Pressure',
parameter_context=core_pressure_ctxt.dump()))
fluorometer_value_ctxt = ParameterContext(
'Fluorometer_Value',
param_type=QuantityType(value_encoding=np.dtype('float32')))
fluorometer_value_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Fluorometer_Value',
parameter_context=fluorometer_value_ctxt.dump()))
fluorometer_gain_ctxt = ParameterContext(
'Fluorometer_Gain',
param_type=QuantityType(value_encoding=np.dtype('int32')))
fluorometer_gain_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Fluorometer_Gain',
parameter_context=fluorometer_gain_ctxt.dump()))
turbidity_value_ctxt = ParameterContext(
'Turbidity_Value',
param_type=QuantityType(value_encoding=np.dtype('float32')))
turbidity_value_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Turbidity_Value',
parameter_context=turbidity_value_ctxt.dump()))
turbidity_gain_ctxt = ParameterContext(
'Turbidity_Gain',
param_type=QuantityType(value_encoding=np.dtype('int32')))
turbidity_gain_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Turbidity_Gain',
parameter_context=turbidity_gain_ctxt.dump()))
optode_oxygen_ctxt = ParameterContext(
'Optode_Oxygen',
param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_oxygen_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Optode_Oxygen',
parameter_context=optode_oxygen_ctxt.dump()))
optode_temp_ctxt = ParameterContext(
'Optode_Temp',
param_type=QuantityType(value_encoding=np.dtype('float32')))
optode_temp_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Optode_Temp', parameter_context=optode_temp_ctxt.dump()))
par_value_ctxt = ParameterContext(
'Par_Value',
param_type=QuantityType(value_encoding=np.dtype('float32')))
par_value_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Par_Value', parameter_context=par_value_ctxt.dump()))
puck_scatter_ctxt = ParameterContext(
'Puck_Scatter',
param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_scatter_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Puck_Scatter',
parameter_context=puck_scatter_ctxt.dump()))
puck_chla_ctxt = ParameterContext(
'Puck_Chla',
param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_chla_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Puck_Chla', parameter_context=puck_chla_ctxt.dump()))
puck_cdom_ctxt = ParameterContext(
'Puck_CDOM',
param_type=QuantityType(value_encoding=np.dtype('int16')))
puck_cdom_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump()))
biosuite_scatter_ctxt = ParameterContext(
'BioSuite_Scatter',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_scatter_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Scatter',
parameter_context=biosuite_scatter_ctxt.dump()))
biosuite_chla_ctxt = ParameterContext(
'BioSuite_Chla',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_chla_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Chla',
parameter_context=biosuite_chla_ctxt.dump()))
biosuite_cdom_ctxt = ParameterContext(
'BioSuite_CDOM',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_cdom_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_CDOM',
parameter_context=biosuite_cdom_ctxt.dump()))
biosuite_temp_ctxt = ParameterContext(
'BioSuite_Temp',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_temp_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Temp',
parameter_context=biosuite_temp_ctxt.dump()))
biosuite_par_ctxt = ParameterContext(
'BioSuite_Par',
param_type=QuantityType(value_encoding=np.dtype('int16')))
biosuite_par_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='BioSuite_Par',
parameter_context=biosuite_par_ctxt.dump()))
flbb_chla_ctxt = ParameterContext(
'FLBB_Chla',
param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_chla_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump()))
flbb_turb_ctxt = ParameterContext(
'FLBB_Turb',
param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_turb_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump()))
flbb_temp_ctxt = ParameterContext(
'FLBB_Temp',
param_type=QuantityType(value_encoding=np.dtype('int16')))
flbb_temp_ctxt.uom = 'unknown'
context_ids.append(
self.dataset_management.create_parameter_context(
name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump()))
return context_ids
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_wpt_y_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_water_cond", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_y_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext(
"u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_hd_fin_ap_inflection_holdoff", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_m_present_time", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext(
"m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="m_leakdetect_voltage_forward", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_bb3slo_b660_scaled", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_science_send_all", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_gps_status", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_water_vx", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_water_vy", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_heading", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_fl3slo_chlor_units", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_hd_fin_ap_gain", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_vacuum", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_min_water_depth", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_gps_lat", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_veh_temp", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="f_fin_offset", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext(
"u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_hd_fin_ap_hardover_holdoff", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_alt_time", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_present_time", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_heading", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_bb3slo_b532_scaled", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_fl3slo_cdom_units", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_fin", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="x_cycle_overrun_in_ms", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_water_pressure", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_hd_fin_ap_igain", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_fl3slo_phyco_units", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_battpos", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_bb3slo_b470_scaled", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_lat", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_gps_lon", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="sci_ctd41cp_timestamp", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_pressure", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_wpt_x_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="c_ballast_pumped", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="x_lmc_xy_source", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_lon", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_avg_speed", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="sci_water_temp", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="u_pitch_ap_gain", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_roll", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_tot_num_inflections", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="m_tot_num_inflections", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_x_lmc", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_pitch_ap_deadband", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_final_water_vy", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_final_water_vx", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_water_depth", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_leakdetect_voltage", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="m_leakdetect_voltage", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext(
"u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=np.dtype("float32"))
)
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(
name="u_pitch_max_delta_battpos", parameter_context=t_ctxt.dump()
)
)
t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_coulomb_amphr", parameter_context=t_ctxt.dump())
)
t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=np.dtype("float32")))
t_ctxt.uom = "unknown"
context_ids.append(
self.dataset_management.create_parameter_context(name="m_pitch", parameter_context=t_ctxt.dump())
)
return context_ids
def _add_location_time_ctxt(self, pdict):
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=numpy.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext(
'lat', param_type=QuantityType(value_encoding=numpy.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext(
'lon', param_type=QuantityType(value_encoding=numpy.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
return pdict
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters['list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters['date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product',
temporal_domain = tdom,
spatial_domain = sdom)
streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp")
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id,
parameter_dictionary=parameter_dictionary)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
#ttool = TaxyTool()
#
#ttool.add_taxonomy_set('data','test data')
pdict = ParameterDictionary()
t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
self.create_stream_and_logger(name='ruv',stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'external_dataset_res':dset,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
def _create_parameter_dictionary(self):
pdict = ParameterDictionary()
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'meters'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
return pdict
def get_context(self, ptype, encoding, fill_value, codeset=None):
ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset)
context = ParameterContext(name='test', param_type=ptype)
context.fill_value = self.types_manager.get_fill_value(fill_value, encoding, ptype)
return context
def get_context(self, ptype, encoding, fill_value, codeset=None):
ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset)
context = ParameterContext(name='test', param_type=ptype)
context.fill_value = self.types_manager.get_fill_value(
fill_value, encoding, ptype)
return context
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = "Christopher Mueller"
dprov.contact.email = "*****@*****.**"
# Create DataSource
dsrc = DataSource(protocol_type="FILE", institution=Institution(), contact=ContactInformation())
dsrc.connection_params["base_data_url"] = ""
dsrc.contact.name = "Tim Giguere"
dsrc.contact.email = "*****@*****.**"
# Create ExternalDataset
ds_name = "slocum_test_dataset"
dset = ExternalDataset(
name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation(),
)
dset.dataset_description.parameters["base_url"] = "test_data/slocum/"
dset.dataset_description.parameters["list_pattern"] = "ru05-2012-021-0-0-sbd.dat"
dset.dataset_description.parameters["date_pattern"] = "%Y %j"
dset.dataset_description.parameters["date_extraction_pattern"] = "ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat"
dset.dataset_description.parameters["temporal_dimension"] = None
dset.dataset_description.parameters["zonal_dimension"] = None
dset.dataset_description.parameters["meridional_dimension"] = None
dset.dataset_description.parameters["vertical_dimension"] = None
dset.dataset_description.parameters["variables"] = [
"c_wpt_y_lmc",
"sci_water_cond",
"m_y_lmc",
"u_hd_fin_ap_inflection_holdoff",
"sci_m_present_time",
"m_leakdetect_voltage_forward",
"sci_bb3slo_b660_scaled",
"c_science_send_all",
"m_gps_status",
"m_water_vx",
"m_water_vy",
"c_heading",
"sci_fl3slo_chlor_units",
"u_hd_fin_ap_gain",
"m_vacuum",
"u_min_water_depth",
"m_gps_lat",
"m_veh_temp",
"f_fin_offset",
"u_hd_fin_ap_hardover_holdoff",
"c_alt_time",
"m_present_time",
"m_heading",
"sci_bb3slo_b532_scaled",
"sci_fl3slo_cdom_units",
"m_fin",
"x_cycle_overrun_in_ms",
"sci_water_pressure",
"u_hd_fin_ap_igain",
"sci_fl3slo_phyco_units",
"m_battpos",
"sci_bb3slo_b470_scaled",
"m_lat",
"m_gps_lon",
"sci_ctd41cp_timestamp",
"m_pressure",
"c_wpt_x_lmc",
"c_ballast_pumped",
"x_lmc_xy_source",
"m_lon",
"m_avg_speed",
"sci_water_temp",
"u_pitch_ap_gain",
"m_roll",
"m_tot_num_inflections",
"m_x_lmc",
"u_pitch_ap_deadband",
"m_final_water_vy",
"m_final_water_vx",
"m_water_depth",
"m_leakdetect_voltage",
"u_pitch_max_delta_battpos",
"m_coulomb_amphr",
"m_pitch",
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name="slocum_model")
dsrc_model.model = "SLOCUM"
dsrc_model.data_handler_module = "N/A"
dsrc_model.data_handler_class = "N/A"
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id
)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
# create temp streamdef so the data product can create the stream
streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp")
# Generate the data product and associate it to the ExternalDataset
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dprod = IonObject(
RT.DataProduct,
name="slocum_parsed_product",
description="parsed slocum product",
temporal_domain=tdom,
spatial_domain=sdom,
)
dproduct_id = dpms_cli.create_data_product(
data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary
)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(
subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True
)
stream_id = stream_id[0]
log.info(
"Created resources: {0}".format(
{
"ExternalDataset": ds_id,
"ExternalDataProvider": ext_dprov_id,
"DataSource": ext_dsrc_id,
"DataSourceModel": ext_dsrc_model_id,
"DataProducer": dproducer_id,
"DataProduct": dproduct_id,
"Stream": stream_id,
}
)
)
# CBM: Use CF standard_names
# ttool = TaxyTool()
#
# ttool.add_taxonomy_set('c_wpt_y_lmc'),
# ttool.add_taxonomy_set('sci_water_cond'),
# ttool.add_taxonomy_set('m_y_lmc'),
# ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'),
# ttool.add_taxonomy_set('sci_m_present_time'),
# ttool.add_taxonomy_set('m_leakdetect_voltage_forward'),
# ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'),
# ttool.add_taxonomy_set('c_science_send_all'),
# ttool.add_taxonomy_set('m_gps_status'),
# ttool.add_taxonomy_set('m_water_vx'),
# ttool.add_taxonomy_set('m_water_vy'),
# ttool.add_taxonomy_set('c_heading'),
# ttool.add_taxonomy_set('sci_fl3slo_chlor_units'),
# ttool.add_taxonomy_set('u_hd_fin_ap_gain'),
# ttool.add_taxonomy_set('m_vacuum'),
# ttool.add_taxonomy_set('u_min_water_depth'),
# ttool.add_taxonomy_set('m_gps_lat'),
# ttool.add_taxonomy_set('m_veh_temp'),
# ttool.add_taxonomy_set('f_fin_offset'),
# ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'),
# ttool.add_taxonomy_set('c_alt_time'),
# ttool.add_taxonomy_set('m_present_time'),
# ttool.add_taxonomy_set('m_heading'),
# ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'),
# ttool.add_taxonomy_set('sci_fl3slo_cdom_units'),
# ttool.add_taxonomy_set('m_fin'),
# ttool.add_taxonomy_set('x_cycle_overrun_in_ms'),
# ttool.add_taxonomy_set('sci_water_pressure'),
# ttool.add_taxonomy_set('u_hd_fin_ap_igain'),
# ttool.add_taxonomy_set('sci_fl3slo_phyco_units'),
# ttool.add_taxonomy_set('m_battpos'),
# ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'),
# ttool.add_taxonomy_set('m_lat'),
# ttool.add_taxonomy_set('m_gps_lon'),
# ttool.add_taxonomy_set('sci_ctd41cp_timestamp'),
# ttool.add_taxonomy_set('m_pressure'),
# ttool.add_taxonomy_set('c_wpt_x_lmc'),
# ttool.add_taxonomy_set('c_ballast_pumped'),
# ttool.add_taxonomy_set('x_lmc_xy_source'),
# ttool.add_taxonomy_set('m_lon'),
# ttool.add_taxonomy_set('m_avg_speed'),
# ttool.add_taxonomy_set('sci_water_temp'),
# ttool.add_taxonomy_set('u_pitch_ap_gain'),
# ttool.add_taxonomy_set('m_roll'),
# ttool.add_taxonomy_set('m_tot_num_inflections'),
# ttool.add_taxonomy_set('m_x_lmc'),
# ttool.add_taxonomy_set('u_pitch_ap_deadband'),
# ttool.add_taxonomy_set('m_final_water_vy'),
# ttool.add_taxonomy_set('m_final_water_vx'),
# ttool.add_taxonomy_set('m_water_depth'),
# ttool.add_taxonomy_set('m_leakdetect_voltage'),
# ttool.add_taxonomy_set('u_pitch_max_delta_battpos'),
# ttool.add_taxonomy_set('m_coulomb_amphr'),
# ttool.add_taxonomy_set('m_pitch'),
pdict = ParameterDictionary()
t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"m_tot_num_inflections", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"m_leakdetect_voltage", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext(
"u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=numpy.dtype("float32"))
)
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
t_ctxt.uom = "unknown"
pdict.add_context(t_ctxt)
# CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
self.create_stream_and_logger(name="slocum", stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG["dh_cfg"] = {
"TESTING": True,
"stream_id": stream_id,
"external_dataset_res": dset,
"param_dictionary": pdict.dump(),
"data_producer_id": dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
"max_records": 20,
}
def build_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_y_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_cond', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_y_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_inflection_holdoff', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_m_present_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage_forward', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b660_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_science_send_all', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_status', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vx', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_vy', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_heading', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_chlor_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_gain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_vacuum', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_min_water_depth', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lat', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_veh_temp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='f_fin_offset', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_hardover_holdoff', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_alt_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_present_time', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_heading', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b532_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_cdom_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_fin', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='x_cycle_overrun_in_ms', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_pressure', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_hd_fin_ap_igain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_fl3slo_phyco_units', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_battpos', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_bb3slo_b470_scaled', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_lat', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_gps_lon', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_ctd41cp_timestamp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_pressure', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_wpt_x_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='c_ballast_pumped', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='x_lmc_xy_source', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_lon', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_avg_speed', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='sci_water_temp', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_gain', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_roll', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_tot_num_inflections', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_x_lmc', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_ap_deadband', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vy', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_final_water_vx', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_water_depth', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_leakdetect_voltage', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='u_pitch_max_delta_battpos', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_coulomb_amphr', parameter_context=t_ctxt.dump()))
t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=np.dtype('float32')))
t_ctxt.uom = 'unknown'
context_ids.append(self.dataset_management.create_parameter_context(name='m_pitch', parameter_context=t_ctxt.dump()))
return context_ids
def adhoc_get_parameter_dictionary(stream_name):
"""
@param stream_name IGNORED in this adhoc function; it returns the same
ParameterDictionary definition always.
@retval corresponding ParameterDictionary.
"""
#@TODO Luke - Maybe we can make this a bit more versatile, we could make this a standard pdict...
pdict = ParameterDictionary()
# ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.float32))
ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
ctxt.uom = 'unknown'
ctxt.fill_value = 0e0
pdict.add_context(ctxt)
ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
ctxt.axis = AxisTypeEnum.TIME
ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(ctxt)
ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
ctxt.axis = AxisTypeEnum.LON
ctxt.uom = 'degree_east'
pdict.add_context(ctxt)
ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
ctxt.axis = AxisTypeEnum.LAT
ctxt.uom = 'degree_north'
pdict.add_context(ctxt)
ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
ctxt.axis = AxisTypeEnum.HEIGHT
ctxt.uom = 'unknown'
pdict.add_context(ctxt)
return pdict
