def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(
node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(
node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
class DensityTransform(TransformFunction):
''' A basic transform that receives input through a subscription,
parses the input from a CTD, extracts the conductivity, pressure and Temperature value and calculates density
according to the defined algorithm. If the transform
has an output_stream it will publish the output on the output stream.
'''
# Make the stream definitions of the transform class attributes... best available option I can think of?
incoming_stream_def = SBE37_CDM_stream_definition()
outgoing_stream_def = L2_density_stream_definition()
def execute(self, granule):
"""Processes incoming data!!!!
"""
# Use the deconstructor to pull data from a granule
psd = PointSupplementStreamParser(
stream_definition=self.incoming_stream_def, stream_granule=granule)
conductivity = psd.get_values('conductivity')
pressure = psd.get_values('pressure')
temperature = psd.get_values('temperature')
longitude = psd.get_values('longitude')
latitude = psd.get_values('latitude')
height = psd.get_values('height')
time = psd.get_values('time')
log.warn('Got conductivity: %s' % str(conductivity))
log.warn('Got pressure: %s' % str(pressure))
log.warn('Got temperature: %s' % str(temperature))
sp = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
sa = SA_from_SP(sp, pressure, longitude, latitude)
density = rho(sa, temperature, pressure)
log.warn('Got density: %s' % str(density))
# Use the constructor to put data into a granule
psc = PointSupplementConstructor(
point_definition=self.outgoing_stream_def,
stream_id=self.streams['output'])
### Assumes the config argument for output streams is known and there is only one 'output'.
### the stream id is part of the metadata which much go in each stream granule - this is awkward to do at the
### application level like this!
for i in xrange(len(density)):
point_id = psc.add_point(time=time[i],
location=(longitude[i], latitude[i],
height[i]))
psc.add_scalar_point_coverage(point_id=point_id,
coverage_id='density',
value=density[i])
return psc.close_stream_granule()
def setUp(self):
# Start container
logging.disable(logging.ERROR)
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# simulate preloading
preload_ion_params(self.container)
logging.disable(logging.NOTSET)
#Instantiate a process to represent the test
process=VisualizationServiceTestProcess()
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process)
self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process)
self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process)
self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process)
self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process)
self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process)
self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process)
self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process)
self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def _create_ctd_input_stream_and_data_product(
self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=self.ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name=data_product_name,
description='ctd stream test')
try:
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
except Exception as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('new ctd_parsed_data_product_id = ',
ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product_id,
persist_data=False,
persist_metadata=False)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id,
PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def test_find_stream_definition(self):
definition = SBE37_CDM_stream_definition()
definition_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=definition_id)
res_id = self.pubsub_cli.find_stream_definition(stream_id=stream_id, id_only=True)
self.assertTrue(res_id==definition_id, 'The returned id did not match the definition_id')
res_obj = self.pubsub_cli.find_stream_definition(stream_id=stream_id, id_only=False)
self.assertTrue(isinstance(res_obj.container, StreamDefinitionContainer),
'The container object is not a stream definition.')
class SalinityTransform(TransformFunction):
'''
L2 Transform for CTD Data.
Input is conductivity temperature and pressure delivered as a single packet.
Output is Practical Salinity as calculated by the Gibbs Seawater package
'''
outgoing_stream_def = L2_practical_salinity_stream_definition()
incoming_stream_def = SBE37_CDM_stream_definition()
def execute(self, granule):
"""Processes incoming data!!!!
"""
# Use the deconstructor to pull data from a granule
psd = PointSupplementStreamParser(stream_definition=self.incoming_stream_def, stream_granule=granule)
conductivity = psd.get_values('conductivity')
pressure = psd.get_values('pressure')
temperature = psd.get_values('temperature')
longitude = psd.get_values('longitude')
latitude = psd.get_values('latitude')
height = psd.get_values('height')
time = psd.get_values('time')
log.warn('Got conductivity: %s' % str(conductivity))
log.warn('Got pressure: %s' % str(pressure))
log.warn('Got temperature: %s' % str(temperature))
salinity = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
log.warn('Got salinity: %s' % str(salinity))
# Use the constructor to put data into a granule
psc = PointSupplementConstructor(point_definition=self.outgoing_stream_def, stream_id=self.streams['output'])
for i in xrange(len(salinity)):
point_id = psc.add_point(time=time[i],location=(longitude[i],latitude[i],height[i]))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='salinity', value=salinity[i])
return psc.close_stream_granule()
def test_strem_def_not_found(self):
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id='nonexistent')
definition = SBE37_CDM_stream_definition()
definition_id = self.pubsub_cli.create_stream_definition(container=definition)
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id='nonexistent')
stream_id = self.pubsub_cli.create_stream()
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id=stream_id)
def make_some_data(self):
import numpy as np
stream_id = 'I am very special'
definition = SBE37_CDM_stream_definition()
definition.stream_resource_id = stream_id
self.couch.create(definition)
total = 200
n = 10 # at most n records per granule
i = 0
while i < total:
r = random.randint(1, n)
psc = PointSupplementConstructor(point_definition=definition,
stream_id=stream_id)
for x in xrange(r):
i += 1
point_id = psc.add_point(time=i, location=(0, 0, 0))
psc.add_scalar_point_coverage(
point_id=point_id,
coverage_id='temperature',
value=np.random.normal(loc=48.0, scale=4.0, size=1)[0])
psc.add_scalar_point_coverage(point_id=point_id,
coverage_id='pressure',
value=np.float32(1.0))
psc.add_scalar_point_coverage(point_id=point_id,
coverage_id='conductivity',
value=np.float32(2.0))
granule = psc.close_stream_granule()
hdf_string = granule.identifiables[
definition.data_stream_id].values
sha1 = hashlib.sha1(hdf_string).hexdigest().upper()
with open(
FileSystem.get_hierarchical_url(FS.CACHE,
'%s.hdf5' % sha1),
'w') as f:
f.write(hdf_string)
granule.identifiables[definition.data_stream_id].values = ''
self.couch.create(granule)
def test_createTransformsThenActivateInstrument(self):
# Set up the preconditions
# ingestion configuration parameters
self.exchange_point_id = 'science_data'
self.number_of_workers = 2
self.hdf_storage = HdfStorage(relative_path='ingest')
self.couch_storage = CouchStorage(datastore_name='test_datastore')
self.XP = 'science_data'
self.exchange_name = 'ingestion_queue'
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
# driver_config = {
# 'svr_addr': "localhost",
# 'cmd_port': 5556,
# 'evt_port': 5557,
# 'dvr_mod': "ion.agents.instrument.drivers.sbe37.sbe37_driver",
# 'dvr_cls': "SBE37Driver",
# 'comms_config': {
# 'addr': 'sbe37-simulator.oceanobservatories.org',
# 'port': 4001,
# }
# }
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product,
PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_raw_data_product,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1ConductivityTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1PressureTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1TemperatureTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new SalinityTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new DensityTransform data process definition: %s"
% ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id)
self.output_products = {}
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_conductivity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_pressure_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_temperature_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_conductivity_id,
ctd_L1_conductivity_dprocdef_id)
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id)
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 conductivity"
)
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity')
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_conductivity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 pressure"
)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure')
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_pressure_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 temperature"
)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature')
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_temperature_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id)
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L2 Salinity"
)
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l2_salinity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L2 Density"
)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Density',
description='transform output pressure')
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l2_density_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L0 all data_process start"
)
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product],
self.output_products)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L0 all data_process return"
)
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start"
)
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_conductivity_dprocdef_id,
[ctd_l0_conductivity_output_dp_id],
{'output': ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return"
)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process start"
)
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id],
{'output': ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process return"
)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process start"
)
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_temperature_dprocdef_id,
[ctd_l0_temperature_output_dp_id],
{'output': ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process return"
)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L2_salinity data_process start"
)
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product],
{'output': ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(
l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L2_salinity data_process return"
)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L2_Density data_process start"
)
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L2_density_dprocdef_id, [ctd_parsed_data_product],
{'output': ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(
l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L2_Density data_process return"
)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
'iaclient',
name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(
" test_createTransformsThenActivateInstrument:: got ia client %s",
str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug(
"test_createTransformsThenActivateInstrument:: initialize %s",
str(retval))
time.sleep(1)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_active %s",
str(reply))
time.sleep(1)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: run %s",
str(reply))
time.sleep(1)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG: 1,
SBE37Parameter.INTERVAL: 5,
SBE37Parameter.TXREALTIME: True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_createTransformsThenActivateInstrument:: go_streaming %s",
str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(15)
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrument: return from go_observatory %s",
str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
log.debug(
"test_createTransformsThenActivateInstrument:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_createTransformsThenActivateInstrument:: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Sampling
#-------------------------------
# cmd = AgentCommand(command='initialize')
# retval = self._ia_client.execute_agent(cmd)
# print retval
# log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
# time.sleep(2)
#
# cmd = AgentCommand(command='go_active')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: go_active %s", str(reply))
# time.sleep(2)
#
# cmd = AgentCommand(command='run')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: run %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample ")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 2")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 2 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 3")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 3 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling go_inactive ")
# cmd = AgentCommand(command='go_inactive')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from go_inactive %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling reset ")
# cmd = AgentCommand(command='reset')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from reset %s", str(reply))
# time.sleep(2)
#
#
# self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
#
#
# #get the dataset id of the ctd_parsed product from the dataproduct ctd_parsed_data_product
# ctd_parsed_data_product_obj = self.dataproductclient.read_data_product(ctd_parsed_data_product)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset id %s", str(ctd_parsed_data_product_obj.dataset_id))
#
# # ask for the dataset bounds from the datasetmgmtsvc
# bounds = self.datasetclient.get_dataset_bounds(ctd_parsed_data_product_obj.dataset_id)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset bounds %s", str(bounds))
# print 'activate_instrument: got dataset bounds %s', str(bounds)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
def on_start(self):
log.debug("VizStreamProducer start")
self.data_source_name = self.CFG.get('name')
self.dataset = self.CFG.get('dataset')
# create a pubsub client and a resource registry client
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
# Dummy instrument related clients
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.IngestClient = IngestionManagementServiceClient(
node=self.container.node)
# create the pubsub client
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
# Additional code for creating a dummy instrument
"""
# Set up the preconditions. Look for an existing ingestion config
while True:
log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.")
ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True)
if len(ingestion_cfgs) > 0:
break
else:
gevent.sleep(1)
"""
# Check to see if the data_product already exists in the system (for e.g re launching the code after a crash)
dp_ids, _ = self.rrclient.find_resources(RT.DataProduct, None,
self.data_source_name, True)
if len(dp_ids) > 0:
data_product_id = dp_ids[0]
print '>>>>>>>>>>>>> Found dp_id = ', data_product_id
else:
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name=self.data_source_name,
description=self.data_source_name,
model_label=self.data_source_name)
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name=self.data_source_name,
description=self.data_source_name,
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=ctd_stream_def)
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name=self.data_source_name,
description='ctd stream test')
data_product_id = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id,
persist_data=True,
persist_metadata=True)
print '>>>>>>>>>>>> New dp_id = ', data_product_id
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id,
PRED.hasStream, None, True)
if self.dataset == 'sinusoidal':
pid = self.container.spawn_process(
name='ctd_test.' + self.data_source_name,
module='ion.processes.data.sinusoidal_stream_publisher',
cls='SinusoidalCtdPublisher',
config={'process': {
'stream_id': stream_ids[0]
}})
else:
pid = self.container.spawn_process(
name='ctd_test.' + self.data_source_name,
module='ion.processes.data.ctd_stream_publisher',
cls='SimpleCtdPublisher',
config={'process': {
'stream_id': stream_ids[0]
}})
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel,
name='ExampleDatasetModel',
description="ExampleDatasetModel",
datset_type="FibSeries")
try:
datasetModel_id = self.damsclient.create_external_dataset_model(
datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s",
str(datasetModel_id))
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent,
name='datasetagent007',
description="datasetagent007",
handler_module=EDA_MOD,
handler_class=EDA_CLS)
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(
datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s",
str(datasetAgent_id))
# Create ExternalDataset
log.debug(
'TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset,
name='ExtDataset',
description="ExtDataset")
try:
extDataset_id = self.damsclient.create_external_dataset(
extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %
ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ",
str(extDataset_id))
#register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(
extDataset_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
container=ctd_stream_def)
log.debug(
"TestExternalDatasetAgentMgmt: new Stream Definition id = %s",
str(ctd_stream_def_id))
log.debug(
"TestExternalDatasetAgentMgmt: Creating new data product with a stream definition"
)
dp_obj = IonObject(RT.DataProduct,
name='eoi dataset data',
description=' stream test')
try:
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s",
str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=extDataset_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s",
str(stream_ids))
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id':
stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id': dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
'taxonomy': tx.dump(), #TODO: Currently does not support sets
'max_records': 4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': EDA_RESOURCE_ID
},
'test_mode': True
}
extDatasetAgentInstance_obj = IonObject(
RT.ExternalDatasetAgentInstance,
name='DatasetAgentInstance',
description="DatasetAgentInstance",
dataset_driver_config=self.DVR_CONFIG,
dataset_agent_config=agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(
external_dataset_agent_instance=extDatasetAgentInstance_obj,
external_dataset_agent_id=datasetAgent_id,
external_dataset_id=extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(extDatasetAgentInstance_obj))
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s",
str(extDatasetAgentInstance_id))
#Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ",
str(id), str(active))
self.damsclient.start_external_dataset_agent_instance(
extDatasetAgentInstance_id)
dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(
extDatasetAgentInstance_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(dataset_agent_instance_obj))
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ",
str(id), str(active))
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id,
process=FakeProcess())
print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s",
str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(
extDatasetAgentInstance_id)
def test_activateInstrumentSample(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'new InstrumentModel id = ', instModel_id
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
container=ctd_stream_def)
print 'new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
try:
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new dp_id = ', data_product_id1
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
print 'Data product streams1 = ', stream_ids
print 'Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
try:
data_product_id2 = self.dpclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new dp_id = ', data_product_id2
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
print 'Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
#self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id,
process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
time.sleep(2)
log.debug(
"test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)"
)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
time.sleep(2)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling acquire_sample ")
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentSample: return from acquire_sample %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling acquire_sample 2")
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentSample: return from acquire_sample 2 %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling acquire_sample 3")
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentSample: return from acquire_sample 3 %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
class SimpleCtdPublisher(StandaloneProcess):
def __init__(self, *args, **kwargs):
super(SimpleCtdPublisher, self).__init__(*args, **kwargs)
#@todo Init stuff
outgoing_stream_def = SBE37_CDM_stream_definition()
def on_start(self):
log.warn('Entering On Start!!!')
# Get the stream(s)
stream_id = self.CFG.get_safe('process.stream_id', {})
self.greenlet_queue = []
# Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided...
if not stream_id:
pubsub_cli = PubsubManagementServiceClient(
node=self.container.node)
stream_def_id = pubsub_cli.create_stream_definition(
name='Producer stream %s' % str(uuid4()),
container=self.outgoing_stream_def)
stream_id = pubsub_cli.create_stream(
name='Example CTD Data',
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_publisher_registrar = StreamPublisherRegistrar(
process=self, node=self.container.node)
# Needed to get the originator's stream_id
self.stream_id = stream_id
self.publisher = self.stream_publisher_registrar.create_publisher(
stream_id=stream_id)
self.last_time = 0
g = Greenlet(self._trigger_func, stream_id)
log.debug('Starting publisher thread for simple ctd data.')
g.start()
log.warn('Publisher Greenlet started in "%s"' %
self.__class__.__name__)
self.greenlet_queue.append(g)
def on_quit(self):
for greenlet in self.greenlet_queue:
greenlet.kill()
super(SimpleCtdPublisher, self).on_quit()
def _trigger_func(self, stream_id):
while True:
length = random.randint(1, 20)
ctd_packet = self._get_ctd_packet(stream_id, length)
log.info('SimpleCtdPublisher sending %d values!' % length)
self.publisher.publish(ctd_packet)
time.sleep(2.0)
def _get_ctd_packet(self, stream_id, length):
c = [random.uniform(0.0, 75.0) for i in xrange(length)]
t = [random.uniform(-1.7, 21.0) for i in xrange(length)]
p = [random.lognormvariate(1, 2) for i in xrange(length)]
lat = [random.uniform(-90.0, 90.0) for i in xrange(length)]
lon = [random.uniform(0.0, 360.0) for i in xrange(length)]
tvar = [self.last_time + i for i in xrange(1, length + 1)]
self.last_time = max(tvar)
ctd_packet = ctd_stream_packet(stream_id=stream_id,
c=c,
t=t,
p=p,
lat=lat,
lon=lon,
time=tvar)
return ctd_packet
class VizTransformGoogleDT(TransformFunction):
"""
This class is used for converting incoming data from CDM format to JSON style Google DataTables
Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed
to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as
a replay transform (one-shot).
[2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream
to create the complete datatable. As the granules come in, they are translated to the datatable
'components'. Components, because we are not creating the actual datatable in this code. That's the job
of the viz service to put all the components of a datatable together in JSON format before sending it
to the client
[3] The time stamp in the incoming stream can't be converted to the datetime object here because
the Raw stream definition only expects regular primitives (strings, floats, ints etc)
"""
outgoing_stream_def = SBE37_RAW_stream_definition()
#outgoing_stream_def = SBE37_CDM_stream_definition()
incoming_stream_def = SBE37_CDM_stream_definition()
def on_start(self):
super(VizTransformGoogleDT,self).on_start()
# init variables
self.varTuple = []
self.total_num_of_records_recvd = 0
# init config. Need to move it to YAML or something
self.realtime_window_size = 100
self.max_google_dt_len = 1024 # Remove this once the decimation has been moved in to the incoming dp
# Note some transform parameters
def execute(self, granule):
log.debug('(Google DT transform): Received Viz Data Packet' )
self.dataDescription = []
self.dataTableContent = []
element_count_id = 0
expected_range = []
# NOTE : Detect somehow that this is a replay stream with a set number of expected granules. Based on this
# calculate the number of expected records and set the self.realtime_window_size bigger or equal to this
# number.
psd = PointSupplementStreamParser(stream_definition=self.incoming_stream_def, stream_granule=granule)
vardict = {}
arrLen = None
for varname in psd.list_field_names():
vardict[varname] = psd.get_values(varname)
arrLen = len(vardict[varname])
#iinit the dataTable
# create data description from the variables in the message
self.dataDescription = [('time', 'datetime', 'time')]
# split the data string to extract variable names
for varname in psd.list_field_names():
if varname == 'time':
continue
self.dataDescription.append((varname, 'number', varname))
# Add the records to the datatable
for i in xrange(arrLen):
varTuple = []
for varname,_,_ in self.dataDescription:
val = float(vardict[varname][i])
if varname == 'time':
#varTuple.append(datetime.fromtimestamp(val))
varTuple.append(val)
else:
varTuple.append(val)
# Append the tuples to the data table
self.dataTableContent.append (varTuple)
# Maintain a sliding window for realtime transform processes
if len(self.dataTableContent) > self.realtime_window_size:
# always pop the first element till window size is what we want
while len(self.dataTableContent) > realtime_window_size:
self.dataTableContent.pop(0)
""" To Do : Do we need to figure out the how many granules have been received for a replay stream ??
if not self.realtime_flag:
# This is the historical view part. Make a note of how many records were received
in_data_stream_id = self.incoming_stream_def.data_stream_id
element_count_id = self.incoming_stream_def.identifiables[in_data_stream_id].element_count_id
# From each granule you can check the constraint on the number of records
expected_range = granule.identifiables[element_count_id].constraint.intervals[0]
# The number of records in a given packet is:
self.total_num_of_records_recvd += packet.identifiables[element_count_id].value
"""
# define an output container of data
# submit the partial datatable to the viz service
rdt = RecordDictionaryTool(taxonomy=tx)
# submit resulting table back using the out stream publisher. The data_product_id is the input dp_id
# responsible for the incoming data
msg = {"viz_product_type": "google_realtime_dt",
"data_product_id": "FAKE_DATAPRODUCT_ID_0000",
"data_table_description": self.dataDescription,
"data_table_content": self.dataTableContent}
rdt['google_dt_components'] = numpy.array([msg])
log.debug('Google DT transform: Sending a granule')
out_granule = build_granule(data_producer_id='google_dt_transform', taxonomy=tx, record_dictionary=rdt)
#self.publish(out_granule)
# clear the tuple for future use
self.varTuple[:] = []
return out_granule
def test_dm_integration(self):
'''
test_salinity_transform
Test full DM Services Integration
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(
node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there were two stream definitions...
###
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = pubsub_management_service.create_stream_definition(
container=ctd_stream_def, name='Simulated CTD data')
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = pubsub_management_service.create_stream_definition(
container=SalinityTransform.outgoing_stream_def,
name='Scalar Salinity data stream')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.ctd_stream_publisher',
'class': 'SimpleCtdPublisher'
}
ctd_sim_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
# one for the salinity transform
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'class': 'SalinityTransform'
}
salinity_transform_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
number_of_workers=1)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
ctd_stream_id = pubsub_management_service.create_stream(
stream_definition_id=ctd_stream_def_id)
# Set up the datasets
ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of this dataset
ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=ctd_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
#---------------------------
# Set up the salinity transform
#---------------------------
# Create the stream
sal_stream_id = pubsub_management_service.create_stream(
stream_definition_id=sal_stream_def_id)
# Set up the datasets
sal_dataset_id = dataset_management_service.create_dataset(
stream_id=sal_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of the salinity as a dataset
sal_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=sal_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Create a subscription as input to the transform
sal_transform_input_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery(stream_ids=[
ctd_stream_id,
]),
exchange_name='salinity_transform_input'
) # how do we make these names??? i.e. Should they be anonymous?
# create the salinity transform
sal_transform_id = transform_management_service.create_transform(
name='example salinity transform',
in_subscription_id=sal_transform_input_subscription_id,
out_streams={
'output': sal_stream_id,
},
process_definition_id=salinity_transform_procdef_id,
# no configuration needed at this time...
)
# start the transform - for a test case it makes sense to do it before starting the producer but it is not required
transform_management_service.activate_transform(
transform_id=sal_transform_id)
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': ctd_stream_id,
}
}
ctd_sim_pid = process_dispatcher.schedule_process(
process_definition_id=ctd_sim_procdef_id,
configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
salinity_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([
sal_stream_id,
]),
exchange_name='salinity_test',
name="test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Salinity data received!')
results.append(message)
if len(results) > 3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
for message in results:
psd = PointSupplementStreamParser(
stream_definition=SalinityTransform.outgoing_stream_def,
stream_granule=message)
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
import numpy
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) >
0.0) # salinity should always be greater than 0
def test_replay_integration(self):
'''
test_replay_integration
'''
import numpy as np
# Keep the import it's used in the vector comparison below even though pycharm says its unused.
cc = self.container
XP = self.XP
assertions = self.assertTrue
### Every thing below here can be run as a script:
log.debug('Got it')
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
datastore_name = 'dm_test_replay_integration'
producer = Publisher(name=(XP, 'stream producer'))
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id=XP,
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
hdf_storage=HdfStorage(),
number_of_workers=1)
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
definition = SBE37_CDM_stream_definition()
data_stream_id = definition.data_stream_id
encoding_id = definition.identifiables[data_stream_id].encoding_id
element_count_id = definition.identifiables[
data_stream_id].element_count_id
stream_def_id = pubsub_management_service.create_stream_definition(
container=definition)
stream_id = pubsub_management_service.create_stream(
stream_definition_id=stream_def_id)
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/dataset_by_id')
ingestion_management_service.create_dataset_configuration(
dataset_id=dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=ingestion_configuration_id)
definition.stream_resource_id = stream_id
packet = _create_packet(definition)
input_file = FileSystem.mktemp()
input_file.write(packet.identifiables[data_stream_id].values)
input_file_path = input_file.name
input_file.close()
fields = [
'conductivity', 'height', 'latitude', 'longitude', 'pressure',
'temperature', 'time'
]
input_vectors = acquire_data([input_file_path], fields, 2).next()
producer.publish(msg=packet, to_name=(XP, '%s.data' % stream_id))
replay_id, replay_stream_id = data_retriever_service.define_replay(
dataset_id)
ar = gevent.event.AsyncResult()
def sub_listen(msg, headers):
assertions(isinstance(msg, StreamGranuleContainer),
'replayed message is not a granule.')
hdf_string = msg.identifiables[data_stream_id].values
sha1 = hashlib.sha1(hdf_string).hexdigest().upper()
assertions(sha1 == msg.identifiables[encoding_id].sha1,
'Checksum failed.')
assertions(
msg.identifiables[element_count_id].value == 1,
'record replay count is incorrect %d.' %
msg.identifiables[element_count_id].value)
output_file = FileSystem.mktemp()
output_file.write(msg.identifiables[data_stream_id].values)
output_file_path = output_file.name
output_file.close()
output_vectors = acquire_data([output_file_path], fields, 2).next()
for field in fields:
comparison = (input_vectors[field]['values'] ==
output_vectors[field]['values'])
assertions(
comparison.all(), 'vector mismatch: %s vs %s' %
(input_vectors[field]['values'],
output_vectors[field]['values']))
FileSystem.unlink(output_file_path)
ar.set(True)
subscriber = Subscriber(name=(XP, 'replay listener'),
callback=sub_listen)
g = gevent.Greenlet(subscriber.listen,
binding='%s.data' % replay_stream_id)
g.start()
data_retriever_service.start_replay(replay_id)
ar.get(timeout=10)
FileSystem.unlink(input_file_path)
class ctd_L0_all(TransformDataProcess):
"""Model for a TransformDataProcess
"""
# Make the stream definitions of the transform class attributes
incoming_stream_def = SBE37_CDM_stream_definition()
outgoing_stream_pressure = L0_pressure_stream_definition()
outgoing_stream_temperature = L0_temperature_stream_definition()
outgoing_stream_conductivity = L0_conductivity_stream_definition()
# Retrieve the id of the OUTPUT stream from the out Data Product for each of the three output streams
# stream_ids, _ = self.clients.resource_registry.find_objects(out_data_product_id, PRED.hasStream, None, True)
#
# log.debug("DataProcessManagementService:create_data_process retrieve out data prod streams: %s", str(stream_ids))
# if not stream_ids:
# raise NotFound("No Stream created for output Data Product " + str(out_data_product_id))
# if len(stream_ids) != 1:
# raise BadRequest("Data Product should only have ONE stream at this time" + str(out_data_product_id))
# self.output_stream_dict[name] = stream_ids[0]
def process(self, packet):
"""Processes incoming data!!!!
"""
# Use the PointSupplementStreamParser to pull data from a granule
psd = PointSupplementStreamParser(
stream_definition=self.incoming_stream_def, stream_granule=packet)
conductivity = psd.get_values('conductivity')
pressure = psd.get_values('pressure')
temperature = psd.get_values('temperature')
longitude = psd.get_values('longitude')
latitude = psd.get_values('latitude')
height = psd.get_values('height')
time = psd.get_values('time')
log.warn('Got conductivity: %s' % str(conductivity))
log.warn('Got pressure: %s' % str(pressure))
log.warn('Got temperature: %s' % str(temperature))
# do L0 scaling here.....
# Use the constructor to put data into a granule
psc_conductivity = PointSupplementConstructor(
point_definition=self.outgoing_stream_conductivity,
stream_id=self.streams['conductivity'])
psc_pressure = PointSupplementConstructor(
point_definition=self.outgoing_stream_pressure,
stream_id=self.streams['pressure'])
psc_temperature = PointSupplementConstructor(
point_definition=self.outgoing_stream_temperature,
stream_id=self.streams['temperature'])
### The stream id is part of the metadata which much go in each stream granule - this is awkward to do at the
### application level like this!
for i in xrange(len(conductivity)):
point_id = psc_conductivity.add_point(time=time[i],
location=(longitude[i],
latitude[i],
height[i]))
psc_conductivity.add_scalar_point_coverage(
point_id=point_id,
coverage_id='conductivity',
value=conductivity[i])
self.conductivity.publish(psc_conductivity.close_stream_granule())
for i in xrange(len(pressure)):
point_id = psc_pressure.add_point(time=time[i],
location=(longitude[i],
latitude[i],
height[i]))
psc_pressure.add_scalar_point_coverage(point_id=point_id,
coverage_id='pressure',
value=pressure[i])
self.pressure.publish(psc_pressure.close_stream_granule())
for i in xrange(len(temperature)):
point_id = psc_temperature.add_point(time=time[i],
location=(longitude[i],
latitude[i],
height[i]))
psc_temperature.add_scalar_point_coverage(
point_id=point_id,
coverage_id='temperature',
value=temperature[i])
self.temperature.publish(psc_temperature.close_stream_granule())
return
def test_lookupTableProcessing(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice and attachment for lookup table
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
driver_config = {
'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'TestDataProcessWithLookupTable: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'TestDataProcessWithLookupTable: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) )
processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) )
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create L0 all data_process start")
try:
in_prods = []
in_prods.append(ctd_parsed_data_product)
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, in_prods, self.output_products)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("TestDataProcessWithLookupTable: create L0 all data_process return")
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
print 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = ', processAttachment