class TestBulkIngest(IonIntegrationTestCase):
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
log.debug('EventSubscriber event received: %s', str(args[0]) )
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
def tearDown(self):
pass
@unittest.skip('Update to agent refactor.')
def test_slocum_data_ingest(self):
HIST_CONSTRAINTS_1 = {}
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_param(params)
self._finished_count = 1
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
# Assert that data was received
self._async_finished_result.get(timeout=15)
self.assertTrue(len(self._finished_events_received) >= 1)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#todo enable after Luke's mor to retrieve, right now must have the Time axis called 'time'
# replay_granule = self.data_retriever.retrieve_last_data_points(self.dataset_id, 10)
# rdt = RecordDictionaryTool.load_from_granule(replay_granule)
#
# comp = rdt['date_pattern'] == numpy.arange(10) + 10
#
# log.debug("TestBulkIngest: comp: %s", comp)
#
# self.assertTrue(comp.all())
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver',
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
log.debug( 'new Stream Definition id = %s', instDevice_id)
log.debug( 'Creating new CDM data product with a stream definition')
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
log.debug( 'Creating new RAW data product with a stream definition')
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug( 'new dp_id = %s', str(ctd_raw_data_product))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
log.debug( 'Data product streams2 = %s', str(stream_ids))
#todo: attaching the taxonomy to the stream is a TEMPORARY measure
# Create taxonomies for both parsed and attach to the stream
RawTax = TaxyTool()
RawTax.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records')
RawTax.add_taxonomy_set('raw_blob','Unlimited length bytes in an array')
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id,
parameter_dictionary)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id,
parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id,
parameter_dictionary
)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_pressure', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_temperature', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_salinity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_density', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from run state: %s", str(state))
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state))
time.sleep(7)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestInstrumentAgentWithTrhph(TrhphTestCase, IonIntegrationTestCase):
"""
R2 instrument agent tests with the TRHPH driver.
"""
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
self.device_address,
self.device_port,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
# make sure the driver is stopped
self.addCleanup(self._reset)
def addCleanup(self, f):
IonIntegrationTestCase.addCleanup(self, f)
def tearDown(self):
try:
IonIntegrationTestCase.tearDown(self)
finally:
TrhphTestCase.tearDown(self)
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for stream_name in PACKET_CONFIG:
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
taxy = get_taxonomy(stream_name)
stream_config = dict(
id=stream_id,
taxonomy=taxy.dump()
)
self._stream_config[stream_name] = stream_config
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(
query=query, exchange_name=exchange_name,
exchange_point='science_data')
pubsub_client.activate_subscription(sub_id)
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._data_greenlets.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
log.info('cleanup: _stop_data_subscribers called.')
for sub in self._data_subscribers:
sub.stop()
for gl in self._data_greenlets:
gl.kill()
def _start_event_subscribers(self):
"""
Create subscribers for agent and driver events.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._no_events and self._no_events == len(self._event_received):
self._async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
event_sub.start()
self._event_subscribers.append(event_sub)
def _stop_event_subscribers(self):
"""
Stop event subscribers on cleanup.
"""
log.info('cleanup: _stop_event_subscribers called.')
for sub in self._event_subscribers:
sub.stop()
def _initialize_and_run(self):
"""
Called explicitly by the tests that do regular operations.
"""
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
def _reset(self):
"""
Set as a clean-up to make sure the agent is reset after each test,
so the driver is stopped.
"""
log.info("_reset called: sending 'reset' command to agent")
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_01_initialize(self):
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_10_states(self):
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def _get_params(self, params):
result = self._ia_client.get_param(params)
log.info('_get_params result: %s' % str(result))
self.assertTrue(isinstance(result, dict))
if params == DriverParameter.ALL:
all_requested_params = TrhphParameter.list()
else:
all_requested_params = params
# check all requested params are in the result
for p in all_requested_params:
self.assertTrue(p in result)
if TrhphParameter.TIME_BETWEEN_BURSTS == p:
seconds = result.get(p)
self.assertTrue(isinstance(seconds, int))
elif TrhphParameter.VERBOSE_MODE == p:
is_data_only = result.get(p)
self.assertTrue(isinstance(is_data_only, bool))
return result
def test_15_get_params(self):
self._initialize_and_run()
self._get_params(DriverParameter.ALL)
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
p2 = TrhphParameter.VERBOSE_MODE
self._get_params([p1, p2])
def _set_params(self, params):
"""
Sets the given parameters, which are assumed to be all valid.
"""
result = self._ia_client.set_param(params)
log.info("set result = %s" % str(result))
if result is None:
# TODO check why self._ia_client.set_param returns None
return
assert isinstance(result, dict)
# check all requested params are in the result
for (p, v) in params.items():
self.assertTrue(p in result)
return result
def _get_verbose_flag_for_set_test(self):
"""
Gets the value to use for the verbose flag in the "set" operations.
If we are testing against the real instrument (self._is_real_instrument
is true), this always returns False because the associated
interfaces with verbose=True are not implemented yet.
Otherwise it returns a random boolean value. Note, in this case, which
means we are testing against the simulator, the actual verbose value
does not have any effect of the other interface elements, so it is ok
to set any value here.
TODO align this when the verbose flag is handled completely,
both in the driver and in the simulator.
"""
if self._is_real_instrument:
log.info("setting verbose=False because _is_real_instrument")
return False
else:
return 0 == random.randint(0, 1)
def test_20_set_params_valid(self):
self._initialize_and_run()
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
new_seconds = random.randint(15, 60)
p2 = TrhphParameter.VERBOSE_MODE
verbose = self._get_verbose_flag_for_set_test()
valid_params = {p1: new_seconds, p2: verbose}
self._set_params(valid_params)
def test_25_get_set_params(self):
self._initialize_and_run()
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
result = self._get_params([p1])
seconds = result[p1]
new_seconds = seconds + 5
if new_seconds > 30 or new_seconds < 15:
new_seconds = 15
p2 = TrhphParameter.VERBOSE_MODE
new_verbose = self._get_verbose_flag_for_set_test()
valid_params = {p1: new_seconds, p2: new_verbose}
log.info("setting: %s" % str(valid_params))
self._set_params(valid_params)
result = self._get_params([p1, p2])
seconds = result[p1]
self.assertEqual(seconds, new_seconds)
verbose = result[p2]
self.assertEqual(verbose, new_verbose)
def test_60_execute_stop_autosample(self):
self._initialize_and_run()
log.info("stopping autosample")
cmd = AgentCommand(command='stop_autosample',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("stop_autosample reply = %s" % str(reply))
def test_70_execute_get_metadata(self):
self._initialize_and_run()
log.info("getting metadata")
cmd = AgentCommand(command='get_metadata',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("get_metadata reply = %s" % str(reply))
self.assertTrue(isinstance(reply.result, dict))
def test_80_execute_diagnostics(self):
self._initialize_and_run()
log.info("executing diagnostics")
num_scans = 11
cmd = AgentCommand(command='diagnostics',
kwargs=dict(num_scans=num_scans, timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("diagnostics reply = %s" % str(reply))
self.assertTrue(isinstance(reply.result, list))
self.assertEqual(len(reply.result), num_scans)
def test_90_execute_get_power_statuses(self):
self._initialize_and_run()
log.info("executing get_power_statuses")
cmd = AgentCommand(command='get_power_statuses',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("get_power_statuses reply = %s" % str(reply))
self.assertTrue(isinstance(reply.result, dict))
def test_99_execute_start_autosample(self):
self._initialize_and_run()
log.info("executing start_autosample")
cmd = AgentCommand(command='start_autosample',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("start_autosample reply = %s" % str(reply))
@unittest.skip('Not running after code updates in other places')
def test_get_params_invalid(self):
self._initialize_and_run()
with self.assertRaises(InstParameterError):
self._get_params(['bad-param'])
@unittest.skip('Not running after code updates in other places')
def test_set_params_invalid(self):
self._initialize_and_run()
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
new_seconds = random.randint(15, 60)
invalid_params = {p1: new_seconds, "bad-param": "dummy-value"}
with self.assertRaises(InstParameterError):
self._set_params(invalid_params)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
# setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
@unittest.skip("TBD")
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug("new InstrumentModel id = %s ", instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
"test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) "
)
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("new Stream Definition id = %s", instDevice_id)
log.debug("Creating new CDM data product with a stream definition")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("test_activateInstrumentSample: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
process_source="some_source_reference",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_conductivity, name="L0_Conductivity"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_pressure, name="L0_Pressure"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_temperature, name="L0_Temperature"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id
)
self.output_products = {}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_conductivity_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_conductivity stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l1_conductivity", stream_ids[0])
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_pressure_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_pressure stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l0_pressure", stream_ids[0])
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_temperature_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_temperature stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l0_temperature", stream_ids[0])
self.loggerpids.append(pid)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
# -------------------------------
log.debug("test_activateInstrumentSample: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [data_product_id1], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
log.debug("Creating new RAW data product with a stream definition")
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug("Data product streams2 = %s", str(stream_ids))
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug("Instrument agent instance obj: = %s", str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command="initialize")
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
time.sleep(1)
log.debug("test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command="go_active")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s", str(reply))
time.sleep(1)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state),
)
cmd = AgentCommand(command="run")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample ")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample 2")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample 2 %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample 3")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample 3 %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s", str(reply))
time.sleep(1)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
@unittest.skip("TBD")
def test_activateInstrumentStream(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug("new InstrumentModel id = %s ", instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
"test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) "
)
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("new Stream Definition id = %s", instDevice_id)
log.debug("Creating new CDM data product with a stream definition")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# simdata_subscription_id = self.pubsubcli.create_subscription(
# query=StreamQuery([stream_ids[0]]),
# exchange_name='Sim_data_queue',
# name='SimDataSubscription',
# description='SimData SubscriptionDescription'
# )
#
#
# def simdata_message_received(message, headers):
# input = str(message)
# log.debug("test_activateInstrumentStream: granule received: %s", input)
#
#
# subscriber_registrar = StreamSubscriberRegistrar(process=self.container, container=self.container)
# simdata_subscriber = subscriber_registrar.create_subscriber(exchange_name='Sim_data_queue', callback=simdata_message_received)
#
# # Start subscribers
# simdata_subscriber.start()
#
# # Activate subscriptions
# self.pubsubcli.activate_subscription(simdata_subscription_id)
log.debug("Creating new RAW data product with a stream definition")
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug("Data product streams2 = %s", str(stream_ids))
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug("test_activateInstrumentStream Instrument agent instance obj: = %s", str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command="initialize")
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
time.sleep(2)
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command="go_active")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state),
)
cmd = AgentCommand(command="run")
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from run state: %s", str(state))
# Make sure the sampling rate and transmission are sane.
params = {SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5, SBE37Parameter.TXREALTIME: True}
self._ia_client.set_param(params)
time.sleep(2)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command="go_streaming")
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state))
time.sleep(5)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command="go_observatory")
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result))
time.sleep(2)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestBulkIngest(IonIntegrationTestCase):
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
log.debug('EventSubscriber event received: %s', str(args[0]) )
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
def tearDown(self):
pass
@unittest.skip('Update to agent refactor.')
def test_slocum_data_ingest(self):
HIST_CONSTRAINTS_1 = {}
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_param(params)
self._finished_count = 1
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
# Assert that data was received
self._async_finished_result.get(timeout=15)
self.assertTrue(len(self._finished_events_received) >= 1)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#todo enable after Luke's mor to retrieve, right now must have the Time axis called 'time'
# replay_granule = self.data_retriever.retrieve_last_data_points(self.dataset_id, 10)
# rdt = RecordDictionaryTool.load_from_granule(replay_granule)
#
# comp = rdt['date_pattern'] == numpy.arange(10) + 10
#
# log.debug("TestBulkIngest: comp: %s", comp)
#
# self.assertTrue(comp.all())
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
self.addCleanup(self.dataproductclient.suspend_data_product_persistence, self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._data_greenlets.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
sub.stop()
for gl in self._data_greenlets:
gl.kill()
def _start_event_subscribers(self):
"""
Create subscribers for agent and driver events.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._no_events and self._no_events == len(self._event_received):
self._async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
event_sub.activate()
self._event_subscribers.append(event_sub)
def _stop_event_subscribers(self):
"""
Stop event subscribers on cleanup.
"""
for sub in self._event_subscribers:
sub.deactivate()
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val*.01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
"""
Test agent state transitions.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set(self):
"""
Test instrument driver get and set interface.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param(SBE37Parameter.ALL)
self.assertParamDict(reply, True)
orig_config = reply
# Retrieve a subset of resource parameters.
params = [
SBE37Parameter.TA0,
SBE37Parameter.INTERVAL,
SBE37Parameter.STORETIME
]
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
SBE37Parameter.TA0 : (orig_params[SBE37Parameter.TA0] * 2),
SBE37Parameter.INTERVAL : (orig_params[SBE37Parameter.INTERVAL] + 1),
SBE37Parameter.STORETIME : (not orig_params[SBE37Parameter.STORETIME])
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
# Reset the parameters back to their original values.
self._ia_client.set_param(orig_params)
reply = self._ia_client.get_param(SBE37Parameter.ALL)
reply.pop(SBE37Parameter.SAMPLENUM)
orig_config.pop(SBE37Parameter.SAMPLENUM)
self.assertParamVals(reply, orig_config)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_poll(self):
"""
Test observatory polling function.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Lets get 3 samples.
self._no_samples = 3
# Poll for a few samples.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
# Assert we got 3 samples.
self._async_data_result.get(timeout=10)
self.assertTrue(len(self._samples_received)==3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_autosample(self):
"""
Test instrument driver execute interface to start and stop streaming
mode.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(15)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert we got some samples.
self._async_data_result.get(timeout=10)
self.assertTrue(len(self._samples_received)>=2)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
"""
Test the ability to retrieve agent and resource parameter and command
capabilities.
"""
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
acmds = [item[1] for item in acmds]
self.assertEqual(acmds, AGT_CMDS)
apars = self._ia_client.get_capabilities(['AGT_PAR'])
apars = [item[1] for item in apars]
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
rcmds = [item[1] for item in rcmds]
self.assertEqual(rcmds, CMDS)
rpars = self._ia_client.get_capabilities(['RES_PAR'])
rpars = [item[1] for item in rpars]
self.assertEqual(rpars, SBE37Parameter.list())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
@unittest.skip('Never written')
def test_errors(self):
"""
Test illegal behavior and replies.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s',str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# OK, I can do this now.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
with self.assertRaises(InstParameterError):
reply = self._ia_client.get_param('bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
@unittest.skip('Direct access test to be finished by adding the telnet client, manual for now.')
def test_direct_access(self):
"""
Test agent direct_access command. This causes creation of
driver process and transition to direct access.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='go_direct_access',
#kwargs={'session_type':DirectAccessTypes.telnet,
kwargs={'session_type':DirectAccessTypes.vsp,
'session_timeout':600,
'inactivity_timeout':600})
retval = self._ia_client.execute_agent(cmd)
print("go_direct_access retval=" + str(retval))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.DIRECT_ACCESS)
# sleep to let tester run telnet client manually
print "test sleeping to run telnet client"
time.sleep(60)
# Halt DA.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
def test_createTransformsThenActivateInstrument(self):
# Set up the preconditions
# ingestion configuration parameters
self.exchange_point_id = 'science_data'
self.number_of_workers = 2
self.hdf_storage = HdfStorage(relative_path='ingest')
self.couch_storage = CouchStorage(datastore_name='test_datastore')
self.XP = 'science_data'
self.exchange_name = 'ingestion_queue'
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.services.mi.instrument_agent", driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
# driver_config = {
# 'svr_addr': "localhost",
# 'cmd_port': 5556,
# 'evt_port': 5557,
# 'dvr_mod': "ion.services.mi.drivers.sbe37_driver",
# 'dvr_cls': "SBE37Driver",
# 'comms_config': {
# 'addr': 'sbe37-simulator.oceanobservatories.org',
# 'port': 4001,
# }
# }
driver_config = {
'dvr_mod' : 'ion.services.mi.drivers.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity',description='transform output L1 conductivity')
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L1_Pressure',description='transform output L1 pressure')
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L1_Temperature',description='transform output L1 temperature')
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity',description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject(RT.DataProduct, name='L2_Density',description='transform output pressure')
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, ctd_parsed_data_product, self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, ctd_l0_conductivity_output_dp_id, {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, ctd_l0_pressure_output_dp_id, {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, ctd_l0_temperature_output_dp_id, {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product, {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, ctd_parsed_data_product, {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
time.sleep(1)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_active %s", str(reply))
time.sleep(1)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: run %s", str(reply))
time.sleep(1)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_streaming %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(15)
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrument: return from go_observatory %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
log.debug("test_createTransformsThenActivateInstrument:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: return from reset %s", str(reply))
time.sleep(2)
#-------------------------------
# Sampling
#-------------------------------
# cmd = AgentCommand(command='initialize')
# retval = self._ia_client.execute_agent(cmd)
# print retval
# log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
# time.sleep(2)
#
# cmd = AgentCommand(command='go_active')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: go_active %s", str(reply))
# time.sleep(2)
#
# cmd = AgentCommand(command='run')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: run %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample ")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 2")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 2 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 3")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 3 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling go_inactive ")
# cmd = AgentCommand(command='go_inactive')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from go_inactive %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling reset ")
# cmd = AgentCommand(command='reset')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from reset %s", str(reply))
# time.sleep(2)
#
#
# self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
#
#
# #get the dataset id of the ctd_parsed product from the dataproduct ctd_parsed_data_product
# ctd_parsed_data_product_obj = self.dataproductclient.read_data_product(ctd_parsed_data_product)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset id %s", str(ctd_parsed_data_product_obj.dataset_id))
#
# # ask for the dataset bounds from the datasetmgmtsvc
# bounds = self.datasetclient.get_dataset_bounds(ctd_parsed_data_product_obj.dataset_id)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset bounds %s", str(bounds))
# print 'activate_instrument: got dataset bounds %s', str(bounds)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
# log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
# log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': {},
'agent': {
'resource_id': self.EDA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError(
'_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id=''):
if not stream_id or stream_id is '':
stream_id = self._pubsub_client.create_stream(name=name,
encoding='ION R2')
pid = self._container_client.spawn_process(
name=name + '_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process': {
'stream_id': stream_id
}})
log.info(
'Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.
format(pid, stream_id))
return stream_id
def _start_finished_event_subscriber(self):
def consume_event(*args, **kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(
self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(
len(self._finished_events_received))
log.debug(
'Called self._async_finished_result.set({0})'.format(
len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(
event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.activate()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.deactivate()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val * .01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
self._finished_count = 3
log.info('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
log.info(
'Send a second unconstrained request for data (\'new data\'), should be rejected'
)
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
config_mods = {}
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config_mods['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
log.info(
'Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_2')
config_mods['constraints'] = self.HIST_CONSTRAINTS_2
# config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2}
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
finished = self._async_finished_result.get(timeout=10)
self.assertEqual(finished, self._finished_count)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config])
reply = self._ia_client.execute(cmd)
self.assertNotEqual(reply.status, 660)
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 3
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_observatory(self):
# Test instrument driver get and set interface.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL')
self.assertParamDict(reply, True)
orig_config = reply
## Retrieve a subset of resource parameters.
params = ['POLLING_INTERVAL']
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set_param(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_param(
['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(type(retval['POLLING_INTERVAL']), int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.get_param,
['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_param({'POLLING_INTERVAL': 10})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.set_param,
{'BAD_PARAM': 'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
self.assertRaises(InstParameterError, self._ia_client.set_param, {
'POLLING_INTERVAL': 20,
'BAD_PARAM': 'bad_val'
})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 20)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._finished_count = 1
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
gevent.sleep(5)
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._async_finished_result.get(timeout=5)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s', str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
self.assertRaises(InstParameterError, self._ia_client.get_param,
'bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class TestInstrumentAgentWithTrhph(TrhphTestCase, MiIntTestCase):
"""
R2 instrument agent tests with the TRHPH driver.
"""
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS,
self.device_address,
self.device_port, DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
# make sure the driver is stopped
self.addCleanup(self._reset)
def addCleanup(self, f):
MiIntTestCase.addCleanup(self, f)
def tearDown(self):
try:
MiIntTestCase.tearDown(self)
finally:
TrhphTestCase.tearDown(self)
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {'addr': 'localhost', 'port': port}
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(
self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, container=self.container)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for stream_name in PACKET_CONFIG:
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
taxy = get_taxonomy(stream_name)
stream_config = dict(id=stream_id, taxonomy=taxy.dump())
self._stream_config[stream_name] = stream_config
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(
exchange_name=exchange_name, callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(
query=query,
exchange_name=exchange_name,
exchange_point='science_data')
pubsub_client.activate_subscription(sub_id)
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._data_greenlets.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
log.info('cleanup: _stop_data_subscribers called.')
for sub in self._data_subscribers:
sub.stop()
for gl in self._data_greenlets:
gl.kill()
def _start_event_subscribers(self):
"""
Create subscribers for agent and driver events.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._no_events and self._no_events == len(
self._event_received):
self._async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent",
callback=consume_event)
event_sub.start()
self._event_subscribers.append(event_sub)
def _stop_event_subscribers(self):
"""
Stop event subscribers on cleanup.
"""
log.info('cleanup: _stop_event_subscribers called.')
for sub in self._event_subscribers:
sub.stop()
def _initialize_and_run(self):
"""
Called explicitly by the tests that do regular operations.
"""
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
def _reset(self):
"""
Set as a clean-up to make sure the agent is reset after each test,
so the driver is stopped.
"""
log.info("_reset called: sending 'reset' command to agent")
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_01_initialize(self):
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_10_states(self):
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_agent_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def _get_params(self, params):
result = self._ia_client.get_param(params)
log.info('_get_params result: %s' % str(result))
self.assertTrue(isinstance(result, dict))
if params == DriverParameter.ALL:
all_requested_params = TrhphParameter.list()
else:
all_requested_params = params
# check all requested params are in the result
for p in all_requested_params:
self.assertTrue(p in result)
if TrhphParameter.TIME_BETWEEN_BURSTS == p:
seconds = result.get(p)
self.assertTrue(isinstance(seconds, int))
elif TrhphParameter.VERBOSE_MODE == p:
is_data_only = result.get(p)
self.assertTrue(isinstance(is_data_only, bool))
return result
def test_15_get_params(self):
self._initialize_and_run()
self._get_params(DriverParameter.ALL)
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
p2 = TrhphParameter.VERBOSE_MODE
self._get_params([p1, p2])
def _set_params(self, params):
"""
Sets the given parameters, which are assumed to be all valid.
"""
result = self._ia_client.set_param(params)
log.info("set result = %s" % str(result))
if result is None:
# TODO check why self._ia_client.set_param returns None
return
assert isinstance(result, dict)
# check all requested params are in the result
for (p, v) in params.items():
self.assertTrue(p in result)
return result
def _get_verbose_flag_for_set_test(self):
"""
Gets the value to use for the verbose flag in the "set" operations.
If we are testing against the real instrument (self._is_real_instrument
is true), this always returns False because the associated
interfaces with verbose=True are not implemented yet.
Otherwise it returns a random boolean value. Note, in this case, which
means we are testing against the simulator, the actual verbose value
does not have any effect of the other interface elements, so it is ok
to set any value here.
TODO align this when the verbose flag is handled completely,
both in the driver and in the simulator.
"""
if self._is_real_instrument:
log.info("setting verbose=False because _is_real_instrument")
return False
else:
return 0 == random.randint(0, 1)
def test_20_set_params_valid(self):
self._initialize_and_run()
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
new_seconds = random.randint(15, 60)
p2 = TrhphParameter.VERBOSE_MODE
verbose = self._get_verbose_flag_for_set_test()
valid_params = {p1: new_seconds, p2: verbose}
self._set_params(valid_params)
def test_25_get_set_params(self):
self._initialize_and_run()
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
result = self._get_params([p1])
seconds = result[p1]
new_seconds = seconds + 5
if new_seconds > 30 or new_seconds < 15:
new_seconds = 15
p2 = TrhphParameter.VERBOSE_MODE
new_verbose = self._get_verbose_flag_for_set_test()
valid_params = {p1: new_seconds, p2: new_verbose}
log.info("setting: %s" % str(valid_params))
self._set_params(valid_params)
result = self._get_params([p1, p2])
seconds = result[p1]
self.assertEqual(seconds, new_seconds)
verbose = result[p2]
self.assertEqual(verbose, new_verbose)
def test_60_execute_stop_autosample(self):
self._initialize_and_run()
log.info("stopping autosample")
cmd = AgentCommand(command='stop_autosample',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("stop_autosample reply = %s" % str(reply))
def test_70_execute_get_metadata(self):
self._initialize_and_run()
log.info("getting metadata")
cmd = AgentCommand(command='get_metadata',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("get_metadata reply = %s" % str(reply))
self.assertTrue(isinstance(reply.result, dict))
def test_80_execute_diagnostics(self):
self._initialize_and_run()
log.info("executing diagnostics")
num_scans = 11
cmd = AgentCommand(command='diagnostics',
kwargs=dict(num_scans=num_scans,
timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("diagnostics reply = %s" % str(reply))
self.assertTrue(isinstance(reply.result, list))
self.assertEqual(len(reply.result), num_scans)
def test_90_execute_get_power_statuses(self):
self._initialize_and_run()
log.info("executing get_power_statuses")
cmd = AgentCommand(command='get_power_statuses',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("get_power_statuses reply = %s" % str(reply))
self.assertTrue(isinstance(reply.result, dict))
def test_99_execute_start_autosample(self):
self._initialize_and_run()
log.info("executing start_autosample")
cmd = AgentCommand(command='start_autosample',
kwargs=dict(timeout=self._timeout))
reply = self._ia_client.execute(cmd)
log.info("start_autosample reply = %s" % str(reply))
@unittest.skip('Not running after code updates in other places')
def test_get_params_invalid(self):
self._initialize_and_run()
with self.assertRaises(InstParameterError):
self._get_params(['bad-param'])
@unittest.skip('Not running after code updates in other places')
def test_set_params_invalid(self):
self._initialize_and_run()
p1 = TrhphParameter.TIME_BETWEEN_BURSTS
new_seconds = random.randint(15, 60)
invalid_params = {p1: new_seconds, "bad-param": "dummy-value"}
with self.assertRaises(InstParameterError):
self._set_params(invalid_params)
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgent.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._data_greenlets.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
sub.stop()
for gl in self._data_greenlets:
gl.kill()
def _start_event_subscribers(self):
"""
Create subscribers for agent and driver events.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._no_events and self._no_events == len(self._event_received):
self._async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
event_sub.activate()
self._event_subscribers.append(event_sub)
def _stop_event_subscribers(self):
"""
Stop event subscribers on cleanup.
"""
for sub in self._event_subscribers:
sub.deactivate()
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val*.01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
"""
Test agent state transitions.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set(self):
"""
Test instrument driver get and set interface.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param(SBE37Parameter.ALL)
self.assertParamDict(reply, True)
orig_config = reply
# Retrieve a subset of resource parameters.
params = [
SBE37Parameter.TA0,
SBE37Parameter.INTERVAL,
SBE37Parameter.STORETIME
]
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
SBE37Parameter.TA0 : (orig_params[SBE37Parameter.TA0] * 2),
SBE37Parameter.INTERVAL : (orig_params[SBE37Parameter.INTERVAL] + 1),
SBE37Parameter.STORETIME : (not orig_params[SBE37Parameter.STORETIME])
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
# Reset the parameters back to their original values.
self._ia_client.set_param(orig_params)
reply = self._ia_client.get_param(SBE37Parameter.ALL)
reply.pop(SBE37Parameter.SAMPLENUM)
orig_config.pop(SBE37Parameter.SAMPLENUM)
self.assertParamVals(reply, orig_config)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_poll(self):
"""
Test observatory polling function.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Lets get 3 samples.
self._no_samples = 3
# Poll for a few samples.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
# Assert we got 3 samples.
self._async_data_result.get(timeout=10)
self.assertTrue(len(self._samples_received)==3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_autosample(self):
"""
Test instrument driver execute interface to start and stop streaming
mode.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(15)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert we got some samples.
self._async_data_result.get(timeout=10)
self.assertTrue(len(self._samples_received)>=2)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
"""
Test the ability to retrieve agent and resource parameter and command
capabilities.
"""
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
acmds = [item[1] for item in acmds]
self.assertEqual(acmds, AGT_CMDS)
apars = self._ia_client.get_capabilities(['AGT_PAR'])
apars = [item[1] for item in apars]
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
rcmds = [item[1] for item in rcmds]
self.assertEqual(rcmds, CMDS)
rpars = self._ia_client.get_capabilities(['RES_PAR'])
rpars = [item[1] for item in rpars]
self.assertEqual(rpars, SBE37Parameter.list())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
@unittest.skip('Never written')
def test_errors(self):
"""
Test illegal behavior and replies.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s',str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# OK, I can do this now.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSampleDict(reply.result)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
with self.assertRaises(InstParameterError):
reply = self._ia_client.get_param('bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
@unittest.skip('Direct access test to be finished by adding the telnet client, manual for now.')
def test_direct_access(self):
"""
Test agent direct_access command. This causes creation of
driver process and transition to direct access.
"""
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='go_direct_access',
#kwargs={'session_type':DirectAccessTypes.telnet,
kwargs={'session_type':DirectAccessTypes.vsp,
'session_timeout':600,
'inactivity_timeout':600})
retval = self._ia_client.execute_agent(cmd)
print("go_direct_access retval=" + str(retval))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.DIRECT_ACCESS)
# sleep to let tester run telnet client manually
print "test sleeping to run telnet client"
time.sleep(60)
# Halt DA.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
},
'packet_config' : {
parsed_stream_name : ('prototype.sci_data.ctd_stream',
'ctd_stream_packet'),
raw_stream_name : None
}
}
# Start container.
self._start_container()
# Establish endpoint with container.
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file.
self._container_client.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# Create parsed stream. The stream name must match one
# used by the driver to label packet data.
parsed_stream_def = ctd_stream_definition(stream_id=None)
parsed_stream_def_id = self._pubsub_client.create_stream_definition(
container=parsed_stream_def)
parsed_stream_id = self._pubsub_client.create_stream(
name=parsed_stream_name,
stream_definition_id=parsed_stream_def_id,
original=True,
encoding='ION R2')
# Create raw stream. The stream name must match one used by the
# driver to label packet data. This stream does not yet have a
# packet definition so will not be published.
raw_stream_def = ctd_stream_definition(stream_id=None)
raw_stream_def_id = self._pubsub_client.create_stream_definition(
container=raw_stream_def)
raw_stream_id = self._pubsub_client.create_stream(name=raw_stream_name,
stream_definition_id=raw_stream_def_id,
original=True,
encoding='ION R2')
# Define stream configuration.
self.stream_config = {
parsed_stream_name : parsed_stream_id,
raw_stream_name : raw_stream_id
}
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
node=self.container.node)
# Create and activate parsed data subscription.
parsed_sub = subscriber_registrar.create_subscriber(exchange_name=\
'parsed_queue', callback=consume)
parsed_sub.start()
parsed_query = StreamQuery(stream_ids=[parsed_stream_id])
parsed_sub_id = self._pubsub_client.create_subscription(\
query=parsed_query, exchange_name='parsed_queue')
self._pubsub_client.activate_subscription(parsed_sub_id)
# Create and activate raw data subscription.
raw_sub = subscriber_registrar.create_subscriber(exchange_name=\
'raw_queue', callback=consume)
raw_sub.start()
raw_query = StreamQuery(stream_ids=[raw_stream_id])
raw_sub_id = self._pubsub_client.create_subscription(\
query=raw_query, exchange_name='raw_queue')
self._pubsub_client.activate_subscription(raw_sub_id)
# Create agent config.
self.agent_config = {
'driver_config' : self.driver_config,
'stream_config' : self.stream_config
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.services.mi.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(name=self._ia_name,
module=self._ia_mod, cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('123xyz', name=self._ia_pid,
process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, [parsed_sub, raw_sub])
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
caps = self._ia_client.get_capabilities()
log.info('Capabilities: %s',str(caps))
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
def test_go_active(self):
"""
Test agent go_active command. This causes a driver process to
launch a connection broker, connect to device hardware, determine
entry state of driver and intialize driver parameters.
"""
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
def test_get_set(self):
"""
Test instrument driver resource get/set interface. This tests
getting and setting driver reousrce paramters in various syntaxes and
validates results including persistence on device hardware.
"""
cmd = AgentCommand(command='initialize')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
get_params = [
(SBE37Channel.CTD, SBE37Parameter.ALL)
]
reply = self._ia_client.get_param(get_params)
time.sleep(2)
self.assertIsInstance(reply, dict)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.TA2)], float)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.PTCA1)], float)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.TCALDATE)], (tuple, list))
# Set up a param dict of the original values.
old_ta2 = reply[(SBE37Channel.CTD, SBE37Parameter.TA2)]
old_ptca1 = reply[(SBE37Channel.CTD, SBE37Parameter.PTCA1)]
old_tcaldate = reply[(SBE37Channel.CTD, SBE37Parameter.TCALDATE)]
orig_params = {
(SBE37Channel.CTD, SBE37Parameter.TA2): old_ta2,
(SBE37Channel.CTD, SBE37Parameter.PTCA1): old_ptca1,
(SBE37Channel.CTD, SBE37Parameter.TCALDATE): old_tcaldate
}
# Set up a param dict of new values.
new_ta2 = old_ta2*2
new_ptcal1 = old_ptca1*2
new_tcaldate = list(old_tcaldate)
new_tcaldate[2] = new_tcaldate[2] + 1
new_params = {
(SBE37Channel.CTD, SBE37Parameter.TA2): new_ta2,
(SBE37Channel.CTD, SBE37Parameter.PTCA1): new_ptcal1,
(SBE37Channel.CTD, SBE37Parameter.TCALDATE): new_tcaldate
}
# Set the params to their new values.
reply = self._ia_client.set_param(new_params)
time.sleep(2)
# Check overall success and success of the individual paramters.
self.assertIsInstance(reply, dict)
# Get the same paramters back from the driver.
get_params = [
(SBE37Channel.CTD, SBE37Parameter.TA2),
(SBE37Channel.CTD, SBE37Parameter.PTCA1),
(SBE37Channel.CTD, SBE37Parameter.TCALDATE)
]
reply = self._ia_client.get_param(get_params)
time.sleep(2)
# Check success, and check that the parameters were set to the
# new values.
self.assertIsInstance(reply, dict)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.TA2)], float)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.PTCA1)], float)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.TCALDATE)], (tuple, list))
self.assertAlmostEqual(reply[(SBE37Channel.CTD, SBE37Parameter.TA2)], new_ta2, delta=abs(0.01*new_ta2))
self.assertAlmostEqual(reply[(SBE37Channel.CTD, SBE37Parameter.PTCA1)], new_ptcal1, delta=abs(0.01*new_ptcal1))
self.assertEqual(reply[(SBE37Channel.CTD, SBE37Parameter.TCALDATE)], new_tcaldate)
# Set the paramters back to their original values.
reply = self._ia_client.set_param(orig_params)
self.assertIsInstance(reply, dict)
# Get the parameters back from the driver.
reply = self._ia_client.get_param(get_params)
# Check overall and individual sucess, and that paramters were
# returned to their original values.
self.assertIsInstance(reply, dict)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.TA2)], float)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.PTCA1)], float)
self.assertIsInstance(reply[(SBE37Channel.CTD, SBE37Parameter.TCALDATE)], (tuple, list))
self.assertAlmostEqual(reply[(SBE37Channel.CTD, SBE37Parameter.TA2)], old_ta2, delta=abs(0.01*old_ta2))
self.assertAlmostEqual(reply[(SBE37Channel.CTD, SBE37Parameter.PTCA1)], old_ptca1, delta=abs(0.01*old_ptca1))
self.assertEqual(reply[(SBE37Channel.CTD, SBE37Parameter.TCALDATE)], old_tcaldate)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
def test_poll(self):
"""
Test instrument driver resource execute interface to do polled
sampling.
"""
cmd = AgentCommand(command='initialize')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
time.sleep(2)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
time.sleep(2)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
def test_autosample(self):
"""
Test instrument driver execute interface to start and stop streaming
mode.
"""
cmd = AgentCommand(command='initialize')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
time.sleep(30)
cmd = AgentCommand(command='go_observatory')
while True:
reply = self._ia_client.execute_agent(cmd)
result = reply.result
if isinstance(result, dict):
if all([val == None for val in result.values()]):
break
time.sleep(2)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
# log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
# log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config
)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError('_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id=''):
if not stream_id or stream_id is '':
stream_id = self._pubsub_client.create_stream(name=name, encoding='ION R2')
pid = self._container_client.spawn_process(
name=name+'_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process':{'stream_id':stream_id}}
)
log.info('Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.format(pid, stream_id))
return stream_id
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.activate()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.deactivate()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val*.01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data(self):
cmd=AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
self._finished_count = 3
log.info('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
config_mods={}
log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1')
config_mods['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_1')
config_mods['constraints']=self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
log.info('Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2')
config_mods['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_2')
config_mods['constraints']=self.HIST_CONSTRAINTS_2
# config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2}
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
finished = self._async_finished_result.get(timeout=10)
self.assertEqual(finished,self._finished_count)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL':5
}
self._ia_client.set_param(params)
self._finished_count = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1')
config['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_1')
config['constraints']=self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config])
reply = self._ia_client.execute(cmd)
self.assertNotEqual(reply.status, 660)
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL':5
}
self._ia_client.set_param(params)
self._finished_count = 3
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_observatory(self):
# Test instrument driver get and set interface.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL')
self.assertParamDict(reply, True)
orig_config = reply
## Retrieve a subset of resource parameters.
params = [
'POLLING_INTERVAL'
]
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL' : (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set_param(self):
cmd=AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_param(['POLLING_INTERVAL','PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval,dict))
self.assertEqual(type(retval['POLLING_INTERVAL']),int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']),list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.get_param,['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_param({'POLLING_INTERVAL':10})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval,dict))
self.assertEqual(retval['POLLING_INTERVAL'],10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.set_param, {'BAD_PARAM':'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
self.assertRaises(InstParameterError, self._ia_client.set_param, {'POLLING_INTERVAL':20,'BAD_PARAM':'bad_val'})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval,dict))
self.assertEqual(retval['POLLING_INTERVAL'],20)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._finished_count = 1
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
gevent.sleep(5)
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._async_finished_result.get(timeout=5)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s',str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
def test_createTransformsThenActivateInstrument(self):
# Set up the preconditions
# ingestion configuration parameters
self.exchange_point_id = 'science_data'
self.number_of_workers = 2
self.hdf_storage = HdfStorage(relative_path='ingest')
self.couch_storage = CouchStorage(datastore_name='test_datastore')
self.XP = 'science_data'
self.exchange_name = 'ingestion_queue'
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
# driver_config = {
# 'svr_addr': "localhost",
# 'cmd_port': 5556,
# 'evt_port': 5557,
# 'dvr_mod': "ion.agents.instrument.drivers.sbe37.sbe37_driver",
# 'dvr_cls': "SBE37Driver",
# 'comms_config': {
# 'addr': 'sbe37-simulator.oceanobservatories.org',
# 'port': 4001,
# }
# }
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product,
PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_raw_data_product,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1ConductivityTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1PressureTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1TemperatureTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new SalinityTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new DensityTransform data process definition: %s"
% ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id)
self.output_products = {}
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_conductivity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_pressure_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_temperature_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_conductivity_id,
ctd_L1_conductivity_dprocdef_id)
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id)
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 conductivity"
)
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity')
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_conductivity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 pressure"
)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure')
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_pressure_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 temperature"
)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature')
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_temperature_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id)
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L2 Salinity"
)
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l2_salinity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L2 Density"
)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Density',
description='transform output pressure')
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l2_density_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L0 all data_process start"
)
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product],
self.output_products)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L0 all data_process return"
)
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start"
)
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_conductivity_dprocdef_id,
[ctd_l0_conductivity_output_dp_id],
{'output': ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return"
)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process start"
)
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id],
{'output': ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process return"
)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process start"
)
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_temperature_dprocdef_id,
[ctd_l0_temperature_output_dp_id],
{'output': ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process return"
)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L2_salinity data_process start"
)
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product],
{'output': ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(
l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L2_salinity data_process return"
)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L2_Density data_process start"
)
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L2_density_dprocdef_id, [ctd_parsed_data_product],
{'output': ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(
l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L2_Density data_process return"
)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
'iaclient',
name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(
" test_createTransformsThenActivateInstrument:: got ia client %s",
str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug(
"test_createTransformsThenActivateInstrument:: initialize %s",
str(retval))
time.sleep(1)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_active %s",
str(reply))
time.sleep(1)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: run %s",
str(reply))
time.sleep(1)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG: 1,
SBE37Parameter.INTERVAL: 5,
SBE37Parameter.TXREALTIME: True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_createTransformsThenActivateInstrument:: go_streaming %s",
str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(15)
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrument: return from go_observatory %s",
str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
log.debug(
"test_createTransformsThenActivateInstrument:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_createTransformsThenActivateInstrument:: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Sampling
#-------------------------------
# cmd = AgentCommand(command='initialize')
# retval = self._ia_client.execute_agent(cmd)
# print retval
# log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
# time.sleep(2)
#
# cmd = AgentCommand(command='go_active')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: go_active %s", str(reply))
# time.sleep(2)
#
# cmd = AgentCommand(command='run')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: run %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample ")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 2")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 2 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 3")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 3 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling go_inactive ")
# cmd = AgentCommand(command='go_inactive')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from go_inactive %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling reset ")
# cmd = AgentCommand(command='reset')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from reset %s", str(reply))
# time.sleep(2)
#
#
# self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
#
#
# #get the dataset id of the ctd_parsed product from the dataproduct ctd_parsed_data_product
# ctd_parsed_data_product_obj = self.dataproductclient.read_data_product(ctd_parsed_data_product)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset id %s", str(ctd_parsed_data_product_obj.dataset_id))
#
# # ask for the dataset bounds from the datasetmgmtsvc
# bounds = self.datasetclient.get_dataset_bounds(ctd_parsed_data_product_obj.dataset_id)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset bounds %s", str(bounds))
# print 'activate_instrument: got dataset bounds %s', str(bounds)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)