class TestBulkIngest(IonIntegrationTestCase):
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
log.debug('EventSubscriber event received: %s', str(args[0]) )
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
def tearDown(self):
pass
@unittest.skip('Update to agent refactor.')
def test_slocum_data_ingest(self):
HIST_CONSTRAINTS_1 = {}
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_param(params)
self._finished_count = 1
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
# Assert that data was received
self._async_finished_result.get(timeout=15)
self.assertTrue(len(self._finished_events_received) >= 1)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#todo enable after Luke's mor to retrieve, right now must have the Time axis called 'time'
# replay_granule = self.data_retriever.retrieve_last_data_points(self.dataset_id, 10)
# rdt = RecordDictionaryTool.load_from_granule(replay_granule)
#
# comp = rdt['date_pattern'] == numpy.arange(10) + 10
#
# log.debug("TestBulkIngest: comp: %s", comp)
#
# self.assertTrue(comp.all())
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
class TestExternalDatasetAgentMgmt(IonIntegrationTestCase):
# DataHandler config
DVR_CONFIG = {"dvr_mod": "ion.agents.data.handlers.base_data_handler", "dvr_cls": "DummyDataHandler"}
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
# @unittest.skip('not yet working. fix activate_data_product_persistence()')
# @unittest.skip()
@unittest.skip("not working")
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(
RT.ExternalDatasetModel,
name="ExampleDatasetModel",
description="ExampleDatasetModel",
datset_type="FibSeries",
)
try:
datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id))
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(
RT.ExternalDatasetAgent,
name="datasetagent007",
description="datasetagent007",
handler_module=EDA_MOD,
handler_class=EDA_CLS,
)
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id))
# Create ExternalDataset
log.debug("TestExternalDatasetAgentMgmt: Create external dataset resource ")
extDataset_obj = IonObject(RT.ExternalDataset, name="ExtDataset", description="ExtDataset")
try:
extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id))
# register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(extDataset_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name="SBE37_CDM", parameter_dictionary_id=pdict_id)
log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id))
log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition")
dp_obj = IonObject(RT.DataProduct, name="eoi dataset data", description=" stream test")
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(
RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1)
# todo fix the problem here....
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids))
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set("data", "external_data")
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG["dh_cfg"] = {
"TESTING": True,
"stream_id": stream_id, # TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
"data_producer_id": dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
"taxonomy": tx.dump(), # TODO: Currently does not support sets
"max_records": 4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
"driver_config": self.DVR_CONFIG,
"stream_config": self._stream_config,
"agent": {"resource_id": EDA_RESOURCE_ID},
"test_mode": True,
}
extDatasetAgentInstance_obj = IonObject(
RT.ExternalDatasetAgentInstance,
name="DatasetAgentInstance",
description="DatasetAgentInstance",
dataset_driver_config=self.DVR_CONFIG,
dataset_agent_config=agent_config,
)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(
external_dataset_agent_instance=extDatasetAgentInstance_obj,
external_dataset_agent_id=datasetAgent_id,
external_dataset_id=extDataset_id,
)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj))
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id))
# Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active)
)
self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id)
dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj))
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active)
)
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess())
print "TestExternalDatasetAgentMgmt: got ia client %s", self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command="initialize")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command="go_active")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command="run")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="pause")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command="resume")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="clear")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command="run")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="pause")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command="clear")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command="run")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="reset")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
def test_dataset_agent_prepare_support(self):
eda_sup = self.damsclient.prepare_external_dataset_agent_support()
eda_obj = IonObject(RT.ExternalDatasetAgent, name="ExternalDatasetAgent")
eda_id = self.damsclient.create_external_dataset_agent(eda_obj)
eda_sup = self.damsclient.prepare_external_dataset_agent_support(external_dataset_agent_id=eda_id)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support()
edai_obj = IonObject(RT.ExternalDatasetAgentInstance, name="ExternalDatasetAgentInstance")
edai_id = self.damsclient.create_external_dataset_agent_instance(edai_obj)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support(
external_dataset_agent_instance_id=edai_id
)
class TestPreloadThenLoadDataset(IonIntegrationTestCase):
""" Uses the preload system to define the ExternalDataset and related resources,
then invokes services to perform the load
"""
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
config = dict(op="load",
scenario="NOSE",
attachments="res/preload/r2_ioc/attachments")
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=config)
self.pubsub = PubsubManagementServiceClient()
self.dams = DataAcquisitionManagementServiceClient()
@unittest.skip("depricated test, now in mi repo")
def test_use_case(self):
# setUp() has already started the container and performed the preload
# self.assert_dataset_loaded('Test External CTD Dataset') # make sure we have the ExternalDataset resources
self.assert_dataset_loaded(
'Unit Test SMB37') # association changed -- now use device name
self.do_listen_for_incoming(
) # listen for any data being received from the dataset
self.do_read_dataset() # call services to load dataset
self.assert_data_received() # check that data was received as expected
self.do_shutdown()
def assert_dataset_loaded(self, name):
rr = self.container.resource_registry
# self.external_dataset = self.find_object_by_name(name, RT.ExternalDataset)
devs, _ = rr.find_resources(RT.InstrumentDevice,
name=name,
id_only=False)
self.assertEquals(len(devs), 1)
self.device = devs[0]
obj, _ = rr.find_objects(subject=self.device._id,
predicate=PRED.hasAgentInstance,
object_type=RT.ExternalDatasetAgentInstance)
self.agent_instance = obj[0]
obj, _ = rr.find_objects(object_type=RT.ExternalDatasetAgent,
predicate=PRED.hasAgentDefinition,
subject=self.agent_instance._id)
self.agent = obj[0]
driver_cfg = self.agent_instance.driver_config
#stream_definition_id = driver_cfg['dh_cfg']['stream_def'] if 'dh_cfg' in driver_cfg else driver_cfg['stream_def']
#self.stream_definition = rr.read(stream_definition_id)
self.data_product = rr.read_object(subject=self.device._id,
predicate=PRED.hasOutputProduct,
object_type=RT.DataProduct)
self.dataset_id = rr.read_object(subject=self.data_product._id,
predicate=PRED.hasDataset,
object_type=RT.Dataset,
id_only=True)
ids, _ = rr.find_objects(subject=self.data_product._id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
self.stream_id = ids[0]
self.route = self.pubsub.read_stream_route(self.stream_id)
def do_listen_for_incoming(self):
subscription_id = self.pubsub.create_subscription(
'validator', data_product_ids=[self.data_product._id])
self.addCleanup(self.pubsub.delete_subscription, subscription_id)
self.granule_capture = []
self.granule_count = 0
def on_granule(msg, route, stream_id):
self.granule_count += 1
if self.granule_count < 5:
self.granule_capture.append(msg)
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub.activate_subscription(subscription_id)
self.addCleanup(self.pubsub.deactivate_subscription, subscription_id)
self.dataset_modified = Event()
def cb2(*args, **kwargs):
self.dataset_modified.set()
# TODO: event isn't using the ExternalDataset, but a different ID for a Dataset
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb2,
origin=self.dataset_id)
es.start()
self.addCleanup(es.stop)
def do_read_dataset(self):
self.dams.start_external_dataset_agent_instance(
self.agent_instance._id)
#
# should i wait for process (above) to start
# before launching client (below)?
#
self.client = None
end = time.time() + MAX_AGENT_START_TIME
while not self.client and time.time() < end:
try:
self.client = ResourceAgentClient(self.device._id,
process=FakeProcess())
except NotFound:
time.sleep(2)
if not self.client:
self.fail(
msg='external dataset agent process did not start in %d seconds'
% MAX_AGENT_START_TIME)
self.client.execute_agent(
AgentCommand(command=ResourceAgentEvent.INITIALIZE))
self.client.execute_agent(
AgentCommand(command=ResourceAgentEvent.GO_ACTIVE))
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.RUN))
self.client.execute_resource(command=AgentCommand(
command=DriverEvent.START_AUTOSAMPLE))
def assert_data_received(self):
#let it go for up to 120 seconds, then stop the agent and reset it
if not self.dataset_modified.is_set():
self.dataset_modified.wait(30)
self.assertTrue(self.granule_count > 2,
msg='granule count = %d' % self.granule_count)
rdt = RecordDictionaryTool.load_from_granule(self.granule_capture[0])
self.assertAlmostEqual(0, rdt['oxygen'][0], delta=0.01)
self.assertAlmostEqual(309.77, rdt['pressure'][0], delta=0.01)
self.assertAlmostEqual(37.9848, rdt['conductivity'][0], delta=0.01)
self.assertAlmostEqual(9.5163, rdt['temp'][0], delta=0.01)
self.assertAlmostEqual(3527207897.0, rdt['time'][0], delta=1)
def do_shutdown(self):
self.dams.stop_external_dataset_agent_instance(self.agent_instance._id)
class TestBulkIngest(IonIntegrationTestCase):
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
log.debug('EventSubscriber event received: %s', str(args[0]) )
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
def tearDown(self):
pass
@unittest.skip('Update to agent refactor.')
def test_slocum_data_ingest(self):
HIST_CONSTRAINTS_1 = {}
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_param(params)
self._finished_count = 1
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
# Assert that data was received
self._async_finished_result.get(timeout=15)
self.assertTrue(len(self._finished_events_received) >= 1)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#todo enable after Luke's mor to retrieve, right now must have the Time axis called 'time'
# replay_granule = self.data_retriever.retrieve_last_data_points(self.dataset_id, 10)
# rdt = RecordDictionaryTool.load_from_granule(replay_granule)
#
# comp = rdt['date_pattern'] == numpy.arange(10) + 10
#
# log.debug("TestBulkIngest: comp: %s", comp)
#
# self.assertTrue(comp.all())
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
self.addCleanup(self.dataproductclient.suspend_data_product_persistence, self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
class TestExternalDatasetAgentMgmt(IonIntegrationTestCase):
# DataHandler config
DVR_CONFIG = {
'dvr_mod': 'ion.agents.data.handlers.base_data_handler',
'dvr_cls': 'DummyDataHandler',
}
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
# @unittest.skip('not yet working. fix activate_data_product_persistence()')
#@unittest.skip()
@unittest.skip('not working')
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel,
name='ExampleDatasetModel',
description="ExampleDatasetModel",
datset_type="FibSeries")
try:
datasetModel_id = self.damsclient.create_external_dataset_model(
datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s",
str(datasetModel_id))
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent,
name='datasetagent007',
description="datasetagent007",
handler_module=EDA_MOD,
handler_class=EDA_CLS)
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(
datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s",
str(datasetAgent_id))
# Create ExternalDataset
log.debug(
'TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset,
name='ExtDataset',
description="ExtDataset")
try:
extDataset_id = self.damsclient.create_external_dataset(
extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %
ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ",
str(extDataset_id))
#register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(
extDataset_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=pdict_id)
log.debug(
"TestExternalDatasetAgentMgmt: new Stream Definition id = %s",
str(ctd_stream_def_id))
log.debug(
"TestExternalDatasetAgentMgmt: Creating new data product with a stream definition"
)
dp_obj = IonObject(RT.DataProduct,
name='eoi dataset data',
description=' stream test')
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s",
str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=extDataset_id,
data_product_id=data_product_id1)
#todo fix the problem here....
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s",
str(stream_ids))
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id':
stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id': dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
'taxonomy': tx.dump(), #TODO: Currently does not support sets
'max_records': 4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': EDA_RESOURCE_ID
},
'test_mode': True
}
extDatasetAgentInstance_obj = IonObject(
RT.ExternalDatasetAgentInstance,
name='DatasetAgentInstance',
description="DatasetAgentInstance",
dataset_driver_config=self.DVR_CONFIG,
dataset_agent_config=agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(
external_dataset_agent_instance=extDatasetAgentInstance_obj,
external_dataset_agent_id=datasetAgent_id,
external_dataset_id=extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(extDatasetAgentInstance_obj))
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s",
str(extDatasetAgentInstance_id))
#Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ",
str(id), str(active))
self.damsclient.start_external_dataset_agent_instance(
extDatasetAgentInstance_id)
dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(
extDatasetAgentInstance_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(dataset_agent_instance_obj))
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ",
str(id), str(active))
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id,
process=FakeProcess())
print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s",
str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(
extDatasetAgentInstance_id)
def test_dataset_agent_prepare_support(self):
eda_sup = self.damsclient.prepare_external_dataset_agent_support()
eda_obj = IonObject(RT.ExternalDatasetAgent,
name="ExternalDatasetAgent")
eda_id = self.damsclient.create_external_dataset_agent(eda_obj)
eda_sup = self.damsclient.prepare_external_dataset_agent_support(
external_dataset_agent_id=eda_id)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support(
)
edai_obj = IonObject(RT.ExternalDatasetAgentInstance,
name="ExternalDatasetAgentInstance")
edai_id = self.damsclient.create_external_dataset_agent_instance(
edai_obj)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support(
external_dataset_agent_instance_id=edai_id)
class TestPreloadThenLoadDataset(IonIntegrationTestCase):
""" Uses the preload system to define the ExternalDataset and related resources,
then invokes services to perform the load
"""
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
config = dict(op="load", scenario="NOSE", attachments="res/preload/r2_ioc/attachments")
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config)
self.pubsub = PubsubManagementServiceClient()
self.dams = DataAcquisitionManagementServiceClient()
@unittest.skip("depricated test, now in mi repo")
def test_use_case(self):
# setUp() has already started the container and performed the preload
# self.assert_dataset_loaded('Test External CTD Dataset') # make sure we have the ExternalDataset resources
self.assert_dataset_loaded('Unit Test SMB37') # association changed -- now use device name
self.do_listen_for_incoming() # listen for any data being received from the dataset
self.do_read_dataset() # call services to load dataset
self.assert_data_received() # check that data was received as expected
self.do_shutdown()
def assert_dataset_loaded(self, name):
rr = self.container.resource_registry
# self.external_dataset = self.find_object_by_name(name, RT.ExternalDataset)
devs, _ = rr.find_resources(RT.InstrumentDevice, name=name, id_only=False)
self.assertEquals(len(devs), 1)
self.device = devs[0]
obj,_ = rr.find_objects(subject=self.device._id, predicate=PRED.hasAgentInstance, object_type=RT.ExternalDatasetAgentInstance)
self.agent_instance = obj[0]
obj,_ = rr.find_objects(object_type=RT.ExternalDatasetAgent, predicate=PRED.hasAgentDefinition, subject=self.agent_instance._id)
self.agent = obj[0]
driver_cfg = self.agent_instance.driver_config
#stream_definition_id = driver_cfg['dh_cfg']['stream_def'] if 'dh_cfg' in driver_cfg else driver_cfg['stream_def']
#self.stream_definition = rr.read(stream_definition_id)
self.data_product = rr.read_object(subject=self.device._id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct)
self.dataset_id = rr.read_object(subject=self.data_product._id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
ids,_ = rr.find_objects(subject=self.data_product._id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
self.stream_id = ids[0]
self.route = self.pubsub.read_stream_route(self.stream_id)
def do_listen_for_incoming(self):
subscription_id = self.pubsub.create_subscription('validator', data_product_ids=[self.data_product._id])
self.addCleanup(self.pubsub.delete_subscription, subscription_id)
self.granule_capture = []
self.granule_count = 0
def on_granule(msg, route, stream_id):
self.granule_count += 1
if self.granule_count < 5:
self.granule_capture.append(msg)
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub.activate_subscription(subscription_id)
self.addCleanup(self.pubsub.deactivate_subscription, subscription_id)
self.dataset_modified = Event()
def cb2(*args, **kwargs):
self.dataset_modified.set()
# TODO: event isn't using the ExternalDataset, but a different ID for a Dataset
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb2, origin=self.dataset_id)
es.start()
self.addCleanup(es.stop)
def do_read_dataset(self):
self.dams.start_external_dataset_agent_instance(self.agent_instance._id)
#
# should i wait for process (above) to start
# before launching client (below)?
#
self.client = None
end = time.time() + MAX_AGENT_START_TIME
while not self.client and time.time() < end:
try:
self.client = ResourceAgentClient(self.device._id, process=FakeProcess())
except NotFound:
time.sleep(2)
if not self.client:
self.fail(msg='external dataset agent process did not start in %d seconds' % MAX_AGENT_START_TIME)
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.INITIALIZE))
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.GO_ACTIVE))
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.RUN))
self.client.execute_resource(command=AgentCommand(command=DriverEvent.START_AUTOSAMPLE))
def assert_data_received(self):
#let it go for up to 120 seconds, then stop the agent and reset it
if not self.dataset_modified.is_set():
self.dataset_modified.wait(30)
self.assertTrue(self.granule_count > 2, msg='granule count = %d'%self.granule_count)
rdt = RecordDictionaryTool.load_from_granule(self.granule_capture[0])
self.assertAlmostEqual(0, rdt['oxygen'][0], delta=0.01)
self.assertAlmostEqual(309.77, rdt['pressure'][0], delta=0.01)
self.assertAlmostEqual(37.9848, rdt['conductivity'][0], delta=0.01)
self.assertAlmostEqual(9.5163, rdt['temp'][0], delta=0.01)
self.assertAlmostEqual(3527207897.0, rdt['time'][0], delta=1)
def do_shutdown(self):
self.dams.stop_external_dataset_agent_instance(self.agent_instance._id)
class TestPreloadThenLoadDataset(IonIntegrationTestCase):
""" replicates the TestHypm_WPF_CTD test (same handler/parser/data file)
but uses the preload system to define the ExternalDataset and related resources,
then invokes services to perform the load
"""
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
config = dict(op="load", scenario="BETA,NOSE", attachments="res/preload/r2_ioc/attachments")
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config)
self.pubsub = PubsubManagementServiceClient()
self.dams = DataAcquisitionManagementServiceClient()
def find_object_by_name(self, name, resource_type):
objects,_ = self.container.resource_registry.find_resources(resource_type)
self.assertTrue(len(objects) >= 1)
# filtered_objs = [obj for obj in objects if obj.name == name]
filtered_objs = []
for obj in objects:
if obj.name==name:
filtered_objs.append(obj)
self.assertEquals(len(filtered_objs), 1, msg='Found %d objects with name %s'%(len(filtered_objs),name))
return filtered_objs[0]
def test_use_case(self):
# setUp() has already started the container and performed the preload
# self.assert_dataset_loaded('Test External CTD Dataset') # make sure we have the ExternalDataset resources
self.assert_dataset_loaded('Unit Test SMB37') # association changed -- now use device name
self.do_listen_for_incoming() # listen for any data being received from the dataset
self.do_read_dataset() # call services to load dataset
self.assert_data_received() # check that data was received as expected
self.do_shutdown()
def assert_dataset_loaded(self, name):
# self.external_dataset = self.find_object_by_name(name, RT.ExternalDataset)
self.device = self.find_object_by_name(name, RT.InstrumentDevice)
rr = self.container.resource_registry
obj,_ = rr.find_objects(subject=self.device._id, predicate=PRED.hasAgentInstance, object_type=RT.ExternalDatasetAgentInstance)
self.agent_instance = obj[0]
obj,_ = rr.find_objects(object_type=RT.ExternalDatasetAgent, predicate=PRED.hasAgentDefinition, subject=self.agent_instance._id)
self.agent = obj[0]
stream_definition_id = self.agent_instance.dataset_driver_config['dh_cfg']['stream_def'] if 'dh_cfg' in self.agent_instance.dataset_driver_config else self.agent_instance.dataset_driver_config['stream_def']
self.stream_definition = rr.read(stream_definition_id)
# data_producer_id = self.agent_instance.dataset_driver_config['dh_cfg']['data_producer_id'] if 'dh_cfg' in self.agent_instance.dataset_driver_config else self.agent_instance.dataset_driver_config['data_producer_id']
# self.data_producer = rr.read(data_producer_id) #subject="", predicate="", object_type="", assoc="", id_only=False)
# self.data_product = rr.read_object(object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, subject=self.external_dataset._id)
self.data_product = rr.read_object(object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, subject=self.device._id)
ids,_ = rr.find_objects(self.data_product._id, PRED.hasStream, RT.Stream, id_only=True)
self.stream_id = ids[0]
self.route = self.pubsub.read_stream_route(self.stream_id)
def do_listen_for_incoming(self):
subscription_id = self.pubsub.create_subscription('validator', data_product_ids=[self.data_product._id])
self.addCleanup(self.pubsub.delete_subscription, subscription_id)
self.granule_capture = []
self.granule_count = 0
def on_granule(msg, route, stream_id):
self.granule_count += 1
if self.granule_count<5:
self.granule_capture.append(msg)
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub.activate_subscription(subscription_id)
self.addCleanup(self.pubsub.deactivate_subscription, subscription_id)
self.dataset_modified = Event()
def cb2(*args, **kwargs):
self.dataset_modified.set()
# TODO: event isn't using the ExternalDataset, but a different ID for a Dataset
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb2, origin=self.device._id)
es.start()
self.addCleanup(es.stop)
def do_read_dataset(self):
self.dams.start_external_dataset_agent_instance(self.agent_instance._id)
#
# should i wait for process (above) to start
# before launching client (below)?
#
self.client = ResourceAgentClient(self.device._id, process=FakeProcess())
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.INITIALIZE))
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.GO_ACTIVE))
self.client.execute_agent(AgentCommand(command=ResourceAgentEvent.RUN))
self.client.execute_resource(command=AgentCommand(command=DriverEvent.START_AUTOSAMPLE))
def assert_data_received(self):
#let it go for up to 120 seconds, then stop the agent and reset it
if not self.dataset_modified.is_set():
self.dataset_modified.wait(30)
self.assertTrue(self.granule_count>2, msg='granule count = %d'%self.granule_count)
rdt = RecordDictionaryTool.load_from_granule(self.granule_capture[0])
self.assertAlmostEqual(0, rdt['oxygen'][0], delta=0.01)
self.assertAlmostEqual(309.77, rdt['pressure'][0], delta=0.01)
self.assertAlmostEqual(37.9848, rdt['conductivity'][0], delta=0.01)
self.assertAlmostEqual(9.5163, rdt['temp'][0], delta=0.01)
self.assertAlmostEqual(1318219097, rdt['time'][0], delta=1)
def do_shutdown(self):
self.dams.stop_external_dataset_agent_instance(self.agent_instance._id)
class DatasetAgentTestCase(IonIntegrationTestCase):
"""
Base class for all coi dataset agent end to end tests
"""
test_config = DatasetAgentTestConfig()
def setUp(self, deploy_file=DEPLOY_FILE):
"""
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
self._dsa_client = None
# Ensure we have a good test configuration
self.test_config.verify()
# Start container.
log.info('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services. %s', deploy_file)
self.container.start_rel_from_url(DEPLOY_FILE)
# Load instrument specific parameters
log.info('Loading additional scenarios')
self._load_params()
# Start data subscribers
self._build_stream_config()
self._start_data_subscribers()
# Start a resource agent client to talk with the instrument agent.
log.info('starting DSA process')
self._dsa_client = self._start_dataset_agent_process()
self.addCleanup(self.assert_reset)
log.info('test setup complete')
###
# Test/Agent Startup Helpers
###
def _load_params(self):
"""
Load specific instrument specific parameters from preload
"""
scenario = None
categories = None
if PRELOAD_CATEGORIES:
categories = ",".join(PRELOAD_CATEGORIES)
# load_parameter_scenarios
if PRELOAD_SCENARIO:
scenario = PRELOAD_SCENARIO
else:
log.warn("No common preload defined. Was this intentional?")
if self.test_config.preload_scenario:
if scenario:
scenario = "%s,%s" % (scenario, self.test_config.preload_scenario)
else:
scenario = self.test_config.preload_scenario
else:
log.warn("No DSA specific preload defined. Was this intentional?")
log.debug("doing preload now: %s", scenario)
if scenario:
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario=scenario,
path="master",
categories=categories,
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
def _start_dataset_agent_process(self):
# Create agent config.
name = self.test_config.instrument_device_name
rr = self.container.resource_registry
log.debug("Start dataset agent process for instrument device: %s", name)
objects,_ = rr.find_resources(RT.InstrumentDevice)
log.debug("Found Instrument Devices: %s", objects)
filtered_objs = [obj for obj in objects if obj.name == name]
if (filtered_objs) == []:
raise ConfigNotFound("No appropriate InstrumentDevice objects loaded")
instrument_device = filtered_objs[0]
log.trace("Found instrument device: %s", instrument_device)
dsa_instance = rr.read_object(subject=instrument_device._id,
predicate=PRED.hasAgentInstance,
object_type=RT.ExternalDatasetAgentInstance)
log.debug("dsa_instance found: %s", dsa_instance)
self._driver_config = dsa_instance.driver_config
self.clear_sample_data()
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
proc_id = self.damsclient.start_external_dataset_agent_instance(dsa_instance._id)
client = ResourceAgentClient(instrument_device._id, process=FakeProcess())
return client
###
# Data file helpers
###
def _get_source_data_file(self, filename):
"""
Search for a sample data file, first check the driver resource directory
then just use the filename as a path. If the file doesn't exists
raise an exception
@param filename name or path of the file to search for
@return full path to the found data file
@raise IonException if the file isn't found
"""
resource_dir = self.test_config.test_resource_dir
source_path = os.path.join(resource_dir, filename)
log.debug("Search for resource file (%s) in %s", filename, resource_dir)
if os.path.isfile(source_path):
log.debug("Found %s in resource directory", filename)
return source_path
log.debug("Search for resource file (%s) in current directory", filename)
if os.path.isfile(filename):
log.debug("Found %s in the current directory", filename)
return filename
raise IonException("Data file %s does not exist", filename)
def create_data_dir(self):
"""
Verify the test data directory is created and exists. Return the path to
the directory.
@return: path to data directory
@raise: ConfigNotFound no harvester config
@raise: IonException if data_dir exists, but not a directory
"""
startup_config = self._driver_config.get('startup_config')
if not startup_config:
raise ConfigNotFound("Driver config missing 'startup_config'")
harvester_config = startup_config.get('harvester')
if not harvester_config:
raise ConfigNotFound("Startup config missing 'harvester' config")
data_dir = harvester_config.get("directory")
if not data_dir:
raise ConfigNotFound("Harvester config missing 'directory'")
if not os.path.exists(data_dir):
log.debug("Creating data dir: %s", data_dir)
os.makedirs(data_dir)
elif not os.path.isdir(data_dir):
raise IonException("'data_dir' is not a directory")
return data_dir
def clear_sample_data(self):
"""
Remove all files from the sample data directory
"""
data_dir = self.create_data_dir()
log.debug("Clean all data from %s", data_dir)
self.remove_all_files(data_dir)
def create_sample_data(self, filename, dest_filename=None):
"""
Search for a data file in the driver resource directory and if the file
is not found there then search using the filename directly. Then copy
the file to the test data directory.
If a dest_filename is supplied it will be renamed in the destination
directory.
@param: filename - filename or path to a data file to copy
@param: dest_filename - name of the file when copied. default to filename
"""
data_dir = self.create_data_dir()
source_path = self._get_source_data_file(filename)
log.debug("DIR: %s", data_dir)
if dest_filename is None:
dest_path = os.path.join(data_dir, os.path.basename(source_path))
else:
dest_path = os.path.join(data_dir, dest_filename)
log.debug("Creating data file src: %s, dest: %s", source_path, dest_path)
shutil.copy2(source_path, dest_path)
def remove_all_files(self, dir_name):
"""
Remove all files from a directory. Raise an exception if the directory contains something
other than files.
@param dir_name directory path to remove files.
@raise RuntimeError if the directory contains anything except files.
"""
for file_name in os.listdir(dir_name):
file_path = os.path.join(dir_name, file_name)
if not os.path.isfile(file_path):
raise RuntimeError("%s is not a file", file_path)
for file_name in os.listdir(dir_name):
file_path = os.path.join(dir_name, file_name)
os.unlink(file_path)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(
event_type=type, callback=consume_event,
origin=IA_RESOURCE_ID)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'ctdpf_parsed'
param_dict_name = 'ctdpf_parsed'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
from pyon.util.containers import create_unique_identifier
stream_name = 'ctdpf_parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
###
# Common assert methods
###
def assert_initialize(self, final_state = ResourceAgentState.STREAMING):
'''
Walk through DSA states to get to streaming mode from uninitialized
'''
state = self._dsa_client.get_agent_state()
with self.assertRaises(Conflict):
res_state = self._dsa_client.get_resource_state()
log.debug("Initialize DataSet agent")
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.info("Sent INITIALIZE; DSA state = %s", state)
log.debug("DataSet agent go active")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent GO_ACTIVE; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.IDLE)
log.debug("DataSet agent run")
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent RUN; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.COMMAND)
if final_state == ResourceAgentState.STREAMING:
self.assert_start_sampling()
def assert_stop_sampling(self):
'''
transition to command. Must be called from streaming
'''
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STREAMING)
log.debug("DataSet agent start sampling")
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._dsa_client.execute_resource(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent START SAMPLING; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.COMMAND)
def assert_start_sampling(self):
'''
transition to sampling. Must be called from command
'''
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
log.debug("DataSet agent start sampling")
cmd = AgentCommand(command='DRIVER_EVENT_START_AUTOSAMPLE')
retval = self._dsa_client.execute_resource(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent START SAMPLING; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.STREAMING)
def assert_reset(self):
'''
Put the instrument back in uninitialized
'''
if self._dsa_client is None:
return
state = self._dsa_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class DatasetAgentTestCase(IonIntegrationTestCase):
"""
Base class for all dataset agent end to end tests
"""
test_config = DatasetAgentTestConfig()
def setUp(self, deploy_file=DEPLOY_FILE):
"""
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
self._dsa_client = None
self.dams = DataAcquisitionManagementServiceClient()
# Ensure we have a good test configuration
self.test_config.verify()
# Start container.
log.info('Staring capability container.')
self._start_container()
self.rr = self.container.resource_registry
# Bring up services in a deploy file (no need to message)
log.info('Starting deploy services. %s', deploy_file)
self.container.start_rel_from_url(DEPLOY_FILE)
# Load instrument specific parameters
log.info('Preload test scenarios')
self._load_params()
# Start a resource agent client to talk with the instrument agent.
log.info('Starting DSA process')
self._dsa_client = self._start_dataset_agent_process()
log.debug("Client created: %s", type(self._dsa_client))
self.addCleanup(self._stop_dataset_agent_process)
log.info('test setup complete')
# Start data subscribers
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
###
# Test/Agent Startup Helpers
###
def _load_params(self):
"""
Do a second round of preload with instrument specific scenarios
"""
scenario = None
categories = None
if PRELOAD_CATEGORIES:
categories = ",".join(PRELOAD_CATEGORIES)
# load_parameter_scenarios
if PRELOAD_SCENARIO:
scenario = PRELOAD_SCENARIO
else:
log.warn("No common preload defined. Was this intentional?")
if self.test_config.preload_scenario:
scenario = "%s,%s" % (scenario, self.test_config.preload_scenario) if scenario else self.test_config.preload_scenario
else:
log.warn("No DSA specific preload defined. Was this intentional?")
if scenario:
preload_config=dict(
op="load",
scenario=scenario,
#path="master",
path=TESTED_DOC,
categories=categories,
clearcols="owner_id,org_ids",
#assets="res/preload/r2_ioc/ooi_assets",
#parseooi="True",
)
log.debug("Starting preload now: config=%s", preload_config)
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", preload_config)
def _start_dataset_agent_process(self):
"""
Launch the agent process and store the configuration. Tried
to emulate the same process used by import_data.py
"""
instrument_device, dsa_instance = self._get_dsa_instance()
self._driver_config = dsa_instance.driver_config
self._update_dsa_config(dsa_instance)
self._update_harvester_config(dsa_instance)
self._dsa_instance = dsa_instance
self.clear_sample_data()
# Return a resource agent client
return self._get_dsa_client(instrument_device, dsa_instance)
def _stop_dataset_agent_process(self):
"""
Stop the dataset agent instance
"""
self.assert_reset()
self.dams.stop_external_dataset_agent_instance(self._dsa_instance._id)
def _get_dsa_instance(self):
"""
Find the dsa instance in preload and return an instance of that object
"""
name = self.test_config.instrument_device_name
log.debug("Start dataset agent process for instrument device: %s", name)
objects,_ = self.rr.find_resources(RT.InstrumentDevice, name=name)
log.debug("Found Instrument Devices: %s", objects)
if not objects:
raise ConfigNotFound("No appropriate InstrumentDevice objects loaded")
instrument_device = objects[0]
log.trace("Found instrument device: %s", instrument_device)
dsa_instance = self.rr.read_object(subject=instrument_device._id,
predicate=PRED.hasAgentInstance,
object_type=RT.ExternalDatasetAgentInstance)
log.info("dsa_instance found: %s", dsa_instance)
return instrument_device, dsa_instance
def _update_harvester_config(self, dsa_instance):
"""
Update the harvester config such that we change the directory to something
we have write permissions.
"""
log.info("dsa agent instance: %s", dsa_instance)
driver_config = dsa_instance.driver_config
log.info("dsa agent driver config: %s", driver_config)
driver_config['startup_config']['harvester']['directory'] = self.test_config.data_dir
log.info("updated driver config: %s", driver_config)
dsa_instance.driver_config = driver_config
self.rr.update(dsa_instance)
def _update_dsa_config(self, dsa_instance):
"""
Update the dsa configuration prior to loading the agent. This is where we can
alter production configurations for use in a controlled test environment.
"""
dsa_obj = self.rr.read_object(
object_type=RT.ExternalDatasetAgent, predicate=PRED.hasAgentDefinition, subject=dsa_instance._id, id_only=False)
log.info("dsa agent definition found: %s", dsa_obj)
# If we don't want to load from an egg then we need to
# alter the driver config read from preload
if self.test_config.mi_repo is not None:
dsa_obj.driver_uri = None
# Strip the custom namespace
dsa_obj.driver_module = ".".join(dsa_obj.driver_module.split('.')[1:])
log.info("saving new dsa agent config: %s", dsa_obj)
self.rr.update(dsa_obj)
if not self.test_config.mi_repo in sys.path: sys.path.insert(0, self.test_config.mi_repo)
log.debug("Driver module: %s", dsa_obj.driver_module)
log.debug("MI Repo: %s", self.test_config.mi_repo)
log.trace("Sys Path: %s", sys.path)
def _get_dsa_client(self, instrument_device, dsa_instance):
"""
Launch the agent and return a client
"""
fake_process = FakeProcess()
fake_process.container = self.container
clients = DataAcquisitionManagementServiceDependentClients(fake_process)
config_builder = ExternalDatasetAgentConfigurationBuilder(clients)
try:
config_builder.set_agent_instance_object(dsa_instance)
self.agent_config = config_builder.prepare()
except Exception as e:
log.error('failed to launch: %s', e, exc_info=True)
raise ServerError('failed to launch')
self._dsa_pid = self.dams.start_external_dataset_agent_instance(dsa_instance._id)
log.debug("_get_dsa_client CFG")
return ResourceAgentClient(instrument_device._id, process=FakeProcess())
dispatcher = ProcessDispatcherServiceClient()
launcher = AgentLauncher(dispatcher)
log.debug("Launching agent process!")
self._dsa_pid = launcher.launch(self.agent_config, config_builder._get_process_definition()._id)
if not self._dsa_pid:
raise ServerError("Launched external dataset agent instance but no process_id")
config_builder.record_launch_parameters(self.agent_config)
launcher.await_launch(10.0)
return ResourceAgentClient(instrument_device._id, process=FakeProcess())
def _get_dsa_object_state(self):
state, _id = self.container.state_repository.get_state(self._dsa_pid)
log.debug("agent_state (%s): %s", self._dsa_pid, state)
driver_state = state.get(DSA_STATE_KEY)
log.debug("driver_state (%s): %s", self._dsa_pid, driver_state)
return driver_state
###
# Data file helpers
###
def _get_source_data_file(self, filename):
"""
Search for a sample data file, first check the driver resource directory
then just use the filename as a path. If the file doesn't exists
raise an exception
@param filename name or path of the file to search for
@return full path to the found data file
@raise IonException if the file isn't found
"""
resource_dir = self.test_config.test_resource_dir
source_path = os.path.join(resource_dir, filename)
log.debug("Search for resource file (%s) in %s", filename, resource_dir)
if os.path.isfile(source_path):
log.debug("Found %s in resource directory", filename)
return source_path
log.debug("Search for resource file (%s) in current directory", filename)
if os.path.isfile(filename):
log.debug("Found %s in the current directory", filename)
return filename
raise IonException("Data file %s does not exist", filename)
def create_data_dir(self):
"""
Verify the test data directory is created and exists. Return the path to
the directory.
@return: path to data directory
@raise: ConfigNotFound no harvester config
@raise: IonException if data_dir exists, but not a directory
"""
startup_config = self._driver_config.get('startup_config')
if not startup_config:
raise ConfigNotFound("Driver config missing 'startup_config'")
harvester_config = startup_config.get('harvester')
if not harvester_config:
raise ConfigNotFound("Startup config missing 'harvester' config")
data_dir = harvester_config.get("directory")
if not data_dir:
raise ConfigNotFound("Harvester config missing 'directory'")
if not os.path.exists(data_dir):
log.debug("Creating data dir: %s", data_dir)
os.makedirs(data_dir)
elif not os.path.isdir(data_dir):
raise IonException("'data_dir' is not a directory")
return data_dir
def clear_sample_data(self):
"""
Remove all files from the sample data directory
"""
data_dir = self.create_data_dir()
log.debug("Clean all data from %s", data_dir)
self.remove_all_files(data_dir)
def create_sample_data(self, filename, dest_filename=None):
"""
Search for a data file in the driver resource directory and if the file
is not found there then search using the filename directly. Then copy
the file to the test data directory.
If a dest_filename is supplied it will be renamed in the destination
directory.
@param: filename - filename or path to a data file to copy
@param: dest_filename - name of the file when copied. default to filename
"""
data_dir = self.create_data_dir()
source_path = self._get_source_data_file(filename)
log.debug("DIR: %s", data_dir)
if dest_filename is None:
dest_path = os.path.join(data_dir, os.path.basename(source_path))
else:
dest_path = os.path.join(data_dir, dest_filename)
log.debug("Creating data file src: %s, dest: %s", source_path, dest_path)
shutil.copy2(source_path, dest_path)
return dest_path
def remove_all_files(self, dir_name):
"""
Remove all files from a directory. Raise an exception if the directory contains something
other than files.
@param dir_name directory path to remove files.
@raise RuntimeError if the directory contains anything except files.
"""
for file_name in os.listdir(dir_name):
file_path = os.path.join(dir_name, file_name)
if not os.path.isfile(file_path):
raise RuntimeError("%s is not a file", file_path)
for file_name in os.listdir(dir_name):
file_path = os.path.join(dir_name, file_name)
os.unlink(file_path)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(
event_type=type, callback=consume_event,
origin="IA_RESOURCE_ID") #TODO
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _start_data_subscribers(self):
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
if self._samples_received.get(stream_id) is None:
self._samples_received[stream_id] = []
log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._samples_received[stream_id].append(message)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = {}
self._stream_id_map = {}
stream_config = self.agent_config['stream_config']
log.info("starting data subscribers")
for stream_name in stream_config.keys():
log.debug("Starting data subscriber for stream '%s'", stream_name)
stream_id = stream_config[stream_name]['stream_id']
self._stream_id_map[stream_name] = stream_id
self._start_data_subscriber(stream_config[stream_name], recv_data)
def _start_data_subscriber(self, config, callback):
"""
Setup and start a data subscriber
"""
exchange_point = config['exchange_point']
stream_id = config['stream_id']
sub = StandaloneStreamSubscriber(exchange_point, callback)
sub.start()
self._data_subscribers.append(sub)
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
sub_id = pubsub_client.create_subscription(name=exchange_point, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def make_data_product(self, pdict_name, dp_name, available_fields=None):
self.pubsub_management = PubsubManagementServiceClient()
if available_fields is None: available_fields = []
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(pdict_name, id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('%s stream_def' % dp_name, parameter_dictionary_id=pdict_id, available_fields=available_fields or None)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = DataProduct(name=dp_name)
dp_obj.temporal_domain = tdom
dp_obj.spatial_domain = sdom
data_product_id = self.data_product_management.create_data_product(dp_obj, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
return data_product_id
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
def get_samples(self, stream_name, sample_count=1, timeout=30):
"""
listen on a stream until 'sample_count' samples are read and return
a list of all samples read. If the required number of samples aren't
read then throw an exception.
Note that this method does not clear the sample queue for the stream.
This should be done explicitly by the caller. However, samples that
are consumed by this method are removed.
@raise SampleTimeout - if the required number of samples aren't read
"""
to = gevent.Timeout(timeout)
to.start()
done = False
result = []
i = 0
log.debug("Fetch %s sample(s) from stream '%s'" % (sample_count, stream_name))
stream_id = self._stream_id_map.get(stream_name)
log.debug("Stream ID Map: %s ", self._stream_id_map)
self.assertIsNotNone(stream_id, msg="Unable to find stream name '%s'" % stream_name)
try:
while(not done):
if (self._samples_received.has_key(stream_id) and
len(self._samples_received.get(stream_id))):
log.trace("get_samples() received sample #%d!", i)
result.append(self._samples_received[stream_id].pop(0))
i += 1
if i >= sample_count:
done = True
else:
log.debug("No samples in %s. Sleep a bit to wait for the data queue to fill up.", stream_name)
gevent.sleep(1)
except Timeout:
log.error("Failed to get %d records from %s. received: %d", sample_count, stream_name, i)
self.fail("Failed to read samples from stream %s", stream_name)
finally:
to.cancel()
return result
def remove_sample_dir(self):
"""
Remove the sample dir and all files
"""
data_dir = self.create_data_dir()
self.clear_sample_data()
os.rmdir(data_dir)
###
# Common assert methods
###
def assertDictEqual(self, d1, d2, msg=None): # assertEqual uses for dicts
for k,v1 in d1.iteritems():
self.assertIn(k, d2, msg)
v2 = d2[k]
if(isinstance(v1, collections.Iterable) and
not isinstance(v1, basestring)):
self.assertItemsEqual(v1, v2, msg)
else:
self.assertEqual(v1, v2, msg)
return True
def assert_initialize(self, final_state = ResourceAgentState.STREAMING):
'''
Walk through DSA states to get to streaming mode from uninitialized
'''
state = self._dsa_client.get_agent_state()
with self.assertRaises(Conflict):
res_state = self._dsa_client.get_resource_state()
log.debug("Initialize DataSet agent")
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.info("Sent INITIALIZE; DSA state = %s", state)
log.debug("DataSet agent go active")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent GO_ACTIVE; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.IDLE)
log.debug("DataSet agent run")
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent RUN; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.COMMAND)
if final_state == ResourceAgentState.STREAMING:
self.assert_start_sampling()
def assert_stop_sampling(self):
'''
transition to command. Must be called from streaming
'''
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STREAMING)
log.debug("DataSet agent start sampling")
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._dsa_client.execute_resource(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent START SAMPLING; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.COMMAND)
def assert_start_sampling(self):
'''
transition to sampling. Must be called from command
'''
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
log.debug("DataSet agent start sampling")
cmd = AgentCommand(command='DRIVER_EVENT_START_AUTOSAMPLE')
retval = self._dsa_client.execute_resource(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent START SAMPLING; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.STREAMING)
def assert_reset(self):
'''
Put the instrument back in uninitialized
'''
if self._dsa_client is None:
return
state = self._dsa_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def assert_data_values(self, granules, dataset_definition_file):
"""
Verify granules match the granules defined in the definition file
"""
rs_file = self._get_source_data_file(dataset_definition_file)
rs = ResultSet(rs_file)
self.assertTrue(rs.verify(granules), msg="Failed data validation. See log for details")
def assert_sample_queue_size(self, stream_name, size):
"""
verify a sample queue is the size we expect it to be.
"""
# Sleep a couple seconds to ensure the
gevent.sleep(2)
stream_id = self._stream_id_map.get(stream_name)
length = 0
if stream_id in self._samples_received:
length = len(self._samples_received[stream_id])
self.assertEqual(length, size, msg="Queue size != expected size (%d != %d)" % (length, size))
def assert_set_pubrate(self, rate):
"""
Set the pubrate for the parsed data stream. Set to 0 for
no buffering
"""
self.assertIsInstance(rate, (int, float))
self.assertGreaterEqual(rate, 0)
expected_pubrate = {self.test_config.stream_name: rate}
retval = self._dsa_client.set_agent({'pubrate': expected_pubrate})
retval = self._dsa_client.get_agent(['pubrate'])
expected_pubrate_result = {'pubrate': expected_pubrate}
self.assertEqual(retval, expected_pubrate_result)
def assert_agent_command(self, command, args=None, timeout=None):
"""
Verify an agent command
@param command: driver command to execute
@param args: kwargs to pass to the agent command object
"""
cmd = AgentCommand(command=command, kwargs=args)
retval = self._dsa_client.execute_agent(cmd, timeout=timeout)
def assert_resource_command(self, command, args=None, timeout=None):
"""
Verify a resource command
@param command: driver command to execute
@param args: kwargs to pass to the agent command object
"""
cmd = AgentCommand(command=command, kwargs=args)
retval = self._dsa_client.execute_resource(cmd)
def assert_schema(self, caps_list):
dd_list = ['display_name','description']
ddt_list = ['display_name','description','type']
ddvt_list = ['display_name','description','visibility','type']
ddak_list = ['display_name','description','args','kwargs']
for x in caps_list:
if isinstance(x,dict):
x.pop('type_')
x = IonObject('AgentCapability', **x)
if x.cap_type == CapabilityType.AGT_CMD:
keys = x.schema.keys()
for y in ddak_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.AGT_PAR:
if x.name != 'example':
keys = x.schema.keys()
for y in ddvt_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.RES_CMD:
keys = x.schema.keys()
self.assertIn('return',keys)
self.assertIn('display_name',keys)
self.assertIn('arguments',keys)
self.assertIn('timeout',keys)
elif x.cap_type == CapabilityType.RES_IFACE:
pass
elif x.cap_type == CapabilityType.RES_PAR:
keys = x.schema.keys()
self.assertIn('get_timeout',keys)
self.assertIn('set_timeout',keys)
self.assertIn('direct_access',keys)
self.assertIn('startup',keys)
self.assertIn('visibility',keys)
elif x.cap_type == CapabilityType.AGT_STATES:
for (k,v) in x.schema.iteritems():
keys = v.keys()
for y in dd_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.ALERT_DEFS:
for (k,v) in x.schema.iteritems():
keys = v.keys()
for y in ddt_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.AGT_CMD_ARGS:
for (k,v) in x.schema.iteritems():
keys = v.keys()
for y in ddt_list:
self.assertIn(y, keys)
def assert_agent_capabilities(self):
"""
Verify capabilities throughout the agent lifecycle
"""
capabilities = {
AgentCapabilityType.AGENT_COMMAND: self._common_agent_commands(ResourceAgentState.UNINITIALIZED),
AgentCapabilityType.AGENT_PARAMETER: self._common_agent_parameters(),
AgentCapabilityType.RESOURCE_COMMAND: None,
AgentCapabilityType.RESOURCE_INTERFACE: None,
AgentCapabilityType.RESOURCE_PARAMETER: None,
}
###
# DSA State INACTIVE
###
log.debug("Initialize DataSet agent")
self.assert_agent_command(ResourceAgentEvent.INITIALIZE)
self.assert_state_change(ResourceAgentState.INACTIVE)
self.assert_capabilities(capabilities)
###
# DSA State IDLE
###
log.debug("DataSet agent go active")
capabilities[AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(ResourceAgentState.IDLE)
self.assert_agent_command(ResourceAgentEvent.GO_ACTIVE)
self.assert_state_change(ResourceAgentState.IDLE)
self.assert_capabilities(capabilities)
###
# DSA State COMMAND
###
log.debug("DataSet agent run")
capabilities[AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(ResourceAgentState.COMMAND)
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = ['DRIVER_EVENT_START_AUTOSAMPLE']
capabilities[AgentCapabilityType.RESOURCE_PARAMETER] = self._common_resource_parameters()
self.assert_agent_command(ResourceAgentEvent.RUN)
self.assert_state_change(ResourceAgentState.COMMAND)
self.assert_capabilities(capabilities)
###
# DSA State STREAMING
###
capabilities[AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(ResourceAgentState.STREAMING)
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = ['DRIVER_EVENT_STOP_AUTOSAMPLE']
capabilities[AgentCapabilityType.RESOURCE_PARAMETER] = self._common_resource_parameters()
self.assert_start_sampling()
self.assert_capabilities(capabilities)
###
# DSA State LOST_CONNECTION
###
capabilities[AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(ResourceAgentState.LOST_CONNECTION)
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = None
capabilities[AgentCapabilityType.RESOURCE_PARAMETER] = None
self.assert_agent_command(ResourceAgentEvent.RESET)
self.assert_state_change(ResourceAgentState.UNINITIALIZED)
self.remove_sample_dir()
self.assert_initialize(final_state=ResourceAgentState.COMMAND)
self.assert_resource_command('DRIVER_EVENT_START_AUTOSAMPLE')
self.assert_state_change(ResourceAgentState.LOST_CONNECTION, 90)
def assert_driver_state(self, expected_state=None):
'''
verify that expected persisted agent state matches was it actually stored
@param expected_state dict expected
'''
state = self._get_dsa_object_state()
if expected_state is None:
self.assertIsNone(expected_state)
else:
self.assertEqual(expected_state, state)
def assert_agent_state_after_restart(self):
'''
Restart the agent. Verify that the agent PID changes. Then verify the new state
matches the old state.
'''
old_pid = self._dsa_pid
old_state = self._get_dsa_object_state()
# Start a resource agent client to talk with the instrument agent.
log.info('Restarting DSA process')
self._stop_dataset_agent_process()
self._dsa_client = self._start_dataset_agent_process()
log.debug("Client created: %s", type(self._dsa_client))
self.addCleanup(self.assert_reset)
self.assert_initialize()
self.assertNotEqual(old_pid, self._dsa_pid)
self.assertEqual(old_state, self._get_dsa_object_state())
# Kick it into autosample and give it time for samples to come in, there shouldn't be any
gevent.sleep(5)
def assert_capabilities(self, capabilities):
'''
Verify that all capabilities are available for a give state
@todo: Currently resource interface not implemented because it requires
a submodule update and some of the submodules are in release
states. So for now, no resource interfaces
@param: dictionary of all the different capability types that are
supposed to be there. i.e.
{
agent_command = ['DO_MY_COMMAND'],
agent_parameter = ['foo'],
resource_command = None,
resource_interface = None,
resource_parameter = None,
}
'''
def sort_capabilities(caps_list):
'''
sort a return value into capability buckets.
@retval agt_cmds, agt_pars, res_cmds, res_iface, res_pars
'''
agt_cmds = []
agt_pars = []
res_cmds = []
res_iface = []
res_pars = []
if len(caps_list)>0 and isinstance(caps_list[0], AgentCapability):
agt_cmds = [x.name for x in caps_list if x.cap_type==CapabilityType.AGT_CMD]
agt_pars = [x.name for x in caps_list if x.cap_type==CapabilityType.AGT_PAR]
res_cmds = [x.name for x in caps_list if x.cap_type==CapabilityType.RES_CMD]
#res_iface = [x.name for x in caps_list if x.cap_type==CapabilityType.RES_IFACE]
res_pars = [x.name for x in caps_list if x.cap_type==CapabilityType.RES_PAR]
elif len(caps_list)>0 and isinstance(caps_list[0], dict):
agt_cmds = [x['name'] for x in caps_list if x['cap_type']==CapabilityType.AGT_CMD]
agt_pars = [x['name'] for x in caps_list if x['cap_type']==CapabilityType.AGT_PAR]
res_cmds = [x['name'] for x in caps_list if x['cap_type']==CapabilityType.RES_CMD]
#res_iface = [x['name'] for x in caps_list if x['cap_type']==CapabilityType.RES_IFACE]
res_pars = [x['name'] for x in caps_list if x['cap_type']==CapabilityType.RES_PAR]
agt_cmds.sort()
agt_pars.sort()
res_cmds.sort()
res_iface.sort()
res_pars.sort()
return agt_cmds, agt_pars, res_cmds, res_iface, res_pars
if not capabilities.get(AgentCapabilityType.AGENT_COMMAND):
capabilities[AgentCapabilityType.AGENT_COMMAND] = []
if not capabilities.get(AgentCapabilityType.AGENT_PARAMETER):
capabilities[AgentCapabilityType.AGENT_PARAMETER] = []
if not capabilities.get(AgentCapabilityType.RESOURCE_COMMAND):
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = []
if not capabilities.get(AgentCapabilityType.RESOURCE_INTERFACE):
capabilities[AgentCapabilityType.RESOURCE_INTERFACE] = []
if not capabilities.get(AgentCapabilityType.RESOURCE_PARAMETER):
capabilities[AgentCapabilityType.RESOURCE_PARAMETER] = []
expected_agent_cmd = capabilities.get(AgentCapabilityType.AGENT_COMMAND)
expected_agent_cmd.sort()
expected_agent_param = self._common_agent_parameters()
expected_agent_param.sort()
expected_res_cmd = capabilities.get(AgentCapabilityType.RESOURCE_COMMAND)
expected_res_cmd.sort()
expected_res_param = capabilities.get(AgentCapabilityType.RESOURCE_PARAMETER)
expected_res_param.sort()
expected_res_int = capabilities.get(AgentCapabilityType.RESOURCE_INTERFACE)
expected_res_int.sort()
# go get the active capabilities
retval = self._dsa_client.get_capabilities()
agt_cmds, agt_pars, res_cmds, res_iface, res_pars = sort_capabilities(retval)
self.assert_schema(retval)
log.debug("Agent Commands: %s ", str(agt_cmds))
log.debug("Compared to: %s", expected_agent_cmd)
log.debug("Agent Parameters: %s ", str(agt_pars))
log.debug("Compared to: %s", expected_agent_param)
log.debug("Resource Commands: %s ", str(res_cmds))
log.debug("Compared to: %s", expected_res_cmd)
log.debug("Resource Interface: %s ", str(res_iface))
log.debug("Compared to: %s", expected_res_int)
log.debug("Resource Parameter: %s ", str(res_pars))
log.debug("Compared to: %s", expected_res_param)
# Compare to what we are supposed to have
self.assertEqual(expected_agent_cmd, agt_cmds)
self.assertEqual(expected_agent_param, agt_pars)
self.assertEqual(expected_res_cmd, res_cmds)
self.assertEqual(expected_res_int, res_iface)
self.assertEqual(expected_res_param, res_pars)
def _common_resource_parameters(self):
'''
list of common resource parameters
@return: list of resource parameters
'''
return ['batched_particle_count', 'publisher_polling_interval', 'records_per_second']
def _common_agent_parameters(self):
'''
list of common agent parameters
@return: list of agent parameters
'''
return ['aggstatus', 'alerts', 'driver_name', 'driver_pid', 'example', 'pubrate', 'streams']
def _common_agent_commands(self, agent_state):
'''
list of common agent parameters for a agent state
@return: list of agent parameters
@raise: KeyError for undefined agent state
'''
capabilities = {
ResourceAgentState.UNINITIALIZED: [
ResourceAgentEvent.GO_ACTIVE,
ResourceAgentEvent.RESET,
],
ResourceAgentState.IDLE: [
ResourceAgentEvent.GO_INACTIVE,
ResourceAgentEvent.RESET,
ResourceAgentEvent.RUN,
],
ResourceAgentState.COMMAND: [
ResourceAgentEvent.CLEAR,
ResourceAgentEvent.RESET,
ResourceAgentEvent.GO_INACTIVE,
ResourceAgentEvent.PAUSE
],
ResourceAgentState.STREAMING: [
ResourceAgentEvent.RESET,
ResourceAgentEvent.GO_INACTIVE
],
ResourceAgentState.LOST_CONNECTION: [
ResourceAgentEvent.RESET,
ResourceAgentEvent.GO_INACTIVE
]
}
return capabilities[agent_state]
def assert_state_change(self, target_agent_state, timeout=10):
"""
Verify the agent and resource states change as expected within the timeout
Fail if the state doesn't change to the expected state.
@param target_agent_state: State we expect the agent to be in
@param timeout: how long to wait for the driver to change states
"""
to = gevent.Timeout(timeout)
to.start()
done = False
agent_state = None
try:
while(not done):
agent_state = self._dsa_client.get_agent_state()
log.error("Current agent state: %s", agent_state)
if(agent_state == target_agent_state):
log.debug("Current state match: %s", agent_state)
done = True
if not done:
log.debug("state mismatch, waiting for state to transition.")
gevent.sleep(1)
except Timeout:
log.error("Failed to transition agent state to %s, current state: %s", target_agent_state, agent_state)
self.fail("Failed to transition state.")
finally:
to.cancel()
class TestExternalDatasetAgentMgmt(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel, name='ExampleDatasetModel', description="ExampleDatasetModel", datset_type="FibSeries" )
try:
datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id) )
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent, name='datasetagent007', description="datasetagent007", handler_module="ion.agents.eoi.external_dataset_agent", handler_class="ExternalDatasetAgent" )
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id) )
# Create ExternalDataset
log.debug('TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset, name='ExtDataset', description="ExtDataset" )
try:
extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id))
#register the dataset as a data producer
self.damsclient.register_external_data_set(extDataset_id)
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': EDA_RESOURCE_ID},
'test_mode' : True
}
extDatasetAgentInstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='DatasetAgentInstance', description="DatasetAgentInstance", dataset_driver_config = DVR_CONFIG, dataset_agent_config = agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj) )
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id) )
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id))
log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition")
dp_obj = IonObject(RT.DataProduct,name='eoi dataset data',description=' stream test')
try:
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1) )
self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids) )
self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id)
dataset_agent_instance_obj= self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj) )
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess())
print 'activate_instrument: got ia client %s', self._dsa_client
log.debug("test_activateInstrument: got dataset client %s", str(self._dsa_client))
cmd=AgentCommand(command='initialize')
_ = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._dsa_client.execute_agent(cmd)
log.info('Send an unconstrained request for data (\'new data\')')
config={'stream_id':'first_new','TESTING':True}
cmd = AgentCommand(command='acquire_data', args=[config])
self._dsa_client.execute(cmd)
log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
config={'stream_id':'second_new','TESTING':True}
cmd = AgentCommand(command='acquire_data', args=[config])
self._dsa_client.execute(cmd)
cmd = AgentCommand(command='reset')
_ = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
class TestExternalDatasetAgentMgmt(IonIntegrationTestCase):
# DataHandler config
DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.base_data_handler',
'dvr_cls' : 'DummyDataHandler',
}
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel, name='ExampleDatasetModel', description="ExampleDatasetModel", datset_type="FibSeries" )
try:
datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id) )
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent, name='datasetagent007', description="datasetagent007", handler_module=EDA_MOD, handler_class=EDA_CLS )
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id) )
# Create ExternalDataset
log.debug('TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset, name='ExtDataset', description="ExtDataset" )
try:
extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id))
#register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(extDataset_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id))
log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition")
dp_obj = IonObject(RT.DataProduct,name='eoi dataset data',description=' stream test')
try:
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1) )
self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids) )
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,#TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id':dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
'taxonomy':tx.dump(), #TODO: Currently does not support sets
'max_records':4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': EDA_RESOURCE_ID},
'test_mode' : True
}
extDatasetAgentInstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='DatasetAgentInstance', description="DatasetAgentInstance", dataset_driver_config = self.DVR_CONFIG, dataset_agent_config = agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj) )
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id) )
#Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active) )
self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id)
dataset_agent_instance_obj= self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj) )
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active) )
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess())
print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
class DatasetAgentTestCase(IonIntegrationTestCase):
"""
Base class for all dataset agent end to end tests
"""
test_config = DatasetAgentTestConfig()
def setUp(self, deploy_file=DEPLOY_FILE):
"""
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
self._dsa_client = None
self.dams = DataAcquisitionManagementServiceClient()
# Ensure we have a good test configuration
self.test_config.verify()
# Start container.
log.info('Staring capability container.')
self._start_container()
self.rr = self.container.resource_registry
# Bring up services in a deploy file (no need to message)
log.info('Starting deploy services. %s', deploy_file)
self.container.start_rel_from_url(DEPLOY_FILE)
# Load instrument specific parameters
log.info('Preload test scenarios')
self._load_params()
# Start a resource agent client to talk with the instrument agent.
log.info('Starting DSA process')
self._dsa_client = self._start_dataset_agent_process()
log.debug("Client created: %s", type(self._dsa_client))
self.addCleanup(self._stop_dataset_agent_process)
log.info('test setup complete')
# Start data subscribers
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
###
# Test/Agent Startup Helpers
###
def _load_params(self):
"""
Do a second round of preload with instrument specific scenarios
"""
scenario = None
categories = None
if PRELOAD_CATEGORIES:
categories = ",".join(PRELOAD_CATEGORIES)
# load_parameter_scenarios
if PRELOAD_SCENARIO:
scenario = PRELOAD_SCENARIO
else:
log.warn("No common preload defined. Was this intentional?")
if self.test_config.preload_scenario:
scenario = "%s,%s" % (
scenario, self.test_config.preload_scenario
) if scenario else self.test_config.preload_scenario
else:
log.warn("No DSA specific preload defined. Was this intentional?")
if scenario:
preload_config = dict(
op="load",
scenario=scenario,
#path="master",
path=TESTED_DOC,
categories=categories,
clearcols="owner_id,org_ids",
#assets="res/preload/r2_ioc/ooi_assets",
#parseooi="True",
)
log.debug("Starting preload now: config=%s", preload_config)
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader", preload_config)
def _start_dataset_agent_process(self):
"""
Launch the agent process and store the configuration. Tried
to emulate the same process used by import_data.py
"""
instrument_device, dsa_instance = self._get_dsa_instance()
self._driver_config = dsa_instance.driver_config
self._update_dsa_config(dsa_instance)
self._update_harvester_config(dsa_instance)
self._dsa_instance = dsa_instance
self.clear_sample_data()
# Return a resource agent client
return self._get_dsa_client(instrument_device, dsa_instance)
def _stop_dataset_agent_process(self):
"""
Stop the dataset agent instance
"""
self.assert_reset()
self.dams.stop_external_dataset_agent_instance(self._dsa_instance._id)
def _get_dsa_instance(self):
"""
Find the dsa instance in preload and return an instance of that object
"""
name = self.test_config.instrument_device_name
log.debug("Start dataset agent process for instrument device: %s",
name)
objects, _ = self.rr.find_resources(RT.InstrumentDevice, name=name)
log.debug("Found Instrument Devices: %s", objects)
if not objects:
raise ConfigNotFound(
"No appropriate InstrumentDevice objects loaded")
instrument_device = objects[0]
log.trace("Found instrument device: %s", instrument_device)
dsa_instance = self.rr.read_object(
subject=instrument_device._id,
predicate=PRED.hasAgentInstance,
object_type=RT.ExternalDatasetAgentInstance)
log.info("dsa_instance found: %s", dsa_instance)
return instrument_device, dsa_instance
def _update_harvester_config(self, dsa_instance):
"""
Update the harvester config such that we change the directory to something
we have write permissions.
"""
log.info("dsa agent instance: %s", dsa_instance)
driver_config = dsa_instance.driver_config
log.info("dsa agent driver config: %s", driver_config)
driver_config['startup_config']['harvester'][
'directory'] = self.test_config.data_dir
log.info("updated driver config: %s", driver_config)
dsa_instance.driver_config = driver_config
self.rr.update(dsa_instance)
def _update_dsa_config(self, dsa_instance):
"""
Update the dsa configuration prior to loading the agent. This is where we can
alter production configurations for use in a controlled test environment.
"""
dsa_obj = self.rr.read_object(object_type=RT.ExternalDatasetAgent,
predicate=PRED.hasAgentDefinition,
subject=dsa_instance._id,
id_only=False)
log.info("dsa agent definition found: %s", dsa_obj)
# If we don't want to load from an egg then we need to
# alter the driver config read from preload
if self.test_config.mi_repo is not None:
dsa_obj.driver_uri = None
# Strip the custom namespace
dsa_obj.driver_module = ".".join(
dsa_obj.driver_module.split('.')[1:])
log.info("saving new dsa agent config: %s", dsa_obj)
self.rr.update(dsa_obj)
if not self.test_config.mi_repo in sys.path:
sys.path.insert(0, self.test_config.mi_repo)
log.debug("Driver module: %s", dsa_obj.driver_module)
log.debug("MI Repo: %s", self.test_config.mi_repo)
log.trace("Sys Path: %s", sys.path)
def _get_dsa_client(self, instrument_device, dsa_instance):
"""
Launch the agent and return a client
"""
fake_process = FakeProcess()
fake_process.container = self.container
clients = DataAcquisitionManagementServiceDependentClients(
fake_process)
config_builder = ExternalDatasetAgentConfigurationBuilder(clients)
try:
config_builder.set_agent_instance_object(dsa_instance)
self.agent_config = config_builder.prepare()
except Exception as e:
log.error('failed to launch: %s', e, exc_info=True)
raise ServerError('failed to launch')
self._dsa_pid = self.dams.start_external_dataset_agent_instance(
dsa_instance._id)
log.debug("_get_dsa_client CFG")
return ResourceAgentClient(instrument_device._id,
process=FakeProcess())
dispatcher = ProcessDispatcherServiceClient()
launcher = AgentLauncher(dispatcher)
log.debug("Launching agent process!")
self._dsa_pid = launcher.launch(
self.agent_config,
config_builder._get_process_definition()._id)
if not self._dsa_pid:
raise ServerError(
"Launched external dataset agent instance but no process_id")
config_builder.record_launch_parameters(self.agent_config)
launcher.await_launch(10.0)
return ResourceAgentClient(instrument_device._id,
process=FakeProcess())
def _get_dsa_object_state(self):
state, _id = self.container.state_repository.get_state(self._dsa_pid)
log.debug("agent_state (%s): %s", self._dsa_pid, state)
driver_state = state.get(DSA_STATE_KEY)
log.debug("driver_state (%s): %s", self._dsa_pid, driver_state)
return driver_state
###
# Data file helpers
###
def _get_source_data_file(self, filename):
"""
Search for a sample data file, first check the driver resource directory
then just use the filename as a path. If the file doesn't exists
raise an exception
@param filename name or path of the file to search for
@return full path to the found data file
@raise IonException if the file isn't found
"""
resource_dir = self.test_config.test_resource_dir
source_path = os.path.join(resource_dir, filename)
log.debug("Search for resource file (%s) in %s", filename,
resource_dir)
if os.path.isfile(source_path):
log.debug("Found %s in resource directory", filename)
return source_path
log.debug("Search for resource file (%s) in current directory",
filename)
if os.path.isfile(filename):
log.debug("Found %s in the current directory", filename)
return filename
raise IonException("Data file %s does not exist", filename)
def create_data_dir(self):
"""
Verify the test data directory is created and exists. Return the path to
the directory.
@return: path to data directory
@raise: ConfigNotFound no harvester config
@raise: IonException if data_dir exists, but not a directory
"""
startup_config = self._driver_config.get('startup_config')
if not startup_config:
raise ConfigNotFound("Driver config missing 'startup_config'")
harvester_config = startup_config.get('harvester')
if not harvester_config:
raise ConfigNotFound("Startup config missing 'harvester' config")
data_dir = harvester_config.get("directory")
if not data_dir:
raise ConfigNotFound("Harvester config missing 'directory'")
if not os.path.exists(data_dir):
log.debug("Creating data dir: %s", data_dir)
os.makedirs(data_dir)
elif not os.path.isdir(data_dir):
raise IonException("'data_dir' is not a directory")
return data_dir
def clear_sample_data(self):
"""
Remove all files from the sample data directory
"""
data_dir = self.create_data_dir()
log.debug("Clean all data from %s", data_dir)
self.remove_all_files(data_dir)
def create_sample_data(self, filename, dest_filename=None):
"""
Search for a data file in the driver resource directory and if the file
is not found there then search using the filename directly. Then copy
the file to the test data directory.
If a dest_filename is supplied it will be renamed in the destination
directory.
@param: filename - filename or path to a data file to copy
@param: dest_filename - name of the file when copied. default to filename
"""
data_dir = self.create_data_dir()
source_path = self._get_source_data_file(filename)
log.debug("DIR: %s", data_dir)
if dest_filename is None:
dest_path = os.path.join(data_dir, os.path.basename(source_path))
else:
dest_path = os.path.join(data_dir, dest_filename)
log.debug("Creating data file src: %s, dest: %s", source_path,
dest_path)
shutil.copy2(source_path, dest_path)
return dest_path
def remove_all_files(self, dir_name):
"""
Remove all files from a directory. Raise an exception if the directory contains something
other than files.
@param dir_name directory path to remove files.
@raise RuntimeError if the directory contains anything except files.
"""
for file_name in os.listdir(dir_name):
file_path = os.path.join(dir_name, file_name)
if not os.path.isfile(file_path):
raise RuntimeError("%s is not a file", file_path)
for file_name in os.listdir(dir_name):
file_path = os.path.join(dir_name, file_name)
os.unlink(file_path)
###############################################################################
# Event helpers.
###############################################################################
def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
"""
Start a subscriber to the instrument agent events.
@param type The type of event to catch.
@count Trigger the async event result when events received reaches this.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._event_count > 0 and \
self._event_count == len(self._events_received):
self._async_event_result.set()
# Event array and async event result.
self._event_count = count
self._events_received = []
self._async_event_result = AsyncResult()
self._event_subscriber = EventSubscriber(
event_type=type, callback=consume_event,
origin="IA_RESOURCE_ID") #TODO
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=5)
def _stop_event_subscriber(self):
"""
Stop event subscribers on cleanup.
"""
self._event_subscriber.stop()
self._event_subscriber = None
###############################################################################
# Data stream helpers.
###############################################################################
def _start_data_subscribers(self):
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
if self._samples_received.get(stream_id) is None:
self._samples_received[stream_id] = []
log.info('Received parsed data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._samples_received[stream_id].append(message)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = {}
self._stream_id_map = {}
stream_config = self.agent_config['stream_config']
log.info("starting data subscribers")
for stream_name in stream_config.keys():
log.debug("Starting data subscriber for stream '%s'", stream_name)
stream_id = stream_config[stream_name]['stream_id']
self._stream_id_map[stream_name] = stream_id
self._start_data_subscriber(stream_config[stream_name], recv_data)
def _start_data_subscriber(self, config, callback):
"""
Setup and start a data subscriber
"""
exchange_point = config['exchange_point']
stream_id = config['stream_id']
sub = StandaloneStreamSubscriber(exchange_point, callback)
sub.start()
self._data_subscribers.append(sub)
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
sub_id = pubsub_client.create_subscription(name=exchange_point,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def make_data_product(self, pdict_name, dp_name, available_fields=None):
self.pubsub_management = PubsubManagementServiceClient()
if available_fields is None: available_fields = []
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
pdict_name, id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition(
'%s stream_def' % dp_name,
parameter_dictionary_id=pdict_id,
available_fields=available_fields or None)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp_obj = DataProduct(name=dp_name)
data_product_id = self.data_product_management.create_data_product(
dp_obj, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
return data_product_id
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber, 'subscription_id'):
try:
pubsub_client.deactivate_subscription(
subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
def get_samples(self, stream_name, sample_count=1, timeout=30):
"""
listen on a stream until 'sample_count' samples are read and return
a list of all samples read. If the required number of samples aren't
read then throw an exception.
Note that this method does not clear the sample queue for the stream.
This should be done explicitly by the caller. However, samples that
are consumed by this method are removed.
@raise SampleTimeout - if the required number of samples aren't read
"""
to = gevent.Timeout(timeout)
to.start()
done = False
result = []
i = 0
log.debug("Fetch %s sample(s) from stream '%s'" %
(sample_count, stream_name))
stream_id = self._stream_id_map.get(stream_name)
log.debug("Stream ID Map: %s ", self._stream_id_map)
self.assertIsNotNone(stream_id,
msg="Unable to find stream name '%s'" %
stream_name)
try:
while (not done):
if (self._samples_received.has_key(stream_id)
and len(self._samples_received.get(stream_id))):
log.trace("get_samples() received sample #%d!", i)
result.append(self._samples_received[stream_id].pop(0))
i += 1
if i >= sample_count:
done = True
else:
log.debug(
"No samples in %s. Sleep a bit to wait for the data queue to fill up.",
stream_name)
gevent.sleep(1)
except Timeout:
log.error("Failed to get %d records from %s. received: %d",
sample_count, stream_name, i)
self.fail("Failed to read samples from stream %s", stream_name)
finally:
to.cancel()
return result
def remove_sample_dir(self):
"""
Remove the sample dir and all files
"""
data_dir = self.create_data_dir()
self.clear_sample_data()
os.rmdir(data_dir)
###
# Common assert methods
###
def assertDictEqual(self, d1, d2, msg=None): # assertEqual uses for dicts
for k, v1 in d1.iteritems():
self.assertIn(k, d2, msg)
v2 = d2[k]
if (isinstance(v1, collections.Iterable)
and not isinstance(v1, basestring)):
self.assertItemsEqual(v1, v2, msg)
else:
self.assertEqual(v1, v2, msg)
return True
def assert_initialize(self, final_state=ResourceAgentState.STREAMING):
'''
Walk through DSA states to get to streaming mode from uninitialized
'''
state = self._dsa_client.get_agent_state()
with self.assertRaises(Conflict):
res_state = self._dsa_client.get_resource_state()
log.debug("Initialize DataSet agent")
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.info("Sent INITIALIZE; DSA state = %s", state)
log.debug("DataSet agent go active")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent GO_ACTIVE; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.IDLE)
log.debug("DataSet agent run")
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent RUN; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.COMMAND)
if final_state == ResourceAgentState.STREAMING:
self.assert_start_sampling()
def assert_stop_sampling(self):
'''
transition to command. Must be called from streaming
'''
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STREAMING)
log.debug("DataSet agent start sampling")
cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE')
retval = self._dsa_client.execute_resource(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent START SAMPLING; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.COMMAND)
def assert_start_sampling(self):
'''
transition to sampling. Must be called from command
'''
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
log.debug("DataSet agent start sampling")
cmd = AgentCommand(command='DRIVER_EVENT_START_AUTOSAMPLE')
retval = self._dsa_client.execute_resource(cmd)
state = self._dsa_client.get_agent_state()
log.info("Sent START SAMPLING; DSA state = %s", state)
self.assertEqual(state, ResourceAgentState.STREAMING)
def assert_reset(self):
'''
Put the instrument back in uninitialized
'''
if self._dsa_client is None:
return
state = self._dsa_client.get_agent_state()
if state != ResourceAgentState.UNINITIALIZED:
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
retval = self._dsa_client.execute_agent(cmd)
state = self._dsa_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
def assert_data_values(self, granules, dataset_definition_file):
"""
Verify granules match the granules defined in the definition file
"""
rs_file = self._get_source_data_file(dataset_definition_file)
rs = ResultSet(rs_file)
self.assertTrue(rs.verify(granules),
msg="Failed data validation. See log for details")
def assert_sample_queue_size(self, stream_name, size):
"""
verify a sample queue is the size we expect it to be.
"""
# Sleep a couple seconds to ensure the
gevent.sleep(2)
stream_id = self._stream_id_map.get(stream_name)
length = 0
if stream_id in self._samples_received:
length = len(self._samples_received[stream_id])
self.assertEqual(length,
size,
msg="Queue size != expected size (%d != %d)" %
(length, size))
def assert_set_pubrate(self, rate):
"""
Set the pubrate for the parsed data stream. Set to 0 for
no buffering
"""
self.assertIsInstance(rate, (int, float))
self.assertGreaterEqual(rate, 0)
expected_pubrate = {self.test_config.stream_name: rate}
retval = self._dsa_client.set_agent({'pubrate': expected_pubrate})
retval = self._dsa_client.get_agent(['pubrate'])
expected_pubrate_result = {'pubrate': expected_pubrate}
self.assertEqual(retval, expected_pubrate_result)
def assert_agent_command(self, command, args=None, timeout=None):
"""
Verify an agent command
@param command: driver command to execute
@param args: kwargs to pass to the agent command object
"""
cmd = AgentCommand(command=command, kwargs=args)
retval = self._dsa_client.execute_agent(cmd, timeout=timeout)
def assert_resource_command(self, command, args=None, timeout=None):
"""
Verify a resource command
@param command: driver command to execute
@param args: kwargs to pass to the agent command object
"""
cmd = AgentCommand(command=command, kwargs=args)
retval = self._dsa_client.execute_resource(cmd)
def assert_schema(self, caps_list):
dd_list = ['display_name', 'description']
ddt_list = ['display_name', 'description', 'type']
ddvt_list = ['display_name', 'description', 'visibility', 'type']
ddak_list = ['display_name', 'description', 'args', 'kwargs']
for x in caps_list:
if isinstance(x, dict):
x.pop('type_')
x = IonObject('AgentCapability', **x)
if x.cap_type == CapabilityType.AGT_CMD:
keys = x.schema.keys()
for y in ddak_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.AGT_PAR:
if x.name != 'example':
keys = x.schema.keys()
for y in ddvt_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.RES_CMD:
keys = x.schema.keys()
self.assertIn('return', keys)
self.assertIn('display_name', keys)
self.assertIn('arguments', keys)
self.assertIn('timeout', keys)
elif x.cap_type == CapabilityType.RES_IFACE:
pass
elif x.cap_type == CapabilityType.RES_PAR:
keys = x.schema.keys()
self.assertIn('get_timeout', keys)
self.assertIn('set_timeout', keys)
self.assertIn('direct_access', keys)
self.assertIn('startup', keys)
self.assertIn('visibility', keys)
elif x.cap_type == CapabilityType.AGT_STATES:
for (k, v) in x.schema.iteritems():
keys = v.keys()
for y in dd_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.ALERT_DEFS:
for (k, v) in x.schema.iteritems():
keys = v.keys()
for y in ddt_list:
self.assertIn(y, keys)
elif x.cap_type == CapabilityType.AGT_CMD_ARGS:
for (k, v) in x.schema.iteritems():
keys = v.keys()
for y in ddt_list:
self.assertIn(y, keys)
def assert_agent_capabilities(self):
"""
Verify capabilities throughout the agent lifecycle
"""
capabilities = {
AgentCapabilityType.AGENT_COMMAND:
self._common_agent_commands(ResourceAgentState.UNINITIALIZED),
AgentCapabilityType.AGENT_PARAMETER:
self._common_agent_parameters(),
AgentCapabilityType.RESOURCE_COMMAND:
None,
AgentCapabilityType.RESOURCE_INTERFACE:
None,
AgentCapabilityType.RESOURCE_PARAMETER:
None,
}
###
# DSA State INACTIVE
###
log.debug("Initialize DataSet agent")
self.assert_agent_command(ResourceAgentEvent.INITIALIZE)
self.assert_state_change(ResourceAgentState.INACTIVE)
self.assert_capabilities(capabilities)
###
# DSA State IDLE
###
log.debug("DataSet agent go active")
capabilities[
AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(
ResourceAgentState.IDLE)
self.assert_agent_command(ResourceAgentEvent.GO_ACTIVE)
self.assert_state_change(ResourceAgentState.IDLE)
self.assert_capabilities(capabilities)
###
# DSA State COMMAND
###
log.debug("DataSet agent run")
capabilities[
AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(
ResourceAgentState.COMMAND)
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = [
'DRIVER_EVENT_START_AUTOSAMPLE'
]
capabilities[AgentCapabilityType.
RESOURCE_PARAMETER] = self._common_resource_parameters()
self.assert_agent_command(ResourceAgentEvent.RUN)
self.assert_state_change(ResourceAgentState.COMMAND)
self.assert_capabilities(capabilities)
###
# DSA State STREAMING
###
capabilities[
AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(
ResourceAgentState.STREAMING)
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = [
'DRIVER_EVENT_STOP_AUTOSAMPLE'
]
capabilities[AgentCapabilityType.
RESOURCE_PARAMETER] = self._common_resource_parameters()
self.assert_start_sampling()
self.assert_capabilities(capabilities)
###
# DSA State LOST_CONNECTION
###
capabilities[
AgentCapabilityType.AGENT_COMMAND] = self._common_agent_commands(
ResourceAgentState.LOST_CONNECTION)
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = None
capabilities[AgentCapabilityType.RESOURCE_PARAMETER] = None
self.assert_agent_command(ResourceAgentEvent.RESET)
self.assert_state_change(ResourceAgentState.UNINITIALIZED)
self.remove_sample_dir()
self.assert_initialize(final_state=ResourceAgentState.COMMAND)
self.assert_resource_command('DRIVER_EVENT_START_AUTOSAMPLE')
self.assert_state_change(ResourceAgentState.LOST_CONNECTION, 90)
def assert_driver_state(self, expected_state=None):
'''
verify that expected persisted agent state matches was it actually stored
@param expected_state dict expected
'''
state = self._get_dsa_object_state()
if expected_state is None:
self.assertIsNone(expected_state)
else:
self.assertEqual(expected_state, state)
def assert_agent_state_after_restart(self):
'''
Restart the agent. Verify that the agent PID changes. Then verify the new state
matches the old state.
'''
old_pid = self._dsa_pid
old_state = self._get_dsa_object_state()
# Start a resource agent client to talk with the instrument agent.
log.info('Restarting DSA process')
self._stop_dataset_agent_process()
self._dsa_client = self._start_dataset_agent_process()
log.debug("Client created: %s", type(self._dsa_client))
self.addCleanup(self.assert_reset)
self.assert_initialize()
self.assertNotEqual(old_pid, self._dsa_pid)
self.assertEqual(old_state, self._get_dsa_object_state())
# Kick it into autosample and give it time for samples to come in, there shouldn't be any
gevent.sleep(5)
def assert_capabilities(self, capabilities):
'''
Verify that all capabilities are available for a give state
@todo: Currently resource interface not implemented because it requires
a submodule update and some of the submodules are in release
states. So for now, no resource interfaces
@param: dictionary of all the different capability types that are
supposed to be there. i.e.
{
agent_command = ['DO_MY_COMMAND'],
agent_parameter = ['foo'],
resource_command = None,
resource_interface = None,
resource_parameter = None,
}
'''
def sort_capabilities(caps_list):
'''
sort a return value into capability buckets.
@retval agt_cmds, agt_pars, res_cmds, res_iface, res_pars
'''
agt_cmds = []
agt_pars = []
res_cmds = []
res_iface = []
res_pars = []
if len(caps_list) > 0 and isinstance(caps_list[0],
AgentCapability):
agt_cmds = [
x.name for x in caps_list
if x.cap_type == CapabilityType.AGT_CMD
]
agt_pars = [
x.name for x in caps_list
if x.cap_type == CapabilityType.AGT_PAR
]
res_cmds = [
x.name for x in caps_list
if x.cap_type == CapabilityType.RES_CMD
]
#res_iface = [x.name for x in caps_list if x.cap_type==CapabilityType.RES_IFACE]
res_pars = [
x.name for x in caps_list
if x.cap_type == CapabilityType.RES_PAR
]
elif len(caps_list) > 0 and isinstance(caps_list[0], dict):
agt_cmds = [
x['name'] for x in caps_list
if x['cap_type'] == CapabilityType.AGT_CMD
]
agt_pars = [
x['name'] for x in caps_list
if x['cap_type'] == CapabilityType.AGT_PAR
]
res_cmds = [
x['name'] for x in caps_list
if x['cap_type'] == CapabilityType.RES_CMD
]
#res_iface = [x['name'] for x in caps_list if x['cap_type']==CapabilityType.RES_IFACE]
res_pars = [
x['name'] for x in caps_list
if x['cap_type'] == CapabilityType.RES_PAR
]
agt_cmds.sort()
agt_pars.sort()
res_cmds.sort()
res_iface.sort()
res_pars.sort()
return agt_cmds, agt_pars, res_cmds, res_iface, res_pars
if not capabilities.get(AgentCapabilityType.AGENT_COMMAND):
capabilities[AgentCapabilityType.AGENT_COMMAND] = []
if not capabilities.get(AgentCapabilityType.AGENT_PARAMETER):
capabilities[AgentCapabilityType.AGENT_PARAMETER] = []
if not capabilities.get(AgentCapabilityType.RESOURCE_COMMAND):
capabilities[AgentCapabilityType.RESOURCE_COMMAND] = []
if not capabilities.get(AgentCapabilityType.RESOURCE_INTERFACE):
capabilities[AgentCapabilityType.RESOURCE_INTERFACE] = []
if not capabilities.get(AgentCapabilityType.RESOURCE_PARAMETER):
capabilities[AgentCapabilityType.RESOURCE_PARAMETER] = []
expected_agent_cmd = capabilities.get(
AgentCapabilityType.AGENT_COMMAND)
expected_agent_cmd.sort()
expected_agent_param = self._common_agent_parameters()
expected_agent_param.sort()
expected_res_cmd = capabilities.get(
AgentCapabilityType.RESOURCE_COMMAND)
expected_res_cmd.sort()
expected_res_param = capabilities.get(
AgentCapabilityType.RESOURCE_PARAMETER)
expected_res_param.sort()
expected_res_int = capabilities.get(
AgentCapabilityType.RESOURCE_INTERFACE)
expected_res_int.sort()
# go get the active capabilities
retval = self._dsa_client.get_capabilities()
agt_cmds, agt_pars, res_cmds, res_iface, res_pars = sort_capabilities(
retval)
self.assert_schema(retval)
log.debug("Agent Commands: %s ", str(agt_cmds))
log.debug("Compared to: %s", expected_agent_cmd)
log.debug("Agent Parameters: %s ", str(agt_pars))
log.debug("Compared to: %s", expected_agent_param)
log.debug("Resource Commands: %s ", str(res_cmds))
log.debug("Compared to: %s", expected_res_cmd)
log.debug("Resource Interface: %s ", str(res_iface))
log.debug("Compared to: %s", expected_res_int)
log.debug("Resource Parameter: %s ", str(res_pars))
log.debug("Compared to: %s", expected_res_param)
# Compare to what we are supposed to have
self.assertEqual(expected_agent_cmd, agt_cmds)
self.assertEqual(expected_agent_param, agt_pars)
self.assertEqual(expected_res_cmd, res_cmds)
self.assertEqual(expected_res_int, res_iface)
self.assertEqual(expected_res_param, res_pars)
def _common_resource_parameters(self):
'''
list of common resource parameters
@return: list of resource parameters
'''
return [
'batched_particle_count', 'publisher_polling_interval',
'records_per_second'
]
def _common_agent_parameters(self):
'''
list of common agent parameters
@return: list of agent parameters
'''
return [
'aggstatus', 'alerts', 'driver_name', 'driver_pid', 'example',
'pubrate', 'streams'
]
def _common_agent_commands(self, agent_state):
'''
list of common agent parameters for a agent state
@return: list of agent parameters
@raise: KeyError for undefined agent state
'''
capabilities = {
ResourceAgentState.UNINITIALIZED: [
ResourceAgentEvent.GO_ACTIVE,
ResourceAgentEvent.RESET,
],
ResourceAgentState.IDLE: [
ResourceAgentEvent.GO_INACTIVE,
ResourceAgentEvent.RESET,
ResourceAgentEvent.RUN,
],
ResourceAgentState.COMMAND: [
ResourceAgentEvent.CLEAR, ResourceAgentEvent.RESET,
ResourceAgentEvent.GO_INACTIVE, ResourceAgentEvent.PAUSE
],
ResourceAgentState.STREAMING:
[ResourceAgentEvent.RESET, ResourceAgentEvent.GO_INACTIVE],
ResourceAgentState.LOST_CONNECTION:
[ResourceAgentEvent.RESET, ResourceAgentEvent.GO_INACTIVE]
}
return capabilities[agent_state]
def assert_state_change(self, target_agent_state, timeout=10):
"""
Verify the agent and resource states change as expected within the timeout
Fail if the state doesn't change to the expected state.
@param target_agent_state: State we expect the agent to be in
@param timeout: how long to wait for the driver to change states
"""
to = gevent.Timeout(timeout)
to.start()
done = False
agent_state = None
try:
while (not done):
agent_state = self._dsa_client.get_agent_state()
log.error("Current agent state: %s", agent_state)
if (agent_state == target_agent_state):
log.debug("Current state match: %s", agent_state)
done = True
if not done:
log.debug(
"state mismatch, waiting for state to transition.")
gevent.sleep(1)
except Timeout:
log.error(
"Failed to transition agent state to %s, current state: %s",
target_agent_state, agent_state)
self.fail("Failed to transition state.")
finally:
to.cancel()
