def test_platform_with_instrument_streaming(self): # # The following is with just a single platform and the single # instrument "SBE37_SIM_08", which corresponds to the one on port 4008. # #load the paramaters and the param dicts necesssary for the VEL3D self._load_params() #create the instrument device/agent/mode self._create_instrument_resources() #create the platform device, agent and instance self._create_platform_configuration('LPJBox_CI_Ben_Hall') self.rrclient.create_association(subject=self.platform_device, predicate=PRED.hasDevice, object=self.instrument_device) self._start_platform() # self.addCleanup(self._stop_platform, p_root) # get everything in command mode: self._ping_agent() self._initialize() _ia_client = ResourceAgentClient(self.instrument_device, process=FakeProcess()) state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) self._go_active() # self._run() gevent.sleep(3) # note that this includes the instrument also getting to the command state # self._stream_instruments() # get client to the instrument: # the i_obj is a DotDict with various pieces captured during the # set-up of the instrument, in particular instrument_device_id #i_obj = self._get_instrument(instr_key) # log.debug("KK creating ResourceAgentClient") # ia_client = ResourceAgentClient(i_obj.instrument_device_id, # process=FakeProcess()) # log.debug("KK got ResourceAgentClient: %s", ia_client) # # # verify the instrument is command state: # state = ia_client.get_agent_state() # log.debug("KK instrument state: %s", state) # self.assertEqual(state, ResourceAgentState.COMMAND) self._reset() self._shutdown()
def get_agent_state(self, resource_id=''): """Return the current resource agent common state. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_agent_state(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type)
def get_agent_state(self, resource_id=''): """Return the current resource agent common state. @param resource_id The id of the resource agennt. @retval A str containing the current agent state. @param resource_id str @retval result str """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_agent_state(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type)
def test_platform_with_instrument_streaming(self): # # The following is with just a single platform and the single # instrument "SBE37_SIM_08", which corresponds to the one on port 4008. # #load the paramaters and the param dicts necesssary for the VEL3D log.debug( "load params------------------------------------------------------------------------------" ) self._load_params() log.debug( " _register_oms_listener------------------------------------------------------------------------------" ) self._register_oms_listener() #create the instrument device/agent/mode log.debug("---------- create_instrument_resources ----------") self._create_instrument_resources() #create the platform device, agent and instance log.debug("---------- create_platform_configuration ----------") self._create_platform_configuration('LPJBox_CI_Ben_Hall') self.rrclient.create_association(subject=self.platform_device_id, predicate=PRED.hasDevice, object=self.instrument_device_id) log.debug("---------- start_platform ----------") self._start_platform() self.addCleanup(self._stop_platform) # get everything in command mode: self._ping_agent() log.debug(" ---------- initialize ----------") self._initialize() _ia_client = ResourceAgentClient(self.instrument_device_id, process=FakeProcess()) state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug(" ---------- go_active ----------") self._go_active() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug("---------- run ----------") self._run() gevent.sleep(2) log.debug(" ---------- _start_resource_monitoring ----------") self._start_resource_monitoring() gevent.sleep(2) # # # verify the instrument is command state: # state = ia_client.get_agent_state() # log.debug(" TestPlatformInstrument get_agent_state: %s", state) # self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring log.debug(" ---------- _stop_resource_monitoring ----------") self._stop_resource_monitoring() gevent.sleep(2) log.debug(" ---------- go_inactive ----------") self._go_inactive() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) self._reset() self._shutdown()
class TestDriverEgg(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def get_streamConfigs(self): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') return raw_config, parsed_config ########################## # # The following tests generate different agent configs and pass them to a common base test script # ########################### @unittest.skip("this test can't be run from coi services. it is missing dependencies") def test_driverLaunchModuleNoURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchNoModuleOnlyURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchBogusModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="bogus", driver_class="Bogus", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) @unittest.skip("Launches an egg 'process' even though the egg download should produce error 404") def test_driverLaunchNoModule404URI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_404, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, False) def test_driverLaunchNoModuleBadEggURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_BAD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, True, False) def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True): """ This method runs a test of launching a driver with a given agent configuration """ # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] # add start/stop for instrument agent gevent.joinall([gevent.spawn(lambda: self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) agent_process_id = ResourceAgentClient._get_agent_process_id(instDevice_id) print "Agent process id is '%s'" % str(agent_process_id) self.assertTrue(agent_process_id) gate = ProcessStateGate(self.processdispatchclient.read_process, agent_process_id, ProcessStateEnum.RUNNING) if not expect_launch: self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" % agent_process_id) return self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % agent_process_id) print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING" #print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=agent_process_id, process=FakeProcess()) print "ResourceAgentClient created: %s" % str(self._ia_client) print "Sending command=ResourceAgentEvent.INITIALIZE" cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) if not expect_command: self.assertRaises(ServerError, self._ia_client.execute_agent, cmd) return retval = self._ia_client.execute_agent(cmd) print "Result of INITIALIZE: %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) print "Sending command=ResourceAgentEvent.RESET" cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "Result of RESET: %s" % str(reply)
class TestAgentPersistence(IonIntegrationTestCase): """ """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config. """ self._ia_client = None log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Create agent config. self._agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True, 'forget_past' : False, 'enable_persistence' : True, 'aparam_pubrate_config' : { 'raw' : 2, 'parsed' : 2 } } self._ia_client = None self._ia_pid = '1234' self.addCleanup(self._verify_agent_reset) self.addCleanup(self.container.state_repository.put_state, self._ia_pid, {}) ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config ############################################################################### # Agent start stop helpers. ############################################################################### def _start_agent(self, bootmode=None): """ """ container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) agent_config = deepcopy(self._agent_config) agent_config['bootmode'] = bootmode self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config, process_id=self._ia_pid) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got instrument agent client %s.', str(self._ia_client)) def _stop_agent(self): """ """ if self._ia_pid: container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) container_client.terminate_process(self._ia_pid) if self._ia_client: self._ia_client = None def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) self._ia_client = None ############################################################################### # Tests. ############################################################################### def test_agent_config_persistence(self): """ test_agent_config_persistence Test that agent parameter configuration is persisted between running instances. """ # Start the agent. self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Confirm the default agent parameters. #{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}} retval = self._ia_client.get_agent(['streams'])['streams'] self.assertIn('raw', retval.keys()) self.assertIn('parsed', retval.keys()) #{'pubrate': {'raw': 0, 'parsed': 0}} retval = self._ia_client.get_agent(['pubrate'])['pubrate'] self.assertIn('raw', retval.keys()) self.assertIn('parsed', retval.keys()) self.assertEqual(retval['raw'], 2) self.assertEqual(retval['parsed'], 2) #{'alerts': []} retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertEqual(retval, []) # Define a few new parameters and set them. # Confirm they are set. alert_def_1 = { 'name' : 'current_warning_interval', 'stream_name' : 'parsed', 'description' : 'Current is below normal range.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 10.0, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def_2 = { 'name' : 'temp_alarm_interval', 'stream_name' : 'parsed', 'description' : 'Temperatoure is critical.', 'alert_type' : StreamAlertType.ALARM, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 20.0, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def3 = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'time_delta' : 180, 'alert_class' : 'LateDataAlert' } orig_alerts = [alert_def_1,alert_def_2, alert_def3] pubrate = { 'parsed' : 10, 'raw' : 20 } params = { 'alerts' : orig_alerts, 'pubrate' : pubrate } # Set the new agent params and confirm. self._ia_client.set_agent(params) params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) pubrate = retval['pubrate'] alerts = retval['alerts'] self.assertIn('raw', pubrate.keys()) self.assertIn('parsed', pubrate.keys()) self.assertEqual(pubrate['parsed'], 10) self.assertEqual(pubrate['raw'], 20) count = 0 for x in alerts: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEqual(count, 3) # Now stop and restart the agent. self._stop_agent() gevent.sleep(15) self._start_agent('restart') # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Confirm the persisted parameters. params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) pubrate = retval['pubrate'] alerts = retval['alerts'] self.assertIn('raw', pubrate.keys()) self.assertIn('parsed', pubrate.keys()) self.assertEqual(pubrate['parsed'], 10) self.assertEqual(pubrate['raw'], 20) count = 0 for x in alerts: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEqual(count, 3) def test_agent_state_persistence(self): """ test_agent_state_persistence Verify that agents can be restored to their prior running state. """ self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) alert_def3 = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'time_delta' : 180, 'alert_class' : 'LateDataAlert' } orig_pubrate = { 'parsed' : 10, 'raw' : 20 } params = { 'alerts' : [alert_def3], 'pubrate' : orig_pubrate } # Set the new agent params and confirm. self._ia_client.set_agent(params) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Acquire sample returns a string, not a particle. The particle # is created by the data handler though. cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = self._ia_client.execute_resource(cmd) # Now stop and restart the agent. self._stop_agent() gevent.sleep(15) self._start_agent('restart') timeout = gevent.Timeout(240) timeout.start() try: while True: state = self._ia_client.get_agent_state() print '## in state: ' + state if state == ResourceAgentState.COMMAND: timeout.cancel() break else: gevent.sleep(1) except gevent.Timeout: self.fail("Could not restore agent state to COMMAND.") params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) alerts = retval['alerts'] pubrate = retval['pubrate'] self.assertEqual(len(alerts), 1) self.assertEqual(alert_def3['name'], alerts[0]['name']) self.assertEqual(pubrate['raw'], 20) self.assertEqual(pubrate['parsed'], 10) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) # Now stop and restart the agent. self._stop_agent() gevent.sleep(15) self._start_agent('restart') timeout = gevent.Timeout(240) timeout.start() try: while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.STOPPED: timeout.cancel() break else: gevent.sleep(1) except gevent.Timeout: self.fail("Could not restore agent state to STOPPED.") retval = self._ia_client.get_agent(params) alerts = retval['alerts'] pubrate = retval['pubrate'] self.assertEqual(len(alerts), 1) self.assertEqual(alert_def3['name'], alerts[0]['name']) self.assertEqual(pubrate['raw'], 20) self.assertEqual(pubrate['parsed'], 10) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_agent_rparam_persistence(self): """ test_agent_rparam_persistence Verify ability to restore device configuration. ### Original values: {'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0} ### Values after set: {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1} ### Restore config: {'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005], 'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649, 'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005], 'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0, 'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05, 'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484, 'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, ' RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, ' PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0, 'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819, 'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492, 'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06, 'RTCA0': 0.9999862} ### Of which we have: {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1} """ self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) params = [ SBE37Parameter.OUTPUTSV, SBE37Parameter.NAVG, SBE37Parameter.TA0 ] retval = self._ia_client.get_resource(params) orig_params = retval new_params = { SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV], SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1, SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2 } #print '########### orig params' #print str(orig_params) self._ia_client.set_resource(new_params) retval = self._ia_client.get_resource(params) self.assertEqual(retval[SBE37Parameter.OUTPUTSV], new_params[SBE37Parameter.OUTPUTSV]) self.assertEqual(retval[SBE37Parameter.NAVG], new_params[SBE37Parameter.NAVG]) delta = max(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0])*.01 self.assertAlmostEqual(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0], delta=delta) #print '########### new params' #print str(retval) # Now stop and restart the agent. self._stop_agent() self._support.stop_pagent() gevent.sleep(10) self._start_pagent() gevent.sleep(10) self._start_agent('restart') timeout = gevent.Timeout(600) timeout.start() try: while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.COMMAND: timeout.cancel() break else: gevent.sleep(3) except gevent.Timeout: self.fail("Could not restore agent state to COMMAND.") # Verify the parameters have been restored as needed. retval = self._ia_client.get_resource(params) #print '########### restored params' #print str(retval) self.assertEqual(retval[SBE37Parameter.OUTPUTSV], new_params[SBE37Parameter.OUTPUTSV]) self.assertEqual(retval[SBE37Parameter.NAVG], new_params[SBE37Parameter.NAVG]) delta = max(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0])*.01 self.assertAlmostEqual(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0], delta=delta) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) @unittest.skip('Making CEI friendly.') def test_cei_launch_mode(self): pdc = ProcessDispatcherServiceClient(node=self.container.node) p_def = ProcessDefinition(name='Agent007') p_def.executable = { 'module' : 'ion.agents.instrument.instrument_agent', 'class' : 'InstrumentAgent' } p_def_id = pdc.create_process_definition(p_def) pid = pdc.create_process(p_def_id) def event_callback(event, *args, **kwargs): print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state]) sub = EventSubscriber(event_type='ProcessLifecycleEvent', callback=event_callback, origin=pid, origin_type='DispatchedProcess') sub.start() agent_config = deepcopy(self._agent_config) agent_config['bootmode'] = 'restart' pdc.schedule_process(p_def_id, process_id=pid, configuration=agent_config) gevent.sleep(5) pdc.cancel_process(pid) gevent.sleep(15) sub.stop()
class TestPrest(IonIntegrationTestCase): """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config, start agent. Start agent client. """ print '#####################' print 'IN SETUP' self._ia_client = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('building stream configuration') # Setup stream config. self._build_stream_config() #log.info('driver uri: %s', DRV_URI) #log.info('device address: %s', DEV_ADDR) #log.info('device port: %s', DEV_PORT) #log.info('work dir: %s', WORK_DIR) # Create agent config. agent_config = { 'driver_config': DVR_CONFIG, 'stream_config': self._stream_config, 'agent': { 'resource_id': IA_RESOURCE_ID }, 'test_mode': True, 'forget_past': True, 'enable_persistence': False } #if org_governance_name is not None: # agent_config['org_governance_name'] = org_governance_name # Start instrument agent. log.info("TestInstrumentAgent.setup(): starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) log.info("Agent setup") ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) self.addCleanup(self._verify_agent_reset) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) log.info('test setup complete') ############################################################################### # Port agent helpers. ############################################################################### def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) ############################################################################### # Event helpers. ############################################################################### def _start_event_subscriber(self, type='ResourceAgentEvent', count=0): """ Start a subscriber to the instrument agent events. @param type The type of event to catch. @count Trigger the async event result when events received reaches this. """ def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._event_count > 0 and \ self._event_count == len(self._events_received): self._async_event_result.set() # Event array and async event result. self._event_count = count self._events_received = [] self._async_event_result = AsyncResult() self._event_subscriber = EventSubscriber(event_type=type, callback=consume_event, origin=IA_RESOURCE_ID) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=5) def _stop_event_subscriber(self): """ Stop event subscribers on cleanup. """ self._event_subscriber.stop() self._event_subscriber = None ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config def _start_data_subscribers(self, count, raw_count): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._data_subscribers = [] self._samples_received = [] self._raw_samples_received = [] self._async_sample_result = AsyncResult() self._async_raw_sample_result = AsyncResult() # A callback for processing subscribed-to data. def recv_data(message, stream_route, stream_id): log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._samples_received.append(message) if len(self._samples_received) == count: self._async_sample_result.set() def recv_raw_data(message, stream_route, stream_id): log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._raw_samples_received.append(message) if len(self._raw_samples_received) == raw_count: self._async_raw_sample_result.set() from pyon.util.containers import create_unique_identifier stream_name = 'parsed' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) stream_name = 'raw' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): for subscriber in self._data_subscribers: pubsub_client = PubsubManagementServiceClient() if hasattr(subscriber, 'subscription_id'): try: pubsub_client.deactivate_subscription( subscriber.subscription_id) except: pass pubsub_client.delete_subscription(subscriber.subscription_id) subscriber.stop() ############################################################################### # Tests. ############################################################################### @unittest.skip('Test should be run manually only.') def test_initialize(self): """ test_initialize Test agent initialize command. This causes creation of driver process and transition to inactive. """ print '#### in test' # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) @unittest.skip('Test should be run manually only.') def test_xx(self): """ """ state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) with self.assertRaises(Conflict): res_state = self._ia_client.get_resource_state() cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print '################################# sbe54 came up in state: ' + state if state == ResourceAgentState.IDLE: cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) elif state == ResourceAgentState.STREAMING: cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE') retval = self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) state = self._ia_client.get_agent_state() print '################################# sbe54 now in state: ' + state gevent.sleep(60 * 2.25) state = self._ia_client.get_agent_state() print '################################# sbe54 now in state: ' + state """ 'DRIVER_EVENT_START_AUTOSAMPLE' self.assertEqual(state, ResourceAgentState.IDLE) res_state = self._ia_client.get_resource_state() self.assertEqual(res_state, DriverProtocolState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) res_state = self._ia_client.get_resource_state() self.assertEqual(res_state, DriverProtocolState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) with self.assertRaises(Conflict): retval = self._ia_client.execute_resource(cmd) """ cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete, index) IndexManagementService._es_call(es.index_delete, index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name='', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name='notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification( notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification( notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id)) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument( self, expected_instrument_device_id='', extended_instrument=None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance( extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) self.assertEqual( extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING) self.assertEqual( extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK) self.assertEqual( extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value)) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_instrument_device_id) self.assertEqual(notification.origin_type, "instrument") self.assertEqual(notification.event_type, 'ResourceLifecycleEvent') def _check_computed_attributes_of_extended_product( self, expected_data_product_id='', extended_data_product=None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_data_product.computed.product_download_size_estimated, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance( extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance( extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance( extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertTrue('ok' in extended_data_product.computed.last_granule. value['quality_flag']) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value)) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_data_product_id) self.assertEqual(notification.origin_type, "data product") self.assertEqual(notification.event_type, 'DetectionEvent') @attr('LOCOINT') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name': 'temperature_warning_interval', 'stream_name': 'parsed', 'value_id': 'temp', 'message': 'Temperature is below the normal range of 50.0 and above.', 'type': StreamAlarmType.WARNING, 'lower_bound': 50.0, 'lower_rel_op': '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5) parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm]) # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received = [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug( 'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s', str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient( instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print())) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals), 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print())) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals), 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateRSNVel3DInstrument, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skip('under construction') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 180}}}) def test_activate_rsn_vel3d(self): log.info("--------------------------------------------------------------------------------------------------------") # load_parameter_scenarios self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='Vel3DMModel', description="Vel3DMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' ) vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') RSN_VEL3D_01 = { 'DEV_ADDR' : "10.180.80.6", 'DEV_PORT' : 2101, 'DATA_PORT' : 1026, 'CMD_PORT' : 1025, 'PA_BINARY' : "port_agent" } # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='Vel3DAgent', description="Vel3DAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ') instDevice_obj = IonObject(RT.InstrumentDevice, name='Vel3DDevice', description="Vel3DDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s " , instDevice_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance', description="Vel3DAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True) parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id) parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True) parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample', temporal_domain = tdom, spatial_domain = sdom) sample_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id) log.debug( 'new dp_id = %s' , sample_data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=sample_data_product_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=sample_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True) log.debug('sample_data_product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for sample_data_product = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('vel3d_b_sample', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering', temporal_domain = tdom, spatial_domain = sdom) eng_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id) log.debug( 'new dp_id = %s' , eng_data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=eng_data_product_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=eng_data_product_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activate_rsn_vel3d Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) def check_state(label, desired_state): actual_state = self._ia_client.get_agent_state() log.debug("%s instrument agent is in state '%s'", label, actual_state) self.assertEqual(desired_state, actual_state) log.debug("test_activate_rsn_vel3d: got ia client %s" , str(self._ia_client)) check_state("just-spawned", ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: initialize %s" , str(retval)) check_state("initialized", ResourceAgentState.INACTIVE) log.debug("test_activate_rsn_vel3d Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return value from go_active %s" , str(reply)) check_state("activated", ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending go_active command %s" , str(state)) # cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) check_state("commanded", ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending run command %s" , str(state)) # cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # gevent.sleep(5) # # cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state)) # # cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.STOPPED, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RESUME) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.IDLE, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RUN) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) log.debug( "test_activate_rsn_vel3d: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return from reset %s" , str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) ) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(sample_data_product_id) self.dpclient.delete_data_product(eng_data_product_id) self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete, index) IndexManagementService._es_call(es.index_delete, index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name='', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name='notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification( notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification( notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id)) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument( self, expected_instrument_device_id='', extended_instrument=None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value)) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_instrument_device_id, notification.origin) self.assertEqual("instrument", notification.origin_type) self.assertEqual('ResourceLifecycleEvent', notification.event_type) def _check_computed_attributes_of_extended_product( self, expected_data_product_id='', extended_data_product=None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_data_product.computed.product_download_size_estimated, ComputedFloatValue) self.assertIsInstance( extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance( extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance( extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance( extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertIn( 'ok', extended_data_product.computed.last_granule.value['quality_flag']) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value)) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_data_product_id, notification.origin) self.assertEqual("data product", notification.origin_type) self.assertEqual('DetectionEvent', notification.event_type) @attr('LOCOINT') #@unittest.skip('refactoring') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 90}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.INACTIVE, state) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.STOPPED, state) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print())) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] all_raw = "".join(raw_vals) # look for 't' entered after a prompt -- ">t" t_commands = all_raw.count(">t") if 10 != t_commands: log.error("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r]) self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands) else: log.debug("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r]) replay_data_parsed = self.dataretrieverclient.retrieve( self.parsed_dataset) self.assertIsInstance(replay_data_parsed, Granule) rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print())) self.assertIn('temp', rdt_parsed) temp_vals = rdt_parsed['temp'] pressure_vals = rdt_parsed['pressure'] if 10 != len(temp_vals): log.error("%s temp_vals: %s", len(temp_vals), temp_vals) self.fail("Expected 10 temp_vals, got %s" % len(temp_vals)) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore #@unittest.skip("TBD") def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' }) instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice print 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' +\ '(SA Req: L4-CI-SA-RQ-241) ' instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) print "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " %\ instDevice_id port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4003, 'data_port': 4000, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self.get_datastore(self.parsed_dataset) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) print "test_activateInstrumentSample: got ia client %s" % str(self._ia_client) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) print "test_activateInstrumentSample: initialize %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) print "(L4-CI-SA-RQ-334): Sending go_active command " cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) print "test_activateInstrument: return value from go_active %s" % str(reply) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result print "(L4-CI-SA-RQ-334): current state after sending go_active command %s" % str(state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) print "test_activateInstrumentSample: run %s" % str(reply) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = self._ia_client.execute_resource(cmd) print "test_activateInstrumentSample: return from sample %s" % str(retval) retval = self._ia_client.execute_resource(cmd) print "test_activateInstrumentSample: return from sample %s" % str(retval) retval = self._ia_client.execute_resource(cmd) print "test_activateInstrumentSample: return from sample %s" % str(retval) print "test_activateInstrumentSample: calling reset " cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "test_activateInstrumentSample: return from reset %s" % str(reply) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT parsed: %s", str(rdt.pretty_print()) ) temp_vals = rdt['temp'] self.assertTrue(len(temp_vals) == 3) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print()) ) raw_vals = rdt['raw'] self.assertTrue(len(raw_vals) == 3) print "l4-ci-sa-rq-138" """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ # #-------------------------------------------------------------------------------- # # Get the extended data product to see if it contains the granules # #-------------------------------------------------------------------------------- # extended_product = self.dpclient.get_data_product_extension(data_product_id1) # self.assertEqual(data_product_id1, extended_product._id) # log.debug( "test_activateInstrumentSample: extended_product.computed.last_granule.value %s", str(extended_product.computed.last_granule.value) ) # log.debug( "test_activateInstrumentSample: extended_product.computed.recent_granules.value %s", str(extended_product.computed.recent_granules.value) ) # log.debug("test_activateInstrumentSample: extended_product.computed.provenance_product_list.value %s", str(extended_product.computed.provenance_product_list.value) ) #------------------------------- # Deactivate loggers #------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid)
def test_xxx(self, host='localhost', port=DATA_PORT, resource_id=IA_RESOURCE_ID, stream_config=None): """ """ if not stream_config: stream_config = self._stream_config log.info("create FakePuckReader") puck_reader = FakePuckReader() log.info("read_puck()") puck_data = puck_reader.read_puck(DEV_ADDR, DEV_PORT, DEV_SERIAL_LINE) driver_config = { 'dvr_mod' : puck_data['dvr_mod'], 'dvr_cls' : puck_data['dvr_cls'], 'workdir' : WORK_DIR, 'process_type' : PROCESS_TYPE, 'comms_config' : { 'addr' : host, 'port' : port } } agent_config = { 'driver_config' : driver_config, 'stream_config' : stream_config, 'agent' : {'resource_id': resource_id}, 'test_mode' : True } log.debug("Starting instrument agent.") ia_pid = self.container_client.spawn_process( name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) #self.addCleanup(self._verify_agent_reset) ia_client = ResourceAgentClient(resource_id, process=FakeProcess()) log.info('Got ia client %s.', str(ia_client)) # We start in uninitialized state. # In this state there is no driver process. state = ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = ia_client.ping_agent() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = ia_client.execute_agent(cmd) state = ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = ia_client.execute_agent(cmd) state = ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = ia_client.execute_agent(cmd) state = ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = ia_client.execute_resource(cmd) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = ia_client.execute_agent(cmd) state = ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class ExternalDatasetAgentTestBase(object): # Agent parameters. EDA_RESOURCE_ID = '123xyz' EDA_NAME = 'ExampleEDA' EDA_MOD = 'ion.agents.data.external_dataset_agent' EDA_CLS = 'ExternalDatasetAgent' """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ def setUp(self): """ Initialize test members. """ #log.warn('Starting the container') # Start container. self._start_container() # Bring up services in a deploy file #log.warn('Starting the rel') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a pubsub client to create streams. # log.warn('Init a pubsub client') self._pubsub_client = PubsubManagementServiceClient(node=self.container.node) # log.warn('Init a ContainerAgentClient') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # Data async and subscription TODO: Replace with new subscriber self._finished_count = None #TODO: Switch to gevent.queue.Queue self._async_finished_result = AsyncResult() self._finished_events_received = [] self._finished_event_subscriber = None self._start_finished_event_subscriber() self.addCleanup(self._stop_finished_event_subscriber) # TODO: Finish dealing with the resources and whatnot # TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources() self._setup_resources() #TG: Setup/configure the granule logger to log granules as they're published # Create agent config. agent_config = { 'driver_config': self.DVR_CONFIG, 'stream_config': {}, 'agent': {'resource_id': self.EDA_RESOURCE_ID}, 'test_mode': True } # Start instrument agent. self._ia_pid = None log.debug('TestInstrumentAgent.setup(): starting EDA.') self._ia_pid = self._container_client.spawn_process( name=self.EDA_NAME, module=self.EDA_MOD, cls=self.EDA_CLS, config=agent_config ) log.info('Agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) ######################################## # Private "setup" functions ######################################## def _setup_resources(self): raise NotImplementedError('_setup_resources must be implemented in the subclass') def create_stream_and_logger(self, name, stream_id='', pdict=None): stream_def_id = '' if not stream_id or stream_id is '': if pdict: stream_def_id = self._pubsub_client.create_stream_definition(parameter_dictionary=pdict.dump(), stream_type='stream') stream_id, route = self._pubsub_client.create_stream(name=name, exchange_point='science_data', stream_definition_id=stream_def_id) else: route = self._pubsub_client.read_stream_route(stream_id=stream_id) stream_def = self._pubsub_client.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id pid = self._container_client.spawn_process( name=name + '_logger', module='ion.processes.data.stream_granule_logger', cls='StreamGranuleLogger', config={'process': {'stream_id': stream_id}} ) log.info('Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.format(pid, stream_id)) return stream_id, route, stream_def_id def _start_finished_event_subscriber(self): def consume_event(*args, **kwargs): if args[0].description == 'TestingFinished': log.debug('TestingFinished event received') self._finished_events_received.append(args[0]) if self._finished_count and self._finished_count == len(self._finished_events_received): log.debug('Finishing test...') self._async_finished_result.set(len(self._finished_events_received)) log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received))) self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event) self._finished_event_subscriber.start() def _stop_finished_event_subscriber(self): if self._finished_event_subscriber: self._finished_event_subscriber.stop() self._finished_event_subscriber = None ######################################## # Custom assertion functions ######################################## def assertListsEqual(self, lst1, lst2): lst1.sort() lst2.sort() return lst1 == lst2 def assertSampleDict(self, val): """ Verify the value is a sample dictionary for the sbe37. """ #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]} self.assertTrue(isinstance(val, dict)) self.assertTrue('c' in val) self.assertTrue('t' in val) self.assertTrue('p' in val) self.assertTrue('time' in val) c = val['c'][0] t = val['t'][0] p = val['p'][0] time = val['time'][0] self.assertTrue(isinstance(c, float)) self.assertTrue(isinstance(t, float)) self.assertTrue(isinstance(p, float)) self.assertTrue(isinstance(time, float)) def assertParamDict(self, pd, all_params=False): """ Verify all device parameters exist and are correct type. """ if all_params: self.assertEqual(set(pd.keys()), set(PARAMS.keys())) for (key, type_val) in PARAMS.iteritems(): if type_val == list or type_val == tuple: self.assertTrue(isinstance(pd[key], (list, tuple))) else: self.assertTrue(isinstance(pd[key], type_val)) else: for (key, val) in pd.iteritems(): self.assertTrue(key in PARAMS) self.assertTrue(isinstance(val, PARAMS[key])) def assertParamVals(self, params, correct_params): """ Verify parameters take the correct values. """ self.assertEqual(set(params.keys()), set(correct_params.keys())) for (key, val) in params.iteritems(): correct_val = correct_params[key] if isinstance(val, float): # Verify to 5% of the larger value. max_val = max(abs(val), abs(correct_val)) self.assertAlmostEqual(val, correct_val, delta=max_val * .01) elif isinstance(val, (list, tuple)): # list of tuple. self.assertEqual(list(val), list(correct_val)) else: # int, bool, str. self.assertEqual(val, correct_val) ######################################## # Test functions ######################################## def test_acquire_data_while_streaming(self): # Test instrument driver execute interface to start and stop streaming mode. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) params = { 'POLLING_INTERVAL': 3 } self._ia_client.set_resource(params) self._finished_count = 1 cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STREAMING) config = get_safe(self.DVR_CONFIG, 'dh_cfg', {}) log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1') config['stream_id'], config['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_1') config['constraints'] = self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config]) self._ia_client.execute_resource(cmd) cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) finished = self._async_finished_result.get(timeout=120) self.assertEqual(finished, self._finished_count) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_acquire_data(self): cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) log.warn('Send an unconstrained request for data (\'new data\')') cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE) self._ia_client.execute_resource(command=cmd) state = self._ia_client.get_agent_state() log.info(state) self.assertEqual(state, ResourceAgentState.COMMAND) self._finished_count = 2 config_mods = {} log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1') config_mods['stream_id'], config_mods['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_1') config_mods['constraints'] = self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config_mods]) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) log.info('Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2') config_mods['stream_id'], config_mods['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_2') config_mods['constraints'] = self.HIST_CONSTRAINTS_2 cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config_mods]) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) finished = self._async_finished_result.get(timeout=120) self.assertEqual(finished, self._finished_count) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_streaming(self): state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) params = { 'POLLING_INTERVAL': 3 } self._ia_client.set_resource(params) self._finished_count = 3 cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STREAMING) #Assert that data was received # self._async_finished_result.get(timeout=600) # self.assertTrue(len(self._finished_events_received) >= 3) cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_command(self): # Test instrument driver get and set interface. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Retrieve all resource parameters. reply = self._ia_client.get_resource(params=['DRIVER_PARAMETER_ALL']) self.assertParamDict(reply, True) ## Retrieve a subset of resource parameters. params = [ 'POLLING_INTERVAL' ] reply = self._ia_client.get_resource(params=params) self.assertParamDict(reply) orig_params = reply # Set a subset of resource parameters. new_params = { 'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2), } self._ia_client.set_resource(params=new_params) check_new_params = self._ia_client.get_resource(params) self.assertParamVals(check_new_params, new_params) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_get_set_resource(self): cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) # Get a couple parameters retval = self._ia_client.get_resource(['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(type(retval['POLLING_INTERVAL']), int) self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list) # Attempt to get a parameter that doesn't exist log.debug('Try getting a non-existent parameter \'BAD_PARAM\'') with self.assertRaises(ServerError): self._ia_client.get_resource(['BAD_PARAM']) # Set the polling_interval to a new value, then get it to make sure it set properly self._ia_client.set_resource({'POLLING_INTERVAL': 10}) retval = self._ia_client.get_resource(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(retval['POLLING_INTERVAL'], 10) # Attempt to set a parameter that doesn't exist log.debug('Try setting a non-existent parameter \'BAD_PARAM\'') with self.assertRaises(ServerError): self._ia_client.set_resource({'BAD_PARAM': 'bad_val'}) # Attempt to set one parameter that does exist, and one that doesn't with self.assertRaises(ServerError): self._ia_client.set_resource({'POLLING_INTERVAL': 20, 'BAD_PARAM': 'bad_val'}) retval = self._ia_client.get_resource(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(retval['POLLING_INTERVAL'], 20) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_initialize(self): # Test agent initialize command. This causes creation of driver process and transition to inactive. # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_states(self): # Test agent state transitions. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STREAMING) cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_capabilities(self): """ Test the ability to retrieve agent and resource parameter and command capabilities in various system states. """ # Test the ability to retrieve agent and resource parameter and command capabilities. acmds = self._ia_client.get_capabilities(['AGT_CMD']) log.debug('Agent Commands: {0}'.format(acmds)) # acmds = [item[1] for item in acmds] self.assertListsEqual(acmds, AGT_CMDS.keys()) apars = self._ia_client.get_capabilities(['AGT_PAR']) log.debug('Agent Parameters: {0}'.format(apars)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) rcmds = self._ia_client.get_capabilities(['RES_CMD']) log.debug('Resource Commands: {0}'.format(rcmds)) # rcmds = [item[1] for item in rcmds] self.assertListsEqual(rcmds, CMDS.keys()) rpars = self._ia_client.get_capabilities(['RES_PAR']) log.debug('Resource Parameters: {0}'.format(rpars)) # rpars = [item[1] for item in rpars] self.assertListsEqual(rpars, PARAMS.keys()) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_errors(self): # Test illegal behavior and replies. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Can't go active in unitialized state. # Status 660 is state error. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) with self.assertRaises(Conflict): self._ia_client.execute_agent(cmd) # Can't command driver in this state. cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE) with self.assertRaises(Conflict): self._ia_client.execute_resource(cmd) #self.assertEqual(reply.status, 660) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # 404 unknown agent command. cmd = AgentCommand(command='kiss_edward') with self.assertRaises(BadRequest): self._ia_client.execute_agent(cmd) # 670 unknown driver command. cmd = AgentCommand(command='acquire_sample_please') with self.assertRaises(ServerError): self._ia_client.execute_resource(cmd) # 630 Parameter error. #self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus') cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin): @classmethod def setUpClass(cls): HelperTestMixin.setUpClass() # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri']) network_definition = RsnOmsUtil.build_network_definition(rsn_oms) network_definition_ser = NetworkUtil.serialize_network_definition( network_definition) if log.isEnabledFor(logging.DEBUG): log.debug("NetworkDefinition serialization:\n%s", network_definition_ser) cls.PLATFORM_CONFIG = { 'platform_id': cls.PLATFORM_ID, 'driver_config': DVR_CONFIG, 'network_definition': network_definition_ser } NetworkUtil._gen_open_diagram( network_definition.pnodes[cls.PLATFORM_ID]) def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self._pubsub_client = PubsubManagementServiceClient( node=self.container.node) # Start data subscribers, add stop to cleanup. # Define stream_config. self._async_data_result = AsyncResult() self._data_greenlets = [] self._stream_config = {} self._samples_received = [] self._data_subscribers = [] self._start_data_subscribers() self.addCleanup(self._stop_data_subscribers) # start event subscriber: self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber() self._agent_config = { 'agent': { 'resource_id': PA_RESOURCE_ID }, 'stream_config': self._stream_config, # pass platform config here 'platform_config': self.PLATFORM_CONFIG } if log.isEnabledFor(logging.TRACE): log.trace("launching with agent_config=%s" % str(self._agent_config)) self._launcher = LauncherFactory.createLauncher() self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config) log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID) # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess()) log.info('Got pa client %s.' % str(self._pa_client)) def tearDown(self): try: self._launcher.cancel_process(self._pid) finally: super(TestPlatformAgent, self).tearDown() def _start_data_subscribers(self): """ """ # Create streams and subscriptions for each stream named in driver. self._stream_config = {} self._data_subscribers = [] # # TODO retrieve appropriate stream definitions; for the moment, using # adhoc_get_stream_names # # A callback for processing subscribed-to data. def consume_data(message, stream_route, stream_id): log.info('Subscriber received data message: %s.' % str(message)) self._samples_received.append(message) self._async_data_result.set() for stream_name in adhoc_get_stream_names(): log.info('creating stream %r ...', stream_name) # TODO use appropriate exchange_point stream_id, stream_route = self._pubsub_client.create_stream( name=stream_name, exchange_point='science_data') log.info('create_stream(%r): stream_id=%r, stream_route=%s', stream_name, stream_id, str(stream_route)) pdict = adhoc_get_parameter_dictionary(stream_name) stream_config = dict(stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump()) self._stream_config[stream_name] = stream_config log.info('_stream_config[%r]= %r', stream_name, stream_config) # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self._pubsub_client.create_subscription( name=exchange_name, stream_ids=[stream_id]) self._pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ for sub in self._data_subscribers: if hasattr(sub, 'subscription_id'): try: self._pubsub_client.deactivate_subscription( sub.subscription_id) except: pass self._pubsub_client.delete_subscription(sub.subscription_id) sub.stop() def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"): """ Starts event subscriber for events of given event_type ("DeviceEvent" by default) and given sub_type ("platform_event" by default). """ def consume_event(evt, *args, **kwargs): # A callback for consuming events. log.info('Event subscriber received evt: %s.', str(evt)) self._events_received.append(evt) self._async_event_result.set(evt) sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event) sub.start() log.info("registered event subscriber for event_type=%r, sub_type=%r", event_type, sub_type) self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) def _stop_event_subscribers(self): """ Stops the event subscribers on cleanup. """ try: for sub in self._event_subscribers: if hasattr(sub, 'subscription_id'): try: self.pubsubcli.deactivate_subscription( sub.subscription_id) except: pass self.pubsubcli.delete_subscription(sub.subscription_id) sub.stop() finally: self._event_subscribers = [] def _get_state(self): state = self._pa_client.get_agent_state() return state def _assert_state(self, state): self.assertEquals(self._get_state(), state) #def _execute_agent(self, cmd, timeout=TIMEOUT): def _execute_agent(self, cmd): log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs) time_start = time.time() #retval = self._pa_client.execute_agent(cmd, timeout=timeout) retval = self._pa_client.execute_agent(cmd) elapsed_time = time.time() - time_start log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval)) return retval def _reset(self): cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.UNINITIALIZED) def _ping_agent(self): retval = self._pa_client.ping_agent() self.assertIsInstance(retval, str) def _ping_resource(self): cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE) if self._get_state() == PlatformAgentState.UNINITIALIZED: # should get ServerError: "Command not handled in current state" with self.assertRaises(ServerError): #self._pa_client.execute_agent(cmd, timeout=TIMEOUT) self._pa_client.execute_agent(cmd) else: # In all other states the command should be accepted: retval = self._execute_agent(cmd) self.assertEquals("PONG", retval.result) def _get_metadata(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_METADATA = %s", md) def _get_ports(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_PORTS = %s", md) def _connect_instrument(self): # # TODO more realistic settings for the connection # port_id = self.PORT_ID instrument_id = self.INSTRUMENT_ID instrument_attributes = self.INSTRUMENT_ATTRIBUTES_AND_VALUES kwargs = dict(port_id=port_id, instrument_id=instrument_id, attributes=instrument_attributes) cmd = AgentCommand(command=PlatformAgentEvent.CONNECT_INSTRUMENT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("CONNECT_INSTRUMENT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], dict) returned_attrs = self._verify_valid_instrument_id( instrument_id, result[port_id]) if isinstance(returned_attrs, dict): for attrName in instrument_attributes: self.assertTrue(attrName in returned_attrs) def _get_connected_instruments(self): port_id = self.PORT_ID kwargs = dict(port_id=port_id, ) cmd = AgentCommand( command=PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("GET_CONNECTED_INSTRUMENTS = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], dict) instrument_id = self.INSTRUMENT_ID self.assertTrue(instrument_id in result[port_id]) def _disconnect_instrument(self): # TODO real settings and corresp verification port_id = self.PORT_ID instrument_id = self.INSTRUMENT_ID kwargs = dict(port_id=port_id, instrument_id=instrument_id) cmd = AgentCommand(command=PlatformAgentEvent.DISCONNECT_INSTRUMENT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("DISCONNECT_INSTRUMENT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], dict) self.assertTrue(instrument_id in result[port_id]) self._verify_instrument_disconnected(instrument_id, result[port_id][instrument_id]) def _turn_on_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("TURN_ON_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_ON) def _turn_off_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("TURN_OFF_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF) def _get_resource(self): attrNames = self.ATTR_NAMES # # OOIION-631: use get_ion_ts() as a basis for using system time, which is # a string. # cur_time = get_ion_ts() from_time = str(int(cur_time) - 50000) # a 50-sec time window kwargs = dict(attr_names=attrNames, from_time=from_time) cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attr_name in attrNames: self._verify_valid_attribute_id(attr_name, attr_values) def _set_resource(self): attrNames = self.ATTR_NAMES writ_attrNames = self.WRITABLE_ATTR_NAMES # do valid settings: # TODO more realistic value depending on attribute's type attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames] log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs) kwargs = dict(attrs=attrs) cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attrName in attrNames: if attrName in writ_attrNames: self._verify_valid_attribute_id(attrName, attr_values) else: self._verify_not_writable_attribute_id(attrName, attr_values) # try invalid settings: # set invalid values to writable attributes: attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames] log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs) kwargs = dict(attrs=attrs) cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attrName in writ_attrNames: self._verify_attribute_value_out_of_range(attrName, attr_values) def _initialize(self): self._assert_state(PlatformAgentState.UNINITIALIZED) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _go_active(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _run(self): cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _pause(self): cmd = AgentCommand(command=PlatformAgentEvent.PAUSE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.STOPPED) def _resume(self): cmd = AgentCommand(command=PlatformAgentEvent.RESUME) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _start_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.MONITORING) def _wait_for_a_data_sample(self): log.info("waiting for reception of a data sample...") # just wait for at least one -- see consume_data self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("Received samples: %s", len(self._samples_received)) def _stop_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _go_inactive(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _get_subplatform_ids(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS) retval = self._execute_agent(cmd) self.assertIsInstance(retval.result, list) self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS) return retval.result def _wait_for_external_event(self): log.info("waiting for reception of an external event...") # just wait for at least one -- see consume_event self._async_event_result.get(timeout=EVENT_TIMEOUT) self.assertTrue(len(self._events_received) >= 1) log.info("Received events: %s", len(self._events_received)) def _check_sync(self): cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC) retval = self._execute_agent(cmd) log.info("CHECK_SYNC result: %s", retval.result) self.assertTrue(retval.result is not None) self.assertEquals(retval.result[0:3], "OK:") return retval.result def test_capabilities(self): agt_cmds_all = [ PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.RESET, PlatformAgentEvent.GO_ACTIVE, PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RUN, PlatformAgentEvent.CLEAR, PlatformAgentEvent.PAUSE, PlatformAgentEvent.RESUME, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.CONNECT_INSTRUMENT, PlatformAgentEvent.DISCONNECT_INSTRUMENT, PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.START_MONITORING, PlatformAgentEvent.STOP_MONITORING, PlatformAgentEvent.CHECK_SYNC, ] def sort_caps(caps): agt_cmds = [] agt_pars = [] res_cmds = [] res_pars = [] if len(caps) > 0 and isinstance(caps[0], AgentCapability): agt_cmds = [ x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD ] agt_pars = [ x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR ] res_cmds = [ x.name for x in caps if x.cap_type == CapabilityType.RES_CMD ] res_pars = [ x.name for x in caps if x.cap_type == CapabilityType.RES_PAR ] elif len(caps) > 0 and isinstance(caps[0], dict): agt_cmds = [ x['name'] for x in caps if x['cap_type'] == CapabilityType.AGT_CMD ] agt_pars = [ x['name'] for x in caps if x['cap_type'] == CapabilityType.AGT_PAR ] res_cmds = [ x['name'] for x in caps if x['cap_type'] == CapabilityType.RES_CMD ] res_pars = [ x['name'] for x in caps if x['cap_type'] == CapabilityType.RES_PAR ] return agt_cmds, agt_pars, res_cmds, res_pars agt_pars_all = ['example' ] # 'cause ResourceAgent defines aparam_example res_pars_all = [] res_cmds_all = [] ################################################################## # UNINITIALIZED ################################################################## self._assert_state(PlatformAgentState.UNINITIALIZED) # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state UNINITIALIZED. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_uninitialized = [ PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states. retval = self._pa_client.get_capabilities(current_state=False) # Validate all capabilities as read from state UNINITIALIZED. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) ################################################################## # INACTIVE ################################################################## self._initialize() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state INACTIVE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_inactive = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GO_ACTIVE, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_inactive) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state INACTIVE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) ################################################################## # IDLE ################################################################## self._go_active() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state IDLE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_idle = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RUN, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_idle) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states as read from IDLE. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state IDLE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) ################################################################## # COMMAND ################################################################## self._run() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state COMMAND agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_command = [ PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RESET, PlatformAgentEvent.PAUSE, PlatformAgentEvent.CLEAR, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.CONNECT_INSTRUMENT, PlatformAgentEvent.DISCONNECT_INSTRUMENT, PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.START_MONITORING, PlatformAgentEvent.CHECK_SYNC, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_command) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) ################################################################## # STOPPED ################################################################## self._pause() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state STOPPED agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_stopped = [ PlatformAgentEvent.RESUME, PlatformAgentEvent.CLEAR, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_stopped) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) # back to COMMAND: self._resume() ################################################################## # MONITORING ################################################################## self._start_resource_monitoring() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state MONITORING agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_monitoring = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.CONNECT_INSTRUMENT, PlatformAgentEvent.DISCONNECT_INSTRUMENT, PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.STOP_MONITORING, PlatformAgentEvent.CHECK_SYNC, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_monitoring) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) # return to COMMAND state: self._stop_resource_monitoring() ################### # ALL CAPABILITIES ################### # Get exposed capabilities in all states as read from state COMMAND. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state COMMAND agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_all) self.assertItemsEqual(res_pars, res_pars_all) self._go_inactive() self._reset() def test_some_state_transitions(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._initialize() # -> INACTIVE self._reset() # -> UNINITIALIZED self._initialize() # -> INACTIVE self._go_active() # -> IDLE self._reset() # -> UNINITIALIZED self._initialize() # -> INACTIVE self._go_active() # -> IDLE self._run() # -> COMMAND self._pause() # -> STOPPED self._resume() # -> COMMAND self._reset() # -> UNINITIALIZED def test_get_set_resources(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._get_resource() self._set_resource() self._go_inactive() self._reset() def test_some_commands(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._ping_agent() self._ping_resource() self._get_metadata() self._get_ports() self._get_subplatform_ids() self._go_inactive() self._reset() def test_resource_monitoring(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._start_resource_monitoring() self._wait_for_a_data_sample() self._stop_resource_monitoring() self._go_inactive() self._reset() def test_external_event_dispatch(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._wait_for_external_event() self._go_inactive() self._reset() def test_connect_disconnect_instrument(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._connect_instrument() self._turn_on_port() self._get_connected_instruments() self._turn_off_port() self._disconnect_instrument() self._go_inactive() self._reset() def test_check_sync(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._check_sync() self._connect_instrument() self._check_sync() self._disconnect_instrument() self._check_sync() self._go_inactive() self._reset()
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin): """ A base class with several conveniences supporting specific platform agent integration tests, see: - ion/agents/platform/test/test_platform_agent_with_rsn.py - ion/services/sa/observatory/test/test_platform_launch.py The platform IDs used here are organized as follows: Node1D -> MJ01C -> LJ01D where -> goes from parent platform to child platform. This is a subset of the whole topology defined in the simulated platform network (network.yml), which in turn is used by the RSN OMS simulator. - 'LJ01D' is the root platform used in test_single_platform - 'Node1D' is the root platform used in test_hierarchy Methods are provided to construct specific platform topologies, but subclasses decide which to use. """ @classmethod def setUpClass(cls): HelperTestMixin.setUpClass() def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient() self.IDS = IdentityManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.org_id = self.RR2.create(any_old(RT.Org)) log.debug("Org created: %s", self.org_id) # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri']) self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms) CIOMSClientFactory.destroy_instance(rsn_oms) # get serialized version for the configuration: self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition) log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser) # set attributes for the platforms: self._platform_attributes = {} for platform_id in self._network_definition.pnodes: pnode = self._network_definition.pnodes[platform_id] dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues()) self._platform_attributes[platform_id] = dic log.trace("_platform_attributes: %s", self._platform_attributes) # set ports for the platforms: self._platform_ports = {} for platform_id in self._network_definition.pnodes: pnode = self._network_definition.pnodes[platform_id] dic = {} for port_id, port in pnode.ports.iteritems(): dic[port_id] = dict(port_id=port_id, network=port.network) self._platform_ports[platform_id] = dic log.trace("_platform_ports: %s", self._platform_attributes) self._async_data_result = AsyncResult() self._data_subscribers = [] self._samples_received = [] self.addCleanup(self._stop_data_subscribers) self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber() ################################################################# # data subscribers handling ################################################################# def _start_data_subscriber(self, stream_name, stream_id): """ Starts data subscriber for the given stream_name and stream_config """ def consume_data(message, stream_route, stream_id): # A callback for processing subscribed-to data. log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r', str(message), stream_name, stream_id) self._samples_received.append(message) self._async_data_result.set() log.info('_start_data_subscriber stream_name=%r stream_id=%r', stream_name, stream_id) # Create subscription for the stream exchange_name = '%s_queue' % stream_name self.container.ex_manager.create_xn_queue(exchange_name).purge() sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id]) self.PSC.activate_subscription(sub_id) sub.subscription_id = sub_id def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ try: for sub in self._data_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._data_subscribers = [] ################################################################# # event subscribers handling ################################################################# def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"): """ Starts event subscriber for events of given event_type ("DeviceEvent" by default) and given sub_type ("platform_event" by default). """ def consume_event(evt, *args, **kwargs): # A callback for consuming events. log.info('Event subscriber received evt: %s.', str(evt)) self._events_received.append(evt) self._async_event_result.set(evt) sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event) sub.start() log.info("registered event subscriber for event_type=%r, sub_type=%r", event_type, sub_type) self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) def _stop_event_subscribers(self): """ Stops the event subscribers on cleanup. """ try: for sub in self._event_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._event_subscribers = [] ################################################################# # config supporting methods ################################################################# def _create_platform_config_builder(self): clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) return pconfig_builder def _generate_parent_with_child_config(self, p_parent, p_child): log.debug("Testing parent platform + child platform as parent config") pconfig_builder = self._create_platform_config_builder() pconfig_builder.set_agent_instance_object(p_parent.platform_agent_instance_obj) parent_config = pconfig_builder.prepare(will_launch=False) self._verify_parent_config(parent_config, p_parent.platform_device_id, p_child.platform_device_id, is_platform=True) self._debug_config(parent_config, "platform_agent_config_%s_->_%s.txt" % ( p_parent.platform_id, p_child.platform_id)) def _generate_platform_with_instrument_config(self, p_obj, i_obj): log.debug("Testing parent platform + child instrument as parent config") pconfig_builder = self._create_platform_config_builder() pconfig_builder.set_agent_instance_object(p_obj.platform_agent_instance_obj) parent_config = pconfig_builder.prepare(will_launch=False) self._verify_parent_config(parent_config, p_obj.platform_device_id, i_obj.instrument_device_id, is_platform=False) self._debug_config(parent_config, "platform_agent_config_%s_->_%s.txt" % ( p_obj.platform_id, i_obj.instrument_device_id)) def _generate_config(self, platform_agent_instance_obj, platform_id, suffix=''): pconfig_builder = self._create_platform_config_builder() pconfig_builder.set_agent_instance_object(platform_agent_instance_obj) config = pconfig_builder.prepare(will_launch=False) self._debug_config(config, "platform_agent_config_%s%s.txt" % (platform_id, suffix)) return config def _get_platform_stream_configs(self): """ This method is an adaptation of get_streamConfigs in test_driver_egg.py """ return [ StreamConfiguration(stream_name='parsed', parameter_dictionary_name='platform_eng_parsed', records_per_granule=2, granule_publish_rate=5) # TODO include a "raw" stream? ] def _get_instrument_stream_configs(self): """ configs copied from test_activate_instrument.py """ return [ StreamConfiguration(stream_name='ctd_raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5), StreamConfiguration(stream_name='ctd_parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5) ] def _create_instrument_config_builder(self): clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC iconfig_builder = InstrumentAgentConfigurationBuilder(clients) return iconfig_builder def _generate_instrument_config(self, instrument_agent_instance_obj, instrument_id, suffix=''): pconfig_builder = self._create_instrument_config_builder() pconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) config = pconfig_builder.prepare(will_launch=False) self._debug_config(config, "instrument_agent_config_%s%s.txt" % (instrument_id, suffix)) return config def _debug_config(self, config, outname): if log.isEnabledFor(logging.DEBUG): import pprint outname = "logs/%s" % outname try: pprint.PrettyPrinter(stream=file(outname, "w")).pprint(config) log.debug("config pretty-printed to %s", outname) except Exception as e: log.warn("error printing config to %s: %s", outname, e) def _verify_child_config(self, config, device_id, is_platform): for key in required_config_keys: self.assertIn(key, config) if is_platform: self.assertEqual(RT.PlatformDevice, config['device_type']) for key in DVR_CONFIG.iterkeys(): self.assertIn(key, config['driver_config']) for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: self.assertEqual(RT.InstrumentDevice, config['device_type']) for key in ['children']: self.assertEqual({}, config[key]) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) def _verify_parent_config(self, config, parent_device_id, child_device_id, is_platform): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(RT.PlatformDevice, config['device_type']) for key in DVR_CONFIG.iterkeys(): self.assertIn(key, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) self._verify_child_config(config['children'][child_device_id], child_device_id, is_platform) def _create_platform_configuration(self, platform_id, parent_platform_id=None): """ This method is an adaptation of test_agent_instance_config in test_instrument_management_service_integration.py @param platform_id @param parent_platform_id @return a DotDict with various of the constructed elements associated to the platform. """ tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # # TODO will each platform have its own param dictionary? # param_dict_name = 'platform_eng_parsed' parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name( param_dict_name, id_only=True) self.parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} driver_config = copy.deepcopy(DVR_CONFIG) driver_config['attributes'] = self._platform_attributes[platform_id] driver_config['ports'] = self._platform_ports[platform_id] log.debug("driver_config: %s", driver_config) # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, { 'driver_config': driver_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation platform_agent_obj = any_old(RT.PlatformAgent, { "stream_configurations": self._get_platform_stream_configs(), 'driver_module': DVR_MOD, 'driver_class': DVR_CLS}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id) ####################################### # dataset log.debug('data product = %s', dp_id) stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True) log.debug('Data product stream_ids = %s', stream_ids) stream_id = stream_ids[0] # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) ####################################### return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id log.debug("Making the structure for a platform agent") # TODO Note: the 'platform_config' entry is a mechanism that the # platform agent expects to know the platform_id and parent_platform_id. # Determine how to finally indicate this info. platform_config = { 'platform_id': platform_id, 'parent_platform_id': parent_platform_id, } child_agent_config = { 'platform_config': platform_config } platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \ _make_platform_agent_structure(child_agent_config) platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) child_config = self._generate_config(platform_agent_instance_child_obj, platform_id) self._verify_child_config(child_config, platform_device_child_id, is_platform=True) self.platform_device_parent_id = platform_device_child_id p_obj = DotDict() p_obj.platform_id = platform_id p_obj.parent_platform_id = parent_platform_id p_obj.agent_config = child_config p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj p_obj.platform_device_id = platform_device_child_id p_obj.platform_agent_instance_id = platform_agent_instance_child_id p_obj.stream_id = stream_id return p_obj def _create_platform(self, platform_id, parent_platform_id=None): """ The main method to create a platform configuration and do other preparations for a given platform. """ p_obj = self._create_platform_configuration(platform_id, parent_platform_id) # start corresponding data subscriber: self._start_data_subscriber(p_obj.platform_agent_instance_id, p_obj.stream_id) return p_obj ################################################################# # platform child-parent linking ################################################################# def _assign_child_to_parent(self, p_child, p_parent): log.debug("assigning child platform %r to parent %r", p_child.platform_id, p_parent.platform_id) self.RR2.assign_platform_device_to_platform_device(p_child.platform_device_id, p_parent.platform_device_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(p_parent.platform_device_id) self.assertNotEqual(0, len(child_device_ids)) self._generate_parent_with_child_config(p_parent, p_child) ################################################################# # instrument ################################################################# def _set_up_pre_environment_for_instrument(self): """ From test_instrument_agent.py Basically, this method launches the port agent and the completes the instrument driver configuration used to properly set up the instrument agent. @return instrument_driver_config """ import sys from ion.agents.instrument.driver_process import DriverProcessType from ion.agents.instrument.driver_process import ZMQEggDriverProcess # A seabird driver. DRV_URI = 'http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.7-py2.7.egg' DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver' DRV_CLS = 'SBE37Driver' WORK_DIR = '/tmp/' DELIM = ['<<', '>>'] instrument_driver_config = { 'dvr_egg' : DRV_URI, 'dvr_mod' : DRV_MOD, 'dvr_cls' : DRV_CLS, 'workdir' : WORK_DIR, 'process_type' : None } # Launch from egg or a local MI repo. LAUNCH_FROM_EGG=True if LAUNCH_FROM_EGG: # Dynamically load the egg into the test path launcher = ZMQEggDriverProcess(instrument_driver_config) egg = launcher._get_egg(DRV_URI) if not egg in sys.path: sys.path.insert(0, egg) instrument_driver_config['process_type'] = (DriverProcessType.EGG,) else: mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo' if not mi_repo in sys.path: sys.path.insert(0, mi_repo) instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,) instrument_driver_config['mi_repo'] = mi_repo DEV_ADDR = CFG.device.sbe37.host DEV_PORT = CFG.device.sbe37.port DATA_PORT = CFG.device.sbe37.port_agent_data_port CMD_PORT = CFG.device.sbe37.port_agent_cmd_port PA_BINARY = CFG.device.sbe37.port_agent_binary self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i', port) self.addCleanup(self._support.stop_pagent) # Configure instrument driver to use port agent port number. instrument_driver_config['comms_config'] = { 'addr': 'localhost', 'port': port, 'cmd_port': CMD_PORT } return instrument_driver_config def _make_instrument_agent_structure(self, org_obj, agent_config=None): if None is agent_config: agent_config = {} # from test_activate_instrument:test_activateInstrumentSample # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) # agent creation instrument_agent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations=self._get_instrument_stream_configs()) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) log.debug('new InstrumentAgent id = %s', instrument_agent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instrument_agent_id) # device creation instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instrument_device_id) log.debug("new InstrumentDevice id = %s ", instrument_device_id) #Create stream alarms alert_def = { 'name' : 'temperature_warning_interval', 'stream_name' : 'ctd_parsed', 'message' : 'Temperature is below the normal range of 50.0 and above.', 'alert_type' : StreamAlertType.WARNING, 'value_id' : 'temp', 'resource_id' : instrument_device_id, 'origin_type' : 'device', 'lower_bound' : 50.0, 'lower_rel_op' : '<', 'alert_class' : 'IntervalAlert' } instrument_driver_config = self._set_up_pre_environment_for_instrument() port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } # instance creation instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config=instrument_driver_config, port_agent_config=port_agent_config, alerts=[alert_def]) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # data products tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(org_obj) # parsed: parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='ctd_parsed', parameter_dictionary_id=parsed_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) # raw: raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='ctd_raw', parameter_dictionary_id=raw_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id2) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) i_obj = DotDict() i_obj.instrument_agent_id = instrument_agent_id i_obj.instrument_device_id = instrument_device_id i_obj.instrument_agent_instance_id = instrument_agent_instance_id return i_obj def verify_instrument_config(self, config, org_obj, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQEggDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def _create_instrument(self): """ The main method to create an instrument configuration. """ iconfig_builder = self._create_instrument_config_builder() org_obj = any_old(RT.Org) log.debug("making the structure for an instrument agent") i_obj = self._make_instrument_agent_structure(org_obj) instrument_agent_instance_obj = self.RR2.read(i_obj.instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) self.verify_instrument_config(instrument_config, org_obj, i_obj.instrument_device_id) self._generate_instrument_config(instrument_agent_instance_obj, i_obj.instrument_agent_instance_id) return i_obj ################################################################# # instrument-platform linking ################################################################# def _assign_instrument_to_platform(self, i_obj, p_obj): log.debug("assigning instrument %r to platform %r", i_obj.instrument_agent_instance_id, p_obj.platform_id) self.RR2.assign_instrument_device_to_platform_device( i_obj.instrument_device_id, p_obj.platform_device_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(p_obj.platform_device_id) self.assertNotEqual(0, len(child_device_ids)) self._generate_platform_with_instrument_config(p_obj, i_obj) ################################################################# # some platform topologies ################################################################# def _create_single_platform(self): """ Creates and prepares a platform corresponding to the platform ID 'LJ01D', which is a leaf in the simulated network. """ p_root = self._create_platform('LJ01D') return p_root def _create_small_hierarchy(self): """ Creates a small platform network consisting of 3 platforms as follows: Node1D -> MJ01C -> LJ01D where -> goes from parent to child. """ p_root = self._create_platform('Node1D') p_child = self._create_platform('MJ01C', parent_platform_id='Node1D') p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C') self._assign_child_to_parent(p_child, p_root) self._assign_child_to_parent(p_grandchild, p_child) self._generate_config(p_root.platform_agent_instance_obj, p_root.platform_id, "_final") return p_root ################################################################# # start / stop platform ################################################################# def _start_platform(self, agent_instance_id): log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id) pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id) gate = ProcessStateGate(self.PDC.read_process, agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) def _stop_platform(self, agent_instance_id): self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id) ################################################################# # start / stop instrument ################################################################# def _start_instrument(self, agent_instance_id): log.debug("about to call start_instrument_agent_instance with id=%s", agent_instance_id) pid = self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id) log.debug("start_instrument_agent_instance returned pid=%s", pid) #wait for start agent_instance_obj = self.IMS.read_instrument_agent_instance(agent_instance_id) gate = ProcessStateGate(self.PDC.read_process, agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(90), "The instrument agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._ia_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("got instrument agent client %s", str(self._ia_client)) def _stop_instrument(self, agent_instance_id): self.IMS.stop_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id) ################################################################# # misc convenience methods ################################################################# def _get_state(self): state = self._pa_client.get_agent_state() return state def _assert_state(self, state): self.assertEquals(self._get_state(), state) def _execute_agent(self, cmd): log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs) time_start = time.time() #retval = self._pa_client.execute_agent(cmd, timeout=timeout) retval = self._pa_client.execute_agent(cmd) elapsed_time = time.time() - time_start log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval)) return retval ################################################################# # commands that concrete tests can call ################################################################# def _ping_agent(self): retval = self._pa_client.ping_agent() self.assertIsInstance(retval, str) def _ping_resource(self): cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE) if self._get_state() == PlatformAgentState.UNINITIALIZED: # should get ServerError: "Command not handled in current state" with self.assertRaises(ServerError): #self._pa_client.execute_agent(cmd, timeout=TIMEOUT) self._pa_client.execute_agent(cmd) else: # In all other states the command should be accepted: retval = self._execute_agent(cmd) self.assertEquals("PONG", retval.result) def _get_metadata(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_METADATA = %s", md) def _get_ports(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_PORTS = %s", md) def _initialize(self): self._assert_state(PlatformAgentState.UNINITIALIZED) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _go_active(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _run(self): cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _start_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.MONITORING) def _wait_for_a_data_sample(self): log.info("waiting for reception of a data sample...") # just wait for at least one -- see consume_data self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("Received samples: %s", len(self._samples_received)) def _wait_for_external_event(self): log.info("waiting for reception of an external event...") # just wait for at least one -- see consume_event self._async_event_result.get(timeout=EVENT_TIMEOUT) self.assertTrue(len(self._events_received) >= 1) log.info("Received events: %s", len(self._events_received)) def _stop_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _pause(self): cmd = AgentCommand(command=PlatformAgentEvent.PAUSE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.STOPPED) def _resume(self): cmd = AgentCommand(command=PlatformAgentEvent.RESUME) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _clear(self): cmd = AgentCommand(command=PlatformAgentEvent.CLEAR) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _go_inactive(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _reset(self): cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.UNINITIALIZED) def _check_sync(self): cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC) retval = self._execute_agent(cmd) log.info("CHECK_SYNC result: %s", retval.result) self.assertTrue(retval.result is not None) self.assertEquals(retval.result[0:3], "OK:") return retval.result
class TestVel3d(IonIntegrationTestCase): """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config, start agent. Start agent client. """ print '#####################' print 'IN SETUP' self._ia_client = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('building stream configuration') # Setup stream config. self._build_stream_config() #log.info('driver uri: %s', DRV_URI) #log.info('device address: %s', DEV_ADDR) #log.info('device port: %s', DEV_PORT) #log.info('work dir: %s', WORK_DIR) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True, 'forget_past' : True, 'enable_persistence' : False } #if org_governance_name is not None: # agent_config['org_governance_name'] = org_governance_name # Start instrument agent. log.info("TestInstrumentAgent.setup(): starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) log.info("Agent setup") ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) #self.addCleanup(self._verify_agent_reset) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) log.info('test setup complete') ############################################################################### # Port agent helpers. ############################################################################### def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) ############################################################################### # Event helpers. ############################################################################### def _start_event_subscriber(self, type='ResourceAgentEvent', count=0): """ Start a subscriber to the instrument agent events. @param type The type of event to catch. @count Trigger the async event result when events received reaches this. """ def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._event_count > 0 and \ self._event_count == len(self._events_received): self._async_event_result.set() # Event array and async event result. self._event_count = count self._events_received = [] self._async_event_result = AsyncResult() self._event_subscriber = EventSubscriber( event_type=type, callback=consume_event, origin=IA_RESOURCE_ID) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=5) def _stop_event_subscriber(self): """ Stop event subscribers on cleanup. """ self._event_subscriber.stop() self._event_subscriber = None ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config def _start_data_subscribers(self, count, raw_count): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._data_subscribers = [] self._samples_received = [] self._raw_samples_received = [] self._async_sample_result = AsyncResult() self._async_raw_sample_result = AsyncResult() # A callback for processing subscribed-to data. def recv_data(message, stream_route, stream_id): log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._samples_received.append(message) if len(self._samples_received) == count: self._async_sample_result.set() def recv_raw_data(message, stream_route, stream_id): log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._raw_samples_received.append(message) if len(self._raw_samples_received) == raw_count: self._async_raw_sample_result.set() from pyon.util.containers import create_unique_identifier stream_name = 'parsed' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) stream_name = 'raw' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): for subscriber in self._data_subscribers: pubsub_client = PubsubManagementServiceClient() if hasattr(subscriber,'subscription_id'): try: pubsub_client.deactivate_subscription(subscriber.subscription_id) except: pass pubsub_client.delete_subscription(subscriber.subscription_id) subscriber.stop() ############################################################################### # tcp helpers. ############################################################################### def _start_tcp_client(self, retval): host = retval.result['ip_address'] port = retval.result['port'] tcp_client = TcpClient(host, port) return tcp_client ############################################################################### # Tests. ############################################################################### @unittest.skip('Test should be run manually only.') def test_initialize(self): """ test_initialize Test agent initialize command. This causes creation of driver process and transition to inactive. """ print '#### in test' # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) @unittest.skip('Test should be run manually only.') def test_xx(self): """ """ state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) with self.assertRaises(Conflict): res_state = self._ia_client.get_resource_state() cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print '################################# mavs4 came up in state: ' + state if state == ResourceAgentState.IDLE: cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) elif state == ResourceAgentState.STREAMING: cmd = AgentCommand(command='DRIVER_EVENT_STOP_AUTOSAMPLE') retval = self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) state = self._ia_client.get_agent_state() print '################################# mavs4 now in state: ' + state cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS, kwargs={'session_type': DirectAccessTypes.telnet, 'session_timeout':600, 'inactivity_timeout':600}) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS) state = self._ia_client.get_agent_state() print '################################# mavs4 now in state: ' + state tcp_client = self._start_tcp_client(retval) self.assertTrue(tcp_client.start_telnet(token=retval.result['token'])) self.assertTrue(tcp_client.send_data('\r\r')) gevent.sleep(180) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS) cmd = AgentCommand(command=ResourceAgentEvent.GO_COMMAND) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND)
class TestDriverEgg(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_streamConfigs(self): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') return raw_config, parsed_config ########################## # # The following tests generate different agent configs and pass them to a common base test script # ########################### @unittest.skip( "this test can't be run from coi services. it is missing dependencies") def test_driverLaunchModuleNoURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchNoModuleOnlyURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchBogusModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="bogus", driver_class="Bogus", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) @unittest.skip( "Launches an egg 'process' even though the egg download should produce error 404" ) def test_driverLaunchNoModule404URI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_404, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, False) def test_driverLaunchNoModuleBadEggURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_BAD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, True, False) def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True): """ This method runs a test of launching a driver with a given agent configuration """ # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self.get_datastore(self.parsed_dataset) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] # add start/stop for instrument agent gevent.joinall([ gevent.spawn( lambda: self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id)) ]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) agent_process_id = ResourceAgentClient._get_agent_process_id( instDevice_id) print "Agent process id is '%s'" % str(agent_process_id) self.assertTrue(agent_process_id) gate = ProcessStateGate(self.processdispatchclient.read_process, agent_process_id, ProcessStateEnum.RUNNING) if not expect_launch: self.assertFalse( gate. await (30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" % agent_process_id) return self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % agent_process_id) print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING" #print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=agent_process_id, process=FakeProcess()) print "ResourceAgentClient created: %s" % str(self._ia_client) print "Sending command=ResourceAgentEvent.INITIALIZE" cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) if not expect_command: self.assertRaises(ServerError, self._ia_client.execute_agent, cmd) return retval = self._ia_client.execute_agent(cmd) print "Result of INITIALIZE: %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) print "Sending command=ResourceAgentEvent.RESET" cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "Result of RESET: %s" % str(reply)
class RunInstrument(MiIntTestCase): """ Main class for communicating with an instrument """ def __init__(self, monitor=False, subscriber=False): self.driver_make = None self.driver_model = None self.driver_name = None self.driver_class = DRIVER_CLASS self.ip_address = None self.data_port = None self.command_port = None self.driver_version = None self._pagent = None self.monitor_window = monitor self.subcriber_window = subscriber self.stream_config = {} self._cleanups = [] def _initialize(self): """ Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config, start agent. Start agent client. """ try: """ Get the information for the driver. This can be read from the yml files; the user can run switch_driver to change the current driver. """ self.fetch_metadata() self.fetch_driver_class() self.fetch_comm_config() if not exists(PIPE_PATH): mkfifo(PIPE_PATH) if not exists(self.metadata.driver_dir()): raise DriverDoesNotExist( "%s/%s/$%s" % (self.metadata.driver_make, self.metadata.driver_model, self.driver_name)) driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF log.info('driver module: %s', driver_module) log.info('driver class: %s', self.driver_class) log.info('device address: %s', self.ip_address) log.info('device data port: %s', self.data_port) log.info('device command port: %s', self.command_port) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) DVR_CONFIG.update({ 'dvr_mod': driver_module, 'dvr_cls': self.driver_class }) """ self._support = DriverIntegrationTestSupport(driver_module, self.driver_class, self.ip_address, self.data_port, DELIM, WORK_DIR) """ # Start port agent, add stop to cleanup (not sure if that's # necessary yet). print( "------------------>>>> Starting Port Agent <<<<------------------" ) self.start_pagent() # Start a monitor window if specified. if self.monitor_window: self.monitor_file = self._pagent.port_agent.logfname strXterm = "xterm -T InstrumentMonitor -sb -rightbar" #pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file pOpenString = strXterm + " -e tail -f " + self.monitor_file x = subprocess.Popen(pOpenString, shell=True) """ DHE: Added self._cleanups to make base classes happy """ self.addCleanup(self.stop_pagent) # Start container. print( "------------------>>>> Starting Capability Container <<<<------------------" ) self._start_container() # Bring up services in a deploy file (no need to message) print( "------------------>>>> Starting Deploy Services <<<<------------------" ) self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Setup stream config. self._build_stream_config() # Create agent config. agent_config = { 'driver_config': DVR_CONFIG, 'stream_config': self.stream_config, 'agent': { 'resource_id': IA_RESOURCE_ID }, 'test_mode': True } # Start instrument agent. self._ia_pid = None print( "------------------>>>> Starting Instrument Agent <<<<------------------" ) container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) if self.subcriber_window: self._start_data_subscribers(6) #self.addCleanup(self._stop_data_subscribers) except: log.error("initialize(): Exception occurred; shutting down.", exc_info=True) return False else: return True ############################################################################### # Port agent helpers. ############################################################################### def start_pagent(self): """ Construct and start the port agent. @retval port Port that was used for connection to agent """ # Create port agent object. comm_config = self.comm_config config = { 'device_addr': comm_config.device_addr, 'device_port': comm_config.device_port, 'command_port': comm_config.command_port, 'data_port': comm_config.data_port, 'process_type': PortAgentProcessType.UNIX, 'log_level': 5, } self._pagent = PortAgentProcess.launch_process(config, timeout=60, test_mode=True) pid = self._pagent.get_pid() port = self._pagent.get_data_port() cmd_port = self._pagent.get_command_port() log.info('Started port agent pid %d listening at port %d', pid, port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr': 'localhost', 'port': port, 'cmd_port': cmd_port } return port def stop_pagent(self): """ Stop the port agent. """ if self._pagent: pid = self._pagent.get_pid() if pid: log.info('Stopping pagent pid %i', pid) self._pagent.stop() else: log.info('No port agent running.') ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self.stream_config = {} streams = { 'parsed': 'ctd_parsed_param_dict', 'raw': 'ctd_raw_param_dict' } for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name( DEFAULT_PARAM_DICT, id_only=True) if (not pd_id): log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) pd = None stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self.stream_config[stream_name] = stream_config def _start_data_subscribers(self, count): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._data_subscribers = [] self._samples_received = [] #self._async_data_result = AsyncResult() strXterm = "xterm -T InstrumentScienceData -sb -rightbar " pOpenString = strXterm + " -e tail -f " + PIPE_PATH subprocess.Popen([ 'xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f', PIPE_PATH ]) #subprocess.Popen(pOpenString) #self.pipeData = open(PIPE_PATH, "w", 1) # A callback for processing subscribed-to data. def recv_data(message, stream_route, stream_id): print 'Received message on ' + str(stream_id) + ' (' + str( stream_route.exchange_point) + ',' + str( stream_route.routing_key) + ')' log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self.pipeData = open(PIPE_PATH, "w", 1) self.pipeData.write(str(message)) self.pipeData.flush() self.pipeData.close() self._samples_received.append(message) #if len(self._samples_received) == count: #self._async_data_result.set() for (stream_name, stream_config) in self._stream_config.iteritems(): stream_id = stream_config['stream_id'] # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_data) sub.start() self._data_subscribers.append(sub) print 'stream_id: %s' % stream_id sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): for subscriber in self._data_subscribers: pubsub_client = PubsubManagementServiceClient() if hasattr(subscriber, 'subscription_id'): try: pubsub_client.deactivate_subscription( subscriber.subscription_id) except: pass pubsub_client.delete_subscription(subscriber.subscription_id) subscriber.stop() def bring_instrument_active(self): """ @brief Bring the agent up to COMMAND state, """ """ DHE: Don't have an event subscriber yet # Set up a subscriber to collect error events. #self._start_event_subscriber('ResourceAgentResourceStateEvent', 6) #self.addCleanup(self._stop_event_subscriber) """ try: state = self._ia_client.get_agent_state() print "AgentState: " + str(state) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print "AgentState: " + str(state) res_state = self._ia_client.get_resource_state() print "DriverState: " + str(res_state) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print "AgentState: " + str(state) res_state = self._ia_client.get_resource_state() print "DriverState: " + str(res_state) """ If the agent is in STREAMING state, it will not accept the run command. """ if state != ResourceAgentState.STREAMING: cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print "AgentState: " + str(state) res_state = self._ia_client.get_resource_state() print "DriverState: " + str(res_state) except: log.error( "bring_instrument_active(): Exception occurred; shutting down.", exc_info=True) return False else: return True """ DHE: Don't have an event subscriber yet so we've received no events. self._async_event_result.get(timeout=2) print "Received: " + str(len(self._events_received)) + " events." """ ############################################################################### # RunInstrument helpers. ############################################################################### def get_capabilities(self): """ @brief Get exposed capabilities in current state. """ retval = self._ia_client.get_capabilities() # Validate capabilities for state UNINITIALIZED. self.agt_cmds = [ x.name for x in retval if x.cap_type == CapabilityType.AGT_CMD ] self.agt_pars = [ x.name for x in retval if x.cap_type == CapabilityType.AGT_PAR ] self.res_cmds = [ x.name for x in retval if x.cap_type == CapabilityType.RES_CMD ] self.res_pars = [ x.name for x in retval if x.cap_type == CapabilityType.RES_PAR ] print "\n------------------>>>> Current Capabilities <<<<------------------" print "Agent Commands: " + str(self.agt_cmds) #print "Agent Parameters: " + str(self.agt_pars) print "Resource Commands: " + str(self.res_cmds) #print "Resource Parameters: " + str(self.res_pars) def send_agent_command(self, command): """ @brief Send a command to the agent. """ DA_WAIT_PERIOD = 60 waiting = False print "Input command: " + str(command) if command == 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS': cmd = AgentCommand(command=command, kwargs={ 'session_type': DirectAccessTypes.telnet, 'session_timeout': 600, 'inactivity_timeout': 600 }) waiting = True else: cmd = AgentCommand(command=command) retval = self._ia_client.execute_agent(cmd) print "Results of command: " + str(retval) while waiting: print "Waiting " + str( DA_WAIT_PERIOD) + " seconds for you to test direct access." gevent.sleep(DA_WAIT_PERIOD) still_waiting = prompt.text('Still waiting? (y/n)') if still_waiting is 'n': waiting = False def send_driver_command(self, command): """ @brief Send a command to the instrument through the instrument agent. First determine whether it's a get or set, which are handled separately. """ if command == DriverEvent.GET: self._get_param() elif command == DriverEvent.SET: self._set_param() else: print "Input command: " + str(command) cmd = AgentCommand(command=command) retval = self._ia_client.execute_resource(cmd) print "Results of command: " + str(retval) def _get_param(self): """ @brief Get a single parameter from the instrument (will be updated to get multiple later). """ _all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL') print "Parameters you can get are: " + str(_all_params) _param_valid = False while _param_valid is False: _param = prompt.text('\nEnter a single parameter') if _param in _all_params: _param_valid = True else: print 'Invalid parameter: ' + _param reply = self._ia_client.get_resource([_param]) print 'Reply is :' + str(reply) def _set_param(self): """ @brief Set a single parameter """ _all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL') print "Parameters you can set are: " + str(_all_params) _param_valid = False while _param_valid is False: _param = prompt.text('\nEnter a single parameter') if _param in _all_params: _param_valid = True else: print 'Invalid parameter: ' + _param _value = prompt.text('Enter value') _value = _value.lower() """ DHE: Need to convert to native types here; can't be string; this is a problem for the UI because we need a way to get the metadata about each param to the UI. """ if _value == 'true': _value = True elif _value == 'false': _value = False param_dict = {_param: _value} self._ia_client.set_resource(param_dict) def fetch_metadata(self): """ @brief collect metadata from the user """ self.metadata = Metadata() self.driver_make = self.metadata.driver_make self.driver_model = self.metadata.driver_model self.driver_name = self.metadata.driver_name if not (self.driver_make and self.driver_model and self.driver_name): self.driver_make = prompt.text('Driver Make', self.driver_make) self.driver_model = prompt.text('Driver Model', self.driver_model) self.driver_name = prompt.text('Driver Name', self.driver_name) if not (self.driver_class): self.driver_class = prompt.text('Driver Class', self.driver_class) self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name) def fetch_comm_config(self): """ @brief collect connection information for the logger from the user """ config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename()) self.comm_config = CommConfig.get_config_from_console(config_path) self.comm_config.display_config() #self.comm_config.get_from_console() self.ip_address = self.comm_config.device_addr self.data_port = self.comm_config.data_port self.command_port = self.comm_config.command_port if not (self.ip_address): self.ip_address = prompt.text('Instrument IP Address', self.ip_address) if not (self.data_port): continuing = True while continuing: sport = prompt.text('Instrument Port', self.data_port) try: self.data_port = int(sport) continuing = False except ValueError as e: print "Error converting port to number: " + str(e) print "Please enter a valid port number.\n" def fetch_driver_class(self): self.driver_class = prompt.text('Driver Class', self.driver_class) def get_user_command(self, text='Enter command'): command = prompt.text(text) return command def run(self): """ @brief Run it. """ print( "------------------>>>> Starting RunInstrument <<<<------------------" ) """ initialize; returns True if successful, else False. """ continuing = self._initialize() """ bring_instrument_active; returns True if successful, else False """ if (continuing): continuing = self.bring_instrument_active() PROMPT = 'Enter command (\'quit\' to exit)' text = PROMPT while continuing: try: """ Get a list of the currently available capabilities """ self.get_capabilities() command = self.get_user_command(text) text = PROMPT if command == 'quit': continuing = False elif command in self.agt_cmds: self.send_agent_command(command) elif command in self.res_cmds: self.send_driver_command(command) else: text = 'Invalid Command: ' + command + '\n' + PROMPT except: log.error("run(): Exception occurred; shutting down.", exc_info=True) continuing = False self.stop_pagent() print( "------------------>>>> Stopping RunInstrument <<<<------------------" )
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin): @classmethod def setUpClass(cls): HelperTestMixin.setUpClass() # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG["oms_uri"]) network_definition = RsnOmsUtil.build_network_definition(rsn_oms) network_definition_ser = NetworkUtil.serialize_network_definition(network_definition) if log.isEnabledFor(logging.DEBUG): log.debug("NetworkDefinition serialization:\n%s", network_definition_ser) cls.PLATFORM_CONFIG = { "platform_id": cls.PLATFORM_ID, "driver_config": DVR_CONFIG, "network_definition": network_definition_ser, } NetworkUtil._gen_open_diagram(network_definition.pnodes[cls.PLATFORM_ID]) def setUp(self): self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self._pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Start data subscribers, add stop to cleanup. # Define stream_config. self._async_data_result = AsyncResult() self._data_greenlets = [] self._stream_config = {} self._samples_received = [] self._data_subscribers = [] self._start_data_subscribers() self.addCleanup(self._stop_data_subscribers) # start event subscriber: self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber() self._agent_config = { "agent": {"resource_id": PA_RESOURCE_ID}, "stream_config": self._stream_config, # pass platform config here "platform_config": self.PLATFORM_CONFIG, } if log.isEnabledFor(logging.TRACE): log.trace("launching with agent_config=%s" % str(self._agent_config)) self._launcher = LauncherFactory.createLauncher() self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config) log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID) # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess()) log.info("Got pa client %s." % str(self._pa_client)) def tearDown(self): try: self._launcher.cancel_process(self._pid) finally: super(TestPlatformAgent, self).tearDown() def _start_data_subscribers(self): """ """ # Create streams and subscriptions for each stream named in driver. self._stream_config = {} self._data_subscribers = [] # # TODO retrieve appropriate stream definitions; for the moment, using # adhoc_get_stream_names # # A callback for processing subscribed-to data. def consume_data(message, stream_route, stream_id): log.info("Subscriber received data message: %s." % str(message)) self._samples_received.append(message) self._async_data_result.set() for stream_name in adhoc_get_stream_names(): log.info("creating stream %r ...", stream_name) # TODO use appropriate exchange_point stream_id, stream_route = self._pubsub_client.create_stream(name=stream_name, exchange_point="science_data") log.info("create_stream(%r): stream_id=%r, stream_route=%s", stream_name, stream_id, str(stream_route)) pdict = adhoc_get_parameter_dictionary(stream_name) stream_config = dict( stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump() ) self._stream_config[stream_name] = stream_config log.info("_stream_config[%r]= %r", stream_name, stream_config) # Create subscriptions for each stream. exchange_name = "%s_queue" % stream_name self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) self._pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ for sub in self._data_subscribers: if hasattr(sub, "subscription_id"): try: self._pubsub_client.deactivate_subscription(sub.subscription_id) except: pass self._pubsub_client.delete_subscription(sub.subscription_id) sub.stop() def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"): """ Starts event subscriber for events of given event_type ("DeviceEvent" by default) and given sub_type ("platform_event" by default). """ def consume_event(evt, *args, **kwargs): # A callback for consuming events. log.info("Event subscriber received evt: %s.", str(evt)) self._events_received.append(evt) self._async_event_result.set(evt) sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event) sub.start() log.info("registered event subscriber for event_type=%r, sub_type=%r", event_type, sub_type) self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) def _stop_event_subscribers(self): """ Stops the event subscribers on cleanup. """ try: for sub in self._event_subscribers: if hasattr(sub, "subscription_id"): try: self.pubsubcli.deactivate_subscription(sub.subscription_id) except: pass self.pubsubcli.delete_subscription(sub.subscription_id) sub.stop() finally: self._event_subscribers = [] def _get_state(self): state = self._pa_client.get_agent_state() return state def _assert_state(self, state): self.assertEquals(self._get_state(), state) # def _execute_agent(self, cmd, timeout=TIMEOUT): def _execute_agent(self, cmd): log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs) time_start = time.time() # retval = self._pa_client.execute_agent(cmd, timeout=timeout) retval = self._pa_client.execute_agent(cmd) elapsed_time = time.time() - time_start log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval)) return retval def _reset(self): cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.UNINITIALIZED) def _ping_agent(self): retval = self._pa_client.ping_agent() self.assertIsInstance(retval, str) def _ping_resource(self): cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE) if self._get_state() == PlatformAgentState.UNINITIALIZED: # should get ServerError: "Command not handled in current state" with self.assertRaises(ServerError): # self._pa_client.execute_agent(cmd, timeout=TIMEOUT) self._pa_client.execute_agent(cmd) else: # In all other states the command should be accepted: retval = self._execute_agent(cmd) self.assertEquals("PONG", retval.result) def _get_metadata(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_METADATA = %s", md) def _get_ports(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_PORTS = %s", md) def _connect_instrument(self): # # TODO more realistic settings for the connection # port_id = self.PORT_ID instrument_id = self.INSTRUMENT_ID instrument_attributes = self.INSTRUMENT_ATTRIBUTES_AND_VALUES kwargs = dict(port_id=port_id, instrument_id=instrument_id, attributes=instrument_attributes) cmd = AgentCommand(command=PlatformAgentEvent.CONNECT_INSTRUMENT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("CONNECT_INSTRUMENT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], dict) returned_attrs = self._verify_valid_instrument_id(instrument_id, result[port_id]) if isinstance(returned_attrs, dict): for attrName in instrument_attributes: self.assertTrue(attrName in returned_attrs) def _get_connected_instruments(self): port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("GET_CONNECTED_INSTRUMENTS = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], dict) instrument_id = self.INSTRUMENT_ID self.assertTrue(instrument_id in result[port_id]) def _disconnect_instrument(self): # TODO real settings and corresp verification port_id = self.PORT_ID instrument_id = self.INSTRUMENT_ID kwargs = dict(port_id=port_id, instrument_id=instrument_id) cmd = AgentCommand(command=PlatformAgentEvent.DISCONNECT_INSTRUMENT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("DISCONNECT_INSTRUMENT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], dict) self.assertTrue(instrument_id in result[port_id]) self._verify_instrument_disconnected(instrument_id, result[port_id][instrument_id]) def _turn_on_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("TURN_ON_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_ON) def _turn_off_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("TURN_OFF_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF) def _get_resource(self): attrNames = self.ATTR_NAMES # # OOIION-631: use get_ion_ts() as a basis for using system time, which is # a string. # cur_time = get_ion_ts() from_time = str(int(cur_time) - 50000) # a 50-sec time window kwargs = dict(attr_names=attrNames, from_time=from_time) cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attr_name in attrNames: self._verify_valid_attribute_id(attr_name, attr_values) def _set_resource(self): attrNames = self.ATTR_NAMES writ_attrNames = self.WRITABLE_ATTR_NAMES # do valid settings: # TODO more realistic value depending on attribute's type attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames] log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs) kwargs = dict(attrs=attrs) cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attrName in attrNames: if attrName in writ_attrNames: self._verify_valid_attribute_id(attrName, attr_values) else: self._verify_not_writable_attribute_id(attrName, attr_values) # try invalid settings: # set invalid values to writable attributes: attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames] log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs) kwargs = dict(attrs=attrs) cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attrName in writ_attrNames: self._verify_attribute_value_out_of_range(attrName, attr_values) def _initialize(self): self._assert_state(PlatformAgentState.UNINITIALIZED) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _go_active(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _run(self): cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _pause(self): cmd = AgentCommand(command=PlatformAgentEvent.PAUSE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.STOPPED) def _resume(self): cmd = AgentCommand(command=PlatformAgentEvent.RESUME) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _start_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.MONITORING) def _wait_for_a_data_sample(self): log.info("waiting for reception of a data sample...") # just wait for at least one -- see consume_data self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("Received samples: %s", len(self._samples_received)) def _stop_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _go_inactive(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _get_subplatform_ids(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS) retval = self._execute_agent(cmd) self.assertIsInstance(retval.result, list) self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS) return retval.result def _wait_for_external_event(self): log.info("waiting for reception of an external event...") # just wait for at least one -- see consume_event self._async_event_result.get(timeout=EVENT_TIMEOUT) self.assertTrue(len(self._events_received) >= 1) log.info("Received events: %s", len(self._events_received)) def _check_sync(self): cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC) retval = self._execute_agent(cmd) log.info("CHECK_SYNC result: %s", retval.result) self.assertTrue(retval.result is not None) self.assertEquals(retval.result[0:3], "OK:") return retval.result def test_capabilities(self): agt_cmds_all = [ PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.RESET, PlatformAgentEvent.GO_ACTIVE, PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RUN, PlatformAgentEvent.CLEAR, PlatformAgentEvent.PAUSE, PlatformAgentEvent.RESUME, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.CONNECT_INSTRUMENT, PlatformAgentEvent.DISCONNECT_INSTRUMENT, PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.START_MONITORING, PlatformAgentEvent.STOP_MONITORING, PlatformAgentEvent.CHECK_SYNC, ] def sort_caps(caps): agt_cmds = [] agt_pars = [] res_cmds = [] res_pars = [] if len(caps) > 0 and isinstance(caps[0], AgentCapability): agt_cmds = [x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD] agt_pars = [x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR] res_cmds = [x.name for x in caps if x.cap_type == CapabilityType.RES_CMD] res_pars = [x.name for x in caps if x.cap_type == CapabilityType.RES_PAR] elif len(caps) > 0 and isinstance(caps[0], dict): agt_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_CMD] agt_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_PAR] res_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_CMD] res_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_PAR] return agt_cmds, agt_pars, res_cmds, res_pars agt_pars_all = ["example"] # 'cause ResourceAgent defines aparam_example res_pars_all = [] res_cmds_all = [] ################################################################## # UNINITIALIZED ################################################################## self._assert_state(PlatformAgentState.UNINITIALIZED) # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state UNINITIALIZED. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_uninitialized = [PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES] self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states. retval = self._pa_client.get_capabilities(current_state=False) # Validate all capabilities as read from state UNINITIALIZED. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) ################################################################## # INACTIVE ################################################################## self._initialize() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state INACTIVE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_inactive = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GO_ACTIVE, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_inactive) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state INACTIVE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) ################################################################## # IDLE ################################################################## self._go_active() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state IDLE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_idle = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RUN, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_idle) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states as read from IDLE. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state IDLE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) ################################################################## # COMMAND ################################################################## self._run() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state COMMAND agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_command = [ PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RESET, PlatformAgentEvent.PAUSE, PlatformAgentEvent.CLEAR, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.CONNECT_INSTRUMENT, PlatformAgentEvent.DISCONNECT_INSTRUMENT, PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.START_MONITORING, PlatformAgentEvent.CHECK_SYNC, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_command) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) ################################################################## # STOPPED ################################################################## self._pause() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state STOPPED agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_stopped = [ PlatformAgentEvent.RESUME, PlatformAgentEvent.CLEAR, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_stopped) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) # back to COMMAND: self._resume() ################################################################## # MONITORING ################################################################## self._start_resource_monitoring() # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state MONITORING agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_monitoring = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.CONNECT_INSTRUMENT, PlatformAgentEvent.DISCONNECT_INSTRUMENT, PlatformAgentEvent.GET_CONNECTED_INSTRUMENTS, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.STOP_MONITORING, PlatformAgentEvent.CHECK_SYNC, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_monitoring) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) # return to COMMAND state: self._stop_resource_monitoring() ################### # ALL CAPABILITIES ################### # Get exposed capabilities in all states as read from state COMMAND. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state COMMAND agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_all) self.assertItemsEqual(res_pars, res_pars_all) self._go_inactive() self._reset() def test_some_state_transitions(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._initialize() # -> INACTIVE self._reset() # -> UNINITIALIZED self._initialize() # -> INACTIVE self._go_active() # -> IDLE self._reset() # -> UNINITIALIZED self._initialize() # -> INACTIVE self._go_active() # -> IDLE self._run() # -> COMMAND self._pause() # -> STOPPED self._resume() # -> COMMAND self._reset() # -> UNINITIALIZED def test_get_set_resources(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._get_resource() self._set_resource() self._go_inactive() self._reset() def test_some_commands(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._ping_agent() self._ping_resource() self._get_metadata() self._get_ports() self._get_subplatform_ids() self._go_inactive() self._reset() def test_resource_monitoring(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._start_resource_monitoring() self._wait_for_a_data_sample() self._stop_resource_monitoring() self._go_inactive() self._reset() def test_external_event_dispatch(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._wait_for_external_event() self._go_inactive() self._reset() def test_connect_disconnect_instrument(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._connect_instrument() self._turn_on_port() self._get_connected_instruments() self._turn_off_port() self._disconnect_instrument() self._go_inactive() self._reset() def test_check_sync(self): self._assert_state(PlatformAgentState.UNINITIALIZED) self._ping_agent() self._initialize() self._go_active() self._run() self._check_sync() self._connect_instrument() self._check_sync() self._disconnect_instrument() self._check_sync() self._go_inactive() self._reset()
class TestAgentCommsAlerts(IonIntegrationTestCase): """ """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config. """ self._ia_client = None log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') self._event_count = 0 self._events_received = [] self._async_event_result = AsyncResult() def consume_event(*args, **kwargs): log.debug('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._event_count > 0 and \ self._event_count == len(self._events_received): self._async_event_result.set() self._event_subscriber = EventSubscriber( event_type='DeviceStatusAlertEvent', callback=consume_event, origin=IA_RESOURCE_ID) self._event_subscriber.start() def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Create agent config. self._agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True, 'aparam_alerts_config' : [state_alert_def, command_alert_def] } self._ia_client = None self._ia_pid = None self.addCleanup(self._verify_agent_reset) ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config ############################################################################### # Agent start stop helpers. ############################################################################### def _start_agent(self): """ """ container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=self._agent_config) log.info('Started instrument agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got instrument agent client %s.', str(self._ia_client)) def _stop_agent(self): """ """ if self._ia_pid: container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) container_client.terminate_process(self._ia_pid) self._ia_pid = None if self._ia_client: self._ia_client = None def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) self._ia_client = None ############################################################################### # Tests. ############################################################################### def test_lost_connection_alert(self): """ test_lost_connection_alert Verify that agents detect lost connection state and issue alert. """ self._event_count = 3 self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Confirm the persisted parameters. retval = self._ia_client.get_agent(['alerts'])['alerts'] """ {'origin': '123xyz', 'status': 1, '_id': 'da03b90d2e064b25bf51ed90b729e82e', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1366749987069', 'sub_type': 3, 'origin_type': 'InstrumentDevice', 'name': 'comms_warning'} """ # Acquire sample returns a string, not a particle. The particle # is created by the data handler though. cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = self._ia_client.execute_resource(cmd) # Blow the port agent out from under the agent. self._support.stop_pagent() # Wait for a while, the supervisor is restarting the port agent. gevent.sleep(5) self._support.start_pagent() timeout = gevent.Timeout(120) timeout.start() try: while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.COMMAND: timeout.cancel() break else: gevent.sleep(2) except Timeout as t: self.fail('Could not reconnect to device.') state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) self._async_event_result.get(timeout=30) """ {'origin': '123xyz', 'status': 1, '_id': '9d7db919a0414741a2aa97f3dc310647', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029709', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} {'origin': '123xyz', 'status': 1, '_id': 'f746cec5e486445e856ef29af8d8d49a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029717', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} {'origin': '123xyz', 'status': 1, '_id': '126b1e20f54f4bd2b84a93584831ea8d', 'description': 'Detected comms failure.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_LOST_CONNECTION'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008062061', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} {'origin': '123xyz', 'status': 1, '_id': '90f5762112dd446b939c6675d34dd61d', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_COMMAND'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008098639', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} """ def test_connect_failed_alert(self): """ test_connect_failed_alert Verify that agents detect failed connections and issue alert. """ self._event_count = 4 # Remove the port agent. self._support.stop_pagent() self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) with self.assertRaises(Exception): cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) # Now start up the port agent. self._support.start_pagent() gevent.sleep(5) # This time it will work. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) self._async_event_result.get(timeout=30) """ {'origin': '123xyz', 'status': 1, '_id': 'ac1ae3ec24e74f65bde24362d689346a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751167', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} {'origin': '123xyz', 'status': 1, '_id': '6af6a6156172481ebc44baad2708ec5c', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751175', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} {'origin': '123xyz', 'status': 1, '_id': 'b493698b46fd4fc1b4c66c53b70ed043', 'description': 'Detected comms failure while connecting.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007756771', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} {'origin': '123xyz', 'status': 1, '_id': 'c01e911ecb1a47c5a04cbbcbfb348a42', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007792905', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} """
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch( host=es_host, port=es_port, timeout=10 ) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete,index) IndexManagementService._es_call(es.index_delete,index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name = '', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name= 'notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) ) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) self.assertEqual(extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING) self.assertEqual(extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK) self.assertEqual(extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_instrument_device_id) self.assertEqual(notification.origin_type, "instrument") self.assertEqual(notification.event_type, 'ResourceLifecycleEvent') def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertTrue( 'ok' in extended_data_product.computed.last_granule.value['quality_flag'] ) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) ) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_data_product_id) self.assertEqual(notification.origin_type, "data product") self.assertEqual(notification.event_type, 'DetectionEvent') @attr('LOCOINT') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name' : 'temperature_warning_interval', 'stream_name' : 'parsed', 'value_id' : 'temp', 'message' : 'Temperature is below the normal range of 50.0 and above.', 'type' : StreamAlarmType.WARNING, 'lower_bound' : 50.0, 'lower_rel_op' : '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm] ) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations = [raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s' , data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources(restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received= [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug('TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s" , str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s" , str(retval)) log.debug( "test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s" , str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print()) ) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals) , 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals ) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals ) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received ) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count ) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print()) ) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals) , 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests.value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent', origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] ) self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent', origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 ) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent', origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] ) self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent', origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 ) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class RunInstrument(MiIntTestCase): """ Main class for communicating with an instrument """ def __init__(self, monitor=False, subscriber=False): self.driver_make = None self.driver_model = None self.driver_name = None self.driver_class = DRIVER_CLASS self.ip_address = None self.data_port = None self.command_port = None self.driver_version = None self._pagent = None self.monitor_window = monitor self.subcriber_window = subscriber self.stream_config = {} self._cleanups = [] def _initialize(self): """ Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config, start agent. Start agent client. """ try: """ Get the information for the driver. This can be read from the yml files; the user can run switch_driver to change the current driver. """ self.fetch_metadata() self.fetch_driver_class() self.fetch_comm_config() if not exists(PIPE_PATH): mkfifo(PIPE_PATH) if not exists(self.metadata.driver_dir()): raise DriverDoesNotExist( "%s/%s/$%s" % (self.metadata.driver_make, self.metadata.driver_model, self.driver_name)) driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF log.info('driver module: %s', driver_module) log.info('driver class: %s', self.driver_class) log.info('device address: %s', self.ip_address) log.info('device data port: %s', self.data_port) log.info('device command port: %s', self.command_port) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) DVR_CONFIG.update({'dvr_mod' : driver_module, 'dvr_cls' : self.driver_class}) """ self._support = DriverIntegrationTestSupport(driver_module, self.driver_class, self.ip_address, self.data_port, DELIM, WORK_DIR) """ # Start port agent, add stop to cleanup (not sure if that's # necessary yet). print( "------------------>>>> Starting Port Agent <<<<------------------" ) self.start_pagent() # Start a monitor window if specified. if self.monitor_window: self.monitor_file = self._pagent.port_agent.logfname strXterm = "xterm -T InstrumentMonitor -sb -rightbar" #pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file pOpenString = strXterm + " -e tail -f " + self.monitor_file x = subprocess.Popen(pOpenString, shell=True) """ DHE: Added self._cleanups to make base classes happy """ self.addCleanup(self.stop_pagent) # Start container. print( "------------------>>>> Starting Capability Container <<<<------------------" ) self._start_container() # Bring up services in a deploy file (no need to message) print( "------------------>>>> Starting Deploy Services <<<<------------------" ) self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Setup stream config. self._build_stream_config() # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. self._ia_pid = None print( "------------------>>>> Starting Instrument Agent <<<<------------------" ) container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) if self.subcriber_window: self._start_data_subscribers(6) #self.addCleanup(self._stop_data_subscribers) except: log.error("initialize(): Exception occurred; shutting down.", exc_info=True) return False else: return True ############################################################################### # Port agent helpers. ############################################################################### def start_pagent(self): """ Construct and start the port agent. @retval port Port that was used for connection to agent """ # Create port agent object. comm_config = self.comm_config config = { 'device_addr' : comm_config.device_addr, 'device_port' : comm_config.device_port, 'command_port': comm_config.command_port, 'data_port': comm_config.data_port, 'process_type': PortAgentProcessType.UNIX, 'log_level': 5, } self._pagent = PortAgentProcess.launch_process(config, timeout = 60, test_mode = True) pid = self._pagent.get_pid() port = self._pagent.get_data_port() log.info('Started port agent pid %d listening at port %d', pid, port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port } return port def stop_pagent(self): """ Stop the port agent. """ if self._pagent: pid = self._pagent.get_pid() if pid: log.info('Stopping pagent pid %i', pid) self._pagent.stop() else: log.info('No port agent running.') ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} streams = { 'parsed' : 'ctd_parsed_param_dict', 'raw' : 'ctd_raw_param_dict' } for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True) if (not pd_id): log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = None stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self.stream_config[stream_name] = stream_config def _start_data_subscribers(self, count): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._data_subscribers = [] self._samples_received = [] #self._async_data_result = AsyncResult() strXterm = "xterm -T InstrumentScienceData -sb -rightbar " pOpenString = strXterm + " -e tail -f " + PIPE_PATH subprocess.Popen(['xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f', PIPE_PATH]) #subprocess.Popen(pOpenString) #self.pipeData = open(PIPE_PATH, "w", 1) # A callback for processing subscribed-to data. def recv_data(message, stream_route, stream_id): print 'Received message on ' + str(stream_id) + ' (' + str(stream_route.exchange_point) + ',' + str(stream_route.routing_key) + ')' log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self.pipeData = open(PIPE_PATH, "w", 1) self.pipeData.write(str(message)) self.pipeData.flush() self.pipeData.close() self._samples_received.append(message) #if len(self._samples_received) == count: #self._async_data_result.set() for (stream_name, stream_config) in self._stream_config.iteritems(): stream_id = stream_config['stream_id'] # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_data) sub.start() self._data_subscribers.append(sub) print 'stream_id: %s' % stream_id sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): for subscriber in self._data_subscribers: pubsub_client = PubsubManagementServiceClient() if hasattr(subscriber,'subscription_id'): try: pubsub_client.deactivate_subscription(subscriber.subscription_id) except: pass pubsub_client.delete_subscription(subscriber.subscription_id) subscriber.stop() def bring_instrument_active(self): """ @brief Bring the agent up to COMMAND state, """ """ DHE: Don't have an event subscriber yet # Set up a subscriber to collect error events. #self._start_event_subscriber('ResourceAgentResourceStateEvent', 6) #self.addCleanup(self._stop_event_subscriber) """ try: state = self._ia_client.get_agent_state() print "AgentState: " + str(state) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print "AgentState: " + str(state) res_state = self._ia_client.get_resource_state() print "DriverState: " + str(res_state) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print "AgentState: " + str(state) res_state = self._ia_client.get_resource_state() print "DriverState: " + str(res_state) """ If the agent is in STREAMING state, it will not accept the run command. """ if state != ResourceAgentState.STREAMING: cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() print "AgentState: " + str(state) res_state = self._ia_client.get_resource_state() print "DriverState: " + str(res_state) except: log.error("bring_instrument_active(): Exception occurred; shutting down.", exc_info=True) return False else: return True """ DHE: Don't have an event subscriber yet so we've received no events. self._async_event_result.get(timeout=2) print "Received: " + str(len(self._events_received)) + " events." """ ############################################################################### # RunInstrument helpers. ############################################################################### def get_capabilities(self): """ @brief Get exposed capabilities in current state. """ retval = self._ia_client.get_capabilities() # Validate capabilities for state UNINITIALIZED. self.agt_cmds = [x.name for x in retval if x.cap_type==CapabilityType.AGT_CMD] self.agt_pars = [x.name for x in retval if x.cap_type==CapabilityType.AGT_PAR] self.res_cmds = [x.name for x in retval if x.cap_type==CapabilityType.RES_CMD] self.res_pars = [x.name for x in retval if x.cap_type==CapabilityType.RES_PAR] print "\n------------------>>>> Current Capabilities <<<<------------------" print "Agent Commands: " + str(self.agt_cmds) #print "Agent Parameters: " + str(self.agt_pars) print "Resource Commands: " + str(self.res_cmds) #print "Resource Parameters: " + str(self.res_pars) def send_agent_command(self, command): """ @brief Send a command to the agent. """ DA_WAIT_PERIOD = 60 waiting = False print "Input command: " + str(command) if command == 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS': cmd = AgentCommand(command = command, kwargs={'session_type': DirectAccessTypes.telnet, 'session_timeout':600, 'inactivity_timeout':600}) waiting = True else: cmd = AgentCommand(command = command) retval = self._ia_client.execute_agent(cmd) print "Results of command: " + str(retval) while waiting: print "Waiting " + str(DA_WAIT_PERIOD) + " seconds for you to test direct access." gevent.sleep(DA_WAIT_PERIOD) still_waiting = prompt.text('Still waiting? (y/n)') if still_waiting is 'n': waiting = False def send_driver_command(self, command): """ @brief Send a command to the instrument through the instrument agent. First determine whether it's a get or set, which are handled separately. """ if command == DriverEvent.GET: self._get_param() elif command == DriverEvent.SET: self._set_param() else: print "Input command: " + str(command) cmd = AgentCommand(command = command) retval = self._ia_client.execute_resource(cmd) print "Results of command: " + str(retval) def _get_param(self): """ @brief Get a single parameter from the instrument (will be updated to get multiple later). """ _all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL') print "Parameters you can get are: " + str(_all_params) _param_valid = False while _param_valid is False: _param = prompt.text('\nEnter a single parameter') if _param in _all_params: _param_valid = True else: print 'Invalid parameter: ' + _param reply = self._ia_client.get_resource([_param]) print 'Reply is :' + str(reply) def _set_param(self): """ @brief Set a single parameter """ _all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL') print "Parameters you can set are: " + str(_all_params) _param_valid = False while _param_valid is False: _param = prompt.text('\nEnter a single parameter') if _param in _all_params: _param_valid = True else: print 'Invalid parameter: ' + _param _value = prompt.text('Enter value') _value = _value.lower() """ DHE: Need to convert to native types here; can't be string; this is a problem for the UI because we need a way to get the metadata about each param to the UI. """ if _value == 'true': _value = True elif _value == 'false': _value = False param_dict = {_param: _value} self._ia_client.set_resource(param_dict) def fetch_metadata(self): """ @brief collect metadata from the user """ self.metadata = Metadata() self.driver_make = self.metadata.driver_make self.driver_model = self.metadata.driver_model self.driver_name = self.metadata.driver_name if not (self.driver_make and self.driver_model and self.driver_name): self.driver_make = prompt.text( 'Driver Make', self.driver_make ) self.driver_model = prompt.text( 'Driver Model', self.driver_model ) self.driver_name = prompt.text( 'Driver Name', self.driver_name ) if not (self.driver_class): self.driver_class = prompt.text( 'Driver Class', self.driver_class ) self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name) def fetch_comm_config(self): """ @brief collect connection information for the logger from the user """ config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename()) self.comm_config = CommConfig.get_config_from_console(config_path) self.comm_config.display_config() #self.comm_config.get_from_console() self.ip_address = self.comm_config.device_addr self.data_port = self.comm_config.data_port self.command_port = self.comm_config.command_port if not (self.ip_address): self.ip_address = prompt.text( 'Instrument IP Address', self.ip_address ) if not (self.data_port): continuing = True while continuing: sport = prompt.text( 'Instrument Port', self.data_port ) try: self.data_port = int(sport) continuing = False except ValueError as e: print "Error converting port to number: " + str(e) print "Please enter a valid port number.\n" def fetch_driver_class(self): self.driver_class = prompt.text( 'Driver Class', self.driver_class ) def get_user_command(self, text='Enter command'): command = prompt.text(text) return command def run(self): """ @brief Run it. """ print( "------------------>>>> Starting RunInstrument <<<<------------------" ) """ initialize; returns True if successful, else False. """ continuing = self._initialize() """ bring_instrument_active; returns True if successful, else False """ if (continuing): continuing = self.bring_instrument_active() PROMPT = 'Enter command (\'quit\' to exit)' text = PROMPT while continuing: try: """ Get a list of the currently available capabilities """ self.get_capabilities() command = self.get_user_command(text) text = PROMPT if command == 'quit': continuing = False elif command in self.agt_cmds: self.send_agent_command(command) elif command in self.res_cmds: self.send_driver_command(command) else: text = 'Invalid Command: ' + command + '\n' + PROMPT except: log.error("run(): Exception occurred; shutting down.", exc_info=True) continuing = False self.stop_pagent() print( "------------------>>>> Stopping RunInstrument <<<<------------------" )
class TestPlatformAgent(IonIntegrationTestCase, HelperTestMixin): @classmethod def setUpClass(cls): HelperTestMixin.setUpClass() def setUp(self): self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self._pubsub_client = PubsubManagementServiceClient(node=self.container.node) self.PLATFORM_CONFIG = {"platform_id": self.PLATFORM_ID, "driver_config": DVR_CONFIG} # Start data suscribers, add stop to cleanup. # Define stream_config. self._async_data_result = AsyncResult() self._data_greenlets = [] self._stream_config = {} self._samples_received = [] self._data_subscribers = [] self._start_data_subscribers() self.addCleanup(self._stop_data_subscribers) self._agent_config = { "agent": {"resource_id": PA_RESOURCE_ID}, "stream_config": self._stream_config, # pass platform config here "platform_config": self.PLATFORM_CONFIG, } log.debug("launching with agent_config=%s", str(self._agent_config)) self._launcher = LauncherFactory.createLauncher() self._pid = self._launcher.launch(self.PLATFORM_ID, self._agent_config) log.debug("LAUNCHED PLATFORM_ID=%r", self.PLATFORM_ID) # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient(PA_RESOURCE_ID, process=FakeProcess()) log.info("Got pa client %s." % str(self._pa_client)) def tearDown(self): try: self._launcher.cancel_process(self._pid) finally: super(TestPlatformAgent, self).tearDown() def _start_data_subscribers(self): """ """ # Create streams and subscriptions for each stream named in driver. self._stream_config = {} self._data_subscribers = [] # # TODO retrieve appropriate stream definitions; for the moment, using # adhoc_get_stream_names # # A callback for processing subscribed-to data. def consume_data(message, stream_route, stream_id): log.info("Subscriber received data message: %s." % str(message)) self._samples_received.append(message) self._async_data_result.set() for stream_name in adhoc_get_stream_names(): log.info("creating stream %r ...", stream_name) # TODO use appropriate exchange_point stream_id, stream_route = self._pubsub_client.create_stream(name=stream_name, exchange_point="science_data") log.info("create_stream(%r): stream_id=%r, stream_route=%s", stream_name, stream_id, str(stream_route)) pdict = adhoc_get_parameter_dictionary(stream_name) stream_config = dict( stream_route=stream_route.routing_key, stream_id=stream_id, parameter_dictionary=pdict.dump() ) self._stream_config[stream_name] = stream_config log.info("_stream_config[%r]= %r", stream_name, stream_config) # Create subscriptions for each stream. exchange_name = "%s_queue" % stream_name self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id]) self._pubsub_client.activate_subscription(sub_id) sub.subscription_id = sub_id def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ for sub in self._data_subscribers: if hasattr(sub, "subscription_id"): try: self._pubsub_client.deactivate_subscription(sub.subscription_id) except: pass self._pubsub_client.delete_subscription(sub.subscription_id) sub.stop() def _get_state(self): state = self._pa_client.get_agent_state() return state def _assert_state(self, state): self.assertEquals(self._get_state(), state) # def _execute_agent(self, cmd, timeout=TIMEOUT): def _execute_agent(self, cmd): log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs) time_start = time.time() # retval = self._pa_client.execute_agent(cmd, timeout=timeout) retval = self._pa_client.execute_agent(cmd) elapsed_time = time.time() - time_start log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval)) return retval def _reset(self): cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.UNINITIALIZED) def _ping_agent(self): retval = self._pa_client.ping_agent() self.assertIsInstance(retval, str) def _ping_resource(self): cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE) if self._get_state() == PlatformAgentState.UNINITIALIZED: # should get ServerError: "Command not handled in current state" with self.assertRaises(ServerError): # self._pa_client.execute_agent(cmd, timeout=TIMEOUT) self._pa_client.execute_agent(cmd) else: # In all other states the command should be accepted: retval = self._execute_agent(cmd) self.assertEquals("PONG", retval.result) def _get_metadata(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_METADATA = %s", md) def _get_ports(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_PORTS = %s", md) def _set_up_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID port_attrName = self.PORT_ATTR_NAME kwargs = dict(port_id=port_id, attributes={port_attrName: self.VALID_PORT_ATTR_VALUE}) cmd = AgentCommand(command=PlatformAgentEvent.SET_UP_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("SET_UP_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertTrue(port_attrName in result[port_id]) def _turn_on_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.TURN_ON_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("TURN_ON_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], bool) def _turn_off_port(self): # TODO real settings and corresp verification port_id = self.PORT_ID kwargs = dict(port_id=port_id) cmd = AgentCommand(command=PlatformAgentEvent.TURN_OFF_PORT, kwargs=kwargs) retval = self._execute_agent(cmd) result = retval.result log.info("TURN_OFF_PORT = %s", result) self.assertIsInstance(result, dict) self.assertTrue(port_id in result) self.assertIsInstance(result[port_id], bool) def _get_resource(self): attrNames = self.ATTR_NAMES cur_time = get_ion_ts() # # OOIION-631 Note: I'm asked to only use get_ion_ts() as a basis for # using system time. However, other associated supporting (and more # "numeric") routines would be convenient. In particular, note the # following operation to subtract a number of seconds for my request: # from_time = str(int(cur_time) - 50000) # a 50-sec time window kwargs = dict(attr_names=attrNames, from_time=from_time) cmd = AgentCommand(command=PlatformAgentEvent.GET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attr_name in attrNames: self._verify_valid_attribute_id(attr_name, attr_values) def _set_resource(self): attrNames = self.ATTR_NAMES writ_attrNames = self.WRITABLE_ATTR_NAMES # do valid settings: # TODO more realistic value depending on attribute's type attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames] log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs) kwargs = dict(attrs=attrs) cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attrName in attrNames: if attrName in writ_attrNames: self._verify_valid_attribute_id(attrName, attr_values) else: self._verify_not_writable_attribute_id(attrName, attr_values) # try invalid settings: # set invalid values to writable attributes: attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames] log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs) kwargs = dict(attrs=attrs) cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs) retval = self._execute_agent(cmd) attr_values = retval.result self.assertIsInstance(attr_values, dict) for attrName in writ_attrNames: self._verify_attribute_value_out_of_range(attrName, attr_values) def _initialize(self): self._assert_state(PlatformAgentState.UNINITIALIZED) # kwargs = dict(plat_config=self.PLATFORM_CONFIG) # cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=kwargs) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _go_active(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _run(self): cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _go_inactive(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _get_subplatform_ids(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_SUBPLATFORM_IDS) retval = self._execute_agent(cmd) self.assertIsInstance(retval.result, list) self.assertTrue(x in retval.result for x in self.SUBPLATFORM_IDS) return retval.result def _start_event_dispatch(self): kwargs = dict(params="TODO set params") cmd = AgentCommand(command=PlatformAgentEvent.START_EVENT_DISPATCH, kwargs=kwargs) retval = self._execute_agent(cmd) self.assertTrue(retval.result is not None) return retval.result def _wait_for_a_data_sample(self): log.info("waiting for reception of a data sample...") self._async_data_result.get(timeout=15) # just wait for at least one -- see consume_data above self.assertTrue(len(self._samples_received) >= 1) def _stop_event_dispatch(self): cmd = AgentCommand(command=PlatformAgentEvent.STOP_EVENT_DISPATCH) retval = self._execute_agent(cmd) self.assertTrue(retval.result is not None) return retval.result def test_capabilities(self): # log.info("test_capabilities starting. Default timeout=%s", TIMEOUT) log.info("test_capabilities starting. Default timeout=%i", CFG.endpoint.receive.timeout) agt_cmds_all = [ PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.RESET, PlatformAgentEvent.GO_ACTIVE, PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RUN, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.SET_UP_PORT, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.START_EVENT_DISPATCH, PlatformAgentEvent.STOP_EVENT_DISPATCH, ] def sort_caps(caps): agt_cmds = [] agt_pars = [] res_cmds = [] res_pars = [] if len(caps) > 0 and isinstance(caps[0], AgentCapability): agt_cmds = [x.name for x in caps if x.cap_type == CapabilityType.AGT_CMD] agt_pars = [x.name for x in caps if x.cap_type == CapabilityType.AGT_PAR] res_cmds = [x.name for x in caps if x.cap_type == CapabilityType.RES_CMD] res_pars = [x.name for x in caps if x.cap_type == CapabilityType.RES_PAR] elif len(caps) > 0 and isinstance(caps[0], dict): agt_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_CMD] agt_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.AGT_PAR] res_cmds = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_CMD] res_pars = [x["name"] for x in caps if x["cap_type"] == CapabilityType.RES_PAR] return agt_cmds, agt_pars, res_cmds, res_pars agt_pars_all = ["example"] # 'cause ResourceAgent defines aparam_example res_pars_all = [] res_cmds_all = [] ################################################################## # UNINITIALIZED ################################################################## self._assert_state(PlatformAgentState.UNINITIALIZED) # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state UNINITIALIZED. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_uninitialized = [PlatformAgentEvent.INITIALIZE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES] self.assertItemsEqual(agt_cmds, agt_cmds_uninitialized) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states. retval = self._pa_client.get_capabilities(current_state=False) # Validate all capabilities as read from state UNINITIALIZED. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) self._initialize() ################################################################## # INACTIVE ################################################################## # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state INACTIVE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_inactive = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GO_ACTIVE, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_inactive) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state INACTIVE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) self._go_active() ################################################################## # IDLE ################################################################## # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities for state IDLE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_idle = [ PlatformAgentEvent.RESET, PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RUN, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, ] self.assertItemsEqual(agt_cmds, agt_cmds_idle) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) # Get exposed capabilities in all states as read from IDLE. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state IDLE. agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, []) self.assertItemsEqual(res_pars, []) self._run() ################################################################## # COMMAND ################################################################## # Get exposed capabilities in current state. retval = self._pa_client.get_capabilities() # Validate capabilities of state COMMAND agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) agt_cmds_command = [ PlatformAgentEvent.GO_INACTIVE, PlatformAgentEvent.RESET, PlatformAgentEvent.GET_METADATA, PlatformAgentEvent.GET_PORTS, PlatformAgentEvent.SET_UP_PORT, PlatformAgentEvent.TURN_ON_PORT, PlatformAgentEvent.TURN_OFF_PORT, PlatformAgentEvent.GET_SUBPLATFORM_IDS, PlatformAgentEvent.GET_RESOURCE_CAPABILITIES, PlatformAgentEvent.PING_RESOURCE, PlatformAgentEvent.GET_RESOURCE, PlatformAgentEvent.SET_RESOURCE, PlatformAgentEvent.START_EVENT_DISPATCH, PlatformAgentEvent.STOP_EVENT_DISPATCH, ] res_cmds_command = [] self.assertItemsEqual(agt_cmds, agt_cmds_command) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_command) self.assertItemsEqual(res_pars, res_pars_all) # Get exposed capabilities in all states as read from state COMMAND. retval = self._pa_client.get_capabilities(False) # Validate all capabilities as read from state COMMAND agt_cmds, agt_pars, res_cmds, res_pars = sort_caps(retval) self.assertItemsEqual(agt_cmds, agt_cmds_all) self.assertItemsEqual(agt_pars, agt_pars_all) self.assertItemsEqual(res_cmds, res_cmds_all) self.assertItemsEqual(res_pars, res_pars_all) self._go_inactive() self._reset() def test_go_active_and_run(self): # log.info("test_go_active_and_run starting. Default timeout=%s", TIMEOUT) log.info("test_capabilities starting. Default timeout=%i", CFG.endpoint.receive.timeout) self._ping_agent() # self._ping_resource() skipping this here while the timeout issue # on r2_light and other builds is investigated. self._initialize() self._go_active() self._run() self._ping_agent() self._ping_resource() self._get_metadata() self._get_ports() self._get_subplatform_ids() self._get_resource() self._set_resource() self._set_up_port() self._turn_on_port() self._start_event_dispatch() self._wait_for_a_data_sample() self._stop_event_dispatch() self._turn_off_port() self._go_inactive() self._reset()
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateRSNVel3DInstrument, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skip('under construction') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 180}}}) def test_activate_rsn_vel3d(self): log.info( "--------------------------------------------------------------------------------------------------------" ) # load_parameter_scenarios self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories= "ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='Vel3DMModel', description="Vel3DMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') vel3d_b_sample = StreamConfiguration( stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration( stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') RSN_VEL3D_01 = { 'DEV_ADDR': "10.180.80.6", 'DEV_PORT': 2101, 'DATA_PORT': 1026, 'CMD_PORT': 1025, 'PA_BINARY': "port_agent" } # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='Vel3DAgent', description="Vel3DAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations=[ raw_config, vel3d_b_sample, vel3d_b_engineering ]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='Vel3DDevice', description="Vel3DDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s ", instDevice_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance', description="Vel3DAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_sample', id_only=True) parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition( name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id) parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_engineering', id_only=True) parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition( name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample') sample_data_product_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id) log.debug('new dp_id = %s', sample_data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=sample_data_product_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=sample_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True) log.debug('sample_data_product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for sample_data_product = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('vel3d_b_sample', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering') eng_data_product_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id) log.debug('new dp_id = %s', eng_data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=eng_data_product_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=eng_data_product_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activate_rsn_vel3d Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) def check_state(label, desired_state): actual_state = self._ia_client.get_agent_state() log.debug("%s instrument agent is in state '%s'", label, actual_state) self.assertEqual(desired_state, actual_state) log.debug("test_activate_rsn_vel3d: got ia client %s", str(self._ia_client)) check_state("just-spawned", ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: initialize %s", str(retval)) check_state("initialized", ResourceAgentState.INACTIVE) log.debug("test_activate_rsn_vel3d Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return value from go_active %s", str(reply)) check_state("activated", ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending go_active command %s", str(state)) # cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: run %s", str(reply)) check_state("commanded", ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending run command %s", str(state)) # cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # gevent.sleep(5) # # cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state)) # # cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.STOPPED, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RESUME) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.IDLE, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RUN) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) log.debug("test_activate_rsn_vel3d: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return from reset %s", str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print())) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(sample_data_product_id) self.dpclient.delete_data_product(eng_data_product_id) self.dpclient.delete_data_product(data_product_id2)
class Test2CAA(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ """ ################################################################### # Internal parameters and container. ################################################################### # Internal parameters. self._terrestrial_platform_id = 'terrestrial_id' self._remote_platform_id = 'remote_id' self._resource_id = 'fake_id' self._xs_name = 'remote1' self._terrestrial_svc_name = 'terrestrial_endpoint' self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name self._remote_svc_name = 'remote_endpoint' self._remote_listen_name = self._remote_svc_name + self._xs_name self._remote_port = 0 self._terrestrial_port = 0 self._te_client = None self._re_client = None self._remote_pid = None self._terrestrial_pid = None # Async test results. self._no_requests = 10 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._telem_evts = [] self._queue_mod_evts = [] self._cmd_tx_evts = [] self._results_recv = {} self._requests_sent = {} self._done_telem_evt = AsyncResult() self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ################################################################### # Start endpoints, agent. ################################################################### self._start_terrestrial() self._start_remote() self._start_agent() ################################################################### # Assign client ports. # This is primarily for test purposes as the IP config in # deployment will be fixed in advance. ################################################################### self.te_client.set_client_port(self._remote_port) check_port = self.te_client.get_client_port() log.debug('Terrestrial client port is: %i', check_port) self.re_client.set_client_port(self._terrestrial_port) check_port = self.re_client.get_client_port() log.debug('Remote client port is: %i', check_port) ################################################################### # Start the event publisher and subscribers. # Used to send fake agent telemetry publications to the endpoints, # and to receive endpoint publications. ################################################################### self._event_publisher = EventPublisher() # Start the event subscriber for remote namespace platform events. # This event could be changed to RemoteNamespaceEvent. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber for remote resource events. self._resource_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=IA_RESOURCE_ID, callback=self.consume_event) self._resource_result_subscriber.start() self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._resource_result_subscriber.stop) # Start the result subscriber for remote service events. self._service_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin='resource_registry' + 'remote1', callback=self.consume_event) self._service_result_subscriber.start() self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._service_result_subscriber.stop) # Start the result subscriber for fake resource results. self._fake_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._fake_result_subscriber.start() self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._fake_result_subscriber.stop) ################################################################### # Start/stop helpers. ################################################################### def _start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port } self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) def _start_terrestrial(self): """ """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._remote_port, 'this_port' : self._terrestrial_port, 'platform_resource_id' : self._terrestrial_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port) def _start_remote(self): """ """ # Create agent config. remote_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._terrestrial_port, 'this_port' : self._remote_port, 'platform_resource_id' : self._remote_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._remote_listen_name } } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') self._remote_pid = self._container_client.spawn_process( name=self._remote_listen_name, module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=remote_endpoint_config) log.debug('Remote endpoint pid=%s.', str(self._remote_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient( process=FakeProcess(), to_name=self._remote_listen_name) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._remote_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._remote_port) ################################################################### # Telemetry publications to start/top endpoint. # (Normally be published by appropriate platform agents.) ################################################################### def terrestrial_link_up(self): """ Publish telemetry available to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.AVAILABLE) def terrestrial_link_down(self): """ Publish telemetry unavailable to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.UNAVAILABLE) def remote_link_up(self): """ Publish telemetry available to the remote endpoint. """ # Publish a link up event to be caught by the remote endpoint. log.debug('Publishing remote telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.AVAILABLE) def remote_link_down(self): """ Publish telemetry unavailable to the remote endpoint. """ # Publish a link down event to be caught by the remote endpoint. log.debug('Publishing remote telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.UNAVAILABLE) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evt.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evt.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evt.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_cmd_evt.set() ################################################################### # Misc helpers. ################################################################### def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id='fake_id', command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd ################################################################### # Tests. ################################################################### def test_queued_fake(self): """ test_queued_fake Test fake resource commands queued prior to linkup. """ # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) # The following error occurs when queue persistence is enabled. # Need to verify correct solution to enable persistent queues. """ 2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8 Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run result = self._run(*self.args, **self.kwargs) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop self._callback(cmd_result) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete self._client.enqueue(result) AttributeError: 'NoneType' object has no attribute 'enqueue' <Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError 2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7 2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6 2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5 2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4 2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3 2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1 2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=# 2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984 2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB ====================================================================== ERROR: test_process_online ---------------------------------------------------------------------- Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner return f(*args, **kw) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online cmd = self.te_client.enqueue_command(cmd) File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request retval, headers = e.send(msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send return self._invite_and_send(to_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send return self._send(to_role, to_role_name, msg, header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send return self._end_point_unit._message_send(msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send return self._send(_msg, _header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send raise ex Conflict: 409 - Object not based on most current version """ def test_process_online(self): """ test_process_online Test fake resource commands queued while link is up. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mods, command transmissions and results. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test fake resource commands queued prior to linkup. Delay remote side linkup substantially to test terrestrial behavior when remote server not initially available. """ for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Actually stop the remote process, since the server is # available even if it hasn't linked up yet and the test is running # in the same container. The test will remember the appropriate # remote port numbers. self._container_client.terminate_process(self._remote_pid) self.terrestrial_link_up() gevent.sleep(10) # Restart remote side and publish remote link up. self._start_remote() self.remote_link_up() # Block for transmission and result events. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_resource_commands(self): """ test_resource_commands """ # Set up to verify the two commands queued. self._no_requests = 2 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests # Use IA client to verify IA. state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # Create and enqueue commands. state_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}) state_cmd = self.te_client.enqueue_command(state_cmd) self._requests_sent[state_cmd.command_id] = state_cmd ping_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='ping_agent', args=[], kwargs={}) ping_cmd = self.te_client.enqueue_command(ping_cmd) self._requests_sent[ping_cmd.command_id] = ping_cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) self.assertEqual(self._results_recv[state_cmd.command_id].result, ResourceAgentState.UNINITIALIZED) self.assertIn('ping from InstrumentAgent', self._results_recv[ping_cmd.command_id].result) def test_service_command_sequence(self): """ test_service_commands """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='create', args=[obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a tuple of strings. #{'result': ['ad183ff26bae4f329ddd85fd69d160a9', #'1-00a308c45fff459c7cda1db9a7314de6'], #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a user info object with the name set. #{'lcstate': 'DEPLOYED_AVAILABLE', #'_rev': '1-851f067bac3c34b2238c0188b3340d0f', #'description': '', #'ts_updated': '1349213207638', #'type_': 'UserInfo', #'contact': <interface.objects.ContactInformation object at 0x10d7df590>, #'_id': '27832d93f4cd4535a75ac75c06e00a7e', #'ts_created': '1349213207638', #'variables': [{'name': '', 'value': ''}], #'name': 'some_name'} self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. read_obj.name = 'some_other_name' cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='update', args=[read_obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='delete', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish link down events. self.terrestrial_link_down() self.remote_link_down()
class TestAgentPersistence(IonIntegrationTestCase): """ """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config. """ self._ia_client = None log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Create agent config. self._agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True, 'forget_past' : False, 'enable_persistence' : True } self._ia_client = None self._ia_pid = '8989' self.addCleanup(self._verify_agent_reset) self.addCleanup(self.container.state_repository.put_state, self._ia_pid, {}) ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config ############################################################################### # Agent start stop helpers. ############################################################################### def _start_agent(self): """ """ container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=self._agent_config, process_id=self._ia_pid) log.info('Started instrument agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got instrument agent client %s.', str(self._ia_client)) def _stop_agent(self): """ """ if self._ia_pid: container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) container_client.terminate_process(self._ia_pid) if self._ia_client: self._ia_client = None def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) self._ia_client = None ############################################################################### # Tests. ############################################################################### def test_agent_config_persistence(self): """ test_agent_config_persistence Test that agent parameter configuration is persisted between running instances. """ # Start the agent. self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Confirm the default agent parameters. #{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}} retval = self._ia_client.get_agent(['streams'])['streams'] self.assertIn('raw', retval.keys()) self.assertIn('parsed', retval.keys()) #{'pubrate': {'raw': 0, 'parsed': 0}} retval = self._ia_client.get_agent(['pubrate'])['pubrate'] self.assertIn('raw', retval.keys()) self.assertIn('parsed', retval.keys()) self.assertEqual(retval['raw'], 0) self.assertEqual(retval['parsed'], 0) #{'alerts': []} retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertEqual(retval, []) # Define a few new parameters and set them. # Confirm they are set. alert_def_1 = { 'name' : 'current_warning_interval', 'stream_name' : 'parsed', 'message' : 'Current is below normal range.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 10.0, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def_2 = { 'name' : 'temp_alarm_interval', 'stream_name' : 'parsed', 'message' : 'Temperatoure is critical.', 'alert_type' : StreamAlertType.ALARM, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 20.0, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def3 = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'message' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'value_id' : None, 'time_delta' : 180, 'alert_class' : 'LateDataAlert' } orig_alerts = [alert_def_1,alert_def_2, alert_def3] pubrate = { 'parsed' : 10, 'raw' : 20 } params = { 'alerts' : orig_alerts, 'pubrate' : pubrate } # Set the new agent params and confirm. self._ia_client.set_agent(params) params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) pubrate = retval['pubrate'] alerts = retval['alerts'] self.assertIn('raw', pubrate.keys()) self.assertIn('parsed', pubrate.keys()) self.assertEqual(pubrate['parsed'], 10) self.assertEqual(pubrate['raw'], 20) count = 0 for x in alerts: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEqual(count, 3) # Now stop and restart the agent. self._stop_agent() gevent.sleep(5) self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Confirm the persisted parameters. params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) pubrate = retval['pubrate'] alerts = retval['alerts'] self.assertIn('raw', pubrate.keys()) self.assertIn('parsed', pubrate.keys()) self.assertEqual(pubrate['parsed'], 10) self.assertEqual(pubrate['raw'], 20) count = 0 for x in alerts: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEqual(count, 3) def test_agent_state_persistence(self): """ test_agent_state_persistence Verify that agents can be restored to their prior running state. """ self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Acquire sample returns a string, not a particle. The particle # is created by the data handler though. cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = self._ia_client.execute_resource(cmd) # Now stop and restart the agent. self._stop_agent() gevent.sleep(3) self._start_agent() state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) # Now stop and restart the agent. self._stop_agent() gevent.sleep(3) self._start_agent() state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name = '', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name= 'notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) ) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_instrument_device_id, notification.origin) self.assertEqual("instrument", notification.origin_type) self.assertEqual('ResourceLifecycleEvent', notification.event_type) def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedFloatValue) self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) ) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_data_product_id, notification.origin) self.assertEqual("data product", notification.origin_type) self.assertEqual('DetectionEvent', notification.event_type) @attr('LOCOINT') #@unittest.skip('refactoring') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 90}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s' , data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s" , str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.INACTIVE, state) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.STOPPED, state) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) for i in xrange(10): monitor = DatasetMonitor(dataset_id=self.parsed_dataset) self._ia_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)) if not monitor.wait(): raise AssertionError('Failed on the %ith granule' % i) monitor.stop() # cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) # for i in xrange(10): # retval = self._ia_client.execute_resource(cmd) # log.debug("test_activateInstrumentSample: return from sample %s" , str(retval)) log.debug( "test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s" , str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) ) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] all_raw = "".join(raw_vals) # look for 't' entered after a prompt -- ">t" t_commands = all_raw.count(">t") if 10 != t_commands: log.error("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r]) self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands) else: log.debug("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r]) replay_data_parsed = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data_parsed, Granule) rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print()) ) self.assertIn('temp', rdt_parsed) temp_vals = rdt_parsed['temp'] pressure_vals = rdt_parsed['pressure'] if 10 != len(temp_vals): log.error("%s temp_vals: %s", len(temp_vals), temp_vals) self.fail("Expected 10 temp_vals, got %s" % len(temp_vals)) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests.value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
def test_platform_with_instrument_streaming(self): # # The following is with just a single platform and the single # instrument "SBE37_SIM_08", which corresponds to the one on port 4008. # #load the paramaters and the param dicts necesssary for the VEL3D log.debug( "load params------------------------------------------------------------------------------") self._load_params() log.debug( " _register_oms_listener------------------------------------------------------------------------------") self._register_oms_listener() #create the instrument device/agent/mode log.debug( "---------- create_instrument_resources ----------" ) self._create_instrument_resources() #create the platform device, agent and instance log.debug( "---------- create_platform_configuration ----------" ) self._create_platform_configuration('LPJBox_CI_Ben_Hall') self.rrclient.create_association(subject=self.platform_device_id, predicate=PRED.hasDevice, object=self.instrument_device_id) log.debug( "---------- start_platform ----------" ) self._start_platform() self.addCleanup(self._stop_platform) # get everything in command mode: self._ping_agent() log.debug( " ---------- initialize ----------" ) self._initialize() _ia_client = ResourceAgentClient(self.instrument_device_id, process=FakeProcess()) state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug( " ---------- go_active ----------" ) self._go_active() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug( "---------- run ----------" ) self._run() gevent.sleep(2) log.debug( " ---------- _start_resource_monitoring ----------" ) self._start_resource_monitoring() gevent.sleep(2) # # # verify the instrument is command state: # state = ia_client.get_agent_state() # log.debug(" TestPlatformInstrument get_agent_state: %s", state) # self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring log.debug( " ---------- _stop_resource_monitoring ----------" ) self._stop_resource_monitoring() gevent.sleep(2) log.debug( " ---------- go_inactive ----------" ) self._go_inactive() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) self._reset() self._shutdown()
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin): """ A base class with several conveniences supporting specific platform agent integration tests, see: - ion/agents/platform/test/test_platform_agent_with_rsn.py - ion/services/sa/observatory/test/test_platform_launch.py The platform IDs used here are organized as follows: Node1D -> MJ01C -> LJ01D where -> goes from parent platform to child platform. This is a subset of the whole topology defined in the simulated platform network (network.yml), which in turn is used by the RSN OMS simulator. - 'LJ01D' is the root platform used in test_single_platform - 'Node1D' is the root platform used in test_hierarchy Methods are provided to construct specific platform topologies, but subclasses decide which to use. """ @classmethod def setUpClass(cls): HelperTestMixin.setUpClass() def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient() self.IDS = IdentityManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.org_id = self.RR2.create(any_old(RT.Org)) log.debug("Org created: %s", self.org_id) # Create InstrumentModel # TODO create multiple models as needed; for the moment assuming all # used instruments are the same model here. instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") self.instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', self.instModel_id) # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri']) self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms) CIOMSClientFactory.destroy_instance(rsn_oms) if log.isEnabledFor(logging.TRACE): # show serialized version for the network definition: network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition) log.trace("NetworkDefinition serialization:\n%s", network_definition_ser) # set attributes for the platforms: self._platform_attributes = {} for platform_id in self._network_definition.pnodes: pnode = self._network_definition.pnodes[platform_id] dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues()) self._platform_attributes[platform_id] = dic log.trace("_platform_attributes: %s", self._platform_attributes) # set ports for the platforms: self._platform_ports = {} for platform_id in self._network_definition.pnodes: pnode = self._network_definition.pnodes[platform_id] dic = {} for port_id, port in pnode.ports.iteritems(): dic[port_id] = dict(port_id=port_id, network=port.network) self._platform_ports[platform_id] = dic log.trace("_platform_ports: %s", self._platform_attributes) self._async_data_result = AsyncResult() self._data_subscribers = [] self._samples_received = [] self.addCleanup(self._stop_data_subscribers) self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber(sub_type="platform_event") # instruments that have been set up: instr_key: i_obj self._setup_instruments = {} ################################################################# # data subscribers handling ################################################################# def _start_data_subscriber(self, stream_name, stream_id): """ Starts data subscriber for the given stream_name and stream_config """ def consume_data(message, stream_route, stream_id): # A callback for processing subscribed-to data. log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r', str(message), stream_name, stream_id) self._samples_received.append(message) self._async_data_result.set() log.info('_start_data_subscriber stream_name=%r stream_id=%r', stream_name, stream_id) # Create subscription for the stream exchange_name = '%s_queue' % stream_name self.container.ex_manager.create_xn_queue(exchange_name).purge() sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id]) self.PSC.activate_subscription(sub_id) sub.subscription_id = sub_id def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ try: for sub in self._data_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._data_subscribers = [] ################################################################# # event subscribers handling ################################################################# def _start_event_subscriber(self, event_type="DeviceEvent", sub_type=None, count=0): """ Starts event subscriber for events of given event_type ("DeviceEvent" by default) and given sub_type ("platform_event" by default). """ def consume_event(evt, *args, **kwargs): # A callback for consuming events. log.info('Event subscriber received evt: %s.', str(evt)) self._events_received.append(evt) if count == 0: self._async_event_result.set(evt) elif count == len(self._events_received): self._async_event_result.set() sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event) sub.start() log.info("registered event subscriber for event_type=%r, sub_type=%r, count=%d", event_type, sub_type, count) self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) def _stop_event_subscribers(self): """ Stops the event subscribers on cleanup. """ try: for sub in self._event_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._event_subscribers = [] ################################################################# # config supporting methods ################################################################# def _get_platform_stream_configs(self): """ This method is an adaptation of get_streamConfigs in test_driver_egg.py """ return [ StreamConfiguration(stream_name='parsed', parameter_dictionary_name='platform_eng_parsed', records_per_granule=2, granule_publish_rate=5) # TODO include a "raw" stream? ] def _get_instrument_stream_configs(self): """ configs copied from test_activate_instrument.py """ return [ StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5), StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5) ] def _verify_child_config(self, config, device_id, is_platform): for key in required_config_keys: self.assertIn(key, config) if is_platform: self.assertEqual(RT.PlatformDevice, config['device_type']) for key in DVR_CONFIG.iterkeys(): self.assertIn(key, config['driver_config']) for key in ['startup_config']: self.assertEqual({}, config[key]) else: self.assertEqual(RT.InstrumentDevice, config['device_type']) for key in ['children']: self.assertEqual({}, config[key]) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) def _verify_parent_config(self, config, parent_device_id, child_device_id, is_platform): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(RT.PlatformDevice, config['device_type']) for key in DVR_CONFIG.iterkeys(): self.assertIn(key, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) self._verify_child_config(config['children'][child_device_id], child_device_id, is_platform) def _create_platform_configuration(self, platform_id, parent_platform_id=None): """ This method is an adaptation of test_agent_instance_config in test_instrument_management_service_integration.py @param platform_id @param parent_platform_id @return a DotDict with various of the constructed elements associated to the platform. """ tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # # TODO will each platform have its own param dictionary? # param_dict_name = 'platform_eng_parsed' parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name( param_dict_name, id_only=True) self.parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} driver_config = copy.deepcopy(DVR_CONFIG) driver_config['attributes'] = self._platform_attributes[platform_id] driver_config['ports'] = self._platform_ports[platform_id] log.debug("driver_config: %s", driver_config) # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, { 'driver_config': driver_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation platform_agent_obj = any_old(RT.PlatformAgent, { "stream_configurations": self._get_platform_stream_configs(), 'driver_module': DVR_MOD, 'driver_class': DVR_CLS}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.delete_data_product, dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id) ####################################### # dataset log.debug('data product = %s', dp_id) stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True) log.debug('Data product stream_ids = %s', stream_ids) stream_id = stream_ids[0] # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) ####################################### return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id log.debug("Making the structure for a platform agent") # TODO Note: the 'platform_config' entry is a mechanism that the # platform agent expects to know the platform_id and parent_platform_id. # Determine how to finally indicate this info. platform_config = { 'platform_id': platform_id, 'parent_platform_id': parent_platform_id, } child_agent_config = { 'platform_config': platform_config } platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \ _make_platform_agent_structure(child_agent_config) platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) self.platform_device_parent_id = platform_device_child_id p_obj = DotDict() p_obj.platform_id = platform_id p_obj.parent_platform_id = parent_platform_id p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj p_obj.platform_device_id = platform_device_child_id p_obj.platform_agent_instance_id = platform_agent_instance_child_id p_obj.stream_id = stream_id p_obj.pid = None # known when process launched return p_obj def _create_platform(self, platform_id, parent_platform_id=None): """ The main method to create a platform configuration and do other preparations for a given platform. """ p_obj = self._create_platform_configuration(platform_id, parent_platform_id) # start corresponding data subscriber: self._start_data_subscriber(p_obj.platform_agent_instance_id, p_obj.stream_id) return p_obj ################################################################# # platform child-parent linking ################################################################# def _assign_child_to_parent(self, p_child, p_parent): log.debug("assigning child platform %r to parent %r", p_child.platform_id, p_parent.platform_id) self.RR2.assign_platform_device_to_platform_device_with_has_device(p_child.platform_device_id, p_parent.platform_device_id) child_device_ids = self.RR2.find_platform_device_ids_of_device_using_has_device(p_parent.platform_device_id) self.assertNotEqual(0, len(child_device_ids)) ################################################################# # instrument ################################################################# def _set_up_pre_environment_for_instrument(self, instr_info): """ Based on test_instrument_agent.py Basically, this method launches a port agent and then completes the instrument driver configuration used to properly set up a particular instrument agent. @param instr_info A value in instruments_dict @return instrument_driver_config """ import sys from ion.agents.instrument.driver_process import DriverProcessType from ion.agents.instrument.driver_process import ZMQEggDriverProcess # A seabird driver. DRV_URI = SBE37_EGG DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver' DRV_CLS = 'SBE37Driver' WORK_DIR = '/tmp/' DELIM = ['<<', '>>'] instrument_driver_config = { 'dvr_egg' : DRV_URI, 'dvr_mod' : DRV_MOD, 'dvr_cls' : DRV_CLS, 'workdir' : WORK_DIR, 'process_type' : None } # Launch from egg or a local MI repo. LAUNCH_FROM_EGG=True if LAUNCH_FROM_EGG: # Dynamically load the egg into the test path launcher = ZMQEggDriverProcess(instrument_driver_config) egg = launcher._get_egg(DRV_URI) if not egg in sys.path: sys.path.insert(0, egg) instrument_driver_config['process_type'] = (DriverProcessType.EGG,) else: mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo' if not mi_repo in sys.path: sys.path.insert(0, mi_repo) instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,) instrument_driver_config['mi_repo'] = mi_repo DEV_ADDR = instr_info['DEV_ADDR'] DEV_PORT = instr_info['DEV_PORT'] DATA_PORT = instr_info['DATA_PORT'] CMD_PORT = instr_info['CMD_PORT'] PA_BINARY = instr_info['PA_BINARY'] support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = support.start_pagent() log.info('Port agent started at port %i', port) self.addCleanup(support.stop_pagent) # Configure instrument driver to use port agent port number. instrument_driver_config['comms_config'] = { 'addr': 'localhost', 'port': port, 'cmd_port': CMD_PORT } return instrument_driver_config def _make_instrument_agent_structure(self, instr_key, org_obj, agent_config=None): if None is agent_config: agent_config = {} instr_info = instruments_dict[instr_key] # initially adapted from test_activate_instrument:test_activateInstrumentSample # agent creation instrument_agent_obj = IonObject(RT.InstrumentAgent, name='agent007_%s' % instr_key, description="SBE37IMAgent_%s" % instr_key, driver_uri=SBE37_EGG, stream_configurations=self._get_instrument_stream_configs()) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) log.debug('new InstrumentAgent id = %s', instrument_agent_id) self.IMS.assign_instrument_model_to_instrument_agent(self.instModel_id, instrument_agent_id) # device creation instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice_%s' % instr_key, description="SBE37IMDevice_%s" % instr_key, serial_number="12345") instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(self.instModel_id, instrument_device_id) log.debug("new InstrumentDevice id = %s ", instrument_device_id) #Create stream alarms temp_alert_def = { 'name' : 'temperature_warning_interval', 'stream_name' : 'parsed', 'message' : 'Temperature is below the normal range of 50.0 and above.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : 50.0, 'lower_rel_op' : '<', 'alert_class' : 'IntervalAlert' } late_data_alert_def = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'message' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'value_id' : None, 'time_delta' : 2, 'alert_class' : 'LateDataAlert' } instrument_driver_config = self._set_up_pre_environment_for_instrument(instr_info) port_agent_config = { 'device_addr': instr_info['DEV_ADDR'], 'device_port': instr_info['DEV_PORT'], 'data_port': instr_info['DATA_PORT'], 'command_port': instr_info['CMD_PORT'], 'binary_path': instr_info['PA_BINARY'], 'process_type': PortAgentProcessType.UNIX, 'port_agent_addr': 'localhost', 'log_level': 5, 'type': PortAgentType.ETHERNET } # instance creation instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance_%s' % instr_key, description="SBE37IMAgentInstance_%s" % instr_key, driver_config=instrument_driver_config, port_agent_config=port_agent_config, alerts=[temp_alert_def, late_data_alert_def]) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # data products tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(org_obj) # parsed: parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='ctd_parsed', parameter_dictionary_id=parsed_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the parsed data for %s' % instr_key, description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) self.addCleanup(self.DP.delete_data_product, data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) # raw: raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='ctd_raw', parameter_dictionary_id=raw_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data for %s' % instr_key, description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) self.addCleanup(self.DP.delete_data_product, data_product_id2) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id2) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) i_obj = DotDict() i_obj.instrument_agent_id = instrument_agent_id i_obj.instrument_device_id = instrument_device_id i_obj.instrument_agent_instance_id = instrument_agent_instance_id i_obj.org_obj = org_obj log.debug("KK CREATED I_obj: %s", i_obj) return i_obj def _create_instrument(self, instr_key): """ The main method to create an instrument configuration. @param instr_key A key in instruments_dict @return instrument_driver_config """ self.assertIn(instr_key, instruments_dict) self.assertNotIn(instr_key, self._setup_instruments) instr_info = instruments_dict[instr_key] log.debug("_create_instrument: creating instrument %r: %s", instr_key, instr_info) org_obj = any_old(RT.Org) log.debug("making the structure for an instrument agent") i_obj = self._make_instrument_agent_structure(instr_key, org_obj) self._setup_instruments[instr_key] = i_obj log.debug("_create_instrument: created instrument %r", instr_key) return i_obj def _get_instrument(self, instr_key): """ Gets the i_obj constructed by _create_instrument(instr_key). """ self.assertIn(instr_key, self._setup_instruments) i_obj = self._setup_instruments[instr_key] return i_obj ################################################################# # instrument-platform linking ################################################################# def _assign_instrument_to_platform(self, i_obj, p_obj): log.debug("assigning instrument %r to platform %r", i_obj.instrument_agent_instance_id, p_obj.platform_id) self.RR2.assign_instrument_device_to_platform_device_with_has_device( i_obj.instrument_device_id, p_obj.platform_device_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device_using_has_device(p_obj.platform_device_id) self.assertNotEqual(0, len(child_device_ids)) ################################################################# # some platform topologies ################################################################# def _create_single_platform(self): """ Creates and prepares a platform corresponding to the platform ID 'LJ01D', which is a leaf in the simulated network. """ p_root = self._create_platform('LJ01D') return p_root def _create_small_hierarchy(self): """ Creates a small platform network consisting of 3 platforms as follows: Node1D -> MJ01C -> LJ01D where -> goes from parent to child. """ p_root = self._create_platform('Node1D') p_child = self._create_platform('MJ01C', parent_platform_id='Node1D') p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C') self._assign_child_to_parent(p_child, p_root) self._assign_child_to_parent(p_grandchild, p_child) return p_root def _create_hierarchy(self, platform_id, p_objs, parent_obj=None): """ Creates a hierarchy of platforms rooted at the given platform. @param platform_id ID of the root platform at this level @param p_objs dict to be updated with (platform_id: p_obj) mappings @param parent_obj platform object of the parent, if any @return platform object for the created root. """ # create the object to be returned: p_obj = self._create_platform(platform_id) # update (platform_id: p_obj) dict: p_objs[platform_id] = p_obj # recursively create child platforms: pnode = self._network_definition.pnodes[platform_id] for sub_platform_id in pnode.subplatforms: self._create_hierarchy(sub_platform_id, p_objs, p_obj) if parent_obj: self._assign_child_to_parent(p_obj, parent_obj) return p_obj def _set_up_single_platform_with_some_instruments(self, instr_keys): """ Sets up single platform with some instruments @param instr_keys Keys of the instruments to be assigned. Must be keys in instruments_dict in base_test_platform_agent_with_rsn @return p_root for subsequent termination """ for instr_key in instr_keys: self.assertIn(instr_key, instruments_dict) p_root = self._create_single_platform() # create and assign instruments: for instr_key in instr_keys: i_obj = self._create_instrument(instr_key) self._assign_instrument_to_platform(i_obj, p_root) return p_root def _set_up_platform_hierarchy_with_some_instruments(self, instr_keys): """ Sets up a multiple-level platform hierarchy with instruments associated to some of the platforms. The platform hierarchy corresponds to the sub-network in the simulated topology rooted at 'Node1B', which at time of writing looks like this: Node1B Node1C Node1D MJ01C LJ01D LV01C PC01B SC01B SF01B LJ01C LV01B LJ01B MJ01B In DEBUG logging level for the platform agent, files like the following are generated under logs/: platform_CFG_received_Node1B.txt platform_CFG_received_MJ01C.txt platform_CFG_received_LJ01D.txt @param instr_keys Keys of the instruments to be assigned. Must be keys in instruments_dict in base_test_platform_agent_with_rsn @return p_root for subsequent termination """ for instr_key in instr_keys: self.assertIn(instr_key, instruments_dict) ##################################### # create platform hierarchy ##################################### log.info("will create platform hierarchy ...") start_time = time.time() root_platform_id = 'Node1B' p_objs = {} p_root = self._create_hierarchy(root_platform_id, p_objs) log.info("platform hierarchy built. Took %.3f secs. " "Root platform=%r, number of platforms=%d: %s", time.time() - start_time, root_platform_id, len(p_objs), p_objs.keys()) self.assertIn(root_platform_id, p_objs) self.assertEquals(13, len(p_objs)) ##################################### # create the indicated instruments ##################################### log.info("will create %d instruments: %s", len(instr_keys), instr_keys) start_time = time.time() i_objs = [] for instr_key in instr_keys: i_obj = self._create_instrument(instr_key) i_objs.append(i_obj) log.debug("instrument created = %r (%s)", i_obj.instrument_agent_instance_id, instr_key) log.info("%d instruments created. Took %.3f secs.", len(instr_keys), time.time() - start_time) ##################################### # assign the instruments ##################################### log.info("will assign instruments ...") start_time = time.time() plats_to_assign_instrs = [ 'LJ01D', 'SF01B', 'LJ01B', 'MJ01B', # leaves 'MJ01C', 'Node1D', 'LV01B', 'Node1C' # intermediate ] # assign one available instrument to a platform; # the assignments are arbitrary. num_assigns = min(len(instr_keys), len(plats_to_assign_instrs)) for ii in range(num_assigns): platform_id = plats_to_assign_instrs[ii] self.assertIn(platform_id, p_objs) p_obj = p_objs[platform_id] i_obj = i_objs[ii] self._assign_instrument_to_platform(i_obj, p_obj) log.debug("instrument %r (%s) assigned to platform %r", i_obj.instrument_agent_instance_id, instr_keys[ii], platform_id) log.info("%d instruments assigned. Took %.3f secs.", num_assigns, time.time() - start_time) return p_root ################################################################# # start / stop platform ################################################################# def _start_platform(self, p_obj): agent_instance_id = p_obj.platform_agent_instance_id log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id) p_obj.pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", p_obj.pid) #wait for start agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id) gate = ProcessStateGate(self.PDC.read_process, agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) def _stop_platform(self, p_obj): try: self.IMS.stop_platform_agent_instance(p_obj.platform_agent_instance_id) except: if log.isEnabledFor(logging.TRACE): log.exception( "platform_id=%r: Exception in IMS.stop_platform_agent_instance with " "platform_agent_instance_id = %r", p_obj.platform_id, p_obj.platform_agent_instance_id) else: log.warn( "platform_id=%r: Exception in IMS.stop_platform_agent_instance with " "platform_agent_instance_id = %r. Perhaps already dead.", p_obj.platform_id, p_obj.platform_agent_instance_id) ################################################################# # misc convenience methods ################################################################# def _create_resource_agent_client(self, resource_id): client = ResourceAgentClient(resource_id, process=FakeProcess()) return client def _get_state(self): state = self._pa_client.get_agent_state() return state def _assert_state(self, state): self.assertEquals(self._get_state(), state) def _execute_agent(self, cmd): log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs) time_start = time.time() #retval = self._pa_client.execute_agent(cmd, timeout=timeout) retval = self._pa_client.execute_agent(cmd) elapsed_time = time.time() - time_start log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval)) return retval ################################################################# # commands that concrete tests can call ################################################################# def _ping_agent(self): retval = self._pa_client.ping_agent() self.assertIsInstance(retval, str) def _ping_resource(self): cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE) if self._get_state() == PlatformAgentState.UNINITIALIZED: # should get ServerError: "Command not handled in current state" with self.assertRaises(ServerError): self._pa_client.execute_agent(cmd) else: # In all other states the command should be accepted: retval = self._execute_agent(cmd) self.assertEquals("PONG", retval.result) def _get_metadata(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_METADATA = %s", md) def _get_ports(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_PORTS = %s", md) def _initialize(self): self._assert_state(PlatformAgentState.UNINITIALIZED) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _go_active(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _run(self): cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _start_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.MONITORING) def _wait_for_a_data_sample(self): log.info("waiting for reception of a data sample...") # just wait for at least one -- see consume_data self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("Received samples: %s", len(self._samples_received)) def _wait_for_external_event(self): log.info("waiting for reception of an external event...") # just wait for at least one -- see consume_event self._async_event_result.get(timeout=EVENT_TIMEOUT) self.assertTrue(len(self._events_received) >= 1) log.info("Received events: %s", len(self._events_received)) def _stop_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _pause(self): cmd = AgentCommand(command=PlatformAgentEvent.PAUSE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.STOPPED) def _resume(self): cmd = AgentCommand(command=PlatformAgentEvent.RESUME) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _clear(self): cmd = AgentCommand(command=PlatformAgentEvent.CLEAR) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _go_inactive(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _reset(self): cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.UNINITIALIZED) def _check_sync(self): cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC) retval = self._execute_agent(cmd) log.info("CHECK_SYNC result: %s", retval.result) self.assertTrue(retval.result is not None) self.assertEquals(retval.result[0:3], "OK:") return retval.result def _stream_instruments(self): from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter for instrument in self._setup_instruments.itervalues(): # instruments that have been set up: instr_key: i_obj # Start a resource agent client to talk with the instrument agent. _ia_client = self._create_resource_agent_client(instrument.instrument_device_id) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = _ia_client.execute_resource(cmd) log.debug('_stream_instruments retval: %s', retval) return def _idle_instruments(self): from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter for instrument in self._setup_instruments.itervalues(): # instruments that have been set up: instr_key: i_obj # Start a resource agent client to talk with the instrument agent. _ia_client = self._create_resource_agent_client(instrument.instrument_device_id) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) with self.assertRaises(Conflict): retval = _ia_client.execute_resource(cmd) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = _ia_client.execute_agent(cmd) state = _ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) return
class Test2CAA(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ """ ################################################################### # Internal parameters and container. ################################################################### # Internal parameters. self._terrestrial_platform_id = 'terrestrial_id' self._remote_platform_id = 'remote_id' self._resource_id = 'fake_id' self._xs_name = 'remote1' self._terrestrial_svc_name = 'terrestrial_endpoint' self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name self._remote_svc_name = 'remote_endpoint' self._remote_listen_name = self._remote_svc_name + self._xs_name self._remote_port = 0 self._terrestrial_port = 0 self._te_client = None self._re_client = None self._remote_pid = None self._terrestrial_pid = None # Async test results. self._no_requests = 10 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._telem_evts = [] self._queue_mod_evts = [] self._cmd_tx_evts = [] self._results_recv = {} self._requests_sent = {} self._done_telem_evt = AsyncResult() self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ################################################################### # Start endpoints, agent. ################################################################### self._start_terrestrial() self._start_remote() self._start_agent() ################################################################### # Assign client ports. # This is primarily for test purposes as the IP config in # deployment will be fixed in advance. ################################################################### self.te_client.set_client_port(self._remote_port) check_port = self.te_client.get_client_port() log.debug('Terrestrial client port is: %i', check_port) self.re_client.set_client_port(self._terrestrial_port) check_port = self.re_client.get_client_port() log.debug('Remote client port is: %i', check_port) ################################################################### # Start the event publisher and subscribers. # Used to send fake agent telemetry publications to the endpoints, # and to receive endpoint publications. ################################################################### self._event_publisher = EventPublisher() # Start the event subscriber for remote namespace platform events. # This event could be changed to RemoteNamespaceEvent. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber for remote resource events. self._resource_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=IA_RESOURCE_ID, callback=self.consume_event) self._resource_result_subscriber.start() self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._resource_result_subscriber.stop) # Start the result subscriber for remote service events. self._service_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin='resource_registry' + 'remote1', callback=self.consume_event) self._service_result_subscriber.start() self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._service_result_subscriber.stop) # Start the result subscriber for fake resource results. self._fake_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._fake_result_subscriber.start() self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._fake_result_subscriber.stop) ################################################################### # Start/stop helpers. ################################################################### def _start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port } self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) def _start_terrestrial(self): """ """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._remote_port, 'this_port' : self._terrestrial_port, 'platform_resource_id' : self._terrestrial_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port) def _start_remote(self): """ """ # Create agent config. remote_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._terrestrial_port, 'this_port' : self._remote_port, 'platform_resource_id' : self._remote_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._remote_listen_name } } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') self._remote_pid = self._container_client.spawn_process( name=self._remote_listen_name, module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=remote_endpoint_config) log.debug('Remote endpoint pid=%s.', str(self._remote_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient( process=FakeProcess(), to_name=self._remote_listen_name) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._remote_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._remote_port) ################################################################### # Telemetry publications to start/top endpoint. # (Normally be published by appropriate platform agents.) ################################################################### def terrestrial_link_up(self): """ Publish telemetry available to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.AVAILABLE) def terrestrial_link_down(self): """ Publish telemetry unavailable to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.UNAVAILABLE) def remote_link_up(self): """ Publish telemetry available to the remote endpoint. """ # Publish a link up event to be caught by the remote endpoint. log.debug('Publishing remote telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.AVAILABLE) def remote_link_down(self): """ Publish telemetry unavailable to the remote endpoint. """ # Publish a link down event to be caught by the remote endpoint. log.debug('Publishing remote telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.UNAVAILABLE) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evt.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evt.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evt.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_cmd_evt.set() ################################################################### # Misc helpers. ################################################################### def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id='fake_id', command=cmdstr, args=['arg1', 23], kwargs={'worktime':3}) return cmd ################################################################### # Tests. ################################################################### def test_queued_fake(self): """ test_queued_fake Test fake resource commands queued prior to linkup. """ # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) # The following error occurs when queue persistence is enabled. # Need to verify correct solution to enable persistent queues. """ 2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8 Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run result = self._run(*self.args, **self.kwargs) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop self._callback(cmd_result) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete self._client.enqueue(result) AttributeError: 'NoneType' object has no attribute 'enqueue' <Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError 2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7 2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6 2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5 2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4 2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3 2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1 2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=# 2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984 2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB ====================================================================== ERROR: test_process_online ---------------------------------------------------------------------- Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner return f(*args, **kw) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online cmd = self.te_client.enqueue_command(cmd) File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request retval, headers = e.send(msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send return self._invite_and_send(to_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send return self._send(to_role, to_role_name, msg, header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send return self._end_point_unit._message_send(msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send return self._send(_msg, _header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send raise ex Conflict: 409 - Object not based on most current version """ def test_process_online(self): """ test_process_online Test fake resource commands queued while link is up. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mods, command transmissions and results. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test fake resource commands queued prior to linkup. Delay remote side linkup substantially to test terrestrial behavior when remote server not initially available. """ for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Actually stop the remote process, since the server is # available even if it hasn't linked up yet and the test is running # in the same container. The test will remember the appropriate # remote port numbers. self._container_client.terminate_process(self._remote_pid) self.terrestrial_link_up() gevent.sleep(10) # Restart remote side and publish remote link up. self._start_remote() self.remote_link_up() # Block for transmission and result events. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_resource_commands(self): """ test_resource_commands """ # Set up to verify the two commands queued. self._no_requests = 2 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests # Use IA client to verify IA. state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # Create and enqueue commands. state_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}) state_cmd = self.te_client.enqueue_command(state_cmd) self._requests_sent[state_cmd.command_id] = state_cmd ping_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='ping_agent', args=[], kwargs={}) ping_cmd = self.te_client.enqueue_command(ping_cmd) self._requests_sent[ping_cmd.command_id] = ping_cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) self.assertEqual(self._results_recv[state_cmd.command_id].result, ResourceAgentState.UNINITIALIZED) self.assertIn('ping from InstrumentAgent', self._results_recv[ping_cmd.command_id].result) def test_service_command_sequence(self): """ test_service_commands """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='create', args=[obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a tuple of strings. #{'result': ['ad183ff26bae4f329ddd85fd69d160a9', #'1-00a308c45fff459c7cda1db9a7314de6'], #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a user info object with the name set. #{'lcstate': 'DEPLOYED_AVAILABLE', #'_rev': '1-851f067bac3c34b2238c0188b3340d0f', #'description': '', #'ts_updated': '1349213207638', #'type_': 'UserInfo', #'contact': <interface.objects.ContactInformation object at 0x10d7df590>, #'_id': '27832d93f4cd4535a75ac75c06e00a7e', #'ts_created': '1349213207638', #'variables': [{'name': '', 'value': ''}], #'name': 'some_name'} self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. read_obj.name = 'some_other_name' cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='update', args=[read_obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='delete', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish link down events. self.terrestrial_link_down() self.remote_link_down()
class TestRemoteEndpoint(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ Start fake terrestrial components and add cleanup. Start terrestrial server and retrieve port. Set internal variables. Start container. Start deployment. Start container agent. Spawn remote endpoint process. Create remote endpoint client and retrieve remote server port. Create event publisher. """ self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close) self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close) self.addCleanup(self._terrestrial_server.stop) self.addCleanup(self._terrestrial_client.stop) self._other_port = self._terrestrial_server.start('*', 0) log.debug('Terrestrial server binding to *:%i', self._other_port) self._other_host = 'localhost' self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._no_telem_events = 0 self._done_evt = AsyncResult() self._done_telem_evts = AsyncResult() self._cmd_tx_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'platform_resource_id' : self._platform_resource_id } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') re_pid = container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(re_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient( process=FakeProcess(), to_name=re_pid) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._this_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._this_port) # Start the event publisher. self._event_publisher = EventPublisher() ###################################################################### # Helpers. ###################################################################### def on_link_up(self): """ Called by a test to simulate turning the link on. """ log.debug('Terrestrial client connecting to localhost:%i.', self._this_port) self._terrestrial_client.start('localhost', self._this_port) # Publish a link up event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status = TelemetryStatusType.AVAILABLE) def on_link_down(self): """ Called by a test to simulate turning the link off. """ self._terrestrial_client.stop() # Publish a link down event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status = TelemetryStatusType.UNAVAILABLE) def consume_req(self, res): """ Consume a terrestrial request setting async event when necessary. """ command_id = res['command_id'] self._results_recv[command_id] = res if len(self._results_recv) == self._no_requests: self._done_evt.set() def consume_ack(self, cmd): """ Consume terrestrial ack setting async event when necessary. """ self._requests_sent[cmd.command_id] = cmd if len(self._requests_sent) == self._no_requests: self._cmd_tx_evt.set() def terrestrial_server_close(self): """ Callback when terrestrial server closes. """ pass def terrestrial_client_close(self): """ Callback when terrestrial client closes. """ pass def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id=self._resource_id, command=cmdstr, args=['arg1', 23], kwargs={'worktime':3}, command_id = str(uuid.uuid4())) return cmd def start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) ###################################################################### # Tests. ###################################################################### def test_process_queued(self): """ test_process_queued Test that queued commands are forwarded to and handled by remote endpoint when link comes up. """ # Create and enqueue some requests. for i in range(self._no_requests): cmd = self.make_fake_command(i) self._terrestrial_client.enqueue(cmd) # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for all the enqueued commands to be acked. # Wait for all the responses to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() # Confirm the results match the commands sent. self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_process_online(self): """ test_process_online Test commands are forwarded and handled while link is up. """ # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for the link to be up. # The remote side does not publish public telemetry events # so we can't wait for that. gevent.sleep(1) # Create and enqueue some requests. for i in range(self._no_requests): cmd = self.make_fake_command(i) self._terrestrial_client.enqueue(cmd) # Wait for all the enqueued commands to be acked. # Wait for all the responses to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() # Confirm the results match the commands sent. self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_terrestrial_late(self): """ test_terrestrial_late Test queued commands are forwarded and handled by remote endpoint when terrestrial side is late to come up. """ # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for the link to be up. # The remote side does not publish public telemetry events # so we can't wait for that. gevent.sleep(1) # Manually stop the terrestrial endpoint. # This will cause it to be unavailable when commands are queued # to simulate stability during asynchronous wake ups. self._terrestrial_server.stop() self._terrestrial_client.stop() # Create and enqueue some requests. for i in range(self._no_requests): cmd = self.make_fake_command(i) self._terrestrial_client.enqueue(cmd) # Remote side awaits the terrestrial waking up. gevent.sleep(3) # Terrestrail endpoint eventually wakes up and starts transmitting. self._terrestrial_client.start('localhost', self._this_port) self._terrestrial_server.start('*', self._other_port) # Wait for all the enqueued commands to be acked. # Wait for all the responses to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() # Confirm the results match the commands sent. self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_service_commands(self): """ test_service_commands Test that real service commands are handled by the remote endpoint. """ # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='create', args=[obj], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = self._results_recv[cmd.command_id]['result'] # Confirm the results are valid. """ Result is a tuple of strings. {'result': ['ad183ff26bae4f329ddd85fd69d160a9', '1-00a308c45fff459c7cda1db9a7314de6'], 'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} """ self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Read user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id]['result'] # Confirm the results are valid. """ Result is a user info object with the name set. {'lcstate': 'DEPLOYED_AVAILABLE', '_rev': '1-851f067bac3c34b2238c0188b3340d0f', 'description': '', 'ts_updated': '1349213207638', 'type_': 'UserInfo', 'contact': <interface.objects.ContactInformation object at 0x10d7df590>, '_id': '27832d93f4cd4535a75ac75c06e00a7e', 'ts_created': '1349213207638', 'variables': [{'name': '', 'value': ''}], 'name': 'some_name'} """ self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Update user object. read_obj.name = 'some_other_name' cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='update', args=[read_obj], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Read user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id]['result'] self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Delete user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='delete', args=[obj_id], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() gevent.sleep(1) def test_resource_commands(self): """ test_resource_commands Test that real resource commands are handled by the remote endpoint. """ # Start the IA and check it's out there and behaving. self.start_agent() state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for the link to be up. # The remote side does not publish public telemetry events # so we can't wait for that. gevent.sleep(1) # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Get agent state via remote endpoint. cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns agent state. state = self._results_recv[cmd.command_id]['result'] self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Ping agent via remote endpoint. cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='ping_agent', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns agent state. ping = self._results_recv[cmd.command_id]['result'] self.assertIn('ping from InstrumentAgent', ping) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() gevent.sleep(1) def test_bad_service_name_resource_id(self): """ test_bad_service_name_resource_id Test for proper exception behavior when a bad service name or resource id is used in a command forwarded to the remote endpoint. """ # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for the link to be up. # The remote side does not publish public telemetry events # so we can't wait for that. gevent.sleep(1) # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='bogus_service', command='create', args=[obj], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns NotFound. result = self._results_recv[cmd.command_id]['result'] self.assertIsInstance(result, NotFound) # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Get agent state via remote endpoint. cmd = IonObject('RemoteCommand', resource_id='bogus_resource_id', svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns NotFound. result = self._results_recv[cmd.command_id]['result'] self.assertIsInstance(result, NotFound) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() gevent.sleep(1) def test_bad_commands(self): """ test_bad_commands Test for correct exception behavior if a bad command name is forwarded to a remote service or resource. """ # Start the IA and check it's out there and behaving. self.start_agent() state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for the link to be up. # The remote side does not publish public telemetry events # so we can't wait for that. gevent.sleep(1) # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='what_the_flunk', args=[obj], kwargs='', command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns BadRequest. result = self._results_recv[cmd.command_id]['result'] self.assertIsInstance(result, BadRequest) # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._done_evt = AsyncResult() self._cmd_tx_evt = AsyncResult() self._requests_sent = {} self._results_recv = {} # Get agent state via remote endpoint. cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='what_the_flunk', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd) # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns NotFound. result = self._results_recv[cmd.command_id]['result'] self.assertIsInstance(result, BadRequest) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() gevent.sleep(1) def test_resource_command_sequence(self): """ test_resource_command_sequence Test for successful completion of a properly ordered sequence of resource commands queued for forwarding to the remote endpoint. """ # Start the IA and check it's out there and behaving. self.start_agent() state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # We execute a sequence of twelve consecutive events. self._no_requests = 12 # Get agent state. cmd1 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd1) # Initialize agent. cmd2 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_agent', args=[AgentCommand(command=ResourceAgentEvent.INITIALIZE)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd2) # Get agent state. cmd3 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd3) # Go active. cmd4 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_agent', args=[AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd4) # Get agent state. cmd5 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd5) # Run. cmd6 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_agent', args=[AgentCommand(command=ResourceAgentEvent.RUN)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd6) # Get agent state. cmd7 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd7) # Acquire sample. cmd8 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_resource', args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd8) # Acquire sample cmd9 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_resource', args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd9) # Acquire sample. cmd10 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_resource', args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd10) # Reset. cmd11 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='execute_agent', args=[AgentCommand(command=ResourceAgentEvent.RESET)], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd11) # Get agent state. cmd12 = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}, command_id = str(uuid.uuid4())) self._terrestrial_client.enqueue(cmd12) # Publish a telemetry available event. # This will cause the endpoint clients to wake up and connect. self.on_link_up() # Wait for command request to be acked. # Wait for response to arrive. self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) # Check results of command sequence. """ 0ccf1e10-eeca-400d-aefe-f9d6888ec963 {'result': 'RESOURCE_AGENT_STATE_INACTIVE', 'command_id': '0ccf1e10-eeca-400d-aefe-f9d6888ec963'} 92531bdf-c2c8-4aa8-817d-5107c7311b37 {'result': <interface.objects.AgentCommandResult object at 0x10d7f11d0>, 'command_id': '92531bdf-c2c8-4aa8-817d-5107c7311b37'} 509934a1-5038-40d8-8014-591e2d8042b6 {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '509934a1-5038-40d8-8014-591e2d8042b6'} 88bacbb7-5366-4d27-9ecf-fff2bec34b2c {'result': <interface.objects.AgentCommandResult object at 0x10d389190>, 'command_id': '88bacbb7-5366-4d27-9ecf-fff2bec34b2c'} f8b4d3fa-a249-439b-8bd4-ac212b6100aa {'result': <interface.objects.AgentCommandResult object at 0x10d3893d0>, 'command_id': 'f8b4d3fa-a249-439b-8bd4-ac212b6100aa'} 8ae98e39-fdb3-4218-ad8f-584620397d9f {'result': <interface.objects.AgentCommandResult object at 0x10d739990>, 'command_id': '8ae98e39-fdb3-4218-ad8f-584620397d9f'} 746364a1-c4c7-400f-96d4-ee36df5dc1a4 {'result': BadRequest('Execute argument "command" not set.',), 'command_id': '746364a1-c4c7-400f-96d4-ee36df5dc1a4'} d516d3d9-e4f9-4ea5-80e0-34639a6377b5 {'result': <interface.objects.AgentCommandResult object at 0x10d3b2350>, 'command_id': 'd516d3d9-e4f9-4ea5-80e0-34639a6377b5'} c7da03f5-59bc-420a-9e10-0a7794266599 {'result': 'RESOURCE_AGENT_STATE_IDLE', 'command_id': 'c7da03f5-59bc-420a-9e10-0a7794266599'} 678d870a-bf18-424a-afb0-f80ecf3277e2 {'result': <interface.objects.AgentCommandResult object at 0x10d739590>, 'command_id': '678d870a-bf18-424a-afb0-f80ecf3277e2'} 750c6a30-56eb-4535-99c2-a81fefab1b1f {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '750c6a30-56eb-4535-99c2-a81fefab1b1f'} c17bd658-3775-4aa3-8844-02df70a0e3c0 {'result': 'RESOURCE_AGENT_STATE_UNINITIALIZED', 'command_id': 'c17bd658-3775-4aa3-8844-02df70a0e3c0'} """ # First result is a state string. result1 = self._results_recv[cmd1.command_id]['result'] self.assertEqual(result1, ResourceAgentState.UNINITIALIZED) # Second result is an empty AgentCommandResult. result2 = self._results_recv[cmd2.command_id]['result'] # Third result is a state string. result3 = self._results_recv[cmd3.command_id]['result'] self.assertEqual(result3, ResourceAgentState.INACTIVE) # Fourth result is an empty AgentCommandResult. result4 = self._results_recv[cmd4.command_id]['result'] # Fifth result is a state string. result5 = self._results_recv[cmd5.command_id]['result'] self.assertEqual(result5, ResourceAgentState.IDLE) # Sixth result is an empty AgentCommandResult. result6 = self._results_recv[cmd6.command_id]['result'] # Seventh result is a state string. result7 = self._results_recv[cmd7.command_id]['result'] self.assertEqual(result7, ResourceAgentState.COMMAND) """ {'raw': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp', 'stream_name': 'raw', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1, ' values': [{'binary': True, 'value_id': 'raw', 'value': 'NzkuNDM3MywxNy4yMDU2NCwgNzYxLjg4NSwgICA2LjIxOTgsIDE1MDYuMzk3LCAwMSBGZWIgMjAwMSwgMDE6MDE6MDA='}], 'driver_timestamp': 3558286748.8039923}, 'parsed': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp', 'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1, 'values': [{'value_id': 'temp', 'value': 79.4373}, {'value_id': 'conductivity', 'value': 17.20564}, {'value_id': 'pressure', 'value': 761.885}], 'driver_timestamp': 3558286748.8039923}} """ # Eigth result is an AgentCommandResult containing a sample. result8 = self._results_recv[cmd8.command_id]['result'] self.assertTrue('parsed',result8.result ) # Ninth result is an AgentCommandResult containing a sample. result9 = self._results_recv[cmd9.command_id]['result'] self.assertTrue('parsed',result9.result ) # Tenth result is an AgentCommandResult containing a sample. result10 = self._results_recv[cmd10.command_id]['result'] self.assertTrue('parsed',result10.result ) # Eleventh result is an empty AgentCommandResult. result11 = self._results_recv[cmd11.command_id]['result'] # Twelth result is a state string. result12 = self._results_recv[cmd12.command_id]['result'] self.assertEqual(result1, ResourceAgentState.UNINITIALIZED) # Publish a telemetry unavailable event. # This will cause the endpoint clients to disconnect and go to sleep. self.on_link_down() gevent.sleep(1)
class TestDriverEgg(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() self.egg_url_good = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg" self.egg_url_bad = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.1a-py2.7.egg" self.egg_url_404 = "http://sddevrepo.oceanobservatories.org/releases/completely_made_up_404.egg" def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_streamConfigs(self): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) return raw_config, parsed_config ########################## # # The following tests generate different agent configs and pass them to a common base test script # ########################### @unittest.skip("this test can't be run from coi services. it is missing dependencies") def test_driverLaunchModuleNoURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", driver_uri=self.egg_url_good, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchNoModuleOnlyURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=self.egg_url_good, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchBogusModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="bogus", driver_class="Bogus", driver_uri=self.egg_url_good, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) @unittest.skip("Launches an egg 'process' even though the egg download should produce error 404") def test_driverLaunchNoModule404URI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=self.egg_url_404, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, False) def test_driverLaunchNoModuleBadEggURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=self.egg_url_bad, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, True, False) def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True): """ This method runs a test of launching a driver with a given agent configuration """ # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self.get_datastore(self.parsed_dataset) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] # add start/stop for instrument agent gevent.joinall([gevent.spawn(lambda: self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) if not expect_launch: self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" % instance_obj.agent_process_id) return self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING" inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) #print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) print "ResourceAgentClient created: %s" % str(self._ia_client) print "Sending command=ResourceAgentEvent.INITIALIZE" cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) if not expect_command: self.assertRaises(ServerError, self._ia_client.execute_agent, cmd) return retval = self._ia_client.execute_agent(cmd) print "Result of INITIALIZE: %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) print "Sending command=ResourceAgentEvent.RESET" cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "Result of RESET: %s" % str(reply)