class TestAgentPersistence(IonIntegrationTestCase): """ """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config. """ self._ia_client = None log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Create agent config. self._agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True, 'forget_past' : False, 'enable_persistence' : True, 'aparam_pubrate_config' : { 'raw' : 2, 'parsed' : 2 } } self._ia_client = None self._ia_pid = '1234' self.addCleanup(self._verify_agent_reset) self.addCleanup(self.container.state_repository.put_state, self._ia_pid, {}) ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config ############################################################################### # Agent start stop helpers. ############################################################################### def _start_agent(self, bootmode=None): """ """ container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) agent_config = deepcopy(self._agent_config) agent_config['bootmode'] = bootmode self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config, process_id=self._ia_pid) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got instrument agent client %s.', str(self._ia_client)) def _stop_agent(self): """ """ if self._ia_pid: container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) container_client.terminate_process(self._ia_pid) if self._ia_client: self._ia_client = None def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) self._ia_client = None ############################################################################### # Tests. ############################################################################### def test_agent_config_persistence(self): """ test_agent_config_persistence Test that agent parameter configuration is persisted between running instances. """ # Start the agent. self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Confirm the default agent parameters. #{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}} retval = self._ia_client.get_agent(['streams'])['streams'] self.assertIn('raw', retval.keys()) self.assertIn('parsed', retval.keys()) #{'pubrate': {'raw': 0, 'parsed': 0}} retval = self._ia_client.get_agent(['pubrate'])['pubrate'] self.assertIn('raw', retval.keys()) self.assertIn('parsed', retval.keys()) self.assertEqual(retval['raw'], 2) self.assertEqual(retval['parsed'], 2) #{'alerts': []} retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertEqual(retval, []) # Define a few new parameters and set them. # Confirm they are set. alert_def_1 = { 'name' : 'current_warning_interval', 'stream_name' : 'parsed', 'description' : 'Current is below normal range.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 10.0, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def_2 = { 'name' : 'temp_alarm_interval', 'stream_name' : 'parsed', 'description' : 'Temperatoure is critical.', 'alert_type' : StreamAlertType.ALARM, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 20.0, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def3 = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'time_delta' : 180, 'alert_class' : 'LateDataAlert' } orig_alerts = [alert_def_1,alert_def_2, alert_def3] pubrate = { 'parsed' : 10, 'raw' : 20 } params = { 'alerts' : orig_alerts, 'pubrate' : pubrate } # Set the new agent params and confirm. self._ia_client.set_agent(params) params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) pubrate = retval['pubrate'] alerts = retval['alerts'] self.assertIn('raw', pubrate.keys()) self.assertIn('parsed', pubrate.keys()) self.assertEqual(pubrate['parsed'], 10) self.assertEqual(pubrate['raw'], 20) count = 0 for x in alerts: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEqual(count, 3) # Now stop and restart the agent. self._stop_agent() gevent.sleep(15) self._start_agent('restart') # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Confirm the persisted parameters. params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) pubrate = retval['pubrate'] alerts = retval['alerts'] self.assertIn('raw', pubrate.keys()) self.assertIn('parsed', pubrate.keys()) self.assertEqual(pubrate['parsed'], 10) self.assertEqual(pubrate['raw'], 20) count = 0 for x in alerts: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEqual(count, 3) def test_agent_state_persistence(self): """ test_agent_state_persistence Verify that agents can be restored to their prior running state. """ self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) alert_def3 = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'time_delta' : 180, 'alert_class' : 'LateDataAlert' } orig_pubrate = { 'parsed' : 10, 'raw' : 20 } params = { 'alerts' : [alert_def3], 'pubrate' : orig_pubrate } # Set the new agent params and confirm. self._ia_client.set_agent(params) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Acquire sample returns a string, not a particle. The particle # is created by the data handler though. cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = self._ia_client.execute_resource(cmd) # Now stop and restart the agent. self._stop_agent() gevent.sleep(15) self._start_agent('restart') timeout = gevent.Timeout(240) timeout.start() try: while True: state = self._ia_client.get_agent_state() print '## in state: ' + state if state == ResourceAgentState.COMMAND: timeout.cancel() break else: gevent.sleep(1) except gevent.Timeout: self.fail("Could not restore agent state to COMMAND.") params = [ 'alerts', 'pubrate' ] retval = self._ia_client.get_agent(params) alerts = retval['alerts'] pubrate = retval['pubrate'] self.assertEqual(len(alerts), 1) self.assertEqual(alert_def3['name'], alerts[0]['name']) self.assertEqual(pubrate['raw'], 20) self.assertEqual(pubrate['parsed'], 10) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) # Now stop and restart the agent. self._stop_agent() gevent.sleep(15) self._start_agent('restart') timeout = gevent.Timeout(240) timeout.start() try: while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.STOPPED: timeout.cancel() break else: gevent.sleep(1) except gevent.Timeout: self.fail("Could not restore agent state to STOPPED.") retval = self._ia_client.get_agent(params) alerts = retval['alerts'] pubrate = retval['pubrate'] self.assertEqual(len(alerts), 1) self.assertEqual(alert_def3['name'], alerts[0]['name']) self.assertEqual(pubrate['raw'], 20) self.assertEqual(pubrate['parsed'], 10) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_agent_rparam_persistence(self): """ test_agent_rparam_persistence Verify ability to restore device configuration. ### Original values: {'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0} ### Values after set: {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1} ### Restore config: {'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005], 'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649, 'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005], 'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0, 'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05, 'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484, 'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, ' RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, ' PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0, 'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819, 'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492, 'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06, 'RTCA0': 0.9999862} ### Of which we have: {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1} """ self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) params = [ SBE37Parameter.OUTPUTSV, SBE37Parameter.NAVG, SBE37Parameter.TA0 ] retval = self._ia_client.get_resource(params) orig_params = retval new_params = { SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV], SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1, SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2 } #print '########### orig params' #print str(orig_params) self._ia_client.set_resource(new_params) retval = self._ia_client.get_resource(params) self.assertEqual(retval[SBE37Parameter.OUTPUTSV], new_params[SBE37Parameter.OUTPUTSV]) self.assertEqual(retval[SBE37Parameter.NAVG], new_params[SBE37Parameter.NAVG]) delta = max(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0])*.01 self.assertAlmostEqual(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0], delta=delta) #print '########### new params' #print str(retval) # Now stop and restart the agent. self._stop_agent() self._support.stop_pagent() gevent.sleep(10) self._start_pagent() gevent.sleep(10) self._start_agent('restart') timeout = gevent.Timeout(600) timeout.start() try: while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.COMMAND: timeout.cancel() break else: gevent.sleep(3) except gevent.Timeout: self.fail("Could not restore agent state to COMMAND.") # Verify the parameters have been restored as needed. retval = self._ia_client.get_resource(params) #print '########### restored params' #print str(retval) self.assertEqual(retval[SBE37Parameter.OUTPUTSV], new_params[SBE37Parameter.OUTPUTSV]) self.assertEqual(retval[SBE37Parameter.NAVG], new_params[SBE37Parameter.NAVG]) delta = max(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0])*.01 self.assertAlmostEqual(retval[SBE37Parameter.TA0], new_params[SBE37Parameter.TA0], delta=delta) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) @unittest.skip('Making CEI friendly.') def test_cei_launch_mode(self): pdc = ProcessDispatcherServiceClient(node=self.container.node) p_def = ProcessDefinition(name='Agent007') p_def.executable = { 'module' : 'ion.agents.instrument.instrument_agent', 'class' : 'InstrumentAgent' } p_def_id = pdc.create_process_definition(p_def) pid = pdc.create_process(p_def_id) def event_callback(event, *args, **kwargs): print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state]) sub = EventSubscriber(event_type='ProcessLifecycleEvent', callback=event_callback, origin=pid, origin_type='DispatchedProcess') sub.start() agent_config = deepcopy(self._agent_config) agent_config['bootmode'] = 'restart' pdc.schedule_process(p_def_id, process_id=pid, configuration=agent_config) gevent.sleep(5) pdc.cancel_process(pid) gevent.sleep(15) sub.stop()
class TestAgentConnectionFailures(IonIntegrationTestCase): """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config, start agent. Start agent client. """ self._ia_client = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Start a resource agent client to talk with the instrument agent. log.info('starting IA process') self._ia_client = start_instrument_agent_process(self.container, self._stream_config) self.addCleanup(self._verify_agent_reset) log.info('test setup complete') ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state(timeout=120.1) if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd,timeout=300) ############################################################################### # Event helpers. ############################################################################### def _start_event_subscriber(self, type='ResourceAgentEvent', count=0): """ Start a subscriber to the instrument agent events. @param type The type of event to catch. @count Trigger the async event result when events received reaches this. """ def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._event_count > 0 and \ self._event_count == len(self._events_received): self._async_event_result.set() # Event array and async event result. self._event_count = count self._events_received = [] self._async_event_result = AsyncResult() self._event_subscriber = EventSubscriber( event_type=type, callback=consume_event, origin=IA_RESOURCE_ID) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=5) def _stop_event_subscriber(self): """ Stop event subscribers on cleanup. """ self._event_subscriber.stop() self._event_subscriber = None ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config def _start_data_subscribers(self, count, raw_count): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._data_subscribers = [] self._samples_received = [] self._raw_samples_received = [] self._async_sample_result = AsyncResult() self._async_raw_sample_result = AsyncResult() # A callback for processing subscribed-to data. def recv_data(message, stream_route, stream_id): log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._samples_received.append(message) if len(self._samples_received) == count: self._async_sample_result.set() def recv_raw_data(message, stream_route, stream_id): log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._raw_samples_received.append(message) if len(self._raw_samples_received) == raw_count: self._async_raw_sample_result.set() from pyon.util.containers import create_unique_identifier stream_name = 'parsed' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id],timeout=120.2) pubsub_client.activate_subscription(sub_id,timeout=120.3) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) stream_name = 'raw' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id],timeout=120.4) pubsub_client.activate_subscription(sub_id,timeout=120.5) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): for subscriber in self._data_subscribers: pubsub_client = PubsubManagementServiceClient() if hasattr(subscriber,'subscription_id'): try: pubsub_client.deactivate_subscription(subscriber.subscription_id,timeout=120.6) except: pass pubsub_client.delete_subscription(subscriber.subscription_id,timeout=120.7) subscriber.stop() ############################################################################### # Socket listen. ############################################################################### def _socket_listen(self, s, prompt, timeout): buf = '' starttime = time.time() while True: try: buf += s.recv(1024) print '##### Listening, got: %s' % buf if prompt and buf.find(prompt) != -1: break except: gevent.sleep(1) finally: delta = time.time() - starttime if delta > timeout: break return buf ############################################################################### # Assert helpers. ############################################################################### def assertSampleDict(self, val): """ Verify the value is a sample dictionary for the sbe37. """ # AgentCommandResult.result['parsed'] """ {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp', 'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1, 'values': [{'value_id': 'temp', 'value': 21.4894}, {'value_id': 'conductivity', 'value': 13.22157}, {'value_id': 'pressure', 'value': 146.186}], 'driver_timestamp': 3556901018.170206} """ self.assertIsInstance(val, dict) self.assertTrue(val.has_key('values')) values_list = val['values'] self.assertTrue(isinstance(values_list, list)) self.assertTrue(len(values_list)==3) ids = ['temp', 'conductivity', 'pressure'] ids_found = [] for x in values_list: self.assertTrue(x.has_key('value_id')) self.assertTrue(x.has_key('value')) ids_found.append(x['value_id']) self.assertTrue(isinstance(x['value'], float)) self.assertItemsEqual(ids, ids_found) self.assertTrue(val.has_key('driver_timestamp')) time = val['driver_timestamp'] self.assertTrue(isinstance(time, float)) def assertParamDict(self, pd, all_params=False): """ Verify all device parameters exist and are correct type. """ if all_params: self.assertEqual(set(pd.keys()), set(PARAMS.keys())) for (key, type_val) in PARAMS.iteritems(): if type_val == list or type_val == tuple: self.assertTrue(isinstance(pd[key], (list, tuple))) else: self.assertTrue(isinstance(pd[key], type_val)) else: for (key, val) in pd.iteritems(): self.assertTrue(PARAMS.has_key(key)) self.assertTrue(isinstance(val, PARAMS[key])) def assertParamVals(self, params, correct_params): """ Verify parameters take the correct values. """ self.assertEqual(set(params.keys()), set(correct_params.keys())) for (key, val) in params.iteritems(): correct_val = correct_params[key] if isinstance(val, float): # Verify to 5% of the larger value. max_val = max(abs(val), abs(correct_val)) self.assertAlmostEqual(val, correct_val, delta=max_val*.01) elif isinstance(val, (list, tuple)): # list of tuple. self.assertEqual(list(val), list(correct_val)) else: # int, bool, str. self.assertEqual(val, correct_val) ############################################################################### # Tests. ############################################################################### def test_lost_connection(self): """ test_lost_connection """ # Set up a subscriber to collect command events. self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1) self.addCleanup(self._stop_event_subscriber) # Start in uninitialized. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Activate. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) # Go into command mode. cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Start streaming. cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # Wait for a while, collect some samples. gevent.sleep(10) # Blow the port agent out from under the agent. self._support.stop_pagent() # Loop until we resyncronize to LOST_CONNECTION/DISCONNECTED. # Test will timeout if this dosn't occur. while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.LOST_CONNECTION: break else: gevent.sleep(1) # Verify the driver has transitioned to disconnected while True: state = self._ia_client.get_resource_state() if state == DriverConnectionState.DISCONNECTED: break else: gevent.sleep(1) # Make sure the lost connection error event arrives. self._async_event_result.get(timeout=CFG.endpoint.receive.timeout) self.assertEqual(len(self._events_received), 1) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) #@unittest.skip('Fails on buildbot for some god unknown reason.') def test_autoreconnect(self): """ test_autoreconnect """ # Set up a subscriber to collect command events. self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1) self.addCleanup(self._stop_event_subscriber) # Start in uninitialized. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Activate. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) # Go into command mode. cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) def poll_func(test): cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) while True: try: gevent.sleep(.5) test._ia_client.execute_resource(cmd) except IonException as ex: # This exception could be ResourceException (broken pipe) # Timeout or Conflict log.info('#### pre shutdown exception: %s, %s', str(type(ex)), str(ex)) break while True: try: gevent.sleep(.5) test._ia_client.execute_resource(cmd) log.info('#### post shutdown got new sample.') break except IonException as ex: # This should be conflict. log.info('#### post shutdown exception: %s, %s', str(type(ex)), str(ex)) timeout = gevent.Timeout(600) timeout.start() try: # Start the command greenlet and let poll for a bit. gl = gevent.spawn(poll_func, self) gevent.sleep(20) # Blow the port agent out from under the agent. self._support.stop_pagent() # Wait for a while, the supervisor is restarting the port agent. gevent.sleep(10) self._support.start_pagent() # Wait for the device to connect and start sampling again. gl.join() gl = None timeout.cancel() except (Exception, gevent.Timeout) as ex: if gl: gl.kill() gl = None self.fail(('Could not reconnect to device: %s, %s', str(type(ex)), str(ex))) def test_connect_failed(self): """ test_connect_failed """ # Stop the port agent. self._support.stop_pagent() # Sleep a bit. gevent.sleep(3) # Start in uninitialized. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Activate. This should fail because there is no port agent to connect to. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) with self.assertRaises(ResourceError): retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) def test_get_set_alerts(self): """ test_get_set_alerts Test specific of get/set alerts, including using result of get to set later. """ state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertItemsEqual(retval, []) alert_def1 = { 'name' : 'temp_warning_interval', 'stream_name' : 'parsed', 'description' : 'Temperature is above normal range.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 10.5, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } alert_def2 = { 'name' : 'temp_alarm_interval', 'stream_name' : 'parsed', 'description' : 'Temperature is way above normal range.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : None, 'lower_rel_op' : None, 'upper_bound' : 15.5, 'upper_rel_op' : '<', 'alert_class' : 'IntervalAlert' } """ Interval alerts are returned from get like this: (value and status fields describe state of the alert) { 'name': 'temp_warning_interval', 'stream_name': 'parsed', 'description': 'Temperature is above normal range.', 'alert_type': 1, 'aggregate_type': 2, 'value_id': 'temp', 'lower_bound': None, 'lower_rel_op': None, 'upper_bound': 10.5, 'upper_rel_op': '<', 'alert_class': 'IntervalAlert', 'status': None, 'value': None } """ alert_def3 = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'time_delta' : 180, 'alert_class' : 'LateDataAlert' } """ Late data alerts are returned from get like this: (value and status fields describe state of the alert) { 'name': 'late_data_warning', 'stream_name': 'parsed', 'description': 'Expected data has not arrived.', 'alert_type': 1, 'aggregate_type': 1, 'value_id': None, 'time_delta': 180, 'alert_class': 'LateDataAlert', 'status': None, 'value': None } """ """ [ {'status': None, 'alert_type': 1, 'name': 'temp_warning_interval', 'upper_bound': 10.5, 'lower_bound': None, 'aggregate_type': 2, 'alert_class': 'IntervalAlert', 'value': None, 'value_id': 'temp', 'lower_rel_op': None, 'upper_rel_op': '<', 'description': 'Temperature is above normal range.'}, {'status': None, 'alert_type': 1, 'name': 'temp_alarm_interval', 'upper_bound': 15.5, 'lower_bound': None, 'aggregate_type': 2, 'alert_class': 'IntervalAlert', 'value': None, 'value_id': 'temp', 'lower_rel_op': None, 'upper_rel_op': '<', 'description': 'Temperature is way above normal range.'}, {'status': None, 'stream_name': 'parsed', 'alert_type': 1, 'name': 'late_data_warning', 'aggregate_type': 1, 'alert_class': 'LateDataAlert', 'value': None, 'time_delta': 180, 'description': 'Expected data has not arrived.'} ] """ orig_alerts = [alert_def1, alert_def2, alert_def3] self._ia_client.set_agent({'alerts' : orig_alerts}) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertTrue(len(retval)==3) alerts = retval self._ia_client.set_agent({'alerts' : ['clear']}) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertItemsEqual(retval, []) self._ia_client.set_agent({'alerts' : alerts}) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertTrue(len(retval)==3) count = 0 for x in retval: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEquals(count, 3) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestAgentCommsAlerts(IonIntegrationTestCase): """ """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config. """ self._ia_client = None log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') self._event_count = 0 self._events_received = [] self._async_event_result = AsyncResult() def consume_event(*args, **kwargs): log.debug('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._event_count > 0 and \ self._event_count == len(self._events_received): self._async_event_result.set() self._event_subscriber = EventSubscriber( event_type='DeviceStatusAlertEvent', callback=consume_event, origin=IA_RESOURCE_ID) self._event_subscriber.start() def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Create agent config. self._agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : self._stream_config, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True, 'aparam_alerts_config' : [state_alert_def, command_alert_def] } self._ia_client = None self._ia_pid = None self.addCleanup(self._verify_agent_reset) ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port, 'cmd_port' : CMD_PORT } ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config ############################################################################### # Agent start stop helpers. ############################################################################### def _start_agent(self): """ """ container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) self._ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=self._agent_config) log.info('Started instrument agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got instrument agent client %s.', str(self._ia_client)) def _stop_agent(self): """ """ if self._ia_pid: container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) container_client.terminate_process(self._ia_pid) self._ia_pid = None if self._ia_client: self._ia_client = None def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state() if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) self._ia_client = None ############################################################################### # Tests. ############################################################################### def test_lost_connection_alert(self): """ test_lost_connection_alert Verify that agents detect lost connection state and issue alert. """ self._event_count = 3 self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Confirm the persisted parameters. retval = self._ia_client.get_agent(['alerts'])['alerts'] """ {'origin': '123xyz', 'status': 1, '_id': 'da03b90d2e064b25bf51ed90b729e82e', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1366749987069', 'sub_type': 3, 'origin_type': 'InstrumentDevice', 'name': 'comms_warning'} """ # Acquire sample returns a string, not a particle. The particle # is created by the data handler though. cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) retval = self._ia_client.execute_resource(cmd) # Blow the port agent out from under the agent. self._support.stop_pagent() # Wait for a while, the supervisor is restarting the port agent. gevent.sleep(5) self._support.start_pagent() timeout = gevent.Timeout(120) timeout.start() try: while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.COMMAND: timeout.cancel() break else: gevent.sleep(2) except Timeout as t: self.fail('Could not reconnect to device.') state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) self._async_event_result.get(timeout=30) """ {'origin': '123xyz', 'status': 1, '_id': '9d7db919a0414741a2aa97f3dc310647', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029709', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} {'origin': '123xyz', 'status': 1, '_id': 'f746cec5e486445e856ef29af8d8d49a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029717', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} {'origin': '123xyz', 'status': 1, '_id': '126b1e20f54f4bd2b84a93584831ea8d', 'description': 'Detected comms failure.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_LOST_CONNECTION'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008062061', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} {'origin': '123xyz', 'status': 1, '_id': '90f5762112dd446b939c6675d34dd61d', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_COMMAND'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008098639', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} """ def test_connect_failed_alert(self): """ test_connect_failed_alert Verify that agents detect failed connections and issue alert. """ self._event_count = 4 # Remove the port agent. self._support.stop_pagent() self._start_agent() # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Ping the agent. retval = self._ia_client.ping_agent() log.info(retval) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Ping the driver proc. retval = self._ia_client.ping_resource() log.info(retval) with self.assertRaises(Exception): cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) # Now start up the port agent. self._support.start_pagent() gevent.sleep(5) # This time it will work. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) self._async_event_result.get(timeout=30) """ {'origin': '123xyz', 'status': 1, '_id': 'ac1ae3ec24e74f65bde24362d689346a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751167', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'} {'origin': '123xyz', 'status': 1, '_id': '6af6a6156172481ebc44baad2708ec5c', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751175', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} {'origin': '123xyz', 'status': 1, '_id': 'b493698b46fd4fc1b4c66c53b70ed043', 'description': 'Detected comms failure while connecting.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007756771', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} {'origin': '123xyz', 'status': 1, '_id': 'c01e911ecb1a47c5a04cbbcbfb348a42', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007792905', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'} """
class TestAgentConnectionFailures(IonIntegrationTestCase): """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): """ Set up driver integration support. Start port agent, add port agent cleanup. Start container. Start deploy services. Define agent config, start agent. Start agent client. """ self._ia_client = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') log.info('Creating driver integration test support:') log.info('driver uri: %s', DRV_URI) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. self._start_pagent() self.addCleanup(self._support.stop_pagent) log.info('building stream configuration') # Setup stream config. self._build_stream_config() # Start a resource agent client to talk with the instrument agent. log.info('starting IA process') self._ia_client = start_instrument_agent_process( self.container, self._stream_config) self.addCleanup(self._verify_agent_reset) log.info('test setup complete') ############################################################################### # Port agent helpers. ############################################################################### def _start_pagent(self): """ Construct and start the port agent. """ port = self._support.start_pagent() log.info('Port agent started at port %i', port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr': 'localhost', 'port': port, 'cmd_port': CMD_PORT } def _verify_agent_reset(self): """ Check agent state and reset if necessary. This called if a test fails and reset hasn't occurred. """ if self._ia_client is None: return state = self._ia_client.get_agent_state(timeout=120.1) if state != ResourceAgentState.UNINITIALIZED: cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd, timeout=300) ############################################################################### # Event helpers. ############################################################################### def _start_event_subscriber(self, type='ResourceAgentEvent', count=0): """ Start a subscriber to the instrument agent events. @param type The type of event to catch. @count Trigger the async event result when events received reaches this. """ def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._event_count > 0 and \ self._event_count == len(self._events_received): self._async_event_result.set() # Event array and async event result. self._event_count = count self._events_received = [] self._async_event_result = AsyncResult() self._event_subscriber = EventSubscriber(event_type=type, callback=consume_event, origin=IA_RESOURCE_ID) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=5) def _stop_event_subscriber(self): """ Stop event subscribers on cleanup. """ self._event_subscriber.stop() self._event_subscriber = None ############################################################################### # Data stream helpers. ############################################################################### def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config def _start_data_subscribers(self, count, raw_count): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._data_subscribers = [] self._samples_received = [] self._raw_samples_received = [] self._async_sample_result = AsyncResult() self._async_raw_sample_result = AsyncResult() # A callback for processing subscribed-to data. def recv_data(message, stream_route, stream_id): log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._samples_received.append(message) if len(self._samples_received) == count: self._async_sample_result.set() def recv_raw_data(message, stream_route, stream_id): log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key) self._raw_samples_received.append(message) if len(self._raw_samples_received) == raw_count: self._async_raw_sample_result.set() from pyon.util.containers import create_unique_identifier stream_name = 'parsed' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id], timeout=120.2) pubsub_client.activate_subscription(sub_id, timeout=120.3) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) stream_name = 'raw' parsed_config = self._stream_config[stream_name] stream_id = parsed_config['stream_id'] exchange_name = create_unique_identifier("%s_queue" % stream_name) self._purge_queue(exchange_name) sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data) sub.start() self._data_subscribers.append(sub) sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id], timeout=120.4) pubsub_client.activate_subscription(sub_id, timeout=120.5) sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice) def _purge_queue(self, queue): xn = self.container.ex_manager.create_xn_queue(queue) xn.purge() def _stop_data_subscribers(self): for subscriber in self._data_subscribers: pubsub_client = PubsubManagementServiceClient() if hasattr(subscriber, 'subscription_id'): try: pubsub_client.deactivate_subscription( subscriber.subscription_id, timeout=120.6) except: pass pubsub_client.delete_subscription(subscriber.subscription_id, timeout=120.7) subscriber.stop() ############################################################################### # Socket listen. ############################################################################### def _socket_listen(self, s, prompt, timeout): buf = '' starttime = time.time() while True: try: buf += s.recv(1024) print '##### Listening, got: %s' % buf if prompt and buf.find(prompt) != -1: break except: gevent.sleep(1) finally: delta = time.time() - starttime if delta > timeout: break return buf ############################################################################### # Assert helpers. ############################################################################### def assertSampleDict(self, val): """ Verify the value is a sample dictionary for the sbe37. """ # AgentCommandResult.result['parsed'] """ {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp', 'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1, 'values': [{'value_id': 'temp', 'value': 21.4894}, {'value_id': 'conductivity', 'value': 13.22157}, {'value_id': 'pressure', 'value': 146.186}], 'driver_timestamp': 3556901018.170206} """ self.assertIsInstance(val, dict) self.assertTrue(val.has_key('values')) values_list = val['values'] self.assertTrue(isinstance(values_list, list)) self.assertTrue(len(values_list) == 3) ids = ['temp', 'conductivity', 'pressure'] ids_found = [] for x in values_list: self.assertTrue(x.has_key('value_id')) self.assertTrue(x.has_key('value')) ids_found.append(x['value_id']) self.assertTrue(isinstance(x['value'], float)) self.assertItemsEqual(ids, ids_found) self.assertTrue(val.has_key('driver_timestamp')) time = val['driver_timestamp'] self.assertTrue(isinstance(time, float)) def assertParamDict(self, pd, all_params=False): """ Verify all device parameters exist and are correct type. """ if all_params: self.assertEqual(set(pd.keys()), set(PARAMS.keys())) for (key, type_val) in PARAMS.iteritems(): if type_val == list or type_val == tuple: self.assertTrue(isinstance(pd[key], (list, tuple))) else: self.assertTrue(isinstance(pd[key], type_val)) else: for (key, val) in pd.iteritems(): self.assertTrue(PARAMS.has_key(key)) self.assertTrue(isinstance(val, PARAMS[key])) def assertParamVals(self, params, correct_params): """ Verify parameters take the correct values. """ self.assertEqual(set(params.keys()), set(correct_params.keys())) for (key, val) in params.iteritems(): correct_val = correct_params[key] if isinstance(val, float): # Verify to 5% of the larger value. max_val = max(abs(val), abs(correct_val)) self.assertAlmostEqual(val, correct_val, delta=max_val * .01) elif isinstance(val, (list, tuple)): # list of tuple. self.assertEqual(list(val), list(correct_val)) else: # int, bool, str. self.assertEqual(val, correct_val) ############################################################################### # Tests. ############################################################################### def test_lost_connection(self): """ test_lost_connection """ # Set up a subscriber to collect command events. self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1) self.addCleanup(self._stop_event_subscriber) # Start in uninitialized. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Activate. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) # Go into command mode. cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Start streaming. cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # Wait for a while, collect some samples. gevent.sleep(10) # Blow the port agent out from under the agent. self._support.stop_pagent() # Loop until we resyncronize to LOST_CONNECTION/DISCONNECTED. # Test will timeout if this dosn't occur. while True: state = self._ia_client.get_agent_state() if state == ResourceAgentState.LOST_CONNECTION: break else: gevent.sleep(1) # Verify the driver has transitioned to disconnected while True: state = self._ia_client.get_resource_state() if state == DriverConnectionState.DISCONNECTED: break else: gevent.sleep(1) # Make sure the lost connection error event arrives. self._async_event_result.get(timeout=CFG.endpoint.receive.timeout) self.assertEqual(len(self._events_received), 1) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) #@unittest.skip('Fails on buildbot for some god unknown reason.') def test_autoreconnect(self): """ test_autoreconnect """ # Set up a subscriber to collect command events. self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1) self.addCleanup(self._stop_event_subscriber) # Start in uninitialized. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Activate. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) # Go into command mode. cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) def poll_func(test): cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) while True: try: gevent.sleep(.5) test._ia_client.execute_resource(cmd) except IonException as ex: # This exception could be ResourceException (broken pipe) # Timeout or Conflict log.info('#### pre shutdown exception: %s, %s', str(type(ex)), str(ex)) break while True: try: gevent.sleep(.5) test._ia_client.execute_resource(cmd) log.info('#### post shutdown got new sample.') break except IonException as ex: # This should be conflict. log.info('#### post shutdown exception: %s, %s', str(type(ex)), str(ex)) timeout = gevent.Timeout(600) timeout.start() try: # Start the command greenlet and let poll for a bit. gl = gevent.spawn(poll_func, self) gevent.sleep(20) # Blow the port agent out from under the agent. self._support.stop_pagent() # Wait for a while, the supervisor is restarting the port agent. gevent.sleep(10) self._support.start_pagent() # Wait for the device to connect and start sampling again. gl.join() gl = None timeout.cancel() except (Exception, gevent.Timeout) as ex: if gl: gl.kill() gl = None self.fail(('Could not reconnect to device: %s, %s', str(type(ex)), str(ex))) def test_connect_failed(self): """ test_connect_failed """ # Stop the port agent. self._support.stop_pagent() # Sleep a bit. gevent.sleep(3) # Start in uninitialized. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Activate. This should fail because there is no port agent to connect to. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) with self.assertRaises(ResourceError): retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) def test_get_set_alerts(self): """ test_get_set_alerts Test specific of get/set alerts, including using result of get to set later. """ state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertItemsEqual(retval, []) alert_def1 = { 'name': 'temp_warning_interval', 'stream_name': 'parsed', 'description': 'Temperature is above normal range.', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_DATA, 'value_id': 'temp', 'lower_bound': None, 'lower_rel_op': None, 'upper_bound': 10.5, 'upper_rel_op': '<', 'alert_class': 'IntervalAlert' } alert_def2 = { 'name': 'temp_alarm_interval', 'stream_name': 'parsed', 'description': 'Temperature is way above normal range.', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_DATA, 'value_id': 'temp', 'lower_bound': None, 'lower_rel_op': None, 'upper_bound': 15.5, 'upper_rel_op': '<', 'alert_class': 'IntervalAlert' } """ Interval alerts are returned from get like this: (value and status fields describe state of the alert) { 'name': 'temp_warning_interval', 'stream_name': 'parsed', 'description': 'Temperature is above normal range.', 'alert_type': 1, 'aggregate_type': 2, 'value_id': 'temp', 'lower_bound': None, 'lower_rel_op': None, 'upper_bound': 10.5, 'upper_rel_op': '<', 'alert_class': 'IntervalAlert', 'status': None, 'value': None } """ alert_def3 = { 'name': 'late_data_warning', 'stream_name': 'parsed', 'description': 'Expected data has not arrived.', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_COMMS, 'time_delta': 180, 'alert_class': 'LateDataAlert' } """ Late data alerts are returned from get like this: (value and status fields describe state of the alert) { 'name': 'late_data_warning', 'stream_name': 'parsed', 'description': 'Expected data has not arrived.', 'alert_type': 1, 'aggregate_type': 1, 'value_id': None, 'time_delta': 180, 'alert_class': 'LateDataAlert', 'status': None, 'value': None } """ """ [ {'status': None, 'alert_type': 1, 'name': 'temp_warning_interval', 'upper_bound': 10.5, 'lower_bound': None, 'aggregate_type': 2, 'alert_class': 'IntervalAlert', 'value': None, 'value_id': 'temp', 'lower_rel_op': None, 'upper_rel_op': '<', 'description': 'Temperature is above normal range.'}, {'status': None, 'alert_type': 1, 'name': 'temp_alarm_interval', 'upper_bound': 15.5, 'lower_bound': None, 'aggregate_type': 2, 'alert_class': 'IntervalAlert', 'value': None, 'value_id': 'temp', 'lower_rel_op': None, 'upper_rel_op': '<', 'description': 'Temperature is way above normal range.'}, {'status': None, 'stream_name': 'parsed', 'alert_type': 1, 'name': 'late_data_warning', 'aggregate_type': 1, 'alert_class': 'LateDataAlert', 'value': None, 'time_delta': 180, 'description': 'Expected data has not arrived.'} ] """ orig_alerts = [alert_def1, alert_def2, alert_def3] self._ia_client.set_agent({'alerts': orig_alerts}) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertTrue(len(retval) == 3) alerts = retval self._ia_client.set_agent({'alerts': ['clear']}) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertItemsEqual(retval, []) self._ia_client.set_agent({'alerts': alerts}) retval = self._ia_client.get_agent(['alerts'])['alerts'] self.assertTrue(len(retval) == 3) count = 0 for x in retval: x.pop('status') x.pop('value') for y in orig_alerts: if x['name'] == y['name']: count += 1 self.assertItemsEqual(x.keys(), y.keys()) self.assertEquals(count, 3) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)