示例#1
0
def start_port_agent(dev_addr=None,
                     dev_port=None,
                     data_port=None,
                     cmd_port=None,
                     pa_binary=None,
                     work_dir=WORK_DIR,
                     delim=DELIM
                     ):
    """
    """
        
    global pagent
    global CFG
    pagent = DriverIntegrationTestSupport(        
        None,
        None,
        dev_addr or CFG.device.sbe37.host,
        dev_port or CFG.device.sbe37.port,
        data_port or CFG.device.sbe37.port_agent_data_port,
        cmd_port or CFG.device.sbe37.port_agent_cmd_port,
        pa_binary or CFG.device.sbe37.port_agent_binary,
        delim,
        work_dir)
    
    pagent.start_pagent()
示例#2
0
def start_port_agent(dev_addr=None,
                     dev_port=None,
                     data_port=None,
                     cmd_port=None,
                     pa_binary=None,
                     work_dir=WORK_DIR,
                     delim=DELIM):
    """
    """

    global pagent
    global CFG
    pagent = DriverIntegrationTestSupport(
        None, None, dev_addr or CFG.device.sbe37.host, dev_port
        or CFG.device.sbe37.port, data_port
        or CFG.device.sbe37.port_agent_data_port, cmd_port
        or CFG.device.sbe37.port_agent_cmd_port, pa_binary
        or CFG.device.sbe37.port_agent_binary, delim, work_dir)

    pagent.start_pagent()
class TrhphDriverProxy(InstrumentDriver):
    """
    An InstrumentDriver serving as a proxy to the driver client
    connecting to the actual TrhphInstrumentDriver implementation.

    Extends InstrumentDriver (instead of TrhphInstrumentDriver) because
    that's the main driver interface in general whereas extending
    TrhphInstrumentDriver would bring unneeded implementation stuff.
    """

    def __init__(self, device_address, device_port):
        """
        Setup test cases.
        """

        driver_module = 'mi.instrument.uw.res_probe.ooicore.trhph_driver'
        driver_class = 'TrhphInstrumentDriver'

        self._support = DriverIntegrationTestSupport(driver_module,
                                                     driver_class,
                                                     device_address,
                                                     device_port)

        # Create and start the port agent.
        mi_logger.info('starting port agent')
        self.comms_config = {
            'addr': 'localhost',
            'port': self._support.start_pagent()}

        # Create and start the driver.
        mi_logger.info('starting driver client')

        ##<update-july-2012>:
        ## start_driver and _dvr_client no longer defined in
        ## DriverIntegrationTestSupport
#        self._support.start_driver()
#        self._dvr_client = self._support._dvr_client
        dvr_config = {
            'comms_config': self.comms_config,
            'dvr_mod': driver_module,
            'dvr_cls': driver_class,
            'workdir' : '/tmp/',
            'process_type': ('ZMQPyClassDriverLauncher',)
        }
        self._start_driver(dvr_config)
        ##</update-july-2012>

    def _start_driver(self, dvr_config):
        ## Part of <update-july-2012>
        ##
        ## Adapted from InstrumentAgent._start_driver(self, dvr_config).
        ##
        """
        Start the driver process and driver client.
        @param dvr_config The driver configuration.
        @raises InstDriverError If the driver or client failed to start properly.
        """

        from ion.agents.instrument.driver_process import DriverProcess

        self._dvr_proc = DriverProcess.get_process(dvr_config, True)
        self._dvr_proc.launch()

        # Verify the driver has started.
        if not self._dvr_proc.getpid():
            log.error('TrhphDriverProxy: error starting driver process.')
            raise InstDriverError('Error starting driver process.')

        def evt_recv(evt):
            """
            Callback to receive asynchronous driver events.
            @param evt The driver event received.
            """
            log.info('TrhphDriverProxy:received driver event %s' % str(evt))

        try:
            driver_client = self._dvr_proc.get_client()
            driver_client.start_messaging(evt_recv)
#            retval = driver_client.cmd_dvr('process_echo', 'Test.')
            self._dvr_client = driver_client

        except Exception, e:
            self._dvr_proc.stop()
            log.error('TrhphDriverProxy: error starting driver client: %s' % e)
            raise InstDriverError('Error starting driver client: %s' % e)

        log.info('TrhphDriverProxy: started driver.')
示例#4
0
class TestIAAlarms(IonIntegrationTestCase):
    """
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Start a resource agent client to talk with the instrument agent.
        log.info('starting IA process')
        self._ia_client = start_instrument_agent_process(self.container, self._stream_config)
        self.addCleanup(self._verify_agent_reset)
        log.info('test setup complete')


    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                        
    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)
            
    ###############################################################################
    # Event helpers.
    ###############################################################################

    def _start_event_subscriber(self, type='StreamAlarmEvent', count=0):
        """
        Start a subscriber to the instrument agent events.
        @param type The type of event to catch.
        @count Trigger the async event result when events received reaches this.
        """
        def consume_event(*args, **kwargs):
            print '#################'
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', 
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()
            
        # Event array and async event result.
        self._event_count = count
        self._events_received = []
        self._async_event_result = AsyncResult()
            
        self._event_subscriber = EventSubscriber(
            event_type=type, callback=consume_event,
            origin=IA_RESOURCE_ID)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=5)

    def _stop_event_subscriber(self):
        """
        Stop event subscribers on cleanup.
        """
        self._event_subscriber.stop()
        self._event_subscriber = None


    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient() 
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        streams = {
            'parsed' : 'ctd_parsed_param_dict',
            'raw'    : 'ctd_raw_param_dict'
        }

        for (stream_name, param_dict_name) in streams.iteritems():
            pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)

            stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
            pd            = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary

            stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)

            stream_config = dict(stream_route=stream_route,
                                 routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
            
            if stream_name == 'parsed':
                
                type = 'IntervalAlarmDef'
                kwargs = {
                    'name' : 'test_sim_warning',
                    'stream_name' : 'parsed',
                    'value_id' : 'temp',
                    'message' : 'Temperature is above test range of 5.0.',
                    'type' : StreamAlarmType.WARNING,
                    'upper_bound' : 5.0,
                    'upper_rel_op' : '<'
                }
                alarm = {}
                alarm['type'] = type
                alarm['kwargs'] = kwargs
                alarms = [alarm]
                stream_config['alarms'] = alarms
            
            self._stream_config[stream_name] = stream_config

    def _start_data_subscribers(self, count):
        """
        """        
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
                
        # Create streams and subscriptions for each stream named in driver.
        self._data_subscribers = []
        self._samples_received = []
        self._async_sample_result = AsyncResult()

        # A callback for processing subscribed-to data.
        def recv_data(message, stream_route, stream_id):
            log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
            #print '######################## stream message:'
            #print str(message)
            self._samples_received.append(message)
            if len(self._samples_received) == count:
                self._async_sample_result.set()

        for (stream_name, stream_config) in self._stream_config.iteritems():
            
            if stream_name == 'parsed':
                
                # Create subscription for parsed stream only.

                stream_id = stream_config['stream_id']
                
                from pyon.util.containers import create_unique_identifier
                # exchange_name = '%s_queue' % stream_name
                exchange_name = create_unique_identifier("%s_queue" %
                        stream_name)
                self._purge_queue(exchange_name)
                sub = StandaloneStreamSubscriber(exchange_name, recv_data)
                sub.start()
                self._data_subscribers.append(sub)
                print 'stream_id: %s' % stream_id
                sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
                pubsub_client.activate_subscription(sub_id)
                sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)

    def _purge_queue(self, queue):
        xn = self.container.ex_manager.create_xn_queue(queue)
        xn.purge()
 
    def _stop_data_subscribers(self):
        for subscriber in self._data_subscribers:
            pubsub_client = PubsubManagementServiceClient()
            if hasattr(subscriber,'subscription_id'):
                try:
                    pubsub_client.deactivate_subscription(subscriber.subscription_id)
                except:
                    pass
                pubsub_client.delete_subscription(subscriber.subscription_id)
            subscriber.stop()

    ###############################################################################
    # Tests.
    ###############################################################################
    
    def test_config(self):
        """
        test_initialize
        Test agent initialize command. This causes creation of
        driver process and transition to inactive.
        """

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        decoder = IonObjectDeserializer(obj_registry=get_obj_registry())

        # Grab the alarms defined in the config.
        retval = decoder.deserialize(self._ia_client.get_agent(['alarms'])['alarms'])

        """
        {'status': None, 'stream_name': 'parsed', 'name': 'test_sim_warning',
        'upper_bound': 5.0, 'expr': 'x<5.0', 'upper_rel_op': '<',
        'lower_rel_op': None, 'type_': 'IntervalAlarmDef', 'value_id': 'temp',
        'lower_bound': None, 'message': 'Temperature is above test range of 5.0.',
        'current_val': None, 'type': 1}
        """
        self.assertEqual(retval[0].type_, 'IntervalAlarmDef')
        self.assertEqual(retval[0].upper_bound, 5.0)
        self.assertEqual(retval[0].expr, 'x<5.0')
        
        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        
    def test_autosample(self):
        """
        test_autosample
        Test instrument driver execute interface to start and stop streaming
        mode. Verify ResourceAgentResourceStateEvents are publsihed.
        """
        
        # Start data subscribers.
        self._start_data_subscribers(5)
        self.addCleanup(self._stop_data_subscribers)    
        
        # Set up a subscriber to collect error events.
        self._start_event_subscriber()
        self.addCleanup(self._stop_event_subscriber)            
        
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)
        
        gevent.sleep(20)
        
        cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)
 
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        #self._async_event_result.get(timeout=CFG.endpoint.receive.timeout)
        #self.assertGreaterEqual(len(self._events_received), 6)

        self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
        self.assertGreaterEqual(len(self._samples_received), 5)

        #for x in self._samples_received:
            

        gevent.sleep(5)
        
        """
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142753e+09]), 2: array([  3.57142753e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 83.66190338], dtype=float32), 8: array([ 27.09519958], dtype=float32), 9: array([ 495.5369873], dtype=float32), 10: None, 11: array([  3.57142753e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438731.584241, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142753e+09]), 2: array([  3.57142753e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 50.97378922], dtype=float32), 8: array([ 17.82060051], dtype=float32), 9: array([ 280.375], dtype=float32), 10: None, 11: array([  3.57142753e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438734.120577, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142754e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 79.61238098], dtype=float32), 8: array([ 99.90670013], dtype=float32), 9: array([ 830.60198975], dtype=float32), 10: None, 11: array([  3.57142754e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438737.154774, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142754e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 81.31384277], dtype=float32), 8: array([ 89.92569733], dtype=float32), 9: array([ 575.98901367], dtype=float32), 10: None, 11: array([  3.57142754e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438739.68893, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142754e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 80.85481262], dtype=float32), 8: array([ 86.46209717], dtype=float32), 9: array([ 563.9329834], dtype=float32), 10: None, 11: array([  3.57142754e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438742.722411, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 42.43975067], dtype=float32), 8: array([ 13.87370014], dtype=float32), 9: array([ 910.49298096], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438745.256714, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142755e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 76.90184021], dtype=float32), 8: array([ 84.85610199], dtype=float32), 9: array([ 742.34002686], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438748.29093, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142755e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 92.28489685], dtype=float32), 8: array([ 19.77809906], dtype=float32), 9: array([ 272.94500732], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438750.841668, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142755e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 60.02323151], dtype=float32), 8: array([-7.79680014], dtype=float32), 9: array([ 418.51400757], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438753.424074, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142756e+09]), 2: array([  3.57142756e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 13.54555035], dtype=float32), 8: array([ 90.14240265], dtype=float32), 9: array([ 105.24099731], dtype=float32), 10: None, 11: array([  3.57142756e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438756.511647, 'provider_metadata_update': {}}
        #######################
        {'origin': '123xyz', 'stream_name': 'parsed', 'type': 'StreamAlaramType.WARNING', 'description': '', 'expr': 'x<5.0', 'value': 27.0952, 'type_': 'StreamWarningAlaramEvent', 'value_id': 'temp', 'base_types': ['StreamAlarmEvent', 'ResourceEvent', 'Event'], 'message': 'Temperature is above test range of 5.0.', '_id': '838337bd75ad4852afa15aac29c82a4c', 'ts_created': '1362438731572', 'sub_type': '', 'origin_type': 'InstrumentDevice', 'name': 'test_sim_warning'}
        {'origin': '123xyz', 'stream_name': 'parsed', 'type': 'StreamAlaramType.ALL_CLEAR', 'description': '', 'expr': 'x<5.0', 'value': -7.7968, 'type_': 'StreamAllClearAlarmEvent', 'value_id': 'temp', 'base_types': ['StreamAlarmEvent', 'ResourceEvent', 'Event'], 'message': 'Alarm is cleared.', '_id': '6a3dc641a3fd4092a203857cf6a70e0c', 'ts_created': '1362438753412', 'sub_type': '', 'origin_type': 'InstrumentDevice', 'name': 'test_sim_warning'}
        {'origin': '123xyz', 'stream_name': 'parsed', 'type': 'StreamAlaramType.WARNING', 'description': '', 'expr': 'x<5.0', 'value': 90.1424, 'type_': 'StreamWarningAlaramEvent', 'value_id': 'temp', 'base_types': ['StreamAlarmEvent', 'ResourceEvent', 'Event'], 'message': 'Temperature is above test range of 5.0.', '_id': 'd45d95a3d94345faa1cf82cb18398f3b', 'ts_created': '1362438756499', 'sub_type': '', 'origin_type': 'InstrumentDevice', 'name': 'test_sim_warning'}
        2013-03-04 15:13:00,746 INFO Dummy-1 pyon.event.event:259 EventSubscriber stopped. Event pattern=#.StreamAlarmEvent.#.*.#.*.123xyz
        """
        
        """
        {'domain': [1],
        'data_producer_id': '123xyz',
        'record_dictionary':
            {1: array([  3.57142756e+09]),
             2: array([  3.57142756e+09]),
             3: array([port_timestamp], dtype=object),
             4: 'ok',
             5: None,
             6: None,
             7: array([ 13.54555035], dtype=float32),
             8: array([ 90.14240265], dtype=float32),
             9: array([ 105.24099731], dtype=float32),
             10: None,
             11: array([  3.57142756e+09]),
             12: None, 13: None},
        'locator': None,
        'type_': 'Granule',
        'param_dictionary': '529b995c1aee4d149bca755aba7de687',
        'creation_timestamp': 1362438756.511647,
        'provider_metadata_update': {}}
        """
        
        print '#######################'
        print '#######################'
        print '#######################'
        for x in self._samples_received:
            #print str(x)
            print str(x.record_dictionary)
            #print str(type(x.record_dictionary))
            print str(x.param_dictionary)
            #print str(x['record_dictionary'][8])
            
        print '#######################'
        for x in self._events_received:
            print str(x)
            
class TestAgentPersistence(IonIntegrationTestCase):
    """
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Create agent config.
        self._agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True,
            'forget_past' : False,
            'enable_persistence' : True,
            'aparam_pubrate_config' :
                {
                    'raw' : 2,
                    'parsed' : 2
                }
        }

        self._ia_client = None
        self._ia_pid = '1234'
        
        self.addCleanup(self._verify_agent_reset)
        self.addCleanup(self.container.state_repository.put_state,
                        self._ia_pid, {})

    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                                    
    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'parsed'
        param_dict_name = 'ctd_parsed_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
        self._stream_config[stream_name] = stream_config

        stream_name = 'raw'
        param_dict_name = 'ctd_raw_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
        self._stream_config[stream_name] = stream_config

    ###############################################################################
    # Agent start stop helpers.
    ###############################################################################

    def _start_agent(self, bootmode=None):
        """
        """
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        
        agent_config = deepcopy(self._agent_config)
        agent_config['bootmode'] = bootmode
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config,
            process_id=self._ia_pid)            
            
        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got instrument agent client %s.', str(self._ia_client))

    def _stop_agent(self):
        """
        """
        if self._ia_pid:
            container_client = ContainerAgentClient(node=self.container.node,
                name=self.container.name)
            container_client.terminate_process(self._ia_pid)
        
        if self._ia_client:
            self._ia_client = None

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)
            self._ia_client = None

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_agent_config_persistence(self):
        """
        test_agent_config_persistence
        Test that agent parameter configuration is persisted between running
        instances.
        """
        
        # Start the agent.
        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Confirm the default agent parameters.
        #{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
        retval = self._ia_client.get_agent(['streams'])['streams']
        self.assertIn('raw', retval.keys())
        self.assertIn('parsed', retval.keys())

        #{'pubrate': {'raw': 0, 'parsed': 0}}
        retval = self._ia_client.get_agent(['pubrate'])['pubrate']
        self.assertIn('raw', retval.keys())
        self.assertIn('parsed', retval.keys())
        self.assertEqual(retval['raw'], 2)
        self.assertEqual(retval['parsed'], 2)
        
        #{'alerts': []}
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertEqual(retval, [])

        # Define a few new parameters and set them.
        # Confirm they are set.
        alert_def_1 = {
            'name' : 'current_warning_interval',
            'stream_name' : 'parsed',
            'description' : 'Current is below normal range.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 10.0,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        alert_def_2 = {
            'name' : 'temp_alarm_interval',
            'stream_name' : 'parsed',
            'description' : 'Temperatoure is critical.',
            'alert_type' : StreamAlertType.ALARM,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 20.0,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        alert_def3 = {
            'name' : 'late_data_warning',
            'stream_name' : 'parsed',
            'description' : 'Expected data has not arrived.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
            'time_delta' : 180,
            'alert_class' : 'LateDataAlert'
        }

        orig_alerts = [alert_def_1,alert_def_2, alert_def3]
        pubrate = {
            'parsed' : 10,
            'raw' : 20
        }
        params = {
            'alerts' : orig_alerts,
            'pubrate' : pubrate
        }
        
        # Set the new agent params and confirm.
        self._ia_client.set_agent(params)
        
        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        pubrate = retval['pubrate']
        alerts = retval['alerts']
        self.assertIn('raw', pubrate.keys())
        self.assertIn('parsed', pubrate.keys())
        self.assertEqual(pubrate['parsed'], 10)
        self.assertEqual(pubrate['raw'], 20)
        count = 0
        for x in alerts:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEqual(count, 3)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(15)
        self._start_agent('restart')

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Confirm the persisted parameters.
        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        
        pubrate = retval['pubrate']
        alerts = retval['alerts']
        self.assertIn('raw', pubrate.keys())
        self.assertIn('parsed', pubrate.keys())
        self.assertEqual(pubrate['parsed'], 10)
        self.assertEqual(pubrate['raw'], 20)
        count = 0
        for x in alerts:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEqual(count, 3)

    def test_agent_state_persistence(self):
        """
        test_agent_state_persistence
        Verify that agents can be restored to their prior running state.
        """

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)


        alert_def3 = {
            'name' : 'late_data_warning',
            'stream_name' : 'parsed',
            'description' : 'Expected data has not arrived.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
            'time_delta' : 180,
            'alert_class' : 'LateDataAlert'
        }

        orig_pubrate = {
            'parsed' : 10,
            'raw' : 20
        }
        params = {
            'alerts' : [alert_def3],
            'pubrate' : orig_pubrate
        }

        # Set the new agent params and confirm.
        self._ia_client.set_agent(params)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Acquire sample returns a string, not a particle.  The particle
        # is created by the data handler though.
        cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(15)
        self._start_agent('restart')

        timeout = gevent.Timeout(240)
        timeout.start()
        try:
            while True:                
                state = self._ia_client.get_agent_state()
                print '## in state: ' + state
                if state == ResourceAgentState.COMMAND:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(1)
        except gevent.Timeout:
            self.fail("Could not restore agent state to COMMAND.")

        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        alerts = retval['alerts']
        pubrate = retval['pubrate']
        self.assertEqual(len(alerts), 1)
        self.assertEqual(alert_def3['name'], alerts[0]['name'])
        self.assertEqual(pubrate['raw'], 20)
        self.assertEqual(pubrate['parsed'], 10)

        cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.STOPPED)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(15)
        self._start_agent('restart')

        timeout = gevent.Timeout(240)
        timeout.start()
        try:
            while True:                
                state = self._ia_client.get_agent_state()
                if state == ResourceAgentState.STOPPED:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(1)
        except gevent.Timeout:
            self.fail("Could not restore agent state to STOPPED.")

        retval = self._ia_client.get_agent(params)
        alerts = retval['alerts']
        pubrate = retval['pubrate']
        self.assertEqual(len(alerts), 1)
        self.assertEqual(alert_def3['name'], alerts[0]['name'])
        self.assertEqual(pubrate['raw'], 20)
        self.assertEqual(pubrate['parsed'], 10)

        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

    def test_agent_rparam_persistence(self):
        """
        test_agent_rparam_persistence
        Verify ability to restore device configuration.
        ### Original values:
        {'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
        
        ### Values after set:
        {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}

        ### Restore config:
        {'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
        'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
        'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
        'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
        'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
        'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
        'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
        RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
        PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
        'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
        'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
        'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
        'RTCA0': 0.9999862}
        
        ### Of which we have:
        {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}        
        """

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        params = [
            SBE37Parameter.OUTPUTSV,
            SBE37Parameter.NAVG,
            SBE37Parameter.TA0
        ]
        retval = self._ia_client.get_resource(params)
        orig_params = retval

        new_params = {
            SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
            SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
            SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
        }
        
        #print '########### orig params'
        #print str(orig_params)
        
        self._ia_client.set_resource(new_params)
        retval = self._ia_client.get_resource(params)
        
        self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
                         new_params[SBE37Parameter.OUTPUTSV])
        self.assertEqual(retval[SBE37Parameter.NAVG],
                         new_params[SBE37Parameter.NAVG])
        delta = max(retval[SBE37Parameter.TA0],
                    new_params[SBE37Parameter.TA0])*.01
        self.assertAlmostEqual(retval[SBE37Parameter.TA0],
                               new_params[SBE37Parameter.TA0], delta=delta)

        #print '########### new params'
        #print str(retval)

        # Now stop and restart the agent.
        self._stop_agent()
        self._support.stop_pagent()
        gevent.sleep(10)
        self._start_pagent()
        gevent.sleep(10)
        self._start_agent('restart')

        timeout = gevent.Timeout(600)
        timeout.start()
        try:
            while True:                
                state = self._ia_client.get_agent_state()
                if state == ResourceAgentState.COMMAND:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(3)
        except gevent.Timeout:
            self.fail("Could not restore agent state to COMMAND.")
        
        # Verify the parameters have been restored as needed.
        retval = self._ia_client.get_resource(params)

        #print '########### restored params'
        #print str(retval)
        
        self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
                         new_params[SBE37Parameter.OUTPUTSV])
        self.assertEqual(retval[SBE37Parameter.NAVG],
                         new_params[SBE37Parameter.NAVG])
        delta = max(retval[SBE37Parameter.TA0],
                    new_params[SBE37Parameter.TA0])*.01
        self.assertAlmostEqual(retval[SBE37Parameter.TA0],
                               new_params[SBE37Parameter.TA0], delta=delta)


        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
            
    @unittest.skip('Making CEI friendly.')
    def test_cei_launch_mode(self):
        
        pdc = ProcessDispatcherServiceClient(node=self.container.node)
        p_def = ProcessDefinition(name='Agent007')
        p_def.executable = {
            'module' : 'ion.agents.instrument.instrument_agent',
            'class' : 'InstrumentAgent'
        }
        p_def_id = pdc.create_process_definition(p_def)
        
        pid = pdc.create_process(p_def_id)
        
        def event_callback(event, *args, **kwargs):
            print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
 
        sub = EventSubscriber(event_type='ProcessLifecycleEvent',
                              callback=event_callback,
                              origin=pid,
                              origin_type='DispatchedProcess')
         
        sub.start()

        agent_config = deepcopy(self._agent_config)
        agent_config['bootmode'] = 'restart'
        pdc.schedule_process(p_def_id, process_id=pid,
                             configuration=agent_config)
        
        gevent.sleep(5)
        
        pdc.cancel_process(pid)
        
        gevent.sleep(15)

        sub.stop()
        
        
class TestIAAlarms(IonIntegrationTestCase):
    """
    """

    ############################################################################
    # Setup, teardown.
    ############################################################################

    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        log.info("Creating driver integration test support:")
        log.info("driver uri: %s", DRV_URI)
        log.info("device address: %s", DEV_ADDR)
        log.info("device port: %s", DEV_PORT)
        log.info("log delimiter: %s", DELIM)
        log.info("work dir: %s", WORK_DIR)
        self._support = DriverIntegrationTestSupport(
            None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR
        )

        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Start container.
        log.info("Staring capability container.")
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        log.info("Staring deploy services.")
        self.container.start_rel_from_url("res/deploy/r2deploy.yml")

        log.info("building stream configuration")
        # Setup stream config.
        self._build_stream_config()

        # Start a resource agent client to talk with the instrument agent.
        log.info("starting IA process")
        self._ia_client = start_instrument_agent_process(self.container, self._stream_config)
        self.addCleanup(self._verify_agent_reset)
        log.info("test setup complete")

    ###############################################################################
    # Port agent helpers.
    ###############################################################################

    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info("Port agent started at port %i", port)

        # Configure driver to use port agent port number.
        DVR_CONFIG["comms_config"] = {"addr": "localhost", "port": port, "cmd_port": CMD_PORT}

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)

    ###############################################################################
    # Event helpers.
    ###############################################################################

    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        streams = {"parsed": "ctd_parsed_param_dict", "raw": "ctd_raw_param_dict"}

        for (stream_name, param_dict_name) in streams.iteritems():
            pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)

            stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
            pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary

            stream_id, stream_route = pubsub_client.create_stream(
                name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
            )

            stream_config = dict(
                stream_route=stream_route,
                routing_key=stream_route.routing_key,
                exchange_point=stream_route.exchange_point,
                stream_id=stream_id,
                stream_definition_ref=stream_def_id,
                parameter_dictionary=pd,
            )

            if stream_name == "parsed":

                type = "IntervalAlarmDef"
                kwargs = {
                    "name": "test_sim_warning",
                    "stream_name": "parsed",
                    "value_id": "temp",
                    "message": "Temperature is above test range of 5.0.",
                    "type": StreamAlarmType.WARNING,
                    "upper_bound": 5.0,
                    "upper_rel_op": "<",
                }
                alarm = {}
                alarm["type"] = type
                alarm["kwargs"] = kwargs
                alarms = [alarm]
                stream_config["alarms"] = alarms

            self._stream_config[stream_name] = stream_config

    def _start_data_subscribers(self, count):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # Create streams and subscriptions for each stream named in driver.
        self._data_subscribers = []
        self._samples_received = []
        self._async_sample_result = AsyncResult()

        # A callback for processing subscribed-to data.
        def recv_data(message, stream_route, stream_id):
            log.info("Received message on %s (%s,%s)", stream_id, stream_route.exchange_point, stream_route.routing_key)
            self._samples_received.append(message)
            if len(self._samples_received) == count:
                self._async_sample_result.set()

        for (stream_name, stream_config) in self._stream_config.iteritems():

            stream_id = stream_config["stream_id"]

            # Create subscriptions for each stream.

            from pyon.util.containers import create_unique_identifier

            # exchange_name = '%s_queue' % stream_name
            exchange_name = create_unique_identifier("%s_queue" % stream_name)
            self._purge_queue(exchange_name)
            sub = StandaloneStreamSubscriber(exchange_name, recv_data)
            sub.start()
            self._data_subscribers.append(sub)
            print "stream_id: %s" % stream_id
            sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
            pubsub_client.activate_subscription(sub_id)
            sub.subscription_id = (
                sub_id
            )  # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)

    def _purge_queue(self, queue):
        xn = self.container.ex_manager.create_xn_queue(queue)
        xn.purge()

    def _stop_data_subscribers(self):
        for subscriber in self._data_subscribers:
            pubsub_client = PubsubManagementServiceClient()
            if hasattr(subscriber, "subscription_id"):
                try:
                    pubsub_client.deactivate_subscription(subscriber.subscription_id)
                except:
                    pass
                pubsub_client.delete_subscription(subscriber.subscription_id)
            subscriber.stop()

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_config(self):
        """
        test_initialize
        Test agent initialize command. This causes creation of
        driver process and transition to inactive.
        """

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        decoder = IonObjectDeserializer(obj_registry=get_obj_registry())

        # Grab the alarms defined in the config.
        retval = decoder.deserialize(self._ia_client.get_agent(["alarms"])["alarms"])

        """
        {'status': None, 'stream_name': 'parsed', 'name': 'test_sim_warning',
        'upper_bound': 5.0, 'expr': 'x<5.0', 'upper_rel_op': '<',
        'lower_rel_op': None, 'type_': 'IntervalAlarmDef', 'value_id': 'temp',
        'lower_bound': None, 'message': 'Temperature is above test range of 5.0.',
        'current_val': None, 'type': 1}
        """
        self.assertEqual(retval[0].type_, "IntervalAlarmDef")
        self.assertEqual(retval[0].upper_bound, 5.0)
        self.assertEqual(retval[0].expr, "x<5.0")

        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

    def test_autosample(self):
        """
        test_autosample
        Test instrument driver execute interface to start and stop streaming
        mode. Verify ResourceAgentResourceStateEvents are publsihed.
        """

        # Start data subscribers.
        # self._start_data_subscribers(6)
        # self.addCleanup(self._stop_data_subscribers)

        # Set up a subscriber to collect error events.
        # self._start_event_subscriber('ResourceAgentResourceStateEvent', 7)
        # self.addCleanup(self._stop_event_subscriber)

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        gevent.sleep(15)

        cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
示例#7
0
class TestAgentPersistence(IonIntegrationTestCase):
    """
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Create agent config.
        self._agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True,
            'forget_past' : False,
            'enable_persistence' : True,
            'aparam_pubrate_config' :
                {
                    'raw' : 2,
                    'parsed' : 2
                }
        }

        self._ia_client = None
        self._ia_pid = '1234'
        
        self.addCleanup(self._verify_agent_reset)
        self.addCleanup(self.container.state_repository.put_state,
                        self._ia_pid, {})

    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                                    
    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'parsed'
        param_dict_name = 'ctd_parsed_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
        self._stream_config[stream_name] = stream_config

        stream_name = 'raw'
        param_dict_name = 'ctd_raw_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
        self._stream_config[stream_name] = stream_config

    ###############################################################################
    # Agent start stop helpers.
    ###############################################################################

    def _start_agent(self, bootmode=None):
        """
        """
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        
        agent_config = deepcopy(self._agent_config)
        agent_config['bootmode'] = bootmode
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config,
            process_id=self._ia_pid)            
            
        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got instrument agent client %s.', str(self._ia_client))

    def _stop_agent(self):
        """
        """
        if self._ia_pid:
            container_client = ContainerAgentClient(node=self.container.node,
                name=self.container.name)
            container_client.terminate_process(self._ia_pid)
        
        if self._ia_client:
            self._ia_client = None

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)
            self._ia_client = None

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_agent_config_persistence(self):
        """
        test_agent_config_persistence
        Test that agent parameter configuration is persisted between running
        instances.
        """
        
        # Start the agent.
        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Confirm the default agent parameters.
        #{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
        retval = self._ia_client.get_agent(['streams'])['streams']
        self.assertIn('raw', retval.keys())
        self.assertIn('parsed', retval.keys())

        #{'pubrate': {'raw': 0, 'parsed': 0}}
        retval = self._ia_client.get_agent(['pubrate'])['pubrate']
        self.assertIn('raw', retval.keys())
        self.assertIn('parsed', retval.keys())
        self.assertEqual(retval['raw'], 2)
        self.assertEqual(retval['parsed'], 2)
        
        #{'alerts': []}
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertEqual(retval, [])

        # Define a few new parameters and set them.
        # Confirm they are set.
        alert_def_1 = {
            'name' : 'current_warning_interval',
            'stream_name' : 'parsed',
            'description' : 'Current is below normal range.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 10.0,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        alert_def_2 = {
            'name' : 'temp_alarm_interval',
            'stream_name' : 'parsed',
            'description' : 'Temperatoure is critical.',
            'alert_type' : StreamAlertType.ALARM,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 20.0,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        alert_def3 = {
            'name' : 'late_data_warning',
            'stream_name' : 'parsed',
            'description' : 'Expected data has not arrived.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
            'time_delta' : 180,
            'alert_class' : 'LateDataAlert'
        }

        orig_alerts = [alert_def_1,alert_def_2, alert_def3]
        pubrate = {
            'parsed' : 10,
            'raw' : 20
        }
        params = {
            'alerts' : orig_alerts,
            'pubrate' : pubrate
        }
        
        # Set the new agent params and confirm.
        self._ia_client.set_agent(params)
        
        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        pubrate = retval['pubrate']
        alerts = retval['alerts']
        self.assertIn('raw', pubrate.keys())
        self.assertIn('parsed', pubrate.keys())
        self.assertEqual(pubrate['parsed'], 10)
        self.assertEqual(pubrate['raw'], 20)
        count = 0
        for x in alerts:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEqual(count, 3)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(15)
        self._start_agent('restart')

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Confirm the persisted parameters.
        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        
        pubrate = retval['pubrate']
        alerts = retval['alerts']
        self.assertIn('raw', pubrate.keys())
        self.assertIn('parsed', pubrate.keys())
        self.assertEqual(pubrate['parsed'], 10)
        self.assertEqual(pubrate['raw'], 20)
        count = 0
        for x in alerts:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEqual(count, 3)

    def test_agent_state_persistence(self):
        """
        test_agent_state_persistence
        Verify that agents can be restored to their prior running state.
        """

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)


        alert_def3 = {
            'name' : 'late_data_warning',
            'stream_name' : 'parsed',
            'description' : 'Expected data has not arrived.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
            'time_delta' : 180,
            'alert_class' : 'LateDataAlert'
        }

        orig_pubrate = {
            'parsed' : 10,
            'raw' : 20
        }
        params = {
            'alerts' : [alert_def3],
            'pubrate' : orig_pubrate
        }

        # Set the new agent params and confirm.
        self._ia_client.set_agent(params)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Acquire sample returns a string, not a particle.  The particle
        # is created by the data handler though.
        cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(15)
        self._start_agent('restart')

        timeout = gevent.Timeout(240)
        timeout.start()
        try:
            while True:                
                state = self._ia_client.get_agent_state()
                print '## in state: ' + state
                if state == ResourceAgentState.COMMAND:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(1)
        except gevent.Timeout:
            self.fail("Could not restore agent state to COMMAND.")

        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        alerts = retval['alerts']
        pubrate = retval['pubrate']
        self.assertEqual(len(alerts), 1)
        self.assertEqual(alert_def3['name'], alerts[0]['name'])
        self.assertEqual(pubrate['raw'], 20)
        self.assertEqual(pubrate['parsed'], 10)

        cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.STOPPED)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(15)
        self._start_agent('restart')

        timeout = gevent.Timeout(240)
        timeout.start()
        try:
            while True:                
                state = self._ia_client.get_agent_state()
                if state == ResourceAgentState.STOPPED:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(1)
        except gevent.Timeout:
            self.fail("Could not restore agent state to STOPPED.")

        retval = self._ia_client.get_agent(params)
        alerts = retval['alerts']
        pubrate = retval['pubrate']
        self.assertEqual(len(alerts), 1)
        self.assertEqual(alert_def3['name'], alerts[0]['name'])
        self.assertEqual(pubrate['raw'], 20)
        self.assertEqual(pubrate['parsed'], 10)

        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

    def test_agent_rparam_persistence(self):
        """
        test_agent_rparam_persistence
        Verify ability to restore device configuration.
        ### Original values:
        {'TA0': -0.0002572242, 'OUTPUTSV': False, 'NAVG': 0}
        
        ### Values after set:
        {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}

        ### Restore config:
        {'PTCA1': 0.6603433, 'WBOTC': 1.2024e-05, 'PCALDATE': [12, 8, 2005],
        'STORETIME': False, 'CPCOR': 9.57e-08, 'PTCA2': 0.00575649,
        'OUTPUTSV': True, 'SAMPLENUM': 0, 'TCALDATE': [8, 11, 2005],
        'OUTPUTSAL': False, 'TA2': -9.717158e-06, 'POFFSET': 0.0,
        'INTERVAL': 19733, 'SYNCWAIT': 0, 'CJ': 3.339261e-05,
        'CI': 0.0001334915, 'CH': 0.1417895, 'TA0': -0.0005144484,
        'TA1': 0.0003138936, 'NAVG': 1, 'TA3': 2.138735e-07, '
        RCALDATE': [8, 11, 2005], 'CG': -0.987093, 'CTCOR': 3.25e-06, '
        PTCB0': 24.6145, 'PTCB1': -0.0009, 'PTCB2': 0.0,
        'CCALDATE': [8, 11, 2005], 'PA0': 5.916199, 'PA1': 0.4851819,
        'PA2': 4.596432e-07, 'SYNCMODE': False, 'PTCA0': 276.2492,
        'TXREALTIME': True, 'RTCA2': -3.022745e-08, 'RTCA1': 1.686132e-06,
        'RTCA0': 0.9999862}
        
        ### Of which we have:
        {'TA0': -0.0005144484, 'OUTPUTSV': True, 'NAVG': 1}        
        """

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        params = [
            SBE37Parameter.OUTPUTSV,
            SBE37Parameter.NAVG,
            SBE37Parameter.TA0
        ]
        retval = self._ia_client.get_resource(params)
        orig_params = retval

        new_params = {
            SBE37Parameter.OUTPUTSV : not orig_params[SBE37Parameter.OUTPUTSV],
            SBE37Parameter.NAVG : orig_params[SBE37Parameter.NAVG] + 1,
            SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 2
        }
        
        #print '########### orig params'
        #print str(orig_params)
        
        self._ia_client.set_resource(new_params)
        retval = self._ia_client.get_resource(params)
        
        self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
                         new_params[SBE37Parameter.OUTPUTSV])
        self.assertEqual(retval[SBE37Parameter.NAVG],
                         new_params[SBE37Parameter.NAVG])
        delta = max(retval[SBE37Parameter.TA0],
                    new_params[SBE37Parameter.TA0])*.01
        self.assertAlmostEqual(retval[SBE37Parameter.TA0],
                               new_params[SBE37Parameter.TA0], delta=delta)

        #print '########### new params'
        #print str(retval)

        # Now stop and restart the agent.
        self._stop_agent()
        self._support.stop_pagent()
        gevent.sleep(10)
        self._start_pagent()
        gevent.sleep(10)
        self._start_agent('restart')

        timeout = gevent.Timeout(600)
        timeout.start()
        try:
            while True:                
                state = self._ia_client.get_agent_state()
                if state == ResourceAgentState.COMMAND:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(3)
        except gevent.Timeout:
            self.fail("Could not restore agent state to COMMAND.")
        
        # Verify the parameters have been restored as needed.
        retval = self._ia_client.get_resource(params)

        #print '########### restored params'
        #print str(retval)
        
        self.assertEqual(retval[SBE37Parameter.OUTPUTSV],
                         new_params[SBE37Parameter.OUTPUTSV])
        self.assertEqual(retval[SBE37Parameter.NAVG],
                         new_params[SBE37Parameter.NAVG])
        delta = max(retval[SBE37Parameter.TA0],
                    new_params[SBE37Parameter.TA0])*.01
        self.assertAlmostEqual(retval[SBE37Parameter.TA0],
                               new_params[SBE37Parameter.TA0], delta=delta)


        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
            
    @unittest.skip('Making CEI friendly.')
    def test_cei_launch_mode(self):
        
        pdc = ProcessDispatcherServiceClient(node=self.container.node)
        p_def = ProcessDefinition(name='Agent007')
        p_def.executable = {
            'module' : 'ion.agents.instrument.instrument_agent',
            'class' : 'InstrumentAgent'
        }
        p_def_id = pdc.create_process_definition(p_def)
        
        pid = pdc.create_process(p_def_id)
        
        def event_callback(event, *args, **kwargs):
            print '######### proc %s in state %s' % (event.origin, ProcessStateEnum._str_map[event.state])
 
        sub = EventSubscriber(event_type='ProcessLifecycleEvent',
                              callback=event_callback,
                              origin=pid,
                              origin_type='DispatchedProcess')
         
        sub.start()

        agent_config = deepcopy(self._agent_config)
        agent_config['bootmode'] = 'restart'
        pdc.schedule_process(p_def_id, process_id=pid,
                             configuration=agent_config)
        
        gevent.sleep(5)
        
        pdc.cancel_process(pid)
        
        gevent.sleep(15)

        sub.stop()
        
        
class TestInstrumentAgentWithTrhph(TrhphTestCase, IonIntegrationTestCase):
    """
    R2 instrument agent tests with the TRHPH driver.
    """

    def setUp(self):
        """
        Initialize test members.
        Start port agent.
        Start container and client.
        Start streams and subscribers.
        Start agent, client.
        """

        TrhphTestCase.setUp(self)

        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     self.device_address,
                                                     self.device_port,
                                                     DELIM,
                                                     WORK_DIR)
        # Start port agent, add stop to cleanup.
        self._pagent = None
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Start container.
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Start data suscribers, add stop to cleanup.
        # Define stream_config.
        self._no_samples = None
        self._async_data_result = AsyncResult()
        self._data_greenlets = []
        self._stream_config = {}
        self._samples_received = []
        self._data_subscribers = []
        self._start_data_subscribers()
        self.addCleanup(self._stop_data_subscribers)

        # Start event subscribers, add stop to cleanup.
        self._no_events = None
        self._async_event_result = AsyncResult()
        self._events_received = []
        self._event_subscribers = []
        self._start_event_subscribers()
        self.addCleanup(self._stop_event_subscribers)

        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }

        # Start instrument agent.
        self._ia_pid = None
        log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
                                                name=self.container.name)
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
                                                      module=IA_MOD,
                                                      cls=IA_CLS,
                                                      config=agent_config)
        log.info('Agent pid=%s.', str(self._ia_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))

        # make sure the driver is stopped
        self.addCleanup(self._reset)

    def addCleanup(self, f):
        IonIntegrationTestCase.addCleanup(self, f)

    def tearDown(self):
        try:
            IonIntegrationTestCase.tearDown(self)
        finally:
            TrhphTestCase.tearDown(self)

    def _start_pagent(self):
        """
        Construct and start the port agent.
        """
        port = self._support.start_pagent()

        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }

    def _start_data_subscribers(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # A callback for processing subscribed-to data.
        def consume_data(message, headers):
            log.info('Subscriber received data message: %s.', str(message))
            self._samples_received.append(message)
            if self._no_samples and self._no_samples == len(self._samples_received):
                self._async_data_result.set()

        # Create a stream subscriber registrar to create subscribers.
        subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
                                                         container=self.container)

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}
        self._data_subscribers = []
        for stream_name in PACKET_CONFIG:
            stream_def = ctd_stream_definition(stream_id=None)
            stream_def_id = pubsub_client.create_stream_definition(
                                                    container=stream_def)
            stream_id = pubsub_client.create_stream(
                        name=stream_name,
                        stream_definition_id=stream_def_id,
                        original=True,
                        encoding='ION R2')

            taxy = get_taxonomy(stream_name)
            stream_config = dict(
                id=stream_id,
                taxonomy=taxy.dump()
            )
            self._stream_config[stream_name] = stream_config

            # Create subscriptions for each stream.
            exchange_name = '%s_queue' % stream_name
            sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
                                                         callback=consume_data)
            self._listen(sub)
            self._data_subscribers.append(sub)
            query = StreamQuery(stream_ids=[stream_id])
            sub_id = pubsub_client.create_subscription(
                                query=query, exchange_name=exchange_name,
                                exchange_point='science_data')
            pubsub_client.activate_subscription(sub_id)

    def _listen(self, sub):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        gl = spawn(sub.listen)
        self._data_greenlets.append(gl)
        sub._ready_event.wait(timeout=5)
        return gl

    def _stop_data_subscribers(self):
        """
        Stop the data subscribers on cleanup.
        """
        log.info('cleanup: _stop_data_subscribers called.')
        for sub in self._data_subscribers:
            sub.stop()
        for gl in self._data_greenlets:
            gl.kill()

    def _start_event_subscribers(self):
        """
        Create subscribers for agent and driver events.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._no_events and self._no_events == len(self._event_received):
                self._async_event_result.set()

        event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
        event_sub.start()
        self._event_subscribers.append(event_sub)

    def _stop_event_subscribers(self):
        """
        Stop event subscribers on cleanup.
        """
        log.info('cleanup: _stop_event_subscribers called.')
        for sub in self._event_subscribers:
            sub.stop()

    def _initialize_and_run(self):
        """
        Called explicitly by the tests that do regular operations.
        """
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)


    def _reset(self):
        """
        Set as a clean-up to make sure the agent is reset after each test,
        so the driver is stopped.
        """
        log.info("_reset called: sending 'reset' command to agent")
        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_01_initialize(self):
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)

        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)

        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_10_states(self):
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)

        cmd = AgentCommand(command='resume')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)

        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def _get_params(self, params):

        result = self._ia_client.get_param(params)
        log.info('_get_params result: %s' % str(result))

        self.assertTrue(isinstance(result, dict))

        if params == DriverParameter.ALL:
            all_requested_params = TrhphParameter.list()
        else:
            all_requested_params = params

        # check all requested params are in the result
        for p in all_requested_params:
            self.assertTrue(p in result)

            if TrhphParameter.TIME_BETWEEN_BURSTS == p:
                seconds = result.get(p)
                self.assertTrue(isinstance(seconds, int))
            elif TrhphParameter.VERBOSE_MODE == p:
                is_data_only = result.get(p)
                self.assertTrue(isinstance(is_data_only, bool))

        return result

    def test_15_get_params(self):
        self._initialize_and_run()

        self._get_params(DriverParameter.ALL)

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        p2 = TrhphParameter.VERBOSE_MODE
        self._get_params([p1, p2])

    def _set_params(self, params):
        """
        Sets the given parameters, which are assumed to be all valid.
        """
        result = self._ia_client.set_param(params)
        log.info("set result = %s" % str(result))

        if result is None:
            # TODO check why self._ia_client.set_param returns None
            return

        assert isinstance(result, dict)

        # check all requested params are in the result
        for (p, v) in params.items():
            self.assertTrue(p in result)

        return result

    def _get_verbose_flag_for_set_test(self):
        """
        Gets the value to use for the verbose flag in the "set" operations.
        If we are testing against the real instrument (self._is_real_instrument
        is true), this always returns False because the associated
        interfaces with verbose=True are not implemented yet.
        Otherwise it returns a random boolean value. Note, in this case, which
        means we are testing against the simulator, the actual verbose value
        does not have any effect of the other interface elements, so it is ok
        to set any value here.
        TODO align this when the verbose flag is handled completely,
        both in the driver and in the simulator.
        """
        if self._is_real_instrument:
            log.info("setting verbose=False because _is_real_instrument")
            return False
        else:
            return 0 == random.randint(0, 1)

    def test_20_set_params_valid(self):
        self._initialize_and_run()

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        new_seconds = random.randint(15, 60)

        p2 = TrhphParameter.VERBOSE_MODE
        verbose = self._get_verbose_flag_for_set_test()

        valid_params = {p1: new_seconds, p2: verbose}

        self._set_params(valid_params)

    def test_25_get_set_params(self):
        self._initialize_and_run()

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        result = self._get_params([p1])
        seconds = result[p1]
        new_seconds = seconds + 5
        if new_seconds > 30 or new_seconds < 15:
            new_seconds = 15

        p2 = TrhphParameter.VERBOSE_MODE
        new_verbose = self._get_verbose_flag_for_set_test()

        valid_params = {p1: new_seconds, p2: new_verbose}

        log.info("setting: %s" % str(valid_params))

        self._set_params(valid_params)

        result = self._get_params([p1, p2])

        seconds = result[p1]
        self.assertEqual(seconds, new_seconds)

        verbose = result[p2]
        self.assertEqual(verbose, new_verbose)

    def test_60_execute_stop_autosample(self):
        self._initialize_and_run()

        log.info("stopping autosample")
        cmd = AgentCommand(command='stop_autosample',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("stop_autosample reply = %s" % str(reply))

    def test_70_execute_get_metadata(self):
        self._initialize_and_run()

        log.info("getting metadata")
        cmd = AgentCommand(command='get_metadata',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("get_metadata reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, dict))

    def test_80_execute_diagnostics(self):
        self._initialize_and_run()

        log.info("executing diagnostics")
        num_scans = 11
        cmd = AgentCommand(command='diagnostics',
                           kwargs=dict(num_scans=num_scans, timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("diagnostics reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, list))
        self.assertEqual(len(reply.result), num_scans)

    def test_90_execute_get_power_statuses(self):
        self._initialize_and_run()

        log.info("executing get_power_statuses")
        cmd = AgentCommand(command='get_power_statuses',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("get_power_statuses reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, dict))

    def test_99_execute_start_autosample(self):
        self._initialize_and_run()

        log.info("executing start_autosample")
        cmd = AgentCommand(command='start_autosample',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("start_autosample reply = %s" % str(reply))

    @unittest.skip('Not running after code updates in other places')
    def test_get_params_invalid(self):
        self._initialize_and_run()
        with self.assertRaises(InstParameterError):
            self._get_params(['bad-param'])

    @unittest.skip('Not running after code updates in other places')
    def test_set_params_invalid(self):
        self._initialize_and_run()
        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        new_seconds = random.randint(15, 60)
        invalid_params = {p1: new_seconds, "bad-param": "dummy-value"}
        with self.assertRaises(InstParameterError):
            self._set_params(invalid_params)
class TestAgentConnectionFailures(IonIntegrationTestCase):
    """
    Test cases for instrument agent class. Functions in this class provide
    instrument agent integration tests and provide a tutorial on use of
    the agent setup and interface.
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Start a resource agent client to talk with the instrument agent.
        log.info('starting IA process')
        self._ia_client = start_instrument_agent_process(self.container, self._stream_config)
        self.addCleanup(self._verify_agent_reset)
        log.info('test setup complete')


    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                        
    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state(timeout=120.1)
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd,timeout=300)
            
    ###############################################################################
    # Event helpers.
    ###############################################################################

    def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
        """
        Start a subscriber to the instrument agent events.
        @param type The type of event to catch.
        @count Trigger the async event result when events received reaches this.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', 
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()
            
        # Event array and async event result.
        self._event_count = count
        self._events_received = []
        self._async_event_result = AsyncResult()
            
        self._event_subscriber = EventSubscriber(
            event_type=type, callback=consume_event,
            origin=IA_RESOURCE_ID)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=5)

    def _stop_event_subscriber(self):
        """
        Stop event subscribers on cleanup.
        """
        self._event_subscriber.stop()
        self._event_subscriber = None

    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        
        encoder = IonObjectSerializer()
        
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'parsed'
        param_dict_name = 'ctd_parsed_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)        
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 parameter_dictionary=pd,
                                 stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config

        stream_name = 'raw'
        param_dict_name = 'ctd_raw_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 parameter_dictionary=pd,
                                 stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config

    def _start_data_subscribers(self, count, raw_count):
        """
        """        
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
                
        # Create streams and subscriptions for each stream named in driver.
        self._data_subscribers = []
        self._samples_received = []
        self._raw_samples_received = []
        self._async_sample_result = AsyncResult()
        self._async_raw_sample_result = AsyncResult()

        # A callback for processing subscribed-to data.
        def recv_data(message, stream_route, stream_id):
            log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
            self._samples_received.append(message)
            if len(self._samples_received) == count:
                self._async_sample_result.set()

        def recv_raw_data(message, stream_route, stream_id):
            log.info('Received raw data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
            self._raw_samples_received.append(message)
            if len(self._raw_samples_received) == raw_count:
                self._async_raw_sample_result.set()

        from pyon.util.containers import create_unique_identifier

        stream_name = 'parsed'
        parsed_config = self._stream_config[stream_name]
        stream_id = parsed_config['stream_id']
        exchange_name = create_unique_identifier("%s_queue" %
                    stream_name)
        self._purge_queue(exchange_name)
        sub = StandaloneStreamSubscriber(exchange_name, recv_data)
        sub.start()
        self._data_subscribers.append(sub)
        sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id],timeout=120.2)
        pubsub_client.activate_subscription(sub_id,timeout=120.3)
        sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
        
        stream_name = 'raw'
        parsed_config = self._stream_config[stream_name]
        stream_id = parsed_config['stream_id']
        exchange_name = create_unique_identifier("%s_queue" %
                    stream_name)
        self._purge_queue(exchange_name)
        sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
        sub.start()
        self._data_subscribers.append(sub)
        sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id],timeout=120.4)
        pubsub_client.activate_subscription(sub_id,timeout=120.5)
        sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)

    def _purge_queue(self, queue):
        xn = self.container.ex_manager.create_xn_queue(queue)
        xn.purge()
 
    def _stop_data_subscribers(self):
        for subscriber in self._data_subscribers:
            pubsub_client = PubsubManagementServiceClient()
            if hasattr(subscriber,'subscription_id'):
                try:
                    pubsub_client.deactivate_subscription(subscriber.subscription_id,timeout=120.6)
                except:
                    pass
                pubsub_client.delete_subscription(subscriber.subscription_id,timeout=120.7)
            subscriber.stop()

    ###############################################################################
    # Socket listen.
    ###############################################################################

    def _socket_listen(self, s, prompt, timeout):

        buf = ''
        starttime = time.time()
        while True:
            try:
                buf += s.recv(1024)
                print '##### Listening, got: %s' % buf
                if prompt and buf.find(prompt) != -1:
                    break
            except:
                gevent.sleep(1)
            
            finally:
                delta = time.time() - starttime
                if delta > timeout:
                    break
        return buf            
                
    ###############################################################################
    # Assert helpers.
    ###############################################################################
        
    def assertSampleDict(self, val):
        """
        Verify the value is a sample dictionary for the sbe37.
        """
        # AgentCommandResult.result['parsed']
        """
        {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
        'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data',
        'pkt_version': 1, 'values':
        [{'value_id': 'temp', 'value': 21.4894},
        {'value_id': 'conductivity', 'value': 13.22157},
        {'value_id': 'pressure', 'value': 146.186}],
        'driver_timestamp': 3556901018.170206}
        """
        
        self.assertIsInstance(val, dict)
        self.assertTrue(val.has_key('values'))
        values_list = val['values']
        self.assertTrue(isinstance(values_list, list))
        self.assertTrue(len(values_list)==3)
        
        ids = ['temp', 'conductivity', 'pressure']
        ids_found = []

        for x in values_list:
            self.assertTrue(x.has_key('value_id'))
            self.assertTrue(x.has_key('value'))
            ids_found.append(x['value_id'])
            self.assertTrue(isinstance(x['value'], float))

        self.assertItemsEqual(ids, ids_found)

        self.assertTrue(val.has_key('driver_timestamp'))
        time = val['driver_timestamp']
        self.assertTrue(isinstance(time, float))
        
    def assertParamDict(self, pd, all_params=False):
        """
        Verify all device parameters exist and are correct type.
        """
        if all_params:
            self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
            for (key, type_val) in PARAMS.iteritems():
                if type_val == list or type_val == tuple:
                    self.assertTrue(isinstance(pd[key], (list, tuple)))
                else:
                    self.assertTrue(isinstance(pd[key], type_val))

        else:
            for (key, val) in pd.iteritems():
                self.assertTrue(PARAMS.has_key(key))
                self.assertTrue(isinstance(val, PARAMS[key]))
        
    def assertParamVals(self, params, correct_params):
        """
        Verify parameters take the correct values.
        """
        self.assertEqual(set(params.keys()), set(correct_params.keys()))
        for (key, val) in params.iteritems():
            correct_val = correct_params[key]
            if isinstance(val, float):
                # Verify to 5% of the larger value.
                max_val = max(abs(val), abs(correct_val))
                self.assertAlmostEqual(val, correct_val, delta=max_val*.01)

            elif isinstance(val, (list, tuple)):
                # list of tuple.
                self.assertEqual(list(val), list(correct_val))
            
            else:
                # int, bool, str.
                self.assertEqual(val, correct_val)

    ###############################################################################
    # Tests.
    ###############################################################################
                
    def test_lost_connection(self):
        """
        test_lost_connection
        """
        
        # Set up a subscriber to collect command events.
        self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1)
        self.addCleanup(self._stop_event_subscriber)    
        
        # Start in uninitialized.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
    
        # Initialize the agent.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Activate.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        # Go into command mode.
        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Start streaming.
        cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)
        
        # Wait for a while, collect some samples.
        gevent.sleep(10)
        
        # Blow the port agent out from under the agent.
        self._support.stop_pagent()
        
        # Loop until we resyncronize to LOST_CONNECTION/DISCONNECTED.
        # Test will timeout if this dosn't occur.
        while True:
            state = self._ia_client.get_agent_state()
            if state == ResourceAgentState.LOST_CONNECTION:
                break
            else:
                gevent.sleep(1)
        
        # Verify the driver has transitioned to disconnected
        while True:
            state = self._ia_client.get_resource_state()
            if state == DriverConnectionState.DISCONNECTED:
                break
            else:
                gevent.sleep(1)

        # Make sure the lost connection error event arrives.
        self._async_event_result.get(timeout=CFG.endpoint.receive.timeout)                        
        self.assertEqual(len(self._events_received), 1)        

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

    #@unittest.skip('Fails on buildbot for some god unknown reason.')
    def test_autoreconnect(self):
        """
        test_autoreconnect
        """
        # Set up a subscriber to collect command events.
        self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent', 1)
        self.addCleanup(self._stop_event_subscriber)    
        
        # Start in uninitialized.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
    
        # Initialize the agent.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Activate.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        # Go into command mode.
        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)


        def poll_func(test):
            cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
            while True:
                try:
                    gevent.sleep(.5)
                    test._ia_client.execute_resource(cmd)
                except IonException as ex:
                    # This exception could be ResourceException (broken pipe)
                    # Timeout or Conflict
                    log.info('#### pre shutdown exception: %s, %s', str(type(ex)), str(ex))
                    break
                    
            while True:
                try:
                    gevent.sleep(.5)
                    test._ia_client.execute_resource(cmd)
                    log.info('#### post shutdown got new sample.')
                    break
                except IonException as ex:
                    # This should be conflict.
                    log.info('#### post shutdown exception: %s, %s', str(type(ex)), str(ex))
                
        timeout = gevent.Timeout(600)
        timeout.start()
        try:

            # Start the command greenlet and let poll for a bit.
            gl = gevent.spawn(poll_func, self)        
            gevent.sleep(20)
        
            # Blow the port agent out from under the agent.
            self._support.stop_pagent()

            # Wait for a while, the supervisor is restarting the port agent.
            gevent.sleep(10)
            self._support.start_pagent()
            
            # Wait for the device to connect and start sampling again.
            gl.join()
            gl = None
            timeout.cancel()
            
        except (Exception, gevent.Timeout) as ex:
            if gl:
                gl.kill()
                gl = None
            self.fail(('Could not reconnect to device: %s,  %s',
                      str(type(ex)), str(ex)))

    def test_connect_failed(self):
        """
        test_connect_failed
        """
        # Stop the port agent.
        self._support.stop_pagent()
        
        # Sleep a bit.
        gevent.sleep(3)
        
        # Start in uninitialized.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
    
        # Initialize the agent.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Activate. This should fail because there is no port agent to connect to.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        with self.assertRaises(ResourceError):
            retval = self._ia_client.execute_agent(cmd)
            
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

    def test_get_set_alerts(self):
        """
        test_get_set_alerts
        Test specific of get/set alerts, including using result of get to
        set later.
        """
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertItemsEqual(retval, [])

        alert_def1 = {
            'name' : 'temp_warning_interval',
            'stream_name' : 'parsed',
            'description' : 'Temperature is above normal range.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 10.5,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }
        
        alert_def2 = {
            'name' : 'temp_alarm_interval',
            'stream_name' : 'parsed',
            'description' : 'Temperature is way above normal range.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 15.5,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }


        """
        Interval alerts are returned from get like this:
        (value and status fields describe state of the alert)
        {
        'name': 'temp_warning_interval',
        'stream_name': 'parsed',
        'description': 'Temperature is above normal range.',
        'alert_type': 1,
        'aggregate_type': 2,
        'value_id': 'temp',
        'lower_bound': None,
        'lower_rel_op': None,
        'upper_bound': 10.5,
        'upper_rel_op': '<',
        'alert_class': 'IntervalAlert',

        'status': None,
        'value': None
        }
        """
        
        alert_def3 = {
            'name' : 'late_data_warning',
            'stream_name' : 'parsed',
            'description' : 'Expected data has not arrived.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
            'time_delta' : 180,
            'alert_class' : 'LateDataAlert'
        }

        """
        Late data alerts are returned from get like this:
        (value and status fields describe state of the alert)
        {
        'name': 'late_data_warning',
        'stream_name': 'parsed',
        'description': 'Expected data has not arrived.',
        'alert_type': 1,
        'aggregate_type': 1,
        'value_id': None,
        'time_delta': 180,
        'alert_class': 'LateDataAlert',
        
        'status': None,
        'value': None
        }
        """
        
        """
        [
            {'status': None,
            'alert_type': 1,
            'name': 'temp_warning_interval',
            'upper_bound': 10.5,
            'lower_bound': None,
            'aggregate_type': 2,
            'alert_class': 'IntervalAlert',
            'value': None,
            'value_id': 'temp',
            'lower_rel_op': None,
            'upper_rel_op': '<',
            'description': 'Temperature is above normal range.'},
            {'status': None,
            'alert_type': 1,
            'name': 'temp_alarm_interval',
            'upper_bound': 15.5,
            'lower_bound': None,
            'aggregate_type': 2,
            'alert_class': 'IntervalAlert',
            'value': None,
            'value_id': 'temp',
            'lower_rel_op': None,
            'upper_rel_op': '<',
            'description': 'Temperature is way above normal range.'},
            {'status': None,
             'stream_name': 'parsed',
             'alert_type': 1,
             'name': 'late_data_warning',
             'aggregate_type': 1,
             'alert_class': 'LateDataAlert',
             'value': None,
             'time_delta': 180,
             'description': 'Expected data has not arrived.'}
        ]
        """
        
        orig_alerts = [alert_def1, alert_def2, alert_def3]
        self._ia_client.set_agent({'alerts' : orig_alerts})
    
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertTrue(len(retval)==3)
        alerts = retval

        self._ia_client.set_agent({'alerts' : ['clear']})        
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertItemsEqual(retval, [])

        self._ia_client.set_agent({'alerts' : alerts})
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertTrue(len(retval)==3)
        
        count = 0
        for x in retval:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEquals(count, 3)
        
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestInstrumentAgent(IonIntegrationTestCase):
    """
    Test cases for instrument agent class. Functions in this class provide
    instrument agent integration tests and provide a tutorial on use of
    the agent setup and interface.
    """ 
    def setUp(self):
        """
        Initialize test members.
        Start port agent.
        Start container and client.
        Start streams and subscribers.
        Start agent, client.
        """
                
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DELIM,
                                                     WORK_DIR)
        # Start port agent, add stop to cleanup.
        self._pagent = None        
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Start data suscribers, add stop to cleanup.
        # Define stream_config.
        self._no_samples = None
        self._async_data_result = AsyncResult()
        self._data_greenlets = []
        self._stream_config = {}
        self._samples_received = []
        self._data_subscribers = []
        self._start_data_subscribers()
        self.addCleanup(self._stop_data_subscribers)

        # Start event subscribers, add stop to cleanup.
        self._no_events = None
        self._async_event_result = AsyncResult()
        self._events_received = []
        self._event_subscribers = []
        self._start_event_subscribers()
        self.addCleanup(self._stop_event_subscribers)

        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = start_fake_instrument_agent(self.container, self._stream_config)


    def _start_pagent(self):
        """
        Construct and start the port agent.
        """
        port = self._support.start_pagent()
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }
        
    def _start_data_subscribers(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # A callback for processing subscribed-to data.
        def consume_data(message, headers):
            log.info('Subscriber received data message: %s.', str(message))
            self._samples_received.append(message)
            if self._no_samples and self._no_samples == len(self._samples_received):
                self._async_data_result.set()
                
        # Create a stream subscriber registrar to create subscribers.
        subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
                                                container=self.container)

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}
        self._data_subscribers = []
        # TODO the following is a mininal adjustment to at least let the test
        # continue:
#        for (stream_name, val) in PACKET_CONFIG.iteritems():
        for stream_name in PACKET_CONFIG:
            stream_def = ctd_stream_definition(stream_id=None)
            stream_def_id = pubsub_client.create_stream_definition(
                                                    container=stream_def)        
            stream_id = pubsub_client.create_stream(
                        name=stream_name,
                        stream_definition_id=stream_def_id,
                        original=True,
                        encoding='ION R2')

            taxy = get_taxonomy(stream_name)
            stream_config = dict(
                id=stream_id,
                taxonomy=taxy.dump()
            )
            self._stream_config[stream_name] = stream_config
#            self._stream_config[stream_name] = stream_id

            # Create subscriptions for each stream.
            exchange_name = '%s_queue' % stream_name
            sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
                                                         callback=consume_data)
            self._listen(sub)
            self._data_subscribers.append(sub)
            query = StreamQuery(stream_ids=[stream_id])
            sub_id = pubsub_client.create_subscription(
                                query=query, exchange_name=exchange_name, exchange_point='science_data')
            pubsub_client.activate_subscription(sub_id)
            
    def _listen(self, sub):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        gl = spawn(sub.listen)
        self._data_greenlets.append(gl)
        sub._ready_event.wait(timeout=5)
        return gl
                                 
    def _stop_data_subscribers(self):
        """
        Stop the data subscribers on cleanup.
        """
        for sub in self._data_subscribers:
            sub.stop()
        for gl in self._data_greenlets:
            gl.kill()
            
    def _start_event_subscribers(self):
        """
        Create subscribers for agent and driver events.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', 
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._no_events and self._no_events == len(self._event_received):
                self._async_event_result.set()
                
        event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
        event_sub.start()
        self._event_subscribers.append(event_sub)
        
    def _stop_event_subscribers(self):
        """
        Stop event subscribers on cleanup.
        """
        for sub in self._event_subscribers:
            sub.stop()
        
    def assertRawSampleDict(self, val):
        """
        Verify the value is a raw sample dictionary for the sbe37.
        """
        #{'stream_name': 'raw', 'blob': ['20.8074,80.04590, 761.394,   33.5658, 1506.076, 01 Feb 2001, 01:01:00'], 'time': [1342392781.61211]}
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('blob'))
        self.assertTrue(val.has_key('time'))
        blob = val['blob'][0]
        time = val['time'][0]
    
        self.assertTrue(isinstance(blob, str))
        self.assertTrue(isinstance(time, float))

    def assertParsedSampleDict(self, val):
        """
        Verify the value is a parsed sample dictionary for the sbe37.
        """
        #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('c'))
        self.assertTrue(val.has_key('t'))
        self.assertTrue(val.has_key('p'))
        self.assertTrue(val.has_key('time'))
        c = val['c'][0]
        t = val['t'][0]
        p = val['p'][0]
        time = val['time'][0]

        self.assertTrue(isinstance(c, float))
        self.assertTrue(isinstance(t, float))
        self.assertTrue(isinstance(p, float))
        self.assertTrue(isinstance(time, float))

    def assertSample(self, sample):
        """
        Verify a sample retrieved from the sbe37, which is expected to be a
        dict of dicts {'raw':raw_sample, 'parsed':parsed_sample}.
        """
        self.assertTrue(isinstance(sample, dict))
        raw_sample = sample['raw']
        parsed_sample = sample['parsed']
        self.assertParsedSampleDict(parsed_sample)
        self.assertRawSampleDict(raw_sample)

    def assertParamDict(self, pd, all_params=False):
        """
        Verify all device parameters exist and are correct type.
        """
        if all_params:
            self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
            for (key, type_val) in PARAMS.iteritems():
                if type_val == list or type_val == tuple:
                    self.assertTrue(isinstance(pd[key], (list, tuple)))
                else:
                    self.assertTrue(isinstance(pd[key], type_val))
                    
        else:
            for (key, val) in pd.iteritems():
                self.assertTrue(PARAMS.has_key(key))
                self.assertTrue(isinstance(val, PARAMS[key]))
    
    def assertParamVals(self, params, correct_params):
        """
        Verify parameters take the correct values.
        """
        self.assertEqual(set(params.keys()), set(correct_params.keys()))
        for (key, val) in params.iteritems():
            correct_val = correct_params[key]
            if isinstance(val, float):
                # Verify to 5% of the larger value.
                max_val = max(abs(val), abs(correct_val))
                self.assertAlmostEqual(val, correct_val, delta=max_val*.01)

            elif isinstance(val, (list, tuple)):
                # list of tuple.
                self.assertEqual(list(val), list(correct_val))
            
            else:
                # int, bool, str.
                self.assertEqual(val, correct_val)

    def test_initialize(self):
        """
        Test agent initialize command. This causes creation of
        driver process and transition to inactive.
        """

        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)

        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
                
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
    def test_states(self):
        """
        Test agent state transitions.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
                
        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)
 
        cmd = AgentCommand(command='resume')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
 
        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)

        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_get_set(self):
        """
        Test instrument driver get and set interface.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # Retrieve all resource parameters.                
        reply = self._ia_client.get_param(SBE37Parameter.ALL)
        self.assertParamDict(reply, True)
        orig_config = reply
        
        # Retrieve a subset of resource parameters.
        params = [
            SBE37Parameter.TA0,
            SBE37Parameter.INTERVAL,
            SBE37Parameter.STORETIME
        ]
        reply = self._ia_client.get_param(params)
        self.assertParamDict(reply)
        orig_params = reply

        # Set a subset of resource parameters.
        new_params = {
            SBE37Parameter.TA0 : (orig_params[SBE37Parameter.TA0] * 2),
            SBE37Parameter.INTERVAL : (orig_params[SBE37Parameter.INTERVAL] + 1),
            SBE37Parameter.STORETIME : (not orig_params[SBE37Parameter.STORETIME])
        }
        self._ia_client.set_param(new_params)
        check_new_params = self._ia_client.get_param(params)
        self.assertParamVals(check_new_params, new_params)
        
        # Reset the parameters back to their original values.
        self._ia_client.set_param(orig_params)
        reply = self._ia_client.get_param(SBE37Parameter.ALL)
        reply.pop(SBE37Parameter.SAMPLENUM)
        orig_config.pop(SBE37Parameter.SAMPLENUM)
        self.assertParamVals(reply, orig_config)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_poll(self):
        """
        Test observatory polling function.
        """
        
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
        
        # Lets call acquire_sample 3 times, so we should get 6 samples =
        # 3 raw samples + 3 parsed samples
        self._no_samples = 6
        
        # Poll for a few samples.
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        log.debug("acquire_sample reply.result=%s" % reply.result)
        self.assertSample(reply.result)

        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSample(reply.result)

        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSample(reply.result)

        # Assert we got 6 samples.
        self._async_data_result.get(timeout=10)
        self.assertEquals(len(self._samples_received), 6)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)        
        
    def test_autosample(self):
        """
        Test instrument driver execute interface to start and stop streaming
        mode.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # Make sure the sampling rate and transmission are sane.                
        params = {
            SBE37Parameter.NAVG : 1,
            SBE37Parameter.INTERVAL : 5,
            SBE37Parameter.TXREALTIME : True
        }
        self._ia_client.set_param(params)

        self._no_samples = 2

        # Begin streaming.                
        cmd = AgentCommand(command='go_streaming')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STREAMING)
 
        # Wait for some samples to roll in.
        gevent.sleep(15)
 
        # Halt streaming.
        cmd = AgentCommand(command='go_observatory')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # Assert we got some samples.
        self._async_data_result.get(timeout=10)
        self.assertTrue(len(self._samples_received)>=2)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_capabilities(self):
        """
        Test the ability to retrieve agent and resource parameter and command
        capabilities.
        """
        acmds = self._ia_client.get_capabilities(['AGT_CMD'])
        acmds = [item[1] for item in acmds]
        self.assertEqual(acmds, AGT_CMDS)
        apars = self._ia_client.get_capabilities(['AGT_PAR'])
        apars = [item[1] for item in apars]
        
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)
        
        rcmds = self._ia_client.get_capabilities(['RES_CMD'])
        rcmds = [item[1] for item in rcmds]
        self.assertEqual(rcmds, CMDS)
        
        rpars = self._ia_client.get_capabilities(['RES_PAR'])
        rpars = [item[1] for item in rpars]
        self.assertEqual(rpars, SBE37Parameter.list())
                
        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
    @unittest.skip('Never written')
    def test_errors(self):
        """
        Test illegal behavior and replies.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        # Can't go active in unitialized state.
        # Status 660 is state error.
        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        log.info('GO ACTIVE CMD %s',str(retval))
        self.assertEquals(retval.status, 660)
        
        # Can't command driver in this state.
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertEqual(reply.status, 660)
        
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # OK, I can do this now.        
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSample(reply.result)

        # 404 unknown agent command.
        cmd = AgentCommand(command='kiss_edward')
        retval = self._ia_client.execute_agent(cmd)
        self.assertEquals(retval.status, 404)
        
        # 670 unknown driver command.
        cmd = AgentCommand(command='acquire_sample_please')
        retval = self._ia_client.execute(cmd)
        self.assertEqual(retval.status, 670)

        # 630 Parameter error.
        with self.assertRaises(InstParameterError):
            reply = self._ia_client.get_param('bogus bogus')

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
        
        
    @unittest.skip('Direct access test to be finished by adding the telnet client, manual for now.')
    def test_direct_access(self):
        """
        Test agent direct_access command. This causes creation of
        driver process and transition to direct access.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='go_direct_access', 
                           #kwargs={'session_type':DirectAccessTypes.telnet,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                                   'session_timeout':600,
                                   'inactivity_timeout':600})
        retval = self._ia_client.execute_agent(cmd)
        print("go_direct_access retval=" + str(retval))       
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.DIRECT_ACCESS)

        # sleep to let tester run telnet client manually
        print "test sleeping to run telnet client"
        time.sleep(60)

        # Halt DA.
        cmd = AgentCommand(command='go_observatory')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin):
    """
    A base class with several conveniences supporting specific platform agent
    integration tests, see:
    - ion/agents/platform/test/test_platform_agent_with_rsn.py
    - ion/services/sa/observatory/test/test_platform_launch.py

    The platform IDs used here are organized as follows:
      Node1D -> MJ01C -> LJ01D

    where -> goes from parent platform to child platform.

    This is a subset of the whole topology defined in the simulated platform
    network (network.yml), which in turn is used by the RSN OMS simulator.

    - 'LJ01D'  is the root platform used in test_single_platform
    - 'Node1D' is the root platform used in test_hierarchy

    Methods are provided to construct specific platform topologies, but
    subclasses decide which to use.
    """

    @classmethod
    def setUpClass(cls):
        HelperTestMixin.setUpClass()

    def setUp(self):
        self._start_container()

        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        self.RR   = ResourceRegistryServiceClient(node=self.container.node)
        self.IMS  = InstrumentManagementServiceClient(node=self.container.node)
        self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
        self.DP   = DataProductManagementServiceClient(node=self.container.node)
        self.PSC  = PubsubManagementServiceClient(node=self.container.node)
        self.PDC  = ProcessDispatcherServiceClient(node=self.container.node)
        self.DSC  = DatasetManagementServiceClient()
        self.IDS  = IdentityManagementServiceClient(node=self.container.node)
        self.RR2  = EnhancedResourceRegistryClient(self.RR)

        self.org_id = self.RR2.create(any_old(RT.Org))
        log.debug("Org created: %s", self.org_id)

        # Use the network definition provided by RSN OMS directly.
        rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
        self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
        CIOMSClientFactory.destroy_instance(rsn_oms)

        # get serialized version for the configuration:
        self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
        log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)

        # set attributes for the platforms:
        self._platform_attributes = {}
        for platform_id in self._network_definition.pnodes:
            pnode = self._network_definition.pnodes[platform_id]
            dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues())
            self._platform_attributes[platform_id] = dic
        log.trace("_platform_attributes: %s", self._platform_attributes)

        # set ports for the platforms:
        self._platform_ports = {}
        for platform_id in self._network_definition.pnodes:
            pnode = self._network_definition.pnodes[platform_id]
            dic = {}
            for port_id, port in pnode.ports.iteritems():
                dic[port_id] = dict(port_id=port_id,
                                    network=port.network)
            self._platform_ports[platform_id] = dic
        log.trace("_platform_ports: %s", self._platform_attributes)

        self._async_data_result = AsyncResult()
        self._data_subscribers = []
        self._samples_received = []
        self.addCleanup(self._stop_data_subscribers)

        self._async_event_result = AsyncResult()
        self._event_subscribers = []
        self._events_received = []
        self.addCleanup(self._stop_event_subscribers)
        self._start_event_subscriber()

    #################################################################
    # data subscribers handling
    #################################################################

    def _start_data_subscriber(self, stream_name, stream_id):
        """
        Starts data subscriber for the given stream_name and stream_config
        """

        def consume_data(message, stream_route, stream_id):
            # A callback for processing subscribed-to data.
            log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r',
                     str(message), stream_name, stream_id)
            self._samples_received.append(message)
            self._async_data_result.set()

        log.info('_start_data_subscriber stream_name=%r stream_id=%r',
                 stream_name, stream_id)

        # Create subscription for the stream
        exchange_name = '%s_queue' % stream_name
        self.container.ex_manager.create_xn_queue(exchange_name).purge()
        sub = StandaloneStreamSubscriber(exchange_name, consume_data)
        sub.start()
        self._data_subscribers.append(sub)
        sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
        self.PSC.activate_subscription(sub_id)
        sub.subscription_id = sub_id

    def _stop_data_subscribers(self):
        """
        Stop the data subscribers on cleanup.
        """
        try:
            for sub in self._data_subscribers:
                if hasattr(sub, 'subscription_id'):
                    try:
                        self.PSC.deactivate_subscription(sub.subscription_id)
                    except:
                        pass
                    self.PSC.delete_subscription(sub.subscription_id)
                sub.stop()
        finally:
            self._data_subscribers = []

    #################################################################
    # event subscribers handling
    #################################################################

    def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
        """
        Starts event subscriber for events of given event_type ("DeviceEvent"
        by default) and given sub_type ("platform_event" by default).
        """

        def consume_event(evt, *args, **kwargs):
            # A callback for consuming events.
            log.info('Event subscriber received evt: %s.', str(evt))
            self._events_received.append(evt)
            self._async_event_result.set(evt)

        sub = EventSubscriber(event_type=event_type,
            sub_type=sub_type,
            callback=consume_event)

        sub.start()
        log.info("registered event subscriber for event_type=%r, sub_type=%r",
            event_type, sub_type)

        self._event_subscribers.append(sub)
        sub._ready_event.wait(timeout=EVENT_TIMEOUT)

    def _stop_event_subscribers(self):
        """
        Stops the event subscribers on cleanup.
        """
        try:
            for sub in self._event_subscribers:
                if hasattr(sub, 'subscription_id'):
                    try:
                        self.PSC.deactivate_subscription(sub.subscription_id)
                    except:
                        pass
                    self.PSC.delete_subscription(sub.subscription_id)
                sub.stop()
        finally:
            self._event_subscribers = []

    #################################################################
    # config supporting methods
    #################################################################

    def _create_platform_config_builder(self):
        clients = DotDict()
        clients.resource_registry  = self.RR
        clients.pubsub_management  = self.PSC
        clients.dataset_management = self.DSC
        pconfig_builder = PlatformAgentConfigurationBuilder(clients)

        # can't do anything without an agent instance obj
        log.debug("Testing that preparing a launcher without agent instance raises an error")
        self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)

        return pconfig_builder

    def _generate_parent_with_child_config(self, p_parent, p_child):
        log.debug("Testing parent platform + child platform as parent config")
        pconfig_builder = self._create_platform_config_builder()
        pconfig_builder.set_agent_instance_object(p_parent.platform_agent_instance_obj)
        parent_config = pconfig_builder.prepare(will_launch=False)
        self._verify_parent_config(parent_config,
                                   p_parent.platform_device_id,
                                   p_child.platform_device_id,
                                   is_platform=True)

        self._debug_config(parent_config,
                           "platform_agent_config_%s_->_%s.txt" % (
                           p_parent.platform_id, p_child.platform_id))

    def _generate_platform_with_instrument_config(self, p_obj, i_obj):
        log.debug("Testing parent platform + child instrument as parent config")
        pconfig_builder = self._create_platform_config_builder()
        pconfig_builder.set_agent_instance_object(p_obj.platform_agent_instance_obj)
        parent_config = pconfig_builder.prepare(will_launch=False)
        self._verify_parent_config(parent_config,
                                   p_obj.platform_device_id,
                                   i_obj.instrument_device_id,
                                   is_platform=False)

        self._debug_config(parent_config,
                           "platform_agent_config_%s_->_%s.txt" % (
                           p_obj.platform_id, i_obj.instrument_device_id))

    def _generate_config(self, platform_agent_instance_obj, platform_id, suffix=''):
        pconfig_builder = self._create_platform_config_builder()
        pconfig_builder.set_agent_instance_object(platform_agent_instance_obj)
        config = pconfig_builder.prepare(will_launch=False)

        self._debug_config(config, "platform_agent_config_%s%s.txt" % (platform_id, suffix))

        return config

    def _get_platform_stream_configs(self):
        """
        This method is an adaptation of get_streamConfigs in
        test_driver_egg.py
        """
        return [
            StreamConfiguration(stream_name='parsed',
                                parameter_dictionary_name='platform_eng_parsed',
                                records_per_granule=2,
                                granule_publish_rate=5)

            # TODO include a "raw" stream?
        ]

    def _get_instrument_stream_configs(self):
        """
        configs copied from test_activate_instrument.py
        """
        return [
            StreamConfiguration(stream_name='ctd_raw',
                                parameter_dictionary_name='ctd_raw_param_dict',
                                records_per_granule=2,
                                granule_publish_rate=5),

            StreamConfiguration(stream_name='ctd_parsed',
                                parameter_dictionary_name='ctd_parsed_param_dict',
                                records_per_granule=2, granule_publish_rate=5)
        ]

    def _create_instrument_config_builder(self):
        clients = DotDict()
        clients.resource_registry  = self.RR
        clients.pubsub_management  = self.PSC
        clients.dataset_management = self.DSC
        iconfig_builder = InstrumentAgentConfigurationBuilder(clients)

        return iconfig_builder

    def _generate_instrument_config(self, instrument_agent_instance_obj, instrument_id, suffix=''):
        pconfig_builder = self._create_instrument_config_builder()
        pconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
        config = pconfig_builder.prepare(will_launch=False)

        self._debug_config(config, "instrument_agent_config_%s%s.txt" % (instrument_id, suffix))

        return config

    def _debug_config(self, config, outname):
        if log.isEnabledFor(logging.DEBUG):
            import pprint
            outname = "logs/%s" % outname
            try:
                pprint.PrettyPrinter(stream=file(outname, "w")).pprint(config)
                log.debug("config pretty-printed to %s", outname)
            except Exception as e:
                log.warn("error printing config to %s: %s", outname, e)

    def _verify_child_config(self, config, device_id, is_platform):
        for key in required_config_keys:
            self.assertIn(key, config)

        if is_platform:
            self.assertEqual(RT.PlatformDevice, config['device_type'])
            for key in DVR_CONFIG.iterkeys():
                self.assertIn(key, config['driver_config'])

            for key in ['children', 'startup_config']:
                self.assertEqual({}, config[key])
        else:
            self.assertEqual(RT.InstrumentDevice, config['device_type'])

            for key in ['children']:
                self.assertEqual({}, config[key])

        self.assertEqual({'resource_id': device_id}, config['agent'])
        self.assertIn('stream_config', config)

    def _verify_parent_config(self, config, parent_device_id,
                              child_device_id, is_platform):
        for key in required_config_keys:
            self.assertIn(key, config)
        self.assertEqual(RT.PlatformDevice, config['device_type'])
        for key in DVR_CONFIG.iterkeys():
            self.assertIn(key, config['driver_config'])
        self.assertEqual({'resource_id': parent_device_id}, config['agent'])
        self.assertIn('stream_config', config)
        for key in ['startup_config']:
            self.assertEqual({}, config[key])

        self.assertIn(child_device_id, config['children'])
        self._verify_child_config(config['children'][child_device_id],
                                  child_device_id, is_platform)

    def _create_platform_configuration(self, platform_id, parent_platform_id=None):
        """
        This method is an adaptation of test_agent_instance_config in
        test_instrument_management_service_integration.py

        @param platform_id
        @param parent_platform_id
        @return a DotDict with various of the constructed elements associated
                to the platform.
        """

        tdom, sdom = time_series_domain()
        sdom = sdom.dump()
        tdom = tdom.dump()

        #
        # TODO will each platform have its own param dictionary?
        #
        param_dict_name = 'platform_eng_parsed'
        parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
            param_dict_name,
            id_only=True)
        self.parsed_stream_def_id = self.PSC.create_stream_definition(
            name='parsed',
            parameter_dictionary_id=parsed_rpdict_id)

        def _make_platform_agent_structure(agent_config=None):
            if None is agent_config: agent_config = {}

            driver_config = copy.deepcopy(DVR_CONFIG)
            driver_config['attributes'] = self._platform_attributes[platform_id]
            driver_config['ports']      = self._platform_ports[platform_id]
            log.debug("driver_config: %s", driver_config)

            # instance creation
            platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
                'driver_config': driver_config})
            platform_agent_instance_obj.agent_config = agent_config
            platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)

            # agent creation
            platform_agent_obj = any_old(RT.PlatformAgent, {
                "stream_configurations": self._get_platform_stream_configs(),
                'driver_module':         DVR_MOD,
                'driver_class':          DVR_CLS})
            platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)

            # device creation
            platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))

            # data product creation
            dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
            dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
            self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
            self.DP.activate_data_product_persistence(data_product_id=dp_id)

            # assignments
            self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
            self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
            self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id)

            #######################################
            # dataset

            log.debug('data product = %s', dp_id)

            stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True)
            log.debug('Data product stream_ids = %s', stream_ids)
            stream_id = stream_ids[0]

            # Retrieve the id of the OUTPUT stream from the out Data Product
            dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
            log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
            #######################################

            return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id

        log.debug("Making the structure for a platform agent")

        # TODO Note: the 'platform_config' entry is a mechanism that the
        # platform agent expects to know the platform_id and parent_platform_id.
        # Determine how to finally indicate this info.
        platform_config = {
            'platform_id':             platform_id,
            'parent_platform_id':      parent_platform_id,
        }

        child_agent_config = {
            'platform_config': platform_config
        }
        platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \
            _make_platform_agent_structure(child_agent_config)

        platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)

        child_config = self._generate_config(platform_agent_instance_child_obj, platform_id)
        self._verify_child_config(child_config, platform_device_child_id,
                                  is_platform=True)

        self.platform_device_parent_id = platform_device_child_id

        p_obj = DotDict()
        p_obj.platform_id = platform_id
        p_obj.parent_platform_id = parent_platform_id
        p_obj.agent_config = child_config
        p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj
        p_obj.platform_device_id = platform_device_child_id
        p_obj.platform_agent_instance_id = platform_agent_instance_child_id
        p_obj.stream_id = stream_id
        return p_obj

    def _create_platform(self, platform_id, parent_platform_id=None):
        """
        The main method to create a platform configuration and do other
        preparations for a given platform.
        """
        p_obj = self._create_platform_configuration(platform_id, parent_platform_id)

        # start corresponding data subscriber:
        self._start_data_subscriber(p_obj.platform_agent_instance_id,
                                    p_obj.stream_id)

        return p_obj

    #################################################################
    # platform child-parent linking
    #################################################################

    def _assign_child_to_parent(self, p_child, p_parent):

        log.debug("assigning child platform %r to parent %r",
                  p_child.platform_id, p_parent.platform_id)

        self.RR2.assign_platform_device_to_platform_device(p_child.platform_device_id,
                                                           p_parent.platform_device_id)
        child_device_ids = self.RR2.find_platform_device_ids_of_device(p_parent.platform_device_id)
        self.assertNotEqual(0, len(child_device_ids))

        self._generate_parent_with_child_config(p_parent, p_child)

    #################################################################
    # instrument
    #################################################################

    def _set_up_pre_environment_for_instrument(self):
        """
        From test_instrument_agent.py

        Basically, this method launches the port agent and the completes the
        instrument driver configuration used to properly set up the
        instrument agent.

        @return instrument_driver_config
        """

        import sys
        from ion.agents.instrument.driver_process import DriverProcessType
        from ion.agents.instrument.driver_process import ZMQEggDriverProcess

        # A seabird driver.
        DRV_URI = 'http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.7-py2.7.egg'
        DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
        DRV_CLS = 'SBE37Driver'

        WORK_DIR = '/tmp/'
        DELIM = ['<<', '>>']

        instrument_driver_config = {
            'dvr_egg' : DRV_URI,
            'dvr_mod' : DRV_MOD,
            'dvr_cls' : DRV_CLS,
            'workdir' : WORK_DIR,
            'process_type' : None
        }

        # Launch from egg or a local MI repo.
        LAUNCH_FROM_EGG=True

        if LAUNCH_FROM_EGG:
            # Dynamically load the egg into the test path
            launcher = ZMQEggDriverProcess(instrument_driver_config)
            egg = launcher._get_egg(DRV_URI)
            if not egg in sys.path: sys.path.insert(0, egg)
            instrument_driver_config['process_type'] = (DriverProcessType.EGG,)

        else:
            mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
            if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
            instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
            instrument_driver_config['mi_repo'] = mi_repo

        DEV_ADDR = CFG.device.sbe37.host
        DEV_PORT = CFG.device.sbe37.port
        DATA_PORT = CFG.device.sbe37.port_agent_data_port
        CMD_PORT = CFG.device.sbe37.port_agent_cmd_port
        PA_BINARY = CFG.device.sbe37.port_agent_binary

        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)

        # Start port agent, add stop to cleanup.
        port = self._support.start_pagent()
        log.info('Port agent started at port %i', port)
        self.addCleanup(self._support.stop_pagent)

        # Configure instrument driver to use port agent port number.
        instrument_driver_config['comms_config'] = {
            'addr':     'localhost',
            'port':     port,
            'cmd_port': CMD_PORT
        }

        return instrument_driver_config

    def _make_instrument_agent_structure(self, org_obj, agent_config=None):
        if None is agent_config: agent_config = {}

        # from test_activate_instrument:test_activateInstrumentSample

        # Create InstrumentModel
        instModel_obj = IonObject(RT.InstrumentModel,
                                  name='SBE37IMModel',
                                  description="SBE37IMModel")
        instModel_id = self.IMS.create_instrument_model(instModel_obj)
        log.debug('new InstrumentModel id = %s ', instModel_id)

        # agent creation
        instrument_agent_obj = IonObject(RT.InstrumentAgent,
                                         name='agent007',
                                         description="SBE37IMAgent",
                                         driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
                                         stream_configurations=self._get_instrument_stream_configs())

        instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
        log.debug('new InstrumentAgent id = %s', instrument_agent_id)

        self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instrument_agent_id)

        # device creation
        instDevice_obj = IonObject(RT.InstrumentDevice,
                                   name='SBE37IMDevice',
                                   description="SBE37IMDevice",
                                   serial_number="12345")
        instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
        self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instrument_device_id)
        log.debug("new InstrumentDevice id = %s ", instrument_device_id)

        #Create stream alarms
        alert_def = {
            'name' : 'temperature_warning_interval',
            'stream_name' : 'ctd_parsed',
            'message' : 'Temperature is below the normal range of 50.0 and above.',
            'alert_type' : StreamAlertType.WARNING,
            'value_id' : 'temp',
            'resource_id' : instrument_device_id,
            'origin_type' : 'device',
            'lower_bound' : 50.0,
            'lower_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        instrument_driver_config = self._set_up_pre_environment_for_instrument()

        port_agent_config = {
            'device_addr':  CFG.device.sbe37.host,
            'device_port':  CFG.device.sbe37.port,
            'process_type': PortAgentProcessType.UNIX,
            'binary_path': "port_agent",
            'port_agent_addr': 'localhost',
            'command_port': CFG.device.sbe37.port_agent_cmd_port,
            'data_port': CFG.device.sbe37.port_agent_data_port,
            'log_level': 5,
            'type': PortAgentType.ETHERNET
        }

        # instance creation
        instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance,
                                                  name='SBE37IMAgentInstance',
                                                  description="SBE37IMAgentInstance",
                                                  driver_config=instrument_driver_config,
                                                  port_agent_config=port_agent_config,
                                                  alerts=[alert_def])

        instrument_agent_instance_obj.agent_config = agent_config

        instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)

        # data products

        tdom, sdom = time_series_domain()
        sdom = sdom.dump()
        tdom = tdom.dump()

        org_id = self.RR2.create(org_obj)

        # parsed:

        parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
        parsed_stream_def_id = self.PSC.create_stream_definition(
            name='ctd_parsed',
            parameter_dictionary_id=parsed_pdict_id)

        dp_obj = IonObject(RT.DataProduct,
                           name='the parsed data',
                           description='ctd stream test',
                           temporal_domain=tdom,
                           spatial_domain=sdom)

        data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
                                                       stream_definition_id=parsed_stream_def_id)
        self.DP.activate_data_product_persistence(data_product_id=data_product_id1)

        self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
                                      data_product_id=data_product_id1)

        # raw:

        raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
        raw_stream_def_id = self.PSC.create_stream_definition(
            name='ctd_raw',
            parameter_dictionary_id=raw_pdict_id)

        dp_obj = IonObject(RT.DataProduct,
                           name='the raw data',
                           description='raw stream test',
                           temporal_domain=tdom,
                           spatial_domain=sdom)

        data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
                                                       stream_definition_id=raw_stream_def_id)

        self.DP.activate_data_product_persistence(data_product_id=data_product_id2)

        self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
                                      data_product_id=data_product_id2)

        # assignments
        self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
        self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
        self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)

        i_obj = DotDict()
        i_obj.instrument_agent_id = instrument_agent_id
        i_obj.instrument_device_id = instrument_device_id
        i_obj.instrument_agent_instance_id = instrument_agent_instance_id

        return i_obj

    def verify_instrument_config(self, config, org_obj, device_id):
        for key in required_config_keys:
            self.assertIn(key, config)
        self.assertEqual(org_obj.name, config['org_name'])
        self.assertEqual(RT.InstrumentDevice, config['device_type'])
        self.assertIn('driver_config', config)
        driver_config = config['driver_config']
        expected_driver_fields = {'process_type': ('ZMQEggDriverLauncher',),
                                  }
        for k, v in expected_driver_fields.iteritems():
            self.assertIn(k, driver_config)
            self.assertEqual(v, driver_config[k])

        self.assertEqual({'resource_id': device_id}, config['agent'])
        self.assertIn('stream_config', config)
        for key in ['children']:
            self.assertEqual({}, config[key])

    def _create_instrument(self):
        """
        The main method to create an instrument configuration.
        """
        iconfig_builder = self._create_instrument_config_builder()

        org_obj = any_old(RT.Org)

        log.debug("making the structure for an instrument agent")
        i_obj = self._make_instrument_agent_structure(org_obj)

        instrument_agent_instance_obj = self.RR2.read(i_obj.instrument_agent_instance_id)

        log.debug("Testing instrument config")
        iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
        instrument_config = iconfig_builder.prepare(will_launch=False)
        self.verify_instrument_config(instrument_config, org_obj,
                                      i_obj.instrument_device_id)

        self._generate_instrument_config(instrument_agent_instance_obj,
                                         i_obj.instrument_agent_instance_id)

        return i_obj

    #################################################################
    # instrument-platform linking
    #################################################################

    def _assign_instrument_to_platform(self, i_obj, p_obj):

        log.debug("assigning instrument %r to platform %r",
                  i_obj.instrument_agent_instance_id, p_obj.platform_id)

        self.RR2.assign_instrument_device_to_platform_device(
            i_obj.instrument_device_id,
            p_obj.platform_device_id)

        child_device_ids = self.RR2.find_instrument_device_ids_of_device(p_obj.platform_device_id)
        self.assertNotEqual(0, len(child_device_ids))

        self._generate_platform_with_instrument_config(p_obj, i_obj)

    #################################################################
    # some platform topologies
    #################################################################

    def _create_single_platform(self):
        """
        Creates and prepares a platform corresponding to the
        platform ID 'LJ01D', which is a leaf in the simulated network.
        """
        p_root = self._create_platform('LJ01D')
        return p_root

    def _create_small_hierarchy(self):
        """
        Creates a small platform network consisting of 3 platforms as follows:
          Node1D -> MJ01C -> LJ01D
        where -> goes from parent to child.
        """

        p_root       = self._create_platform('Node1D')
        p_child      = self._create_platform('MJ01C', parent_platform_id='Node1D')
        p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C')

        self._assign_child_to_parent(p_child, p_root)
        self._assign_child_to_parent(p_grandchild, p_child)

        self._generate_config(p_root.platform_agent_instance_obj, p_root.platform_id, "_final")

        return p_root

    #################################################################
    # start / stop platform
    #################################################################

    def _start_platform(self, agent_instance_id):
        log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
        pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
        log.debug("start_platform_agent_instance returned pid=%s", pid)

        #wait for start
        agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
        gate = ProcessStateGate(self.PDC.read_process,
                                agent_instance_obj.agent_process_id,
                                ProcessStateEnum.RUNNING)
        self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")

        # Start a resource agent client to talk with the agent.
        self._pa_client = ResourceAgentClient('paclient',
                                              name=agent_instance_obj.agent_process_id,
                                              process=FakeProcess())
        log.debug("got platform agent client %s", str(self._pa_client))

    def _stop_platform(self, agent_instance_id):
        self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)

    #################################################################
    # start / stop instrument
    #################################################################

    def _start_instrument(self, agent_instance_id):
        log.debug("about to call start_instrument_agent_instance with id=%s", agent_instance_id)
        pid = self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
        log.debug("start_instrument_agent_instance returned pid=%s", pid)

        #wait for start
        agent_instance_obj = self.IMS.read_instrument_agent_instance(agent_instance_id)
        gate = ProcessStateGate(self.PDC.read_process,
                                agent_instance_obj.agent_process_id,
                                ProcessStateEnum.RUNNING)
        self.assertTrue(gate.await(90), "The instrument agent instance did not spawn in 90 seconds")

        # Start a resource agent client to talk with the agent.
        self._ia_client = ResourceAgentClient('paclient',
                                              name=agent_instance_obj.agent_process_id,
                                              process=FakeProcess())
        log.debug("got instrument agent client %s", str(self._ia_client))

    def _stop_instrument(self, agent_instance_id):
        self.IMS.stop_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)

    #################################################################
    # misc convenience methods
    #################################################################

    def _get_state(self):
        state = self._pa_client.get_agent_state()
        return state

    def _assert_state(self, state):
        self.assertEquals(self._get_state(), state)

    def _execute_agent(self, cmd):
        log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
        time_start = time.time()
        #retval = self._pa_client.execute_agent(cmd, timeout=timeout)
        retval = self._pa_client.execute_agent(cmd)
        elapsed_time = time.time() - time_start
        log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
                 cmd.command, elapsed_time, str(retval))
        return retval

    #################################################################
    # commands that concrete tests can call
    #################################################################

    def _ping_agent(self):
        retval = self._pa_client.ping_agent()
        self.assertIsInstance(retval, str)

    def _ping_resource(self):
        cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
        if self._get_state() == PlatformAgentState.UNINITIALIZED:
            # should get ServerError: "Command not handled in current state"
            with self.assertRaises(ServerError):
                #self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
                self._pa_client.execute_agent(cmd)
        else:
            # In all other states the command should be accepted:
            retval = self._execute_agent(cmd)
            self.assertEquals("PONG", retval.result)

    def _get_metadata(self):
        cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
        retval = self._execute_agent(cmd)
        md = retval.result
        self.assertIsInstance(md, dict)
        # TODO verify possible subset of required entries in the dict.
        log.info("GET_METADATA = %s", md)

    def _get_ports(self):
        cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
        retval = self._execute_agent(cmd)
        md = retval.result
        self.assertIsInstance(md, dict)
        # TODO verify possible subset of required entries in the dict.
        log.info("GET_PORTS = %s", md)

    def _initialize(self):
        self._assert_state(PlatformAgentState.UNINITIALIZED)
        cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.INACTIVE)

    def _go_active(self):
        cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.IDLE)

    def _run(self):
        cmd = AgentCommand(command=PlatformAgentEvent.RUN)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.COMMAND)

    def _start_resource_monitoring(self):
        cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.MONITORING)

    def _wait_for_a_data_sample(self):
        log.info("waiting for reception of a data sample...")
        # just wait for at least one -- see consume_data
        self._async_data_result.get(timeout=DATA_TIMEOUT)
        self.assertTrue(len(self._samples_received) >= 1)
        log.info("Received samples: %s", len(self._samples_received))

    def _wait_for_external_event(self):
        log.info("waiting for reception of an external event...")
        # just wait for at least one -- see consume_event
        self._async_event_result.get(timeout=EVENT_TIMEOUT)
        self.assertTrue(len(self._events_received) >= 1)
        log.info("Received events: %s", len(self._events_received))

    def _stop_resource_monitoring(self):
        cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.COMMAND)

    def _pause(self):
        cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.STOPPED)

    def _resume(self):
        cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.COMMAND)

    def _clear(self):
        cmd = AgentCommand(command=PlatformAgentEvent.CLEAR)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.IDLE)

    def _go_inactive(self):
        cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.INACTIVE)

    def _reset(self):
        cmd = AgentCommand(command=PlatformAgentEvent.RESET)
        retval = self._execute_agent(cmd)
        self._assert_state(PlatformAgentState.UNINITIALIZED)

    def _check_sync(self):
        cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
        retval = self._execute_agent(cmd)
        log.info("CHECK_SYNC result: %s", retval.result)
        self.assertTrue(retval.result is not None)
        self.assertEquals(retval.result[0:3], "OK:")
        return retval.result
示例#12
0
class TestAgentConnectionFailures(IonIntegrationTestCase):
    """
    Test cases for instrument agent class. Functions in this class provide
    instrument agent integration tests and provide a tutorial on use of
    the agent setup and interface.
    """

    ############################################################################
    # Setup, teardown.
    ############################################################################

    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        # Start container.
        log.info('Staring capability container.')
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR,
                                                     DEV_PORT, DATA_PORT,
                                                     CMD_PORT, PA_BINARY,
                                                     DELIM, WORK_DIR)

        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Start a resource agent client to talk with the instrument agent.
        log.info('starting IA process')
        self._ia_client = start_instrument_agent_process(
            self.container, self._stream_config)
        self.addCleanup(self._verify_agent_reset)
        log.info('test setup complete')

    ###############################################################################
    # Port agent helpers.
    ###############################################################################

    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i', port)

        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr': 'localhost',
            'port': port,
            'cmd_port': CMD_PORT
        }

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state(timeout=120.1)
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd, timeout=300)

    ###############################################################################
    # Event helpers.
    ###############################################################################

    def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
        """
        Start a subscriber to the instrument agent events.
        @param type The type of event to catch.
        @count Trigger the async event result when events received reaches this.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()

        # Event array and async event result.
        self._event_count = count
        self._events_received = []
        self._async_event_result = AsyncResult()

        self._event_subscriber = EventSubscriber(event_type=type,
                                                 callback=consume_event,
                                                 origin=IA_RESOURCE_ID)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=5)

    def _stop_event_subscriber(self):
        """
        Stop event subscribers on cleanup.
        """
        self._event_subscriber.stop()
        self._event_subscriber = None

    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()

        encoder = IonObjectSerializer()

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'parsed'
        param_dict_name = 'ctd_parsed_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(
            param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(
            name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(
            name=stream_name,
            exchange_point='science_data',
            stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                             exchange_point=stream_route.exchange_point,
                             stream_id=stream_id,
                             parameter_dictionary=pd,
                             stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config

        stream_name = 'raw'
        param_dict_name = 'ctd_raw_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(
            param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(
            name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(
            name=stream_name,
            exchange_point='science_data',
            stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                             exchange_point=stream_route.exchange_point,
                             stream_id=stream_id,
                             parameter_dictionary=pd,
                             stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config

    def _start_data_subscribers(self, count, raw_count):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # Create streams and subscriptions for each stream named in driver.
        self._data_subscribers = []
        self._samples_received = []
        self._raw_samples_received = []
        self._async_sample_result = AsyncResult()
        self._async_raw_sample_result = AsyncResult()

        # A callback for processing subscribed-to data.
        def recv_data(message, stream_route, stream_id):
            log.info('Received parsed data on %s (%s,%s)', stream_id,
                     stream_route.exchange_point, stream_route.routing_key)
            self._samples_received.append(message)
            if len(self._samples_received) == count:
                self._async_sample_result.set()

        def recv_raw_data(message, stream_route, stream_id):
            log.info('Received raw data on %s (%s,%s)', stream_id,
                     stream_route.exchange_point, stream_route.routing_key)
            self._raw_samples_received.append(message)
            if len(self._raw_samples_received) == raw_count:
                self._async_raw_sample_result.set()

        from pyon.util.containers import create_unique_identifier

        stream_name = 'parsed'
        parsed_config = self._stream_config[stream_name]
        stream_id = parsed_config['stream_id']
        exchange_name = create_unique_identifier("%s_queue" % stream_name)
        self._purge_queue(exchange_name)
        sub = StandaloneStreamSubscriber(exchange_name, recv_data)
        sub.start()
        self._data_subscribers.append(sub)
        sub_id = pubsub_client.create_subscription(name=exchange_name,
                                                   stream_ids=[stream_id],
                                                   timeout=120.2)
        pubsub_client.activate_subscription(sub_id, timeout=120.3)
        sub.subscription_id = sub_id  # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)

        stream_name = 'raw'
        parsed_config = self._stream_config[stream_name]
        stream_id = parsed_config['stream_id']
        exchange_name = create_unique_identifier("%s_queue" % stream_name)
        self._purge_queue(exchange_name)
        sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
        sub.start()
        self._data_subscribers.append(sub)
        sub_id = pubsub_client.create_subscription(name=exchange_name,
                                                   stream_ids=[stream_id],
                                                   timeout=120.4)
        pubsub_client.activate_subscription(sub_id, timeout=120.5)
        sub.subscription_id = sub_id  # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)

    def _purge_queue(self, queue):
        xn = self.container.ex_manager.create_xn_queue(queue)
        xn.purge()

    def _stop_data_subscribers(self):
        for subscriber in self._data_subscribers:
            pubsub_client = PubsubManagementServiceClient()
            if hasattr(subscriber, 'subscription_id'):
                try:
                    pubsub_client.deactivate_subscription(
                        subscriber.subscription_id, timeout=120.6)
                except:
                    pass
                pubsub_client.delete_subscription(subscriber.subscription_id,
                                                  timeout=120.7)
            subscriber.stop()

    ###############################################################################
    # Socket listen.
    ###############################################################################

    def _socket_listen(self, s, prompt, timeout):

        buf = ''
        starttime = time.time()
        while True:
            try:
                buf += s.recv(1024)
                print '##### Listening, got: %s' % buf
                if prompt and buf.find(prompt) != -1:
                    break
            except:
                gevent.sleep(1)

            finally:
                delta = time.time() - starttime
                if delta > timeout:
                    break
        return buf

    ###############################################################################
    # Assert helpers.
    ###############################################################################

    def assertSampleDict(self, val):
        """
        Verify the value is a sample dictionary for the sbe37.
        """
        # AgentCommandResult.result['parsed']
        """
        {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
        'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data',
        'pkt_version': 1, 'values':
        [{'value_id': 'temp', 'value': 21.4894},
        {'value_id': 'conductivity', 'value': 13.22157},
        {'value_id': 'pressure', 'value': 146.186}],
        'driver_timestamp': 3556901018.170206}
        """

        self.assertIsInstance(val, dict)
        self.assertTrue(val.has_key('values'))
        values_list = val['values']
        self.assertTrue(isinstance(values_list, list))
        self.assertTrue(len(values_list) == 3)

        ids = ['temp', 'conductivity', 'pressure']
        ids_found = []

        for x in values_list:
            self.assertTrue(x.has_key('value_id'))
            self.assertTrue(x.has_key('value'))
            ids_found.append(x['value_id'])
            self.assertTrue(isinstance(x['value'], float))

        self.assertItemsEqual(ids, ids_found)

        self.assertTrue(val.has_key('driver_timestamp'))
        time = val['driver_timestamp']
        self.assertTrue(isinstance(time, float))

    def assertParamDict(self, pd, all_params=False):
        """
        Verify all device parameters exist and are correct type.
        """
        if all_params:
            self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
            for (key, type_val) in PARAMS.iteritems():
                if type_val == list or type_val == tuple:
                    self.assertTrue(isinstance(pd[key], (list, tuple)))
                else:
                    self.assertTrue(isinstance(pd[key], type_val))

        else:
            for (key, val) in pd.iteritems():
                self.assertTrue(PARAMS.has_key(key))
                self.assertTrue(isinstance(val, PARAMS[key]))

    def assertParamVals(self, params, correct_params):
        """
        Verify parameters take the correct values.
        """
        self.assertEqual(set(params.keys()), set(correct_params.keys()))
        for (key, val) in params.iteritems():
            correct_val = correct_params[key]
            if isinstance(val, float):
                # Verify to 5% of the larger value.
                max_val = max(abs(val), abs(correct_val))
                self.assertAlmostEqual(val, correct_val, delta=max_val * .01)

            elif isinstance(val, (list, tuple)):
                # list of tuple.
                self.assertEqual(list(val), list(correct_val))

            else:
                # int, bool, str.
                self.assertEqual(val, correct_val)

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_lost_connection(self):
        """
        test_lost_connection
        """

        # Set up a subscriber to collect command events.
        self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent',
                                     1)
        self.addCleanup(self._stop_event_subscriber)

        # Start in uninitialized.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        # Initialize the agent.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Activate.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        # Go into command mode.
        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Start streaming.
        cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        # Wait for a while, collect some samples.
        gevent.sleep(10)

        # Blow the port agent out from under the agent.
        self._support.stop_pagent()

        # Loop until we resyncronize to LOST_CONNECTION/DISCONNECTED.
        # Test will timeout if this dosn't occur.
        while True:
            state = self._ia_client.get_agent_state()
            if state == ResourceAgentState.LOST_CONNECTION:
                break
            else:
                gevent.sleep(1)

        # Verify the driver has transitioned to disconnected
        while True:
            state = self._ia_client.get_resource_state()
            if state == DriverConnectionState.DISCONNECTED:
                break
            else:
                gevent.sleep(1)

        # Make sure the lost connection error event arrives.
        self._async_event_result.get(timeout=CFG.endpoint.receive.timeout)
        self.assertEqual(len(self._events_received), 1)

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

    #@unittest.skip('Fails on buildbot for some god unknown reason.')
    def test_autoreconnect(self):
        """
        test_autoreconnect
        """
        # Set up a subscriber to collect command events.
        self._start_event_subscriber('ResourceAgentConnectionLostErrorEvent',
                                     1)
        self.addCleanup(self._stop_event_subscriber)

        # Start in uninitialized.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        # Initialize the agent.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Activate.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        # Go into command mode.
        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        def poll_func(test):
            cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
            while True:
                try:
                    gevent.sleep(.5)
                    test._ia_client.execute_resource(cmd)
                except IonException as ex:
                    # This exception could be ResourceException (broken pipe)
                    # Timeout or Conflict
                    log.info('#### pre shutdown exception: %s, %s',
                             str(type(ex)), str(ex))
                    break

            while True:
                try:
                    gevent.sleep(.5)
                    test._ia_client.execute_resource(cmd)
                    log.info('#### post shutdown got new sample.')
                    break
                except IonException as ex:
                    # This should be conflict.
                    log.info('#### post shutdown exception: %s, %s',
                             str(type(ex)), str(ex))

        timeout = gevent.Timeout(600)
        timeout.start()
        try:

            # Start the command greenlet and let poll for a bit.
            gl = gevent.spawn(poll_func, self)
            gevent.sleep(20)

            # Blow the port agent out from under the agent.
            self._support.stop_pagent()

            # Wait for a while, the supervisor is restarting the port agent.
            gevent.sleep(10)
            self._support.start_pagent()

            # Wait for the device to connect and start sampling again.
            gl.join()
            gl = None
            timeout.cancel()

        except (Exception, gevent.Timeout) as ex:
            if gl:
                gl.kill()
                gl = None
            self.fail(('Could not reconnect to device: %s,  %s', str(type(ex)),
                       str(ex)))

    def test_connect_failed(self):
        """
        test_connect_failed
        """
        # Stop the port agent.
        self._support.stop_pagent()

        # Sleep a bit.
        gevent.sleep(3)

        # Start in uninitialized.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        # Initialize the agent.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Activate. This should fail because there is no port agent to connect to.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        with self.assertRaises(ResourceError):
            retval = self._ia_client.execute_agent(cmd)

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

    def test_get_set_alerts(self):
        """
        test_get_set_alerts
        Test specific of get/set alerts, including using result of get to
        set later.
        """
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertItemsEqual(retval, [])

        alert_def1 = {
            'name': 'temp_warning_interval',
            'stream_name': 'parsed',
            'description': 'Temperature is above normal range.',
            'alert_type': StreamAlertType.WARNING,
            'aggregate_type': AggregateStatusType.AGGREGATE_DATA,
            'value_id': 'temp',
            'lower_bound': None,
            'lower_rel_op': None,
            'upper_bound': 10.5,
            'upper_rel_op': '<',
            'alert_class': 'IntervalAlert'
        }

        alert_def2 = {
            'name': 'temp_alarm_interval',
            'stream_name': 'parsed',
            'description': 'Temperature is way above normal range.',
            'alert_type': StreamAlertType.WARNING,
            'aggregate_type': AggregateStatusType.AGGREGATE_DATA,
            'value_id': 'temp',
            'lower_bound': None,
            'lower_rel_op': None,
            'upper_bound': 15.5,
            'upper_rel_op': '<',
            'alert_class': 'IntervalAlert'
        }
        """
        Interval alerts are returned from get like this:
        (value and status fields describe state of the alert)
        {
        'name': 'temp_warning_interval',
        'stream_name': 'parsed',
        'description': 'Temperature is above normal range.',
        'alert_type': 1,
        'aggregate_type': 2,
        'value_id': 'temp',
        'lower_bound': None,
        'lower_rel_op': None,
        'upper_bound': 10.5,
        'upper_rel_op': '<',
        'alert_class': 'IntervalAlert',

        'status': None,
        'value': None
        }
        """

        alert_def3 = {
            'name': 'late_data_warning',
            'stream_name': 'parsed',
            'description': 'Expected data has not arrived.',
            'alert_type': StreamAlertType.WARNING,
            'aggregate_type': AggregateStatusType.AGGREGATE_COMMS,
            'time_delta': 180,
            'alert_class': 'LateDataAlert'
        }
        """
        Late data alerts are returned from get like this:
        (value and status fields describe state of the alert)
        {
        'name': 'late_data_warning',
        'stream_name': 'parsed',
        'description': 'Expected data has not arrived.',
        'alert_type': 1,
        'aggregate_type': 1,
        'value_id': None,
        'time_delta': 180,
        'alert_class': 'LateDataAlert',
        
        'status': None,
        'value': None
        }
        """
        """
        [
            {'status': None,
            'alert_type': 1,
            'name': 'temp_warning_interval',
            'upper_bound': 10.5,
            'lower_bound': None,
            'aggregate_type': 2,
            'alert_class': 'IntervalAlert',
            'value': None,
            'value_id': 'temp',
            'lower_rel_op': None,
            'upper_rel_op': '<',
            'description': 'Temperature is above normal range.'},
            {'status': None,
            'alert_type': 1,
            'name': 'temp_alarm_interval',
            'upper_bound': 15.5,
            'lower_bound': None,
            'aggregate_type': 2,
            'alert_class': 'IntervalAlert',
            'value': None,
            'value_id': 'temp',
            'lower_rel_op': None,
            'upper_rel_op': '<',
            'description': 'Temperature is way above normal range.'},
            {'status': None,
             'stream_name': 'parsed',
             'alert_type': 1,
             'name': 'late_data_warning',
             'aggregate_type': 1,
             'alert_class': 'LateDataAlert',
             'value': None,
             'time_delta': 180,
             'description': 'Expected data has not arrived.'}
        ]
        """

        orig_alerts = [alert_def1, alert_def2, alert_def3]
        self._ia_client.set_agent({'alerts': orig_alerts})

        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertTrue(len(retval) == 3)
        alerts = retval

        self._ia_client.set_agent({'alerts': ['clear']})
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertItemsEqual(retval, [])

        self._ia_client.set_agent({'alerts': alerts})
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertTrue(len(retval) == 3)

        count = 0
        for x in retval:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEquals(count, 3)

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
示例#13
0
class TestRemoteClient(IonIntegrationTestCase):
    """
    Test cases for 2CAA remote clients.
    """
    def setUp(self):
        """
        Setup the test parameters.
        Start the container and retrieve client.
        Start the endpoints and resource agent.
        Start publisher and subscribers.
        """
        ###################################################################
        # Internal parameters and container.
        ###################################################################

        # Internal parameters.
        self._terrestrial_platform_id = 'terrestrial_id'
        self._remote_platform_id = 'remote_id'
        self._resource_id = 'fake_id'
        self._xs_name = 'remote1'
        self._terrestrial_svc_name = 'terrestrial_endpoint'
        self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
        self._remote_svc_name = 'remote_endpoint'
        self._remote_listen_name = self._remote_svc_name + self._xs_name
        self._remote_port = 0
        self._terrestrial_port = 0
        self._te_client = None
        self._re_client = None
        self._remote_pid = None
        self._terrestrial_pid = None

        # Async test results.
        self._no_requests = 10
        self._no_telem_evts = 2
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._telem_evts = []
        self._queue_mod_evts = []
        self._cmd_tx_evts = []
        self._results_recv = {}
        self._requests_sent = {}
        self._done_telem_evt = AsyncResult()
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()

        # Start container.
        log.debug('Staring capability container.')
        self._start_container()

        # Bring up services in a deploy file (no need to message).
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Create a container client.
        log.debug('Creating container client.')
        self._container_client = ContainerAgentClient(node=self.container.node,
                                                      name=self.container.name)

        ###################################################################
        # Start endpoints, agent.
        ###################################################################

        self._start_terrestrial()
        self._start_remote()
        self._start_agent()

        ###################################################################
        # Assign client ports.
        # This is primarily for test purposes as the IP config in
        # deployment will be fixed in advance.
        ###################################################################

        self.te_client.set_client_port(self._remote_port)
        check_port = self.te_client.get_client_port()
        log.debug('Terrestrial client port is: %i', check_port)

        self.re_client.set_client_port(self._terrestrial_port)
        check_port = self.re_client.get_client_port()
        log.debug('Remote client port is: %i', check_port)

        ###################################################################
        # Start the event publisher and subscribers.
        # Used to send fake agent telemetry publications to the endpoints,
        # and to receive endpoint publications.
        ###################################################################
        self._event_publisher = EventPublisher()

        # Start the event subscriber for remote namespace platform events.
        # This event could be changed to RemoteNamespaceEvent.
        self._event_subscriber = EventSubscriber(event_type='PlatformEvent',
                                                 callback=self.consume_event,
                                                 origin=self._xs_name)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(
            timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._event_subscriber.stop)

        # Start the result subscriber for remote resource events.
        self._resource_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=IA_RESOURCE_ID,
            callback=self.consume_event)
        self._resource_result_subscriber.start()
        self._resource_result_subscriber._ready_event.wait(
            timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._resource_result_subscriber.stop)

        # Start the result subscriber for remote service events.
        self._service_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin='resource_registry' + 'remote1',
            callback=self.consume_event)
        self._service_result_subscriber.start()
        self._service_result_subscriber._ready_event.wait(
            timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._service_result_subscriber.stop)

        # Start the result subscriber for fake resource results.
        self._fake_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=self._resource_id,
            callback=self.consume_event)
        self._fake_result_subscriber.start()
        self._fake_result_subscriber._ready_event.wait(
            timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._fake_result_subscriber.stop)

    ###################################################################
    # Start/stop helpers.
    ###################################################################

    def _start_agent(self):
        """
        Start an instrument agent and client.
        """

        log.info('Creating driver integration test support:')
        log.info('driver module: %s', DRV_MOD)
        log.info('driver class: %s', DRV_CLS)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS,
                                                     DEV_ADDR, DEV_PORT,
                                                     DATA_PORT, CMD_PORT,
                                                     PA_BINARY, DELIM,
                                                     WORK_DIR)

        # Start port agent, add stop to cleanup.
        port = self._support.start_pagent()
        log.info('Port agent started at port %i', port)

        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {'addr': 'localhost', 'port': port}
        self.addCleanup(self._support.stop_pagent)

        # Create agent config.
        agent_config = {
            'driver_config': DVR_CONFIG,
            'stream_config': {},
            'agent': {
                'resource_id': IA_RESOURCE_ID
            },
            'test_mode': True
        }

        # Start instrument agent.
        log.debug("Starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
                                                name=self.container.name)

        ia_pid = container_client.spawn_process(name=IA_NAME,
                                                module=IA_MOD,
                                                cls=IA_CLS,
                                                config=agent_config)

        log.info('Agent pid=%s.', str(ia_pid))

        # Start a resource agent client to talk with the instrument agent.

        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
                                              process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))

    def _start_terrestrial(self):
        """
        Start up the terrestrial endpoint.
        """
        # Create terrestrial config.
        terrestrial_endpoint_config = {
            'other_host': 'localhost',
            'other_port': self._remote_port,
            'this_port': self._terrestrial_port,
            'platform_resource_id': self._terrestrial_platform_id,
            'xs_name': self._xs_name,
            'process': {
                'listen_name': self._terrestrial_listen_name
            }
        }

        # Spawn the terrestrial enpoint process.
        log.debug('Spawning terrestrial endpoint process.')
        self._terrestrial_pid = self._container_client.spawn_process(
            name=self._terrestrial_listen_name,
            module='ion.services.sa.tcaa.terrestrial_endpoint',
            cls='TerrestrialEndpoint',
            config=terrestrial_endpoint_config)
        log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))

        # Create a terrestrial client.
        self.te_client = TerrestrialEndpointClient(
            process=FakeProcess(), to_name=self._terrestrial_listen_name)
        log.debug('Got te client %s.', str(self.te_client))
        self._terrestrial_port = self.te_client.get_port()
        log.debug('Terrestrial port is: %i', self._terrestrial_port)

    def _start_remote(self):
        """
        Start up the remote endpoint.
        """
        # Create agent config.
        remote_endpoint_config = {
            'other_host': 'localhost',
            'other_port': self._terrestrial_port,
            'this_port': self._remote_port,
            'platform_resource_id': self._remote_platform_id,
            'xs_name': self._xs_name,
            'process': {
                'listen_name': self._remote_listen_name
            }
        }

        # Spawn the remote enpoint process.
        log.debug('Spawning remote endpoint process.')
        self._remote_pid = self._container_client.spawn_process(
            name=self._remote_listen_name,
            module='ion.services.sa.tcaa.remote_endpoint',
            cls='RemoteEndpoint',
            config=remote_endpoint_config)
        log.debug('Remote endpoint pid=%s.', str(self._remote_pid))

        # Create an endpoint client.
        self.re_client = RemoteEndpointClient(process=FakeProcess(),
                                              to_name=self._remote_listen_name)
        log.debug('Got re client %s.', str(self.re_client))

        # Remember the remote port.
        self._remote_port = self.re_client.get_port()
        log.debug('The remote port is: %i.', self._remote_port)

    ###################################################################
    # Telemetry publications to start/top endpoint.
    # (Normally be published by appropriate platform agents.)
    ###################################################################

    def terrestrial_link_up(self):
        """
        Publish telemetry available to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry available event.')
        self._event_publisher.publish_event(
            event_type='PlatformTelemetryEvent',
            origin=self._terrestrial_platform_id,
            status=TelemetryStatusType.AVAILABLE)

    def terrestrial_link_down(self):
        """
        Publish telemetry unavailable to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry unavailable event.')
        self._event_publisher.publish_event(
            event_type='PlatformTelemetryEvent',
            origin=self._terrestrial_platform_id,
            status=TelemetryStatusType.UNAVAILABLE)

    def remote_link_up(self):
        """
        Publish telemetry available to the remote endpoint.
        """
        # Publish a link up event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry available event.')
        self._event_publisher.publish_event(
            event_type='PlatformTelemetryEvent',
            origin=self._remote_platform_id,
            status=TelemetryStatusType.AVAILABLE)

    def remote_link_down(self):
        """
        Publish telemetry unavailable to the remote endpoint.
        """
        # Publish a link down event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry unavailable event.')
        self._event_publisher.publish_event(
            event_type='PlatformTelemetryEvent',
            origin=self._remote_platform_id,
            status=TelemetryStatusType.UNAVAILABLE)

    def consume_event(self, evt, *args, **kwargs):
        """
        Test callback for events.
        """
        log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt),
                  str(args), str(kwargs))

        if evt.type_ == 'PublicPlatformTelemetryEvent':
            self._telem_evts.append(evt)
            if self._no_telem_evts > 0 and self._no_telem_evts == len(
                    self._telem_evts):
                self._done_telem_evt.set()

        elif evt.type_ == 'RemoteQueueModifiedEvent':
            self._queue_mod_evts.append(evt)
            if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(
                    self._queue_mod_evts):
                self._done_queue_mod_evt.set()

        elif evt.type_ == 'RemoteCommandTransmittedEvent':
            self._cmd_tx_evts.append(evt)
            if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(
                    self._cmd_tx_evts):
                self._done_cmd_tx_evt.set()

        elif evt.type_ == 'RemoteCommandResult':
            cmd = evt.command
            self._results_recv[cmd.command_id] = cmd
            if len(self._results_recv) == self._no_requests:
                self._done_cmd_evt.set()

    ###################################################################
    # Misc helpers.
    ###################################################################

    def make_fake_command(self, no):
        """
        Build a fake command for use in tests.
        """

        cmdstr = 'fake_cmd_%i' % no
        cmd = IonObject('RemoteCommand',
                        resource_id='fake_id',
                        command=cmdstr,
                        args=['arg1', 23],
                        kwargs={'kwargs1': 'someval'})
        return cmd

    ###################################################################
    # Tests.
    ###################################################################

    def test_resource_client_online(self):
        """
        test_recourse_client_online
        Test the client transparently forwards commands to the remote
        resource while link is up.
        """

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        remote_client = RemoteClient(iface=IResourceAgent,
                                     xs_name=self._xs_name,
                                     resource_id='fake_id',
                                     process=FakeProcess())

        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent()
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()

        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                              self._results_recv.keys())

    #@unittest.skip('For some reason this bastard wont run on the builder.')
    def test_resource_client_blocking(self):
        """
        test_resource_client_blocking
        Test the client can block on remote resource command results.
        """

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        remote_client = RemoteClient(iface=IResourceAgent,
                                     xs_name=self._xs_name,
                                     resource_id=IA_RESOURCE_ID,
                                     process=FakeProcess())

        # Queue up a series of fake commands to be handled by the remote side.
        """
        {'time_completed': 1350421095.804607, 'resource_id': '123xyz',
        'time_queued': 1350421095.623531, 'args': [], 'type_': 'RemoteCommand',
        'command': 'ping_agent', 'result': 'ping from InstrumentAgent
        (name=Agent007,id=Edwards-MacBook-Pro_local_10126.35,type=agent),
        time: 1350421095757', 'kwargs': {}, 'svc_name': '',
        'command_id': '76be11b4-a22c-49de-89cd-4e019463d7c9'}
        """
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent(
                remote_timeout=CFG.endpoint.receive.timeout)
            self.assertEqual(cmd.resource_id, IA_RESOURCE_ID)
            self.assertEqual(cmd.command, 'ping_agent')
            self.assertIn('ping from InstrumentAgent', cmd.result)

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()

        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

    #@unittest.skip('For some reason this bastard wont run on the builder.')
    def test_service_client_blocking(self):
        """
        test_service_client_blocking
        Test the client can command remote services and block on their
        results.
        """

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Create remote client for the resource registry.
        remote_client = RemoteClient(iface=IResourceRegistryService,
                                     xs_name=self._xs_name,
                                     svc_name='resource_registry',
                                     process=FakeProcess())

        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        result = remote_client.create(
            obj, remote_timeout=CFG.endpoint.receive.timeout)

        # Returns obj_id, obj_rev.
        obj_id, obj_rev = result.result

        # Confirm the results are valid.

        #Result is a tuple of strings.
        #{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
        #'1-00a308c45fff459c7cda1db9a7314de6'],
        #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}

        self.assertIsInstance(obj_id, str)
        self.assertNotEqual(obj_id, '')
        self.assertIsInstance(obj_rev, str)
        self.assertNotEqual(obj_rev, '')

        # Read user object.
        result = remote_client.read(
            obj_id, remote_timeout=CFG.endpoint.receive.timeout)

        # Returns read_obj.
        read_obj = result.result

        # Confirm the results are valid.

        #Result is a user info object with the name set.
        #{'lcstate': 'DEPLOYED_AVAILABLE',
        #'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
        #'description': '',
        #'ts_updated': '1349213207638',
        #'type_': 'UserInfo',
        #'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
        #'_id': '27832d93f4cd4535a75ac75c06e00a7e',
        #'ts_created': '1349213207638',
        #'variables': [{'name': '', 'value': ''}],
        #'name': 'some_name'}

        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_name')

        # Update user object.
        read_obj.name = 'some_other_name'
        result = remote_client.update(
            read_obj, remote_timeout=CFG.endpoint.receive.timeout)

        # Returns nothing.

        # Read user object.
        result = remote_client.read(
            obj_id, remote_timeout=CFG.endpoint.receive.timeout)

        # Returns read_obj.
        read_obj = result.result

        # Confirm results are valid.
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_other_name')

        # Delete user object.
        result = remote_client.delete(
            obj_id, remote_timeout=CFG.endpoint.receive.timeout)

        # Returns nothing.

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()

        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

    def test_queue_manipulators(self):
        """
        test_queue_manipulators
        Test ability to instpect and manipulate the command queue corresponding
        to this resource or service.
        """

        remote_client = RemoteClient(iface=IResourceAgent,
                                     xs_name=self._xs_name,
                                     resource_id='fake_id',
                                     process=FakeProcess())

        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent(link=False)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        queue = remote_client.get_queue()
        self.assertEqual(len(queue), self._no_requests)

        popped = remote_client.clear_queue()
        self.assertEqual(len(popped), self._no_requests)

        self._requests_sent = {}

        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent(link=False)
            self._requests_sent[cmd.command_id] = cmd

        # Pop the last three commands.
        cmd_ids = self._requests_sent.keys()[:3]
        poped = []
        for x in cmd_ids:
            poped.append(remote_client.pop_queue(x))
            self._requests_sent.pop(x)

        queue = remote_client.get_queue()
        self.assertEqual(len(queue), self._no_requests - 3)
        self.assertEqual(len(poped), 3)

        self._no_requests = self._no_requests - 3
        self._no_cmd_tx_evts = self._no_requests

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        pending = remote_client.get_pending()
        for x in pending:
            self.assertIn(x.command_id, self._requests_sent.keys())
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()

        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                              self._results_recv.keys())

    def test_errors(self):
        """
        test_errors
        Test various error conditions.
        """

        # Constructed without a xs name.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(iface=IResourceAgent,
                                         resource_id=IA_RESOURCE_ID,
                                         process=FakeProcess())

        # Constructed without an interface.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(xs_name=self._xs_name,
                                         resource_id=IA_RESOURCE_ID,
                                         process=FakeProcess())

        # Construct with invalid interface.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient('Bogus_interface',
                                         xs_name=self._xs_name,
                                         resource_id=IA_RESOURCE_ID,
                                         process=FakeProcess())

        # Construct with no resource or service specified.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(iface=IResourceAgent,
                                         xs_name=self._xs_name,
                                         process=FakeProcess())

        # Construct with both resource and service specified.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(iface=IResourceAgent,
                                         xs_name=self._xs_name,
                                         resource_id=IA_RESOURCE_ID,
                                         svc_name='resource_registry',
                                         process=FakeProcess())

        # Create a valid resource client.
        remote_client = RemoteClient(iface=IResourceAgent,
                                     xs_name=self._xs_name,
                                     resource_id=IA_RESOURCE_ID,
                                     process=FakeProcess())

        # Send a command while link is down.
        with self.assertRaises(Conflict):
            result = remote_client.ping_agent()

        # Test port manipulators refused by remote proxy client.
        with self.assertRaises(BadRequest):
            remote_client.get_port()
        with self.assertRaises(BadRequest):
            remote_client.set_client_port()
        with self.assertRaises(BadRequest):
            remote_client.get_client_port()

    def test_interfaces(self):
        """
        test_interfaces
        Test that the client declare the correct interfaces.
        """
        remote_client = RemoteClient(iface=IResourceAgent,
                                     xs_name=self._xs_name,
                                     resource_id='fake_id',
                                     process=FakeProcess())

        interfaces = providedBy(remote_client)
        self.assertIn(IResourceAgent, interfaces)
        self.assertIn(ITerrestrialEndpoint, interfaces)
示例#14
0
class Test2CAA(IonIntegrationTestCase):
    """
    Test cases for 2CAA terrestrial endpoint.
    """
    def setUp(self):
        """
        """
        
        ###################################################################
        # Internal parameters and container.
        ###################################################################
        
        # Internal parameters.        
        self._terrestrial_platform_id = 'terrestrial_id'
        self._remote_platform_id = 'remote_id'
        self._resource_id = 'fake_id'
        self._xs_name = 'remote1'
        self._terrestrial_svc_name = 'terrestrial_endpoint'
        self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
        self._remote_svc_name = 'remote_endpoint'
        self._remote_listen_name = self._remote_svc_name + self._xs_name
        self._remote_port = 0
        self._terrestrial_port = 0
        self._te_client = None
        self._re_client = None
        self._remote_pid = None
        self._terrestrial_pid = None

        # Async test results.
        self._no_requests = 10
        self._no_telem_evts = 2
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._telem_evts = []
        self._queue_mod_evts = []
        self._cmd_tx_evts = []
        self._results_recv = {}
        self._requests_sent = {}
        self._done_telem_evt = AsyncResult()
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()

        # Start container.
        log.debug('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message).
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Create a container client.
        log.debug('Creating container client.')
        self._container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        
        ###################################################################
        # Start endpoints, agent.
        ###################################################################

        self._start_terrestrial()
        self._start_remote()
        self._start_agent()
            
        ###################################################################
        # Assign client ports.
        # This is primarily for test purposes as the IP config in
        # deployment will be fixed in advance.
        ###################################################################
        
        self.te_client.set_client_port(self._remote_port)
        check_port = self.te_client.get_client_port()
        log.debug('Terrestrial client port is: %i', check_port)
    
        self.re_client.set_client_port(self._terrestrial_port)
        check_port = self.re_client.get_client_port()
        log.debug('Remote client port is: %i', check_port)
        
        ###################################################################
        # Start the event publisher and subscribers.
        # Used to send fake agent telemetry publications to the endpoints,
        # and to receive endpoint publications.
        ###################################################################
        self._event_publisher = EventPublisher()

        # Start the event subscriber for remote namespace platform events.
        # This event could be changed to RemoteNamespaceEvent.
        self._event_subscriber = EventSubscriber(
            event_type='PlatformEvent',
            callback=self.consume_event,
            origin=self._xs_name)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._event_subscriber.stop)

        # Start the result subscriber for remote resource events.
        self._resource_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=IA_RESOURCE_ID,
            callback=self.consume_event)
        self._resource_result_subscriber.start()
        self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._resource_result_subscriber.stop)

        # Start the result subscriber for remote service events.
        self._service_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin='resource_registry' + 'remote1',
            callback=self.consume_event)
        self._service_result_subscriber.start()
        self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._service_result_subscriber.stop)

        # Start the result subscriber for fake resource results.      
        self._fake_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=self._resource_id,
            callback=self.consume_event)
        self._fake_result_subscriber.start()
        self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._fake_result_subscriber.stop)

    ###################################################################
    # Start/stop helpers.
    ###################################################################

    def _start_agent(self):
        """
        Start an instrument agent and client.
        """
        
        log.info('Creating driver integration test support:')
        log.info('driver module: %s', DRV_MOD)
        log.info('driver class: %s', DRV_CLS)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)        
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }
        self.addCleanup(self._support.stop_pagent)    
                        
        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : {},
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }
    
        # Start instrument agent.
        log.debug("Starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
    
        ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config)
    
        log.info('Agent pid=%s.', str(ia_pid))
    
        # Start a resource agent client to talk with the instrument agent.
    
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))
        
    def _start_terrestrial(self):
        """
        """
        # Create terrestrial config.
        terrestrial_endpoint_config = {
            'other_host' : 'localhost',
            'other_port' : self._remote_port,
            'this_port' : self._terrestrial_port,
            'platform_resource_id' : self._terrestrial_platform_id,
            'xs_name' : self._xs_name,
            'process' : {
                'listen_name' : self._terrestrial_listen_name
            }
        }
        
        # Spawn the terrestrial enpoint process.
        log.debug('Spawning terrestrial endpoint process.')
        self._terrestrial_pid = self._container_client.spawn_process(
            name=self._terrestrial_listen_name,
            module='ion.services.sa.tcaa.terrestrial_endpoint',
            cls='TerrestrialEndpoint',
            config=terrestrial_endpoint_config)
        log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))

        # Create a terrestrial client.
        self.te_client = TerrestrialEndpointClient(
            process=FakeProcess(),
            to_name=self._terrestrial_listen_name)
        log.debug('Got te client %s.', str(self.te_client))
        self._terrestrial_port = self.te_client.get_port()
        log.debug('Terrestrial port is: %i', self._terrestrial_port)
        
    def _start_remote(self):
        """
        """        
        # Create agent config.
        remote_endpoint_config = {
            'other_host' : 'localhost',
            'other_port' : self._terrestrial_port,
            'this_port' : self._remote_port,
            'platform_resource_id' : self._remote_platform_id,
            'xs_name' : self._xs_name,
            'process' : {
                'listen_name' : self._remote_listen_name
            }
        }
        
        # Spawn the remote enpoint process.
        log.debug('Spawning remote endpoint process.')
        self._remote_pid = self._container_client.spawn_process(
            name=self._remote_listen_name,
            module='ion.services.sa.tcaa.remote_endpoint',
            cls='RemoteEndpoint',
            config=remote_endpoint_config)
        log.debug('Remote endpoint pid=%s.', str(self._remote_pid))

        # Create an endpoint client.
        self.re_client = RemoteEndpointClient(
            process=FakeProcess(),
            to_name=self._remote_listen_name)
        log.debug('Got re client %s.', str(self.re_client))
        
        # Remember the remote port.
        self._remote_port = self.re_client.get_port()
        log.debug('The remote port is: %i.', self._remote_port)
        
        
    ###################################################################
    # Telemetry publications to start/top endpoint.
    # (Normally be published by appropriate platform agents.)
    ###################################################################

    def terrestrial_link_up(self):
        """
        Publish telemetry available to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry available event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._terrestrial_platform_id,
                            status = TelemetryStatusType.AVAILABLE)
        
    def terrestrial_link_down(self):
        """
        Publish telemetry unavailable to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry unavailable event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._terrestrial_platform_id,
                            status = TelemetryStatusType.UNAVAILABLE)
    
    def remote_link_up(self):
        """
        Publish telemetry available to the remote endpoint.
        """
        # Publish a link up event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry available event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._remote_platform_id,
                            status = TelemetryStatusType.AVAILABLE)
    
    def remote_link_down(self):
        """
        Publish telemetry unavailable to the remote endpoint.
        """
        # Publish a link down event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry unavailable event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._remote_platform_id,
                            status = TelemetryStatusType.UNAVAILABLE)
    
    def consume_event(self, evt, *args, **kwargs):
        """
        Test callback for events.
        """
        log.debug('Test got event: %s, args: %s, kwargs: %s',
                  str(evt), str(args), str(kwargs))
        
        if evt.type_ == 'PublicPlatformTelemetryEvent':
            self._telem_evts.append(evt)
            if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
                self._done_telem_evt.set()
                    
        elif evt.type_ == 'RemoteQueueModifiedEvent':
            self._queue_mod_evts.append(evt)
            if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
                self._done_queue_mod_evt.set()
            
        elif evt.type_ == 'RemoteCommandTransmittedEvent':
            self._cmd_tx_evts.append(evt)
            if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
                self._done_cmd_tx_evt.set()
        
        elif evt.type_ == 'RemoteCommandResult':
            cmd = evt.command
            self._results_recv[cmd.command_id] = cmd
            if len(self._results_recv) == self._no_requests:
                self._done_cmd_evt.set()
        
    ###################################################################
    # Misc helpers.
    ###################################################################
        
    def make_fake_command(self, no):
        """
        Build a fake command for use in tests.
        """
            
        cmdstr = 'fake_cmd_%i' % no
        cmd = IonObject('RemoteCommand',
                             resource_id='fake_id',
                             command=cmdstr,
                             args=['arg1', 23],
                             kwargs={'worktime':3})
        return cmd
        
    ###################################################################
    # Tests.
    ###################################################################

    def test_queued_fake(self):
        """
        test_queued_fake
        Test fake resource commands queued prior to linkup.
        """
                
        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            cmd = self.te_client.enqueue_command(cmd)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()
        
        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    # The following error occurs when queue persistence is enabled.
    # Need to verify correct solution to enable persistent queues.
    """
    2012-11-06 14:27:13,517 INFO     pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
    Traceback (most recent call last):
      File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
        result = self._run(*self.args, **self.kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
        self._callback(cmd_result)
      File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
        self._client.enqueue(result)
    AttributeError: 'NoneType' object has no attribute 'enqueue'
    <Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
    
    2012-11-06 14:27:13,586 INFO     pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
    2012-11-06 14:27:13,665 INFO     pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
    2012-11-06 14:27:13,739 INFO     pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
    2012-11-06 14:27:13,807 INFO     pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
    2012-11-06 14:27:13,874 INFO     pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
    2012-11-06 14:27:13,941 INFO     pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
    2012-11-06 14:27:13,945 INFO     pyon.event.event EventSubscriber stopped. Event pattern=#
    2012-11-06 14:27:14,124 INFO     pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
    2012-11-06 14:27:14,399 INFO     pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
    
    ======================================================================
    ERROR: test_process_online
    ----------------------------------------------------------------------
    Traceback (most recent call last):
      File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
        return f(*args, **kw)
      File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
        cmd = self.te_client.enqueue_command(cmd)
      File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
        return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
        return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
        retval, headers = e.send(msg, headers=headers, timeout=timeout)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
        result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
        return self._invite_and_send(to_role, msg, headers, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
        return self._send(to_role, to_role_name, msg, header, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
        return self._end_point_unit._message_send(msg, headers, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
        return ProcessRPCRequestEndpointUnit.send(self, msg, headers,  **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
        return self._send(_msg, _header, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
        raise ex
    Conflict: 409 - Object not based on most current version
    
    """

    def test_process_online(self):
        """
        test_process_online
        Test fake resource commands queued while link is up.
        """
                
        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            cmd = self.te_client.enqueue_command(cmd)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mods, command transmissions and results.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    def test_remote_late(self):
        """
        test_remote_late
        Test fake resource commands queued prior to linkup.
        Delay remote side linkup substantially to test terrestrial
        behavior when remote server not initially available.
        """
                
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            cmd = self.te_client.enqueue_command(cmd)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Actually stop the remote process, since the server is
        # available even if it hasn't linked up yet and the test is running
        # in the same container. The test will remember the appropriate
        # remote port numbers.
        self._container_client.terminate_process(self._remote_pid)
        self.terrestrial_link_up()
        gevent.sleep(10)
        
        # Restart remote side and publish remote link up.
        self._start_remote()
        self.remote_link_up()
        
        # Block for transmission and result events.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())
        
    def test_resource_commands(self):
        """
        test_resource_commands
        """

        # Set up to verify the two commands queued.        
        self._no_requests = 2
        self._no_telem_evts = 2
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests

        # Use IA client to verify IA.
        state = self._ia_client.get_agent_state()
        log.debug('Agent state is: %s', state)
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        retval = self._ia_client.ping_agent()
        log.debug('Agent ping is: %s', str(retval))
        self.assertIn('ping from InstrumentAgent', retval)

        # Create and enqueue commands.
        state_cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={})
        state_cmd = self.te_client.enqueue_command(state_cmd)
        self._requests_sent[state_cmd.command_id] = state_cmd

        ping_cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='ping_agent',
                             args=[],
                             kwargs={})
        ping_cmd = self.te_client.enqueue_command(ping_cmd)
        self._requests_sent[ping_cmd.command_id] = ping_cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()

        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())        
        
        self.assertEqual(self._results_recv[state_cmd.command_id].result,
                         ResourceAgentState.UNINITIALIZED)
        self.assertIn('ping from InstrumentAgent',
                      self._results_recv[ping_cmd.command_id].result)
        
    def test_service_command_sequence(self):
        """
        test_service_commands
        """
        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []        
        self._cmd_tx_evts = []
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='create',
                             args=[obj],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns obj_id, obj_rev.
        obj_id, obj_rev = self._results_recv[cmd.command_id].result
        
        # Confirm the results are valid.
        
        #Result is a tuple of strings.
        #{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
        #'1-00a308c45fff459c7cda1db9a7314de6'],
        #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
        
        self.assertIsInstance(obj_id, str)
        self.assertNotEqual(obj_id, '')
        self.assertIsInstance(obj_rev, str)
        self.assertNotEqual(obj_rev, '')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []        
        self._cmd_tx_evts = []
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='read',
                             args=[obj_id],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        
                
        # Returns read_obj.
        read_obj = self._results_recv[cmd.command_id].result
        
        # Confirm the results are valid.
        
        #Result is a user info object with the name set.
        #{'lcstate': 'DEPLOYED_AVAILABLE',
        #'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
        #'description': '',
        #'ts_updated': '1349213207638',
        #'type_': 'UserInfo',
        #'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
        #'_id': '27832d93f4cd4535a75ac75c06e00a7e',
        #'ts_created': '1349213207638',
        #'variables': [{'name': '', 'value': ''}],
        #'name': 'some_name'}
        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_name')

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []
        self._cmd_tx_evts = []        
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        read_obj.name = 'some_other_name'
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='update',
                             args=[read_obj],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        

        # Returns nothing.

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []
        self._cmd_tx_evts = []        
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='read',
                             args=[obj_id],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        
        
        # Returns read_obj.
        read_obj = self._results_recv[cmd.command_id].result
        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_other_name')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []
        self._cmd_tx_evts = []        
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='delete',
                             args=[obj_id],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        

        # Returns nothing.
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
class TestInstrumentDataIngestion(IonIntegrationTestCase):
    """
    Tests for data ingestion from an instrument. It is basically the
    test_poll from test_instrument_agent (as of 7/23/12) with
    the additional preparation of the ingestion management service and the
    verification that the published granules are persisted.
    """

    def setUp(self):

        self.resource_registry = ResourceRegistryServiceClient()
        self.ingestion_management = IngestionManagementServiceClient()


        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DELIM,
                                                     WORK_DIR)
        # Start port agent, add stop to cleanup.
        self._pagent = None        
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Start data suscribers, add stop to cleanup.
        # Define stream_config.
        self._no_samples = None
        self._async_data_result = AsyncResult()
        self._data_greenlets = []
        self._stream_config = {}
        self._samples_received = []
        self._data_subscribers = []
        self._start_data_subscribers()
        self.addCleanup(self._stop_data_subscribers)

        self.prepare_ingestion()

        # Start event subscribers, add stop to cleanup.
        self._no_events = None
        self._async_event_result = AsyncResult()
        self._events_received = []

        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }
        
        # Start instrument agent.
        self._ia_pid = None
        log.debug("TestInstrumentDataIngestion.setup(): starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
                                                name=self.container.name)
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
                                                      module=IA_MOD, 
                                                      cls=IA_CLS, 
                                                      config=agent_config)      
        log.info('Agent pid=%s.', str(self._ia_pid))
        
        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))        
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """
        port = self._support.start_pagent()
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }
        
    def _start_data_subscribers(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # A callback for processing subscribed-to data.
        def consume_data(message, headers):
            log.info('Subscriber received data message: type(message)=%s.', str(type(message)))
            log.info('Subscriber received data message: %s.', str(message))
            self._samples_received.append(message)
            if self._no_samples and self._no_samples == len(self._samples_received):
                self._async_data_result.set()
                
        # Create a stream subscriber registrar to create subscribers.
        subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
                                                container=self.container)

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}
        self._data_subscribers = []
        for stream_name in PACKET_CONFIG:
            stream_def = ctd_stream_definition(stream_id=None)
            stream_def_id = pubsub_client.create_stream_definition(
                                                    container=stream_def)        
            stream_id = pubsub_client.create_stream(
                        name=stream_name,
                        stream_definition_id=stream_def_id,
                        original=True,
                        encoding='ION R2')

            taxy = get_taxonomy(stream_name)
            stream_config = dict(
                id=stream_id,
                taxonomy=taxy.dump()
            )
            self._stream_config[stream_name] = stream_config

            # Create subscriptions for each stream.
            exchange_name = '%s_queue' % stream_name
            sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
                                                         callback=consume_data)
            self._listen(sub)
            self._data_subscribers.append(sub)
            query = StreamQuery(stream_ids=[stream_id])
            sub_id = pubsub_client.create_subscription(
                                query=query, exchange_name=exchange_name, exchange_point='science_data')
            pubsub_client.activate_subscription(sub_id)
            
    def _listen(self, sub):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        gl = spawn(sub.listen)
        self._data_greenlets.append(gl)
        sub._ready_event.wait(timeout=5)
        return gl
                                 
    def _stop_data_subscribers(self):
        """
        Stop the data subscribers on cleanup.
        """
        for sub in self._data_subscribers:
            sub.stop()
        for gl in self._data_greenlets:
            gl.kill()
            
    def assertRawSampleDict(self, val):
        """
        Verify the value is a raw sample dictionary for the sbe37.
        """
        #{'stream_name': 'raw', 'blob': ['20.8074,80.04590, 761.394,   33.5658, 1506.076, 01 Feb 2001, 01:01:00'], 'time': [1342392781.61211]}
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('blob'))
        self.assertTrue(val.has_key('time'))
        blob = val['blob'][0]
        time = val['time'][0]
    
        self.assertTrue(isinstance(blob, str))
        self.assertTrue(isinstance(time, float))

    def assertParsedSampleDict(self, val):
        """
        Verify the value is a parsed sample dictionary for the sbe37.
        """
        #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('c'))
        self.assertTrue(val.has_key('t'))
        self.assertTrue(val.has_key('p'))
        self.assertTrue(val.has_key('time'))
        c = val['c'][0]
        t = val['t'][0]
        p = val['p'][0]
        time = val['time'][0]

        self.assertTrue(isinstance(c, float))
        self.assertTrue(isinstance(t, float))
        self.assertTrue(isinstance(p, float))
        self.assertTrue(isinstance(time, float))

    def assertSample(self, sample):
        """
        Verify a sample retrieved from the sbe37, which is expected to be a
        dict of dicts {'raw':raw_sample, 'parsed':parsed_sample}.
        """
        self.assertTrue(isinstance(sample, dict))
        raw_sample = sample['raw']
        parsed_sample = sample['parsed']
        self.assertParsedSampleDict(parsed_sample)
        self.assertRawSampleDict(raw_sample)

    def get_ingestion_config(self):
        #
        # From test_dm_end_2_end.py as of 7/23/12
        #

        #--------------------------------------------------------------------------------
        # Grab the ingestion configuration from the resource registry
        #--------------------------------------------------------------------------------
        # The ingestion configuration should have been created by the bootstrap service
        # which is configured through r2deploy.yml

        ingest_configs, _ = self.resource_registry.find_resources(
            restype=RT.IngestionConfiguration,id_only=True)
        return ingest_configs[0]

    def prepare_ingestion(self):
        #
        # Takes pieces from test_dm_end_2_end.py as of 7/23/12
        #

        # Get the ingestion configuration from the resource registry
        self.ingest_config_id = ingest_config_id = self.get_ingestion_config()

        # to keep the (stream_id, dataset_id) associated with each stream_name
        self.dataset_ids = {}

        for stream_name, stream_config in self._stream_config.iteritems():
            stream_id = stream_config['id']

            dataset_id = self.ingestion_management.persist_data_stream(
                stream_id=stream_id,
                ingestion_configuration_id=ingest_config_id)

            log.info("persisting stream_name=%s (stream_id=%s): dataset_id=%s" % (
                stream_name, stream_id, dataset_id))

            self.assertTrue(self.ingestion_management.is_persisted(stream_id))

            self.dataset_ids[stream_name] = (stream_id, dataset_id)

    def verify_granules_persisted(self):
        #
        # takes elements from ingestion_management_test.py as of 7/23/12
        #
        ingest_config_id = self.ingest_config_id
        for stream_name, (stream_id, dataset_id) in self.dataset_ids.iteritems():

            assoc = self.resource_registry.find_associations(
                subject=ingest_config_id, predicate=PRED.hasSubscription)

            sub = self.resource_registry.read(assoc[0].o)

            self.assertTrue(sub.is_active)

            dataset = self.resource_registry.read(dataset_id)
            self.assertIsInstance(dataset, DataSet)

            log.info("Data persisted for stream_name=%s (stream_id=%s, "
                     "dataset_id=%s) dataset=%s" % (stream_name, stream_id, dataset_id, dataset))


    def test_poll_and_verify_granules_persisted(self):
        #
        # As test_instrument_agent.py:TestInstrumentAgent.test_poll with
        # verification that data are persisted.
        #
        self._test_poll()
        self.verify_granules_persisted()

    def _test_poll(self):
        #
        # Same as test_instrument_agent.py:TestInstrumentAgent.test_poll as of 7/23/12
        #

        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
        
        # Lets call acquire_sample 3 times, so we should get 6 samples =
        # 3 raw samples + 3 parsed samples
        self._no_samples = 6
        
        # Poll for a few samples.
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        log.debug("acquire_sample reply.result=%s" % reply.result)
        self.assertSample(reply.result)

        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSample(reply.result)

        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSample(reply.result)

        # Assert we got 6 samples.
        self._async_data_result.get(timeout=10)
        self.assertEquals(len(self._samples_received), 6)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class InstrumentAgentTestDA():
    """
    Test cases for instrument agent class. Functions in this class provide
    instrument agent integration tests for Direct Access mode and provide 
    a tutorial on use of the agent setup and interface.
    """
    def _setup(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None, None, DEV_ADDR,
                                                     DEV_PORT, DATA_PORT,
                                                     CMD_PORT, PA_BINARY,
                                                     DELIM, WORK_DIR)

        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Start container.
        log.info('Staring capability container.')
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Start a resource agent client to talk with the instrument agent.
        log.info('starting IA process')
        self._ia_client = start_instrument_agent_process(self.container)
        self.addCleanup(self._verify_agent_reset)
        log.info('test setup complete')

    ###############################################################################
    # Port agent helpers.
    ###############################################################################

    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i', port)

        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr': 'localhost',
            'port': port,
            'cmd_port': CMD_PORT
        }

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)

    ###############################################################################
    # Event helpers.
    ###############################################################################

    def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
        """
        Start a subscriber to the instrument agent events.
        @param type The type of event to catch.
        @count Trigger the async event result when events received reaches this.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()

        # Event array and async event result.
        self._event_count = count
        self._events_received = []
        self._async_event_result = AsyncResult()

        self._event_subscriber = EventSubscriber(event_type=type,
                                                 callback=consume_event,
                                                 origin=IA_RESOURCE_ID)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=5)

    def _stop_event_subscriber(self):
        """
        Stop event subscribers on cleanup.
        """
        self._event_subscriber.stop()
        self._event_subscriber = None

    ###############################################################################
    # tcp helpers.
    ###############################################################################

    def _start_tcp_client(self, retval):
        host = retval.result['ip_address']
        port = retval.result['port']
        tcp_client = TcpClient(host, port)
        return tcp_client

    ###############################################################################
    # Assert helpers.
    ###############################################################################

    def assertInstrumentAgentState(self, expected_state, timeout=0):
        end_time = time.time() + timeout

        while (True):
            state = self._ia_client.get_agent_state()
            log.debug(
                "assertInstrumentAgentState: IA state = %s, expected state = %s"
                % (state, expected_state))
            if state == expected_state:
                return True
            if time.time() >= end_time:
                self.fail(
                    "assertInstrumentAgentState: IA failed to transition to %s state"
                    % expected_state)
            gevent.sleep(1)

    def assertSetInstrumentState(self, command, new_state):
        if type(command) == str:
            log.debug("assertSetInstrumentState: building command for %s" %
                      command)
            cmd = AgentCommand(command=command)
        else:
            cmd = command
        retval = self._ia_client.execute_agent(cmd)
        self.assertInstrumentAgentState(new_state)
        return retval

    ###############################################################################
    # Tests.
    #
    # response from IA for start DA looks similar to the following:
    #
    # {'status': 0, 'type_': 'AgentCommandResult', 'command': 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS',
    # 'result': {'token': 'F2B6EED3-F926-4B3B-AE80-4F8DE79276F3', 'ip_address': 'Edwards-MacBook-Pro.local', 'port': 8000},
    # 'ts_execute': '1344889063861', 'command_id': ''}
    ###############################################################################

    def test_direct_access_vsp_IA_shutdown(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown from IA. 
        """
        def start_and_stop_DA():
            retval = self.assertSetInstrumentState(
                cmd, ResourceAgentState.DIRECT_ACCESS)
            tcp_client = self._start_tcp_client(retval)
            self.assertTrue(tcp_client.send_data('ts\r\n'))
            self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                          ResourceAgentState.COMMAND)

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.vsp,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        # test for starting DA, making connection with TCP client, sending/recving commands to/from instrument, and DA shut down
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.send_data('ts\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ts
        
        -0.1964,85.12299, 697.168,   39.5241, 1506.965, 01 Feb 2001, 01:01:00
        """

        sample_pattern = r'^#? *(-?\d+\.\d+), *(-?\d+\.\d+), *(-?\d+\.\d+)'
        sample_pattern += r'(, *(-?\d+\.\d+))?(, *(-?\d+\.\d+))?'
        sample_pattern += r'(, *(\d+) +([a-zA-Z]+) +(\d+), *(\d+):(\d+):(\d+))?'
        sample_pattern += r'(, *(\d+)-(\d+)-(\d+), *(\d+):(\d+):(\d+))?'
        sample_regex = re.compile(sample_pattern)
        lines = response.split('\r\n')

        sample_count = 0
        for x in lines:
            if sample_regex.match(x):
                sample_count += 1
        #self.assertEqual(sample_count, 1)

        self.assertTrue(tcp_client.send_data('ds\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ds
        SBE37-SMP V 2.6 SERIAL NO. 2165   01 Feb 2001  01:01:00
        not logging: received stop command
        sample interval = 23195 seconds
        samplenumber = 0, free = 200000
        do not transmit real-time data
        do not output salinity with each sample
        do not output sound velocity with each sample
        do not store time with each sample
        number of samples to average = 0
        reference pressure = 0.0 db
        serial sync mode disabled
        wait time after serial sync sampling = 0 seconds
        internal pump is installed
        temperature = 7.54 deg C
        WARNING: LOW BATTERY VOLTAGE!!
        """

        self.assertNotEqual(response.find('SBE37-SMP'), -1)
        self.assertNotEqual(response.find('sample interval'), -1)
        self.assertNotEqual(response.find('samplenumber'), -1)
        self.assertNotEqual(response.find('number of samples to average'), -1)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                      ResourceAgentState.COMMAND)

        # test for starting and shutting down DA w/o ever connecting TCP client
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                      ResourceAgentState.COMMAND)

        # test for race condition in DA when shut down by IA after sending something from TCP client
        for i in range(0, 10):
            start_and_stop_DA()

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_telnet_IA_shutdown(self):
        """
        Test agent direct_access mode for telnet when shutdown from IA. 
        """
        def start_and_stop_DA():
            retval = self.assertSetInstrumentState(
                cmd, ResourceAgentState.DIRECT_ACCESS)
            tcp_client = self._start_tcp_client(retval)
            self.assertTrue(
                tcp_client.start_telnet(token=retval.result['token']))
            self.assertTrue(tcp_client.send_data('ts\r\n'))
            self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                          ResourceAgentState.COMMAND)

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.telnet,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        # test for starting DA, making connection with TCP/telnet client, sending/recving commands to/from instrument, and DA shut down
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(token=retval.result['token']))

        self.assertTrue(tcp_client.send_data('ts\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ts
        
        -0.1964,85.12299, 697.168,   39.5241, 1506.965, 01 Feb 2001, 01:01:00
        """

        sample_pattern = r'^#? *(-?\d+\.\d+), *(-?\d+\.\d+), *(-?\d+\.\d+)'
        sample_pattern += r'(, *(-?\d+\.\d+))?(, *(-?\d+\.\d+))?'
        sample_pattern += r'(, *(\d+) +([a-zA-Z]+) +(\d+), *(\d+):(\d+):(\d+))?'
        sample_pattern += r'(, *(\d+)-(\d+)-(\d+), *(\d+):(\d+):(\d+))?'
        sample_regex = re.compile(sample_pattern)
        lines = response.split('\r\n')

        sample_count = 0
        for x in lines:
            if sample_regex.match(x):
                sample_count += 1
        #self.assertEqual(sample_count, 1)

        self.assertTrue(tcp_client.send_data('ds\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ds
        SBE37-SMP V 2.6 SERIAL NO. 2165   01 Feb 2001  01:01:00
        not logging: received stop command
        sample interval = 23195 seconds
        samplenumber = 0, free = 200000
        do not transmit real-time data
        do not output salinity with each sample
        do not output sound velocity with each sample
        do not store time with each sample
        number of samples to average = 0
        reference pressure = 0.0 db
        serial sync mode disabled
        wait time after serial sync sampling = 0 seconds
        internal pump is installed
        temperature = 7.54 deg C
        WARNING: LOW BATTERY VOLTAGE!!
        """

        self.assertNotEqual(response.find('SBE37-SMP'), -1)
        self.assertNotEqual(response.find('sample interval'), -1)
        self.assertNotEqual(response.find('samplenumber'), -1)
        self.assertNotEqual(response.find('number of samples to average'), -1)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                      ResourceAgentState.COMMAND)

        # test for starting and shutting down DA w/o ever connecting TCP client
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                      ResourceAgentState.COMMAND)

        # test for starting and shutting down DA w/o ever entering a username
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                      ResourceAgentState.COMMAND)

        # test for starting and shutting down DA w/o ever entering a token
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND,
                                      ResourceAgentState.COMMAND)

        # test for race condition in DA when shut down by IA after sending something from TCP client
        for i in range(0, 10):
            start_and_stop_DA()

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_telnet_login_failure(self):
        """
        Test agent direct_access mode for telnet when login fails. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.telnet,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        # test that DA session quits when bad token is sent
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertFalse(tcp_client.start_telnet(token='some junk'))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

    def test_direct_access_telnet_session_setup_failure(self):
        """
        Test agent direct_access mode for telnet when session setup fails. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.telnet,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        # test that DA session quits when bad token is sent
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(
            tcp_client.start_telnet(token=retval.result['token'],
                                    handshake=False))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

    def test_direct_access_vsp_client_disconnect(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by tcp client disconnect. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.vsp,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.send_data('ts\r\n'))

        tcp_client.disconnect()

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_telnet_client_disconnect(self):
        """
        Test agent direct_access mode for telnet when shutdown by tcp client disconnect. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.telnet,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(retval.result['token']))

        self.assertTrue(tcp_client.send_data('ts\r\n'))

        tcp_client.disconnect()

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and disconnecting client w/o ever entering a username
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        tcp_client.disconnect()
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and disconnecting client w/o ever entering a token
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        tcp_client.disconnect()
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and disconnecting client w/o ever negotiating the telnet session
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(
            tcp_client.start_telnet(token=retval.result['token'],
                                    handshake=False))
        tcp_client.disconnect()
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_vsp_session_timeout(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by session timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.vsp,
                               'session_timeout': 10,
                               'inactivity_timeout': 600
                           })

        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        self._start_tcp_client(retval)

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)

        # test for starting and session timeout w/o ever connecting TCP client
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_telnet_session_timeout(self):
        """
        Test agent direct_access mode for telnet when shutdown by session timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.telnet,
                               'session_timeout': 10,
                               'inactivity_timeout': 600
                           })

        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(retval.result['token']))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)

        # test for starting and session timeout w/o ever entering a username
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and session timeout w/o ever entering a token
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and session timeout w/o ever negotiating the telnet session
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(
            tcp_client.start_telnet(token=retval.result['token'],
                                    handshake=False))
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_vsp_inactivity_timeout(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by inactivity timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.vsp,
                               'session_timeout': 600,
                               'inactivity_timeout': 10
                           })

        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        self._start_tcp_client(retval)

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)

        # test for starting and inactivity timeout w/o ever connecting TCP client
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    def test_direct_access_telnet_inactivity_timeout(self):
        """
        Test agent direct_access mode for telnet when shutdown by inactivity timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)

        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE,
                                      ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE,
                                      ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN,
                                      ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.telnet,
                               'session_timeout': 600,
                               'inactivity_timeout': 10
                           })

        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(retval.result['token']))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)

        # test for starting and inactivity timeout w/o ever entering a username
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and inactivity timeout w/o ever entering a token
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        # test for starting and inactivity timeout w/o ever negotiating the telnet session
        retval = self.assertSetInstrumentState(
            cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(
            tcp_client.start_telnet(token=retval.result['token'],
                                    handshake=False))
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)

        self.assertSetInstrumentState(ResourceAgentEvent.RESET,
                                      ResourceAgentState.UNINITIALIZED)

    @unittest.skip('A manual test for timing purposes.')
    def test_exit_da_timing(self):
        """
        test_exit_da_timing
        Test time it takes to leave direct access and return to command mode.
        """

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={
                               'session_type': DirectAccessTypes.vsp,
                               'session_timeout': 600,
                               'inactivity_timeout': 600
                           })

        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        starttime = time.time()

        cmd = AgentCommand(command=ResourceAgentEvent.GO_COMMAND)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        delta = time.time() - starttime

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        print '######## exiting direct access takes: %f seconds' % delta
class TestAgentPersistence(IonIntegrationTestCase):
    """
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Create agent config.
        self._agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True,
            'forget_past' : False,
            'enable_persistence' : True
        }

        self._ia_client = None
        self._ia_pid = '8989'
        
        self.addCleanup(self._verify_agent_reset)
        self.addCleanup(self.container.state_repository.put_state,
                        self._ia_pid, {})

    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                                    
    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'parsed'
        param_dict_name = 'ctd_parsed_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(stream_route=stream_route,
                                 routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
        self._stream_config[stream_name] = stream_config

        stream_name = 'raw'
        param_dict_name = 'ctd_raw_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(stream_route=stream_route,
                                 routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
        self._stream_config[stream_name] = stream_config

    ###############################################################################
    # Agent start stop helpers.
    ###############################################################################

    def _start_agent(self):
        """
        """
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
            
        pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=self._agent_config,
            process_id=self._ia_pid)
        log.info('Started instrument agent pid=%s.', str(self._ia_pid))
        
        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got instrument agent client %s.', str(self._ia_client))

    def _stop_agent(self):
        """
        """
        if self._ia_pid:
            container_client = ContainerAgentClient(node=self.container.node,
                name=self.container.name)
            container_client.terminate_process(self._ia_pid)
        
        if self._ia_client:
            self._ia_client = None

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)
            self._ia_client = None

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_agent_config_persistence(self):
        """
        test_agent_config_persistence
        Test that agent parameter configuration is persisted between running
        instances.
        """
        
        # Start the agent.
        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Confirm the default agent parameters.
        #{'streams': {'raw': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'raw', 'lat', 'driver_timestamp', 'preferred_timestamp', 'lon', 'internal_timestamp', 'time'], 'parsed': ['quality_flag', 'ingestion_timestamp', 'port_timestamp', 'pressure', 'lat', 'driver_timestamp', 'conductivity', 'preferred_timestamp', 'temp', 'density', 'salinity', 'lon', 'internal_timestamp', 'time']}}
        retval = self._ia_client.get_agent(['streams'])['streams']
        self.assertIn('raw', retval.keys())
        self.assertIn('parsed', retval.keys())

        #{'pubrate': {'raw': 0, 'parsed': 0}}
        retval = self._ia_client.get_agent(['pubrate'])['pubrate']
        self.assertIn('raw', retval.keys())
        self.assertIn('parsed', retval.keys())
        self.assertEqual(retval['raw'], 0)
        self.assertEqual(retval['parsed'], 0)
        
        #{'alerts': []}
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        self.assertEqual(retval, [])

        # Define a few new parameters and set them.
        # Confirm they are set.
        alert_def_1 = {
            'name' : 'current_warning_interval',
            'stream_name' : 'parsed',
            'message' : 'Current is below normal range.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 10.0,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        alert_def_2 = {
            'name' : 'temp_alarm_interval',
            'stream_name' : 'parsed',
            'message' : 'Temperatoure is critical.',
            'alert_type' : StreamAlertType.ALARM,
            'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
            'value_id' : 'temp',
            'lower_bound' : None,
            'lower_rel_op' : None,
            'upper_bound' : 20.0,
            'upper_rel_op' : '<',
            'alert_class' : 'IntervalAlert'
        }

        alert_def3 = {
            'name' : 'late_data_warning',
            'stream_name' : 'parsed',
            'message' : 'Expected data has not arrived.',
            'alert_type' : StreamAlertType.WARNING,
            'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
            'value_id' : None,
            'time_delta' : 180,
            'alert_class' : 'LateDataAlert'
        }

        orig_alerts = [alert_def_1,alert_def_2, alert_def3]
        pubrate = {
            'parsed' : 10,
            'raw' : 20
        }
        params = {
            'alerts' : orig_alerts,
            'pubrate' : pubrate
        }
        
        # Set the new agent params and confirm.
        self._ia_client.set_agent(params)
        
        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        pubrate = retval['pubrate']
        alerts = retval['alerts']
        self.assertIn('raw', pubrate.keys())
        self.assertIn('parsed', pubrate.keys())
        self.assertEqual(pubrate['parsed'], 10)
        self.assertEqual(pubrate['raw'], 20)
        count = 0
        for x in alerts:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEqual(count, 3)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(5)
        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Confirm the persisted parameters.
        params = [
            'alerts',
            'pubrate'
        ]
        retval = self._ia_client.get_agent(params)
        
        pubrate = retval['pubrate']
        alerts = retval['alerts']
        self.assertIn('raw', pubrate.keys())
        self.assertIn('parsed', pubrate.keys())
        self.assertEqual(pubrate['parsed'], 10)
        self.assertEqual(pubrate['raw'], 20)
        count = 0
        for x in alerts:
            x.pop('status')
            x.pop('value')
            for y in orig_alerts:
                if x['name'] == y['name']:
                    count += 1
                    self.assertItemsEqual(x.keys(), y.keys())
        self.assertEqual(count, 3)
       
    def test_agent_state_persistence(self):
        """
        test_agent_state_persistence
        Verify that agents can be restored to their prior running state.
        """

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Acquire sample returns a string, not a particle.  The particle
        # is created by the data handler though.
        cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(3)
        self._start_agent()

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.STOPPED)

        # Now stop and restart the agent.
        self._stop_agent()
        gevent.sleep(3)
        self._start_agent()

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.STOPPED)

        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
    def _set_up_pre_environment_for_instrument(self, instr_info):
        """
        Based on test_instrument_agent.py

        Basically, this method launches a port agent and then completes the
        instrument driver configuration used to properly set up a particular
        instrument agent.

        @param instr_info  A value in instruments_dict
        @return instrument_driver_config
        """

        import sys
        from ion.agents.instrument.driver_process import DriverProcessType
        from ion.agents.instrument.driver_process import ZMQEggDriverProcess

        # A seabird driver.
        DRV_URI = SBE37_EGG
        DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
        DRV_CLS = 'SBE37Driver'

        WORK_DIR = '/tmp/'
        DELIM = ['<<', '>>']

        instrument_driver_config = {
            'dvr_egg' : DRV_URI,
            'dvr_mod' : DRV_MOD,
            'dvr_cls' : DRV_CLS,
            'workdir' : WORK_DIR,
            'process_type' : None
        }

        # Launch from egg or a local MI repo.
        LAUNCH_FROM_EGG=True

        if LAUNCH_FROM_EGG:
            # Dynamically load the egg into the test path
            launcher = ZMQEggDriverProcess(instrument_driver_config)
            egg = launcher._get_egg(DRV_URI)
            if not egg in sys.path: sys.path.insert(0, egg)
            instrument_driver_config['process_type'] = (DriverProcessType.EGG,)

        else:
            mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
            if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
            instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
            instrument_driver_config['mi_repo'] = mi_repo

        DEV_ADDR  = instr_info['DEV_ADDR']
        DEV_PORT  = instr_info['DEV_PORT']
        DATA_PORT = instr_info['DATA_PORT']
        CMD_PORT  = instr_info['CMD_PORT']
        PA_BINARY = instr_info['PA_BINARY']

        support = DriverIntegrationTestSupport(None,
                                               None,
                                               DEV_ADDR,
                                               DEV_PORT,
                                               DATA_PORT,
                                               CMD_PORT,
                                               PA_BINARY,
                                               DELIM,
                                               WORK_DIR)

        # Start port agent, add stop to cleanup.
        port = support.start_pagent()
        log.info('Port agent started at port %i', port)
        self.addCleanup(support.stop_pagent)

        # Configure instrument driver to use port agent port number.
        instrument_driver_config['comms_config'] = {
            'addr':     'localhost',
            'port':     port,
            'cmd_port': CMD_PORT
        }

        return instrument_driver_config
示例#19
0
class TestInstrumentAgent(IonIntegrationTestCase):
    """
    Test cases for instrument agent class. Functions in this class provide
    instrument agent integration tests and provide a tutorial on use of
    the agent setup and interface.
    """ 
    def setUp(self):
        """
        Initialize test members.
        Start port agent.
        Start container and client.
        Start streams and subscribers.
        Start agent, client.
        """
                
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DELIM,
                                                     WORK_DIR)
        # Start port agent, add stop to cleanup.
        self._pagent = None        
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        self.container.start_rel_from_url('res/deploy/r2dm.yml')

        # Start data suscribers, add stop to cleanup.
        # Define stream_config.
        self._no_samples = None
        self._async_data_result = AsyncResult()
        self._data_greenlets = []
        self._stream_config = {}
        self._samples_received = []
        self._data_subscribers = []
        self._start_data_subscribers()
        self.addCleanup(self._stop_data_subscribers)

        # Start event subscribers, add stop to cleanup.
        self._no_events = None
        self._async_event_result = AsyncResult()
        self._events_received = []
        self._event_subscribers = []
        self._start_event_subscribers()
        self.addCleanup(self._stop_event_subscribers)
                
        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }
        
        # Start instrument agent.
        self._ia_pid = None
        log.debug("TestInstrumentAgent.setup(): starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
                                                name=self.container.name)
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
                                                      module=IA_MOD, 
                                                      cls=IA_CLS, 
                                                      config=agent_config)      
        log.info('Agent pid=%s.', str(self._ia_pid))
        
        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))        
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """
        port = self._support.start_pagent()
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }
        
    def _start_data_subscribers(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # A callback for processing subscribed-to data.
        def consume_data(message, headers):
            log.info('Subscriber received data message: %s.', str(message))
            self._samples_received.append(message)
            if self._no_samples and self._no_samples == len(self._samples_received):
                self._async_data_result.set()
                
        # Create a stream subscriber registrar to create subscribers.
        subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
                                                node=self.container.node)

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}
        self._data_subscribers = []
        for (stream_name, val) in PACKET_CONFIG.iteritems():
            stream_def = ctd_stream_definition(stream_id=None)
            stream_def_id = pubsub_client.create_stream_definition(
                                                    container=stream_def)        
            stream_id = pubsub_client.create_stream(
                        name=stream_name,
                        stream_definition_id=stream_def_id,
                        original=True,
                        encoding='ION R2')
            self._stream_config[stream_name] = stream_id
            
            # Create subscriptions for each stream.
            exchange_name = '%s_queue' % stream_name
            sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
                                                         callback=consume_data)
            self._listen(sub)
            self._data_subscribers.append(sub)
            query = StreamQuery(stream_ids=[stream_id])
            sub_id = pubsub_client.create_subscription(\
                                query=query, exchange_name=exchange_name)
            pubsub_client.activate_subscription(sub_id)
            
    def _listen(self, sub):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        gl = spawn(sub.listen)
        self._data_greenlets.append(gl)
        sub._ready_event.wait(timeout=5)
        return gl
                                 
    def _stop_data_subscribers(self):
        """
        Stop the data subscribers on cleanup.
        """
        for sub in self._data_subscribers:
            sub.stop()
        for gl in self._data_greenlets:
            gl.kill()
            
    def _start_event_subscribers(self):
        """
        Create subscribers for agent and driver events.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', 
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._no_events and self._no_events == len(self._event_received):
                self._async_event_result.set()
                
        event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
        event_sub.activate()
        self._event_subscribers.append(event_sub)
        
    def _stop_event_subscribers(self):
        """
        Stop event subscribers on cleanup.
        """
        for sub in self._event_subscribers:
            sub.deactivate()
        
    def assertSampleDict(self, val):
        """
        Verify the value is a sample dictionary for the sbe37.
        """
        #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}        
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('c'))
        self.assertTrue(val.has_key('t'))
        self.assertTrue(val.has_key('p'))
        self.assertTrue(val.has_key('time'))
        c = val['c'][0]
        t = val['t'][0]
        p = val['p'][0]
        time = val['time'][0]
    
        self.assertTrue(isinstance(c, float))
        self.assertTrue(isinstance(t, float))
        self.assertTrue(isinstance(p, float))
        self.assertTrue(isinstance(time, float))

    def assertParamDict(self, pd, all_params=False):
        """
        Verify all device parameters exist and are correct type.
        """
        if all_params:
            self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
            for (key, type_val) in PARAMS.iteritems():
                if type_val == list or type_val == tuple:
                    self.assertTrue(isinstance(pd[key], (list, tuple)))
                else:
                    self.assertTrue(isinstance(pd[key], type_val))
                    
        else:
            for (key, val) in pd.iteritems():
                self.assertTrue(PARAMS.has_key(key))
                self.assertTrue(isinstance(val, PARAMS[key]))
    
    def assertParamVals(self, params, correct_params):
        """
        Verify parameters take the correct values.
        """
        self.assertEqual(set(params.keys()), set(correct_params.keys()))
        for (key, val) in params.iteritems():
            correct_val = correct_params[key]
            if isinstance(val, float):
                # Verify to 5% of the larger value.
                max_val = max(abs(val), abs(correct_val))
                self.assertAlmostEqual(val, correct_val, delta=max_val*.01)

            elif isinstance(val, (list, tuple)):
                # list of tuple.
                self.assertEqual(list(val), list(correct_val))
            
            else:
                # int, bool, str.
                self.assertEqual(val, correct_val)

    def test_initialize(self):
        """
        Test agent initialize command. This causes creation of
        driver process and transition to inactive.
        """

        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)

        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
                
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
    def test_states(self):
        """
        Test agent state transitions.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
                
        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)
 
        cmd = AgentCommand(command='resume')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
 
        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)

        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_get_set(self):
        """
        Test instrument driver get and set interface.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # Retrieve all resource parameters.                
        reply = self._ia_client.get_param(SBE37Parameter.ALL)
        self.assertParamDict(reply, True)
        orig_config = reply
        
        # Retrieve a subset of resource parameters.
        params = [
            SBE37Parameter.TA0,
            SBE37Parameter.INTERVAL,
            SBE37Parameter.STORETIME
        ]
        reply = self._ia_client.get_param(params)
        self.assertParamDict(reply)
        orig_params = reply

        # Set a subset of resource parameters.
        new_params = {
            SBE37Parameter.TA0 : (orig_params[SBE37Parameter.TA0] * 2),
            SBE37Parameter.INTERVAL : (orig_params[SBE37Parameter.INTERVAL] + 1),
            SBE37Parameter.STORETIME : (not orig_params[SBE37Parameter.STORETIME])
        }
        self._ia_client.set_param(new_params)
        check_new_params = self._ia_client.get_param(params)
        self.assertParamVals(check_new_params, new_params)
        
        # Reset the parameters back to their original values.
        self._ia_client.set_param(orig_params)
        reply = self._ia_client.get_param(SBE37Parameter.ALL)
        reply.pop(SBE37Parameter.SAMPLENUM)
        orig_config.pop(SBE37Parameter.SAMPLENUM)
        self.assertParamVals(reply, orig_config)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_poll(self):
        """
        Test observatory polling function.
        """
        
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
        
        # Lets get 3 samples.
        self._no_samples = 3
        
        # Poll for a few samples.
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSampleDict(reply.result)

        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSampleDict(reply.result)

        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSampleDict(reply.result)        

        # Assert we got 3 samples.
        self._async_data_result.get(timeout=10)
        self.assertTrue(len(self._samples_received)==3)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)        
        
    def test_autosample(self):
        """
        Test instrument driver execute interface to start and stop streaming
        mode.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # Make sure the sampling rate and transmission are sane.                
        params = {
            SBE37Parameter.NAVG : 1,
            SBE37Parameter.INTERVAL : 5,
            SBE37Parameter.TXREALTIME : True
        }
        self._ia_client.set_param(params)

        self._no_samples = 2

        # Begin streaming.                
        cmd = AgentCommand(command='go_streaming')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STREAMING)
 
        # Wait for some samples to roll in.
        gevent.sleep(15)
 
        # Halt streaming.
        cmd = AgentCommand(command='go_observatory')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # Assert we got some samples.
        self._async_data_result.get(timeout=10)
        self.assertTrue(len(self._samples_received)>=2)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_capabilities(self):
        """
        Test the ability to retrieve agent and resource parameter and command
        capabilities.
        """
        acmds = self._ia_client.get_capabilities(['AGT_CMD'])
        acmds = [item[1] for item in acmds]
        self.assertEqual(acmds, AGT_CMDS)
        apars = self._ia_client.get_capabilities(['AGT_PAR'])
        apars = [item[1] for item in apars]
        
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)
        
        rcmds = self._ia_client.get_capabilities(['RES_CMD'])
        rcmds = [item[1] for item in rcmds]
        self.assertEqual(rcmds, CMDS)
        
        rpars = self._ia_client.get_capabilities(['RES_PAR'])
        rpars = [item[1] for item in rpars]
        self.assertEqual(rpars, SBE37Parameter.list())
                
        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
    @unittest.skip('Never written')
    def test_errors(self):
        """
        Test illegal behavior and replies.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        # Can't go active in unitialized state.
        # Status 660 is state error.
        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        log.info('GO ACTIVE CMD %s',str(retval))
        self.assertEquals(retval.status, 660)
        
        # Can't command driver in this state.
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertEqual(reply.status, 660)
        
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        # OK, I can do this now.        
        cmd = AgentCommand(command='acquire_sample')
        reply = self._ia_client.execute(cmd)
        self.assertSampleDict(reply.result)

        # 404 unknown agent command.
        cmd = AgentCommand(command='kiss_edward')
        retval = self._ia_client.execute_agent(cmd)
        self.assertEquals(retval.status, 404)
        
        # 670 unknown driver command.
        cmd = AgentCommand(command='acquire_sample_please')
        retval = self._ia_client.execute(cmd)
        self.assertEqual(retval.status, 670)

        # 630 Parameter error.
        with self.assertRaises(InstParameterError):
            reply = self._ia_client.get_param('bogus bogus')

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
        
        
    @unittest.skip('Direct access test to be finished by adding the telnet client, manual for now.')
    def test_direct_access(self):
        """
        Test agent direct_access command. This causes creation of
        driver process and transition to direct access.
        """
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)
        
        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='go_direct_access', 
                           #kwargs={'session_type':DirectAccessTypes.telnet,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                                   'session_timeout':600,
                                   'inactivity_timeout':600})
        retval = self._ia_client.execute_agent(cmd)
        print("go_direct_access retval=" + str(retval))       
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.DIRECT_ACCESS)

        # sleep to let tester run telnet client manually
        print "test sleeping to run telnet client"
        time.sleep(60)

        # Halt DA.
        cmd = AgentCommand(command='go_observatory')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_current_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
示例#20
0
class TestAgentCommsAlerts(IonIntegrationTestCase):
    """
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(None,
                                                     None,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
                
        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        self._event_count = 0
        self._events_received = []
        self._async_event_result = AsyncResult()

        def consume_event(*args, **kwargs):
            log.debug('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
                     str(args), str(kwargs), str(args[0]))
            
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()
                        
        self._event_subscriber = EventSubscriber(
            event_type='DeviceStatusAlertEvent', callback=consume_event,
            origin=IA_RESOURCE_ID)
        
        self._event_subscriber.start()

        def stop_subscriber():
            self._event_subscriber.stop()
            self._event_subscriber = None
        self.addCleanup(stop_subscriber)

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Create agent config.
        self._agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : self._stream_config,
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True,
            'aparam_alerts_config' : [state_alert_def, command_alert_def]
        }

        self._ia_client = None
        self._ia_pid = None
        
        self.addCleanup(self._verify_agent_reset)

    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                                    
    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        
        encoder = IonObjectSerializer()
        
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'parsed'
        param_dict_name = 'ctd_parsed_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)        
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 parameter_dictionary=pd,
                                 stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config

        stream_name = 'raw'
        param_dict_name = 'ctd_raw_param_dict'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 parameter_dictionary=pd,
                                 stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config
        
    ###############################################################################
    # Agent start stop helpers.
    ###############################################################################

    def _start_agent(self):
        """
        """
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=self._agent_config)
        log.info('Started instrument agent pid=%s.', str(self._ia_pid))
        
        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = None
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got instrument agent client %s.', str(self._ia_client))

    def _stop_agent(self):
        """
        """
        if self._ia_pid:
            container_client = ContainerAgentClient(node=self.container.node,
                name=self.container.name)
            container_client.terminate_process(self._ia_pid)
            self._ia_pid = None
            
        if self._ia_client:
            self._ia_client = None

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)
            self._ia_client = None

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_lost_connection_alert(self):
        """
        test_lost_connection_alert
        Verify that agents detect lost connection state and issue alert.
        """

        self._event_count = 3

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Confirm the persisted parameters.
        retval = self._ia_client.get_agent(['alerts'])['alerts']
        """
        {'origin': '123xyz', 'status': 1, '_id': 'da03b90d2e064b25bf51ed90b729e82e',
        'description': 'The alert is cleared.', 'time_stamps': [],
        'type_': 'DeviceStatusAlertEvent', 'valid_values': [],
        'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '',
        'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'],
        'stream_name': '', 'ts_created': '1366749987069', 'sub_type': 3,
        'origin_type': 'InstrumentDevice', 'name': 'comms_warning'}
        """

        # Acquire sample returns a string, not a particle.  The particle
        # is created by the data handler though.
        cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        # Blow the port agent out from under the agent.
        self._support.stop_pagent()

        # Wait for a while, the supervisor is restarting the port agent.
        gevent.sleep(5)
        self._support.start_pagent()

        timeout = gevent.Timeout(120)
        timeout.start()
        try:
            while True:
                state = self._ia_client.get_agent_state()
                if state == ResourceAgentState.COMMAND:
                    timeout.cancel()
                    break
                else:
                    gevent.sleep(2)
            
        except Timeout as t:
            self.fail('Could not reconnect to device.')
        
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        self._async_event_result.get(timeout=30) 
        """
        {'origin': '123xyz', 'status': 1, '_id': '9d7db919a0414741a2aa97f3dc310647', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029709', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
        {'origin': '123xyz', 'status': 1, '_id': 'f746cec5e486445e856ef29af8d8d49a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008029717', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
        {'origin': '123xyz', 'status': 1, '_id': '126b1e20f54f4bd2b84a93584831ea8d', 'description': 'Detected comms failure.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_LOST_CONNECTION'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008062061', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
        {'origin': '123xyz', 'status': 1, '_id': '90f5762112dd446b939c6675d34dd61d', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_COMMAND'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367008098639', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
        """
        
    def test_connect_failed_alert(self):
        """
        test_connect_failed_alert
        Verify that agents detect failed connections and issue alert.
        """

        self._event_count = 4

        # Remove the port agent.
        self._support.stop_pagent()

        self._start_agent()

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        with self.assertRaises(Exception):
            cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
            retval = self._ia_client.execute_agent(cmd)

        # Now start up the port agent.
        self._support.start_pagent()
        gevent.sleep(5)

        # This time it will work.
        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        self._async_event_result.get(timeout=30)
        """
        {'origin': '123xyz', 'status': 1, '_id': 'ac1ae3ec24e74f65bde24362d689346a', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': ['RESOURCE_AGENT_STATE_INACTIVE'], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751167', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_state_warning'}
        {'origin': '123xyz', 'status': 1, '_id': '6af6a6156172481ebc44baad2708ec5c', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007751175', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
        {'origin': '123xyz', 'status': 1, '_id': 'b493698b46fd4fc1b4c66c53b70ed043', 'description': 'Detected comms failure while connecting.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007756771', 'sub_type': 'WARNING', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
        {'origin': '123xyz', 'status': 1, '_id': 'c01e911ecb1a47c5a04cbbcbfb348a42', 'description': 'The alert is cleared.', 'time_stamps': [], 'type_': 'DeviceStatusAlertEvent', 'valid_values': [], 'values': [None], 'value_id': '', 'base_types': ['DeviceStatusEvent', 'DeviceEvent', 'Event'], 'stream_name': '', 'ts_created': '1367007792905', 'sub_type': 'ALL_CLEAR', 'origin_type': 'InstrumentDevice', 'name': 'comms_command_warning'}
        """
        
示例#21
0
class TestPuck(IonIntegrationTestCase):
    """
    """
    
    ############################################################################
    # Setup, teardown.
    ############################################################################
        
    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        
        log.info('Creating driver integration test support:')
        log.info('driver module: %s', DRV_MOD)
        log.info('driver class: %s', DRV_CLS)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        self._support.start_pagent()
        self.addCleanup(self._support.stop_pagent)    
        
        # Start container.
        log.info('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.debug("Starting container client.")
        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)

        self._build_stream_config()
        
    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient() 
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        streams = {
            'parsed' : 'ctd_parsed_param_dict',
            'raw' : 'ctd_raw_param_dict'
        }

        for (stream_name, param_dict_name) in streams.iteritems():
            pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)

            stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
            pd            = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary

            stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)

            stream_config = dict(stream_route=stream_route,
                                 routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 stream_definition_ref=stream_def_id,
                                 parameter_dictionary=pd)
            self._stream_config[stream_name] = stream_config
                     
    ###############################################################################
    # Tests.
    ###############################################################################

    def test_xxx(self, host='localhost', port=DATA_PORT,
                 resource_id=IA_RESOURCE_ID, stream_config=None):
        """
        """
        
        if not stream_config:
            stream_config = self._stream_config

        log.info("create FakePuckReader")
        puck_reader = FakePuckReader()
        log.info("read_puck()")
        puck_data = puck_reader.read_puck(DEV_ADDR, DEV_PORT, DEV_SERIAL_LINE)

        driver_config = {
            'dvr_mod' : puck_data['dvr_mod'],
            'dvr_cls' : puck_data['dvr_cls'],
            'workdir' : WORK_DIR,
            'process_type' : PROCESS_TYPE,
            'comms_config' : {
                'addr' : host,
                'port' : port
            }
        }
        
        agent_config = {
            'driver_config' : driver_config,
            'stream_config' : stream_config,
            'agent'         : {'resource_id': resource_id},
            'test_mode'     : True            
        }

        log.debug("Starting instrument agent.")
        ia_pid = self.container_client.spawn_process(
            name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config)
        #self.addCleanup(self._verify_agent_reset)

        ia_client = ResourceAgentClient(resource_id, process=FakeProcess())
        log.info('Got ia client %s.', str(ia_client))
    
        # We start in uninitialized state.
        # In this state there is no driver process.
        state = ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
        
        # Ping the agent.
        retval = ia_client.ping_agent()
        log.info(retval)
    
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = ia_client.execute_agent(cmd)
        state = ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = ia_client.execute_agent(cmd)
        state = ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = ia_client.execute_agent(cmd)
        state = ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
        retval = ia_client.execute_resource(cmd)
        
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = ia_client.execute_agent(cmd)
        state = ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class InstrumentAgentTestDA():
    """
    Test cases for instrument agent class. Functions in this class provide
    instrument agent integration tests for Direct Access mode and provide 
    a tutorial on use of the agent setup and interface.
    """
    
    def _setup(self):

        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        log.info('LAUNCH_FROM_EGG: %s', LAUNCH_FROM_EGG)
        self._support = DriverIntegrationTestSupport(None,
            None,
            DEV_ADDR,
            DEV_PORT,
            DATA_PORT,
            CMD_PORT,
            PA_BINARY,
            DELIM,
            WORK_DIR)

        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Start container.
        log.info('Staring capability container.')
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Start a resource agent client to talk with the instrument agent.
        log.info('starting IA process')
        self._ia_client = start_instrument_agent_process(self.container)
        self.addCleanup(self._verify_agent_reset)
        log.info('test setup complete')


    ###############################################################################
    # Port agent helpers.
    ###############################################################################
        
    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
                        
    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)
            
    ###############################################################################
    # Event helpers.
    ###############################################################################

    def _start_event_subscriber(self, type='ResourceAgentEvent', count=0):
        """
        Start a subscriber to the instrument agent events.
        @param type The type of event to catch.
        @count Trigger the async event result when events received reaches this.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', 
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()
            
        # Event array and async event result.
        self._event_count = count
        self._events_received = []
        self._async_event_result = AsyncResult()
            
        self._event_subscriber = EventSubscriber(
            event_type=type, callback=consume_event,
            origin=IA_RESOURCE_ID)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=5)

    def _stop_event_subscriber(self):
        """
        Stop event subscribers on cleanup.
        """
        self._event_subscriber.stop()
        self._event_subscriber = None

    ###############################################################################
    # tcp helpers.
    ###############################################################################        

    def _start_tcp_client(self, retval):
        host = retval.result['ip_address']
        port = retval.result['port']
        tcp_client = TcpClient(host, port)
        return tcp_client
        

    ###############################################################################
    # Assert helpers.
    ###############################################################################        

    def assertInstrumentAgentState(self, expected_state, timeout=0):
        end_time = time.time() + timeout
        
        while (True):
            state = self._ia_client.get_agent_state()
            log.debug("assertInstrumentAgentState: IA state = %s, expected state = %s" %(state, expected_state))
            if state == expected_state:
                return True
            if time.time() >= end_time:
                self.fail("assertInstrumentAgentState: IA failed to transition to %s state" %expected_state)
            gevent.sleep(1)
                
    
    def assertSetInstrumentState(self, command, new_state):
        if type(command) == str:
            log.debug("assertSetInstrumentState: building command for %s" %command)
            cmd = AgentCommand(command=command)
        else:
            cmd = command
        retval = self._ia_client.execute_agent(cmd)
        self.assertInstrumentAgentState(new_state)
        return retval
        

    ###############################################################################
    # Tests.
    #
    # response from IA for start DA looks similar to the following:
    #
    # {'status': 0, 'type_': 'AgentCommandResult', 'command': 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS',
    # 'result': {'token': 'F2B6EED3-F926-4B3B-AE80-4F8DE79276F3', 'ip_address': 'Edwards-MacBook-Pro.local', 'port': 8000},
    # 'ts_execute': '1344889063861', 'command_id': ''}
    ###############################################################################
    
    
    def test_direct_access_vsp_IA_shutdown(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown from IA. 
        """
        def start_and_stop_DA():
            retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
            tcp_client = self._start_tcp_client(retval)
            self.assertTrue(tcp_client.send_data('ts\r\n'))
            self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':600,
                           'inactivity_timeout':600})
        
        # test for starting DA, making connection with TCP client, sending/recving commands to/from instrument, and DA shut down
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertTrue(tcp_client.send_data('ts\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ts
        
        -0.1964,85.12299, 697.168,   39.5241, 1506.965, 01 Feb 2001, 01:01:00
        """
        
        sample_pattern = r'^#? *(-?\d+\.\d+), *(-?\d+\.\d+), *(-?\d+\.\d+)'
        sample_pattern += r'(, *(-?\d+\.\d+))?(, *(-?\d+\.\d+))?'
        sample_pattern += r'(, *(\d+) +([a-zA-Z]+) +(\d+), *(\d+):(\d+):(\d+))?'
        sample_pattern += r'(, *(\d+)-(\d+)-(\d+), *(\d+):(\d+):(\d+))?'
        sample_regex = re.compile(sample_pattern)
        lines = response.split('\r\n')
        
        sample_count = 0
        for x in lines:
            if sample_regex.match(x):
                sample_count += 1
        #self.assertEqual(sample_count, 1)

        self.assertTrue(tcp_client.send_data('ds\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ds
        SBE37-SMP V 2.6 SERIAL NO. 2165   01 Feb 2001  01:01:00
        not logging: received stop command
        sample interval = 23195 seconds
        samplenumber = 0, free = 200000
        do not transmit real-time data
        do not output salinity with each sample
        do not output sound velocity with each sample
        do not store time with each sample
        number of samples to average = 0
        reference pressure = 0.0 db
        serial sync mode disabled
        wait time after serial sync sampling = 0 seconds
        internal pump is installed
        temperature = 7.54 deg C
        WARNING: LOW BATTERY VOLTAGE!!
        """

        self.assertNotEqual(response.find('SBE37-SMP'), -1)
        self.assertNotEqual(response.find('sample interval'), -1)
        self.assertNotEqual(response.find('samplenumber'), -1)
        self.assertNotEqual(response.find('number of samples to average'), -1)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)
        
        # test for starting and shutting down DA w/o ever connecting TCP client 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)
        
        # test for race condition in DA when shut down by IA after sending something from TCP client
        for i in range(0, 10):
            start_and_stop_DA()
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_telnet_IA_shutdown(self):
        """
        Test agent direct_access mode for telnet when shutdown from IA. 
        """

        def start_and_stop_DA():
            retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
            tcp_client = self._start_tcp_client(retval)
            self.assertTrue(tcp_client.start_telnet(token=retval.result['token']))
            self.assertTrue(tcp_client.send_data('ts\r\n'))
            self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':600,
                           'inactivity_timeout':600})

        # test for starting DA, making connection with TCP/telnet client, sending/recving commands to/from instrument, and DA shut down
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertTrue(tcp_client.start_telnet(token=retval.result['token']))
        
        self.assertTrue(tcp_client.send_data('ts\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ts
        
        -0.1964,85.12299, 697.168,   39.5241, 1506.965, 01 Feb 2001, 01:01:00
        """
        
        sample_pattern = r'^#? *(-?\d+\.\d+), *(-?\d+\.\d+), *(-?\d+\.\d+)'
        sample_pattern += r'(, *(-?\d+\.\d+))?(, *(-?\d+\.\d+))?'
        sample_pattern += r'(, *(\d+) +([a-zA-Z]+) +(\d+), *(\d+):(\d+):(\d+))?'
        sample_pattern += r'(, *(\d+)-(\d+)-(\d+), *(\d+):(\d+):(\d+))?'
        sample_regex = re.compile(sample_pattern)
        lines = response.split('\r\n')
        
        sample_count = 0
        for x in lines:
            if sample_regex.match(x):
                sample_count += 1
        #self.assertEqual(sample_count, 1)

        self.assertTrue(tcp_client.send_data('ds\r\n'))
        _, response = tcp_client.expect(None, 3)

        # CORRECT PATTERN TO MATCH IN RESPONSE BUFFER:
        """
        ds
        SBE37-SMP V 2.6 SERIAL NO. 2165   01 Feb 2001  01:01:00
        not logging: received stop command
        sample interval = 23195 seconds
        samplenumber = 0, free = 200000
        do not transmit real-time data
        do not output salinity with each sample
        do not output sound velocity with each sample
        do not store time with each sample
        number of samples to average = 0
        reference pressure = 0.0 db
        serial sync mode disabled
        wait time after serial sync sampling = 0 seconds
        internal pump is installed
        temperature = 7.54 deg C
        WARNING: LOW BATTERY VOLTAGE!!
        """

        self.assertNotEqual(response.find('SBE37-SMP'), -1)
        self.assertNotEqual(response.find('sample interval'), -1)
        self.assertNotEqual(response.find('samplenumber'), -1)
        self.assertNotEqual(response.find('number of samples to average'), -1)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)
        
        # test for starting and shutting down DA w/o ever connecting TCP client 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)
        
        # test for starting and shutting down DA w/o ever entering a username 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)
        
        # test for starting and shutting down DA w/o ever entering a token 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        self.assertSetInstrumentState(ResourceAgentEvent.GO_COMMAND, ResourceAgentState.COMMAND)
        
        # test for race condition in DA when shut down by IA after sending something from TCP client
        for i in range(0, 10):
            start_and_stop_DA()
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_telnet_login_failure(self):
        """
        Test agent direct_access mode for telnet when login fails. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':600,
                           'inactivity_timeout':600})

        # test that DA session quits when bad token is sent
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertFalse(tcp_client.start_telnet(token='some junk'))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        

    def test_direct_access_telnet_session_setup_failure(self):
        """
        Test agent direct_access mode for telnet when session setup fails. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':600,
                           'inactivity_timeout':600})

        # test that DA session quits when bad token is sent
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertTrue(tcp_client.start_telnet(token=retval.result['token'], handshake=False))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        

    def test_direct_access_vsp_client_disconnect(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by tcp client disconnect. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':600,
                           'inactivity_timeout':600})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertTrue(tcp_client.send_data('ts\r\n'))
        
        tcp_client.disconnect()

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
                
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_telnet_client_disconnect(self):
        """
        Test agent direct_access mode for telnet when shutdown by tcp client disconnect. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':600,
                           'inactivity_timeout':600})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(retval.result['token']))

        self.assertTrue(tcp_client.send_data('ts\r\n'))
        
        tcp_client.disconnect()

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
                
        # test for starting and disconnecting client w/o ever entering a username 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        tcp_client.disconnect()
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        # test for starting and disconnecting client w/o ever entering a token 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        tcp_client.disconnect()
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        # test for starting and disconnecting client w/o ever negotiating the telnet session 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet(token=retval.result['token'], handshake=False))
        tcp_client.disconnect()
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_vsp_session_timeout(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by session timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':10,
                           'inactivity_timeout':600})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        self._start_tcp_client(retval)

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        # test for starting and session timeout w/o ever connecting TCP client 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_telnet_session_timeout(self):
        """
        Test agent direct_access mode for telnet when shutdown by session timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':10,
                           'inactivity_timeout':600})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(retval.result['token']))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        # test for starting and session timeout w/o ever entering a username 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        # test for starting and session timeout w/o ever entering a token 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        # test for starting and session timeout w/o ever negotiating the telnet session 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet(token=retval.result['token'], handshake=False))
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_vsp_inactivity_timeout(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by inactivity timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':600,
                           'inactivity_timeout':10})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        self._start_tcp_client(retval)

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        # test for starting and inactivity timeout w/o ever connecting TCP client 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    def test_direct_access_telnet_inactivity_timeout(self):
        """
        Test agent direct_access mode for telnet when shutdown by inactivity timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':600,
                           'inactivity_timeout':10})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)

        self.assertTrue(tcp_client.start_telnet(retval.result['token']))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        # test for starting and inactivity timeout w/o ever entering a username 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        # test for starting and inactivity timeout w/o ever entering a token 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet())
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        # test for starting and inactivity timeout w/o ever negotiating the telnet session 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        tcp_client = self._start_tcp_client(retval)
        self.assertTrue(tcp_client.start_telnet(token=retval.result['token'], handshake=False))
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        
    @unittest.skip('A manual test for timing purposes.')        
    def test_exit_da_timing(self):
        """
        test_exit_da_timing
        Test time it takes to leave direct access and return to command mode.
        """

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
    
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':600,
                           'inactivity_timeout':600})
        
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.DIRECT_ACCESS)
                
        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        starttime = time.time()
        
        cmd = AgentCommand(command=ResourceAgentEvent.GO_COMMAND)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)
        
        delta = time.time() - starttime
        
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        print '######## exiting direct access takes: %f seconds' % delta
class TestInstrumentAgentWithTrhph(TrhphTestCase, MiIntTestCase):
    """
    R2 instrument agent tests with the TRHPH driver.
    """
    def setUp(self):
        """
        Initialize test members.
        Start port agent.
        Start container and client.
        Start streams and subscribers.
        Start agent, client.
        """

        TrhphTestCase.setUp(self)

        self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS,
                                                     self.device_address,
                                                     self.device_port, DELIM,
                                                     WORK_DIR)
        # Start port agent, add stop to cleanup.
        self._pagent = None
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Start container.
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Start data suscribers, add stop to cleanup.
        # Define stream_config.
        self._no_samples = None
        self._async_data_result = AsyncResult()
        self._data_greenlets = []
        self._stream_config = {}
        self._samples_received = []
        self._data_subscribers = []
        self._start_data_subscribers()
        self.addCleanup(self._stop_data_subscribers)

        # Start event subscribers, add stop to cleanup.
        self._no_events = None
        self._async_event_result = AsyncResult()
        self._events_received = []
        self._event_subscribers = []
        self._start_event_subscribers()
        self.addCleanup(self._stop_event_subscribers)

        # Create agent config.
        agent_config = {
            'driver_config': DVR_CONFIG,
            'stream_config': self._stream_config,
            'agent': {
                'resource_id': IA_RESOURCE_ID
            },
            'test_mode': True
        }

        # Start instrument agent.
        self._ia_pid = None
        log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
                                                name=self.container.name)
        self._ia_pid = container_client.spawn_process(name=IA_NAME,
                                                      module=IA_MOD,
                                                      cls=IA_CLS,
                                                      config=agent_config)
        log.info('Agent pid=%s.', str(self._ia_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
                                              process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))

        # make sure the driver is stopped
        self.addCleanup(self._reset)

    def addCleanup(self, f):
        MiIntTestCase.addCleanup(self, f)

    def tearDown(self):
        try:
            MiIntTestCase.tearDown(self)
        finally:
            TrhphTestCase.tearDown(self)

    def _start_pagent(self):
        """
        Construct and start the port agent.
        """
        port = self._support.start_pagent()

        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {'addr': 'localhost', 'port': port}

    def _start_data_subscribers(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # A callback for processing subscribed-to data.
        def consume_data(message, headers):
            log.info('Subscriber received data message: %s.', str(message))
            self._samples_received.append(message)
            if self._no_samples and self._no_samples == len(
                    self._samples_received):
                self._async_data_result.set()

        # Create a stream subscriber registrar to create subscribers.
        subscriber_registrar = StreamSubscriberRegistrar(
            process=self.container, container=self.container)

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}
        self._data_subscribers = []
        for stream_name in PACKET_CONFIG:
            stream_def = ctd_stream_definition(stream_id=None)
            stream_def_id = pubsub_client.create_stream_definition(
                container=stream_def)
            stream_id = pubsub_client.create_stream(
                name=stream_name,
                stream_definition_id=stream_def_id,
                original=True,
                encoding='ION R2')

            taxy = get_taxonomy(stream_name)
            stream_config = dict(id=stream_id, taxonomy=taxy.dump())
            self._stream_config[stream_name] = stream_config

            # Create subscriptions for each stream.
            exchange_name = '%s_queue' % stream_name
            sub = subscriber_registrar.create_subscriber(
                exchange_name=exchange_name, callback=consume_data)
            self._listen(sub)
            self._data_subscribers.append(sub)
            query = StreamQuery(stream_ids=[stream_id])
            sub_id = pubsub_client.create_subscription(
                query=query,
                exchange_name=exchange_name,
                exchange_point='science_data')
            pubsub_client.activate_subscription(sub_id)

    def _listen(self, sub):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        gl = spawn(sub.listen)
        self._data_greenlets.append(gl)
        sub._ready_event.wait(timeout=5)
        return gl

    def _stop_data_subscribers(self):
        """
        Stop the data subscribers on cleanup.
        """
        log.info('cleanup: _stop_data_subscribers called.')
        for sub in self._data_subscribers:
            sub.stop()
        for gl in self._data_greenlets:
            gl.kill()

    def _start_event_subscribers(self):
        """
        Create subscribers for agent and driver events.
        """
        def consume_event(*args, **kwargs):
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._no_events and self._no_events == len(
                    self._event_received):
                self._async_event_result.set()

        event_sub = EventSubscriber(event_type="DeviceEvent",
                                    callback=consume_event)
        event_sub.start()
        self._event_subscribers.append(event_sub)

    def _stop_event_subscribers(self):
        """
        Stop event subscribers on cleanup.
        """
        log.info('cleanup: _stop_event_subscribers called.')
        for sub in self._event_subscribers:
            sub.stop()

    def _initialize_and_run(self):
        """
        Called explicitly by the tests that do regular operations.
        """
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

    def _reset(self):
        """
        Set as a clean-up to make sure the agent is reset after each test,
        so the driver is stopped.
        """
        log.info("_reset called: sending 'reset' command to agent")
        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_01_initialize(self):
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)

        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)

        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def test_10_states(self):
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

        cmd = AgentCommand(command='initialize')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.INACTIVE)

        cmd = AgentCommand(command='go_active')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)

        cmd = AgentCommand(command='resume')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='run')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.OBSERVATORY)

        cmd = AgentCommand(command='pause')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.STOPPED)

        cmd = AgentCommand(command='clear')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.IDLE)

        cmd = AgentCommand(command='reset')
        retval = self._ia_client.execute_agent(cmd)
        cmd = AgentCommand(command='get_agent_state')
        retval = self._ia_client.execute_agent(cmd)
        state = retval.result
        self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)

    def _get_params(self, params):

        result = self._ia_client.get_param(params)
        log.info('_get_params result: %s' % str(result))

        self.assertTrue(isinstance(result, dict))

        if params == DriverParameter.ALL:
            all_requested_params = TrhphParameter.list()
        else:
            all_requested_params = params

        # check all requested params are in the result
        for p in all_requested_params:
            self.assertTrue(p in result)

            if TrhphParameter.TIME_BETWEEN_BURSTS == p:
                seconds = result.get(p)
                self.assertTrue(isinstance(seconds, int))
            elif TrhphParameter.VERBOSE_MODE == p:
                is_data_only = result.get(p)
                self.assertTrue(isinstance(is_data_only, bool))

        return result

    def test_15_get_params(self):
        self._initialize_and_run()

        self._get_params(DriverParameter.ALL)

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        p2 = TrhphParameter.VERBOSE_MODE
        self._get_params([p1, p2])

    def _set_params(self, params):
        """
        Sets the given parameters, which are assumed to be all valid.
        """
        result = self._ia_client.set_param(params)
        log.info("set result = %s" % str(result))

        if result is None:
            # TODO check why self._ia_client.set_param returns None
            return

        assert isinstance(result, dict)

        # check all requested params are in the result
        for (p, v) in params.items():
            self.assertTrue(p in result)

        return result

    def _get_verbose_flag_for_set_test(self):
        """
        Gets the value to use for the verbose flag in the "set" operations.
        If we are testing against the real instrument (self._is_real_instrument
        is true), this always returns False because the associated
        interfaces with verbose=True are not implemented yet.
        Otherwise it returns a random boolean value. Note, in this case, which
        means we are testing against the simulator, the actual verbose value
        does not have any effect of the other interface elements, so it is ok
        to set any value here.
        TODO align this when the verbose flag is handled completely,
        both in the driver and in the simulator.
        """
        if self._is_real_instrument:
            log.info("setting verbose=False because _is_real_instrument")
            return False
        else:
            return 0 == random.randint(0, 1)

    def test_20_set_params_valid(self):
        self._initialize_and_run()

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        new_seconds = random.randint(15, 60)

        p2 = TrhphParameter.VERBOSE_MODE
        verbose = self._get_verbose_flag_for_set_test()

        valid_params = {p1: new_seconds, p2: verbose}

        self._set_params(valid_params)

    def test_25_get_set_params(self):
        self._initialize_and_run()

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        result = self._get_params([p1])
        seconds = result[p1]
        new_seconds = seconds + 5
        if new_seconds > 30 or new_seconds < 15:
            new_seconds = 15

        p2 = TrhphParameter.VERBOSE_MODE
        new_verbose = self._get_verbose_flag_for_set_test()

        valid_params = {p1: new_seconds, p2: new_verbose}

        log.info("setting: %s" % str(valid_params))

        self._set_params(valid_params)

        result = self._get_params([p1, p2])

        seconds = result[p1]
        self.assertEqual(seconds, new_seconds)

        verbose = result[p2]
        self.assertEqual(verbose, new_verbose)

    def test_60_execute_stop_autosample(self):
        self._initialize_and_run()

        log.info("stopping autosample")
        cmd = AgentCommand(command='stop_autosample',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("stop_autosample reply = %s" % str(reply))

    def test_70_execute_get_metadata(self):
        self._initialize_and_run()

        log.info("getting metadata")
        cmd = AgentCommand(command='get_metadata',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("get_metadata reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, dict))

    def test_80_execute_diagnostics(self):
        self._initialize_and_run()

        log.info("executing diagnostics")
        num_scans = 11
        cmd = AgentCommand(command='diagnostics',
                           kwargs=dict(num_scans=num_scans,
                                       timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("diagnostics reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, list))
        self.assertEqual(len(reply.result), num_scans)

    def test_90_execute_get_power_statuses(self):
        self._initialize_and_run()

        log.info("executing get_power_statuses")
        cmd = AgentCommand(command='get_power_statuses',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("get_power_statuses reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, dict))

    def test_99_execute_start_autosample(self):
        self._initialize_and_run()

        log.info("executing start_autosample")
        cmd = AgentCommand(command='start_autosample',
                           kwargs=dict(timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("start_autosample reply = %s" % str(reply))

    @unittest.skip('Not running after code updates in other places')
    def test_get_params_invalid(self):
        self._initialize_and_run()
        with self.assertRaises(InstParameterError):
            self._get_params(['bad-param'])

    @unittest.skip('Not running after code updates in other places')
    def test_set_params_invalid(self):
        self._initialize_and_run()
        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        new_seconds = random.randint(15, 60)
        invalid_params = {p1: new_seconds, "bad-param": "dummy-value"}
        with self.assertRaises(InstParameterError):
            self._set_params(invalid_params)
示例#24
0
class TestSBE37Driver(unittest.TestCase):    
    """
    Integration tests for the sbe37 driver. This class tests and shows
    use patterns for the sbe37 driver as a zmq driver process.
    """    
    def setUp(self):
        """
        Setup test cases.
        """
        self.device_addr = DEV_ADDR
        self.device_port = DEV_PORT
        self.work_dir = WORK_DIR
        self.delim = DELIM
        
        self.driver_class = DVR_CLS
        self.driver_module = DVR_MOD
        self._support = DriverIntegrationTestSupport(self.driver_module,
                                                     self.driver_class,
                                                     self.device_addr,
                                                     self.device_port,
                                                     self.delim,
                                                     self.work_dir)

        # Clear driver event list.
        self._events = []

        # The port agent object. Used to start and stop the port agent.
        self._pagent = None
        
        # The driver process popen object.
        self._dvr_proc = None
        
        # The driver client.
        self._dvr_client = None

        # Create and start the port agent.
        mi_logger.info('start')
        COMMS_CONFIG['port'] = self._support.start_pagent()
        self.addCleanup(self._support.stop_pagent)    

        # Create and start the driver.
        self._support.start_driver()
        self.addCleanup(self._support.stop_driver)

        # Grab some variables from support that we need
        self._dvr_client = self._support._dvr_client
        self._dvr_proc = self._support._dvr_proc
        self._pagent = self._support._pagent
        self._events = self._support._events

    def assertSampleDict(self, val):
        """
        Verify the value is a sample dictionary for the sbe37.
        """
        #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}        
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('c'))
        self.assertTrue(val.has_key('t'))
        self.assertTrue(val.has_key('p'))
        self.assertTrue(val.has_key('time'))
        c = val['c'][0]
        t = val['t'][0]
        p = val['p'][0]
        time = val['time'][0]
    
        self.assertTrue(isinstance(c, float))
        self.assertTrue(isinstance(t, float))
        self.assertTrue(isinstance(p, float))
        self.assertTrue(isinstance(time, float))
    
    def assertParamDict(self, pd, all_params=False):
        """
        Verify all device parameters exist and are correct type.
        """
        if all_params:
            self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
            #print str(pd)
            #print str(PARAMS)
            for (key, type_val) in PARAMS.iteritems():
                #print key
                self.assertTrue(isinstance(pd[key], type_val))
        else:
            for (key, val) in pd.iteritems():
                self.assertTrue(PARAMS.has_key(key))
                self.assertTrue(isinstance(val, PARAMS[key]))
    
    def assertParamVals(self, params, correct_params):
        """
        Verify parameters take the correct values.
        """
        self.assertEqual(set(params.keys()), set(correct_params.keys()))
        for (key, val) in params.iteritems():
            correct_val = correct_params[key]
            if isinstance(val, float):
                # Verify to 5% of the larger value.
                max_val = max(abs(val), abs(correct_val))
                self.assertAlmostEqual(val, correct_val, delta=max_val*.01)

            else:
                # int, bool, str, or tuple of same
                self.assertEqual(val, correct_val)
    
    def test_process(self):
        """
        Test for correct launch of driver process and communications, including
        asynchronous driver events.
        """

        # Verify processes exist.
        self.assertNotEqual(self._dvr_proc, None)
        drv_pid = self._dvr_proc.pid
        self.assertTrue(isinstance(drv_pid, int))
        
        self.assertNotEqual(self._pagent, None)
        pagent_pid = self._pagent.get_pid()
        self.assertTrue(isinstance(pagent_pid, int))
        
        # Send a test message to the process interface, confirm result.
        msg = 'I am a ZMQ message going to the process.'
        reply = self._dvr_client.cmd_dvr('process_echo', msg)
        self.assertEqual(reply,'process_echo: '+msg)
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)
        
        # Send a test message to the driver interface, confirm result.
        msg = 'I am a ZMQ message going to the driver.'
        reply = self._dvr_client.cmd_dvr('driver_echo', msg)
        self.assertEqual(reply, 'driver_echo: '+msg)
        
        # Test the event thread publishes and client side picks up events.
        events = [
            'I am important event #1!',
            'And I am important event #2!'
            ]
        reply = self._dvr_client.cmd_dvr('test_events', events=events)
        gevent.sleep(1)
        
        # Confirm the events received are as expected.
        self.assertEqual(self._events, events)

        # Test the exception mechanism.
        with self.assertRaises(InstrumentException):
            exception_str = 'Oh no, something bad happened!'
            reply = self._dvr_client.cmd_dvr('test_exceptions', exception_str)
        
        # Verify we received a driver error event.
        gevent.sleep(1)
        error_events = [evt for evt in self._events if isinstance(evt, dict) and evt['type']==DriverAsyncEvent.ERROR]
        self.assertTrue(len(error_events) == 1)
        
    def test_config(self):
        """
        Test to configure the driver process for device comms and transition
        to disconnected state.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver returned state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)
        
    def test_connect(self):
        """
        Test configuring and connecting to the device through the port
        agent. Discover device state.
        """
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')
    
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)
        
    def test_get_set(self):
        """
        Test device parameter access.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        reply = self._dvr_client.cmd_dvr('connect')
                
        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)
                
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Get all device parameters. Confirm all expected keys are retrived
        # and have correct type.
        reply = self._dvr_client.cmd_dvr('get', SBE37Parameter.ALL)
        self.assertParamDict(reply, True)

        # Remember original configuration.
        orig_config = reply
        
        # Grab a subset of parameters.
        params = [
            SBE37Parameter.TA0,
            SBE37Parameter.INTERVAL,
            SBE37Parameter.STORETIME,
            SBE37Parameter.TCALDATE
            ]
        reply = self._dvr_client.cmd_dvr('get', params)
        self.assertParamDict(reply)        

        # Remember the original subset.
        orig_params = reply
        
        # Construct new parameters to set.
        old_date = orig_params[SBE37Parameter.TCALDATE]
        new_params = {
            SBE37Parameter.TA0 : orig_params[SBE37Parameter.TA0] * 1.2,
            SBE37Parameter.INTERVAL : orig_params[SBE37Parameter.INTERVAL] + 1,
            SBE37Parameter.STORETIME : not orig_params[SBE37Parameter.STORETIME],
            SBE37Parameter.TCALDATE : (old_date[0], old_date[1], old_date[2] + 1)
        }

        # Set parameters and verify.
        reply = self._dvr_client.cmd_dvr('set', new_params)
        reply = self._dvr_client.cmd_dvr('get', params)
        self.assertParamVals(reply, new_params)
        
        # Restore original parameters and verify.
        reply = self._dvr_client.cmd_dvr('set', orig_params)
        reply = self._dvr_client.cmd_dvr('get', params)
        self.assertParamVals(reply, orig_params)

        # Retrieve the configuration and ensure it matches the original.
        # Remove samplenum as it is switched by autosample and storetime.
        reply = self._dvr_client.cmd_dvr('get', SBE37Parameter.ALL)
        reply.pop('SAMPLENUM')
        orig_config.pop('SAMPLENUM')
        self.assertParamVals(reply, orig_config)

        # Disconnect from the port agent.
        reply = self._dvr_client.cmd_dvr('disconnect')
        
        # Test the driver is disconnected.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)
        
        # Deconfigure the driver.
        reply = self._dvr_client.cmd_dvr('initialize')
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)        
    
    def test_poll(self):
        """
        Test sample polling commands and events.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        reply = self._dvr_client.cmd_dvr('connect')
                
        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)
                
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)
        
        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)
        
        # Confirm that 3 samples arrived as published events.
        gevent.sleep(1)
        sample_events = [evt for evt in self._events if evt['type']==DriverAsyncEvent.SAMPLE]
        self.assertEqual(len(sample_events), 3)

        # Disconnect from the port agent.
        reply = self._dvr_client.cmd_dvr('disconnect')
        
        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)
        
        # Deconfigure the driver.
        reply = self._dvr_client.cmd_dvr('initialize')
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_autosample(self):
        """
        Test autosample mode.
        """
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)
        
        # Make sure the device parameters are set to sample frequently.
        params = {
            SBE37Parameter.NAVG : 1,
            SBE37Parameter.INTERVAL : 5
        }
        reply = self._dvr_client.cmd_dvr('set', params)
        
        reply = self._dvr_client.cmd_dvr('execute_start_autosample')

        # Test the driver is in autosample mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.AUTOSAMPLE)
        
        # Wait for a few samples to roll in.
        gevent.sleep(30)
        
        # Return to command mode. Catch timeouts and retry if necessary.
        count = 0
        while True:
            try:
                reply = self._dvr_client.cmd_dvr('execute_stop_autosample')
            
            except InstrumentTimeoutException:
                count += 1
                if count >= 5:
                    self.fail('Could not wakeup device to leave autosample mode.')

            else:
                break

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Verify we received at least 2 samples.
        sample_events = [evt for evt in self._events if evt['type']==DriverAsyncEvent.SAMPLE]
        self.assertTrue(len(sample_events) >= 2)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')
    
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    @unittest.skip('Not supported by simulator and very long (> 5 min).')
    def test_test(self):
        """
        Test the hardware testing mode.
        """
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        start_time = time.time()
        reply = self._dvr_client.cmd_dvr('execute_test')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.TEST)
        
        while state != SBE37ProtocolState.COMMAND:
            gevent.sleep(5)
            elapsed = time.time() - start_time
            mi_logger.info('Device testing %f seconds elapsed.' % elapsed)
            state = self._dvr_client.cmd_dvr('get_current_state')

        # Verify we received the test result and it passed.
        test_results = [evt for evt in self._events if evt['type']==DriverAsyncEvent.TEST_RESULT]
        self.assertTrue(len(test_results) == 1)
        self.assertEqual(test_results[0]['value']['success'], 'Passed')

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')
    
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_errors(self):
        """
        Test response to erroneous commands and parameters.
        """
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Assert for an unknown driver command.
        with self.assertRaises(InstrumentCommandException):
            reply = self._dvr_client.cmd_dvr('bogus_command')

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_acquire_sample')

        # Assert we forgot the comms parameter.
        with self.assertRaises(InstrumentParameterException):
            reply = self._dvr_client.cmd_dvr('configure')

        # Assert we send a bad config object (not a dict).
        with self.assertRaises(InstrumentParameterException):
            BOGUS_CONFIG = 'not a config dict'            
            reply = self._dvr_client.cmd_dvr('configure', BOGUS_CONFIG)
            
        # Assert we send a bad config object (missing addr value).
        with self.assertRaises(InstrumentParameterException):
            BOGUS_CONFIG = COMMS_CONFIG.copy()
            BOGUS_CONFIG.pop('addr')
            reply = self._dvr_client.cmd_dvr('configure', BOGUS_CONFIG)

        # Assert we send a bad config object (bad addr value).
        with self.assertRaises(InstrumentParameterException):
            BOGUS_CONFIG = COMMS_CONFIG.copy()
            BOGUS_CONFIG['addr'] = ''
            reply = self._dvr_client.cmd_dvr('configure', BOGUS_CONFIG)
        
        # Configure for comms.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_acquire_sample')

        reply = self._dvr_client.cmd_dvr('connect')
                
        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
                
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_stop_autosample')
        
        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('connect')

        # Get all device parameters. Confirm all expected keys are retrived
        # and have correct type.
        reply = self._dvr_client.cmd_dvr('get', SBE37Parameter.ALL)
        self.assertParamDict(reply, True)
        
        # Assert get fails without a parameter.
        with self.assertRaises(InstrumentParameterException):
            reply = self._dvr_client.cmd_dvr('get')
            
        # Assert get fails without a bad parameter (not ALL or a list).
        with self.assertRaises(InstrumentParameterException):
            bogus_params = 'I am a bogus param list.'
            reply = self._dvr_client.cmd_dvr('get', bogus_params)
            
        # Assert get fails without a bad parameter (not ALL or a list).
        #with self.assertRaises(InvalidParameterValueError):
        with self.assertRaises(InstrumentParameterException):
            bogus_params = [
                'a bogus parameter name',
                SBE37Parameter.INTERVAL,
                SBE37Parameter.STORETIME,
                SBE37Parameter.TCALDATE
                ]
            reply = self._dvr_client.cmd_dvr('get', bogus_params)        
        
        # Assert we cannot set a bogus parameter.
        with self.assertRaises(InstrumentParameterException):
            bogus_params = {
                'a bogus parameter name' : 'bogus value'
            }
            reply = self._dvr_client.cmd_dvr('set', bogus_params)
            
        # Assert we cannot set a real parameter to a bogus value.
        with self.assertRaises(InstrumentParameterException):
            bogus_params = {
                SBE37Parameter.INTERVAL : 'bogus value'
            }
            reply = self._dvr_client.cmd_dvr('set', bogus_params)
        
        # Disconnect from the port agent.
        reply = self._dvr_client.cmd_dvr('disconnect')
        
        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)
        
        # Deconfigure the driver.
        reply = self._dvr_client.cmd_dvr('initialize')
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)
    
    @unittest.skip('Not supported by simulator.')
    def test_discover_autosample(self):
        """
        Test the device can discover autosample mode.
        """
        
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)
        
        # Make sure the device parameters are set to sample frequently.
        params = {
            SBE37Parameter.NAVG : 1,
            SBE37Parameter.INTERVAL : 5
        }
        reply = self._dvr_client.cmd_dvr('set', params)
        
        reply = self._dvr_client.cmd_dvr('execute_start_autosample')

        # Test the driver is in autosample mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.AUTOSAMPLE)
    
        # Let a sample or two come in.
        gevent.sleep(30)
    
        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')
    
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Wait briefly before we restart the comms.
        gevent.sleep(10)
    
        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        count = 0
        while True:
            try:        
                reply = self._dvr_client.cmd_dvr('discover')

            except InstrumentTimeoutException:
                count += 1
                if count >=5:
                    self.fail('Could not discover device state.')

            else:
                break

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.AUTOSAMPLE)

        # Let a sample or two come in.
        # This device takes awhile to begin transmitting again after you
        # prompt it in autosample mode.
        gevent.sleep(30)

        # Return to command mode. Catch timeouts and retry if necessary.
        count = 0
        while True:
            try:
                reply = self._dvr_client.cmd_dvr('execute_stop_autosample')
            
            except InstrumentTimeoutException:
                count += 1
                if count >= 5:
                    self.fail('Could not wakeup device to leave autosample mode.')

            else:
                break

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')
    
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)
示例#25
0
class TestIAAlarms(IonIntegrationTestCase):
    """
    """

    ############################################################################
    # Setup, teardown.
    ############################################################################

    def setUp(self):
        """
        Set up driver integration support.
        Start port agent, add port agent cleanup.
        Start container.
        Start deploy services.
        Define agent config, start agent.
        Start agent client.
        """
        self._ia_client = None

        log.info('Creating driver integration test support:')
        log.info('driver uri: %s', DRV_URI)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)
        self._support = DriverIntegrationTestSupport(
            None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY,
            DELIM, WORK_DIR)

        # Start port agent, add stop to cleanup.
        self._start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Start container.
        log.info('Staring capability container.')
        self._start_container()

        # Bring up services in a deploy file (no need to message)
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        log.info('building stream configuration')
        # Setup stream config.
        self._build_stream_config()

        # Start a resource agent client to talk with the instrument agent.
        log.info('starting IA process')
        self._ia_client = start_instrument_agent_process(
            self.container, self._stream_config)
        self.addCleanup(self._verify_agent_reset)
        log.info('test setup complete')

    ###############################################################################
    # Port agent helpers.
    ###############################################################################

    def _start_pagent(self):
        """
        Construct and start the port agent.
        """

        port = self._support.start_pagent()
        log.info('Port agent started at port %i', port)

        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr': 'localhost',
            'port': port,
            'cmd_port': CMD_PORT
        }

    def _verify_agent_reset(self):
        """
        Check agent state and reset if necessary.
        This called if a test fails and reset hasn't occurred.
        """
        if self._ia_client is None:
            return

        state = self._ia_client.get_agent_state()
        if state != ResourceAgentState.UNINITIALIZED:
            cmd = AgentCommand(command=ResourceAgentEvent.RESET)
            retval = self._ia_client.execute_agent(cmd)

    ###############################################################################
    # Event helpers.
    ###############################################################################

    def _start_event_subscriber(self, type='StreamAlarmEvent', count=0):
        """
        Start a subscriber to the instrument agent events.
        @param type The type of event to catch.
        @count Trigger the async event result when events received reaches this.
        """

        def consume_event(*args, **kwargs):
            print '#################'
            log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
                     str(args), str(kwargs), str(args[0]))
            self._events_received.append(args[0])
            if self._event_count > 0 and \
                self._event_count == len(self._events_received):
                self._async_event_result.set()

        # Event array and async event result.
        self._event_count = count
        self._events_received = []
        self._async_event_result = AsyncResult()

        self._event_subscriber = EventSubscriber(
            event_type=type, callback=consume_event, origin=IA_RESOURCE_ID)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=5)

    def _stop_event_subscriber(self):
        """
        Stop event subscribers on cleanup.
        """
        self._event_subscriber.stop()
        self._event_subscriber = None

    ###############################################################################
    # Data stream helpers.
    ###############################################################################

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()
        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        streams = {
            'parsed': 'ctd_parsed_param_dict',
            'raw': 'ctd_raw_param_dict'
        }

        for (stream_name, param_dict_name) in streams.iteritems():
            pd_id = dataset_management.read_parameter_dictionary_by_name(
                param_dict_name, id_only=True)

            stream_def_id = pubsub_client.create_stream_definition(
                name=stream_name, parameter_dictionary_id=pd_id)
            pd = pubsub_client.read_stream_definition(
                stream_def_id).parameter_dictionary

            stream_id, stream_route = pubsub_client.create_stream(
                name=stream_name,
                exchange_point='science_data',
                stream_definition_id=stream_def_id)

            stream_config = dict(
                stream_route=stream_route,
                routing_key=stream_route.routing_key,
                exchange_point=stream_route.exchange_point,
                stream_id=stream_id,
                stream_definition_ref=stream_def_id,
                parameter_dictionary=pd)

            if stream_name == 'parsed':

                type = 'IntervalAlarmDef'
                kwargs = {
                    'name': 'test_sim_warning',
                    'stream_name': 'parsed',
                    'value_id': 'temp',
                    'message': 'Temperature is above test range of 5.0.',
                    'type': StreamAlarmType.WARNING,
                    'upper_bound': 5.0,
                    'upper_rel_op': '<'
                }
                alarm = {}
                alarm['type'] = type
                alarm['kwargs'] = kwargs
                alarms = [alarm]
                stream_config['alarms'] = alarms

            self._stream_config[stream_name] = stream_config

    def _start_data_subscribers(self, count):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)

        # Create streams and subscriptions for each stream named in driver.
        self._data_subscribers = []
        self._samples_received = []
        self._async_sample_result = AsyncResult()

        # A callback for processing subscribed-to data.
        def recv_data(message, stream_route, stream_id):
            log.info('Received message on %s (%s,%s)', stream_id,
                     stream_route.exchange_point, stream_route.routing_key)
            #print '######################## stream message:'
            #print str(message)
            self._samples_received.append(message)
            if len(self._samples_received) == count:
                self._async_sample_result.set()

        for (stream_name, stream_config) in self._stream_config.iteritems():

            if stream_name == 'parsed':

                # Create subscription for parsed stream only.

                stream_id = stream_config['stream_id']

                from pyon.util.containers import create_unique_identifier
                # exchange_name = '%s_queue' % stream_name
                exchange_name = create_unique_identifier(
                    "%s_queue" % stream_name)
                self._purge_queue(exchange_name)
                sub = StandaloneStreamSubscriber(exchange_name, recv_data)
                sub.start()
                self._data_subscribers.append(sub)
                print 'stream_id: %s' % stream_id
                sub_id = pubsub_client.create_subscription(
                    name=exchange_name, stream_ids=[stream_id])
                pubsub_client.activate_subscription(sub_id)
                sub.subscription_id = sub_id  # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)

    def _purge_queue(self, queue):
        xn = self.container.ex_manager.create_xn_queue(queue)
        xn.purge()

    def _stop_data_subscribers(self):
        for subscriber in self._data_subscribers:
            pubsub_client = PubsubManagementServiceClient()
            if hasattr(subscriber, 'subscription_id'):
                try:
                    pubsub_client.deactivate_subscription(
                        subscriber.subscription_id)
                except:
                    pass
                pubsub_client.delete_subscription(subscriber.subscription_id)
            subscriber.stop()

    ###############################################################################
    # Tests.
    ###############################################################################

    def test_config(self):
        """
        test_initialize
        Test agent initialize command. This causes creation of
        driver process and transition to inactive.
        """

        # We start in uninitialized state.
        # In this state there is no driver process.
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        # Ping the agent.
        retval = self._ia_client.ping_agent()
        log.info(retval)

        # Initialize the agent.
        # The agent is spawned with a driver config, but you can pass one in
        # optinally with the initialize command. This validates the driver
        # config, launches a driver process and connects to it via messaging.
        # If successful, we switch to the inactive state.
        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        # Ping the driver proc.
        retval = self._ia_client.ping_resource()
        log.info(retval)

        decoder = IonObjectDeserializer(obj_registry=get_obj_registry())

        # Grab the alarms defined in the config.
        retval = decoder.deserialize(
            self._ia_client.get_agent(['alarms'])['alarms'])
        """
        {'status': None, 'stream_name': 'parsed', 'name': 'test_sim_warning',
        'upper_bound': 5.0, 'expr': 'x<5.0', 'upper_rel_op': '<',
        'lower_rel_op': None, 'type_': 'IntervalAlarmDef', 'value_id': 'temp',
        'lower_bound': None, 'message': 'Temperature is above test range of 5.0.',
        'current_val': None, 'type': 1}
        """
        self.assertEqual(retval[0].type_, 'IntervalAlarmDef')
        self.assertEqual(retval[0].upper_bound, 5.0)
        self.assertEqual(retval[0].expr, 'x<5.0')

        # Reset the agent. This causes the driver messaging to be stopped,
        # the driver process to end and switches us back to uninitialized.
        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

    def test_autosample(self):
        """
        test_autosample
        Test instrument driver execute interface to start and stop streaming
        mode. Verify ResourceAgentResourceStateEvents are publsihed.
        """

        # Start data subscribers.
        self._start_data_subscribers(5)
        self.addCleanup(self._stop_data_subscribers)

        # Set up a subscriber to collect error events.
        self._start_event_subscriber()
        self.addCleanup(self._stop_event_subscriber)

        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.INACTIVE)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.IDLE)

        cmd = AgentCommand(command=ResourceAgentEvent.RUN)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        gevent.sleep(20)

        cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
        retval = self._ia_client.execute_resource(cmd)

        cmd = AgentCommand(command=ResourceAgentEvent.RESET)
        retval = self._ia_client.execute_agent(cmd)
        state = self._ia_client.get_agent_state()
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        #self._async_event_result.get(timeout=CFG.endpoint.receive.timeout)
        #self.assertGreaterEqual(len(self._events_received), 6)

        self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
        self.assertGreaterEqual(len(self._samples_received), 5)

        #for x in self._samples_received:

        gevent.sleep(5)
        """
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142753e+09]), 2: array([  3.57142753e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 83.66190338], dtype=float32), 8: array([ 27.09519958], dtype=float32), 9: array([ 495.5369873], dtype=float32), 10: None, 11: array([  3.57142753e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438731.584241, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142753e+09]), 2: array([  3.57142753e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 50.97378922], dtype=float32), 8: array([ 17.82060051], dtype=float32), 9: array([ 280.375], dtype=float32), 10: None, 11: array([  3.57142753e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438734.120577, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142754e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 79.61238098], dtype=float32), 8: array([ 99.90670013], dtype=float32), 9: array([ 830.60198975], dtype=float32), 10: None, 11: array([  3.57142754e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438737.154774, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142754e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 81.31384277], dtype=float32), 8: array([ 89.92569733], dtype=float32), 9: array([ 575.98901367], dtype=float32), 10: None, 11: array([  3.57142754e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438739.68893, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142754e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 80.85481262], dtype=float32), 8: array([ 86.46209717], dtype=float32), 9: array([ 563.9329834], dtype=float32), 10: None, 11: array([  3.57142754e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438742.722411, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142754e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 42.43975067], dtype=float32), 8: array([ 13.87370014], dtype=float32), 9: array([ 910.49298096], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438745.256714, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142755e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 76.90184021], dtype=float32), 8: array([ 84.85610199], dtype=float32), 9: array([ 742.34002686], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438748.29093, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142755e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 92.28489685], dtype=float32), 8: array([ 19.77809906], dtype=float32), 9: array([ 272.94500732], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438750.841668, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142755e+09]), 2: array([  3.57142755e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 60.02323151], dtype=float32), 8: array([-7.79680014], dtype=float32), 9: array([ 418.51400757], dtype=float32), 10: None, 11: array([  3.57142755e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438753.424074, 'provider_metadata_update': {}}
        {'domain': [1], 'data_producer_id': '123xyz', 'record_dictionary': {1: array([  3.57142756e+09]), 2: array([  3.57142756e+09]), 3: array([port_timestamp], dtype=object), 4: 'ok', 5: None, 6: None, 7: array([ 13.54555035], dtype=float32), 8: array([ 90.14240265], dtype=float32), 9: array([ 105.24099731], dtype=float32), 10: None, 11: array([  3.57142756e+09]), 12: None, 13: None}, 'locator': None, 'type_': 'Granule', 'param_dictionary': '529b995c1aee4d149bca755aba7de687', 'creation_timestamp': 1362438756.511647, 'provider_metadata_update': {}}
        #######################
        {'origin': '123xyz', 'stream_name': 'parsed', 'type': 'StreamAlaramType.WARNING', 'description': '', 'expr': 'x<5.0', 'value': 27.0952, 'type_': 'StreamWarningAlaramEvent', 'value_id': 'temp', 'base_types': ['StreamAlarmEvent', 'ResourceEvent', 'Event'], 'message': 'Temperature is above test range of 5.0.', '_id': '838337bd75ad4852afa15aac29c82a4c', 'ts_created': '1362438731572', 'sub_type': '', 'origin_type': 'InstrumentDevice', 'name': 'test_sim_warning'}
        {'origin': '123xyz', 'stream_name': 'parsed', 'type': 'StreamAlaramType.ALL_CLEAR', 'description': '', 'expr': 'x<5.0', 'value': -7.7968, 'type_': 'StreamAllClearAlarmEvent', 'value_id': 'temp', 'base_types': ['StreamAlarmEvent', 'ResourceEvent', 'Event'], 'message': 'Alarm is cleared.', '_id': '6a3dc641a3fd4092a203857cf6a70e0c', 'ts_created': '1362438753412', 'sub_type': '', 'origin_type': 'InstrumentDevice', 'name': 'test_sim_warning'}
        {'origin': '123xyz', 'stream_name': 'parsed', 'type': 'StreamAlaramType.WARNING', 'description': '', 'expr': 'x<5.0', 'value': 90.1424, 'type_': 'StreamWarningAlaramEvent', 'value_id': 'temp', 'base_types': ['StreamAlarmEvent', 'ResourceEvent', 'Event'], 'message': 'Temperature is above test range of 5.0.', '_id': 'd45d95a3d94345faa1cf82cb18398f3b', 'ts_created': '1362438756499', 'sub_type': '', 'origin_type': 'InstrumentDevice', 'name': 'test_sim_warning'}
        2013-03-04 15:13:00,746 INFO Dummy-1 pyon.event.event:259 EventSubscriber stopped. Event pattern=#.StreamAlarmEvent.#.*.#.*.123xyz
        """
        """
        {'domain': [1],
        'data_producer_id': '123xyz',
        'record_dictionary':
            {1: array([  3.57142756e+09]),
             2: array([  3.57142756e+09]),
             3: array([port_timestamp], dtype=object),
             4: 'ok',
             5: None,
             6: None,
             7: array([ 13.54555035], dtype=float32),
             8: array([ 90.14240265], dtype=float32),
             9: array([ 105.24099731], dtype=float32),
             10: None,
             11: array([  3.57142756e+09]),
             12: None, 13: None},
        'locator': None,
        'type_': 'Granule',
        'param_dictionary': '529b995c1aee4d149bca755aba7de687',
        'creation_timestamp': 1362438756.511647,
        'provider_metadata_update': {}}
        """

        print '#######################'
        print '#######################'
        print '#######################'
        for x in self._samples_received:
            #print str(x)
            print str(x.record_dictionary)
            #print str(type(x.record_dictionary))
            print str(x.param_dictionary)
            #print str(x['record_dictionary'][8])

        print '#######################'
        for x in self._events_received:
            print str(x)
示例#26
0
class Test2CAA(IonIntegrationTestCase):
    """
    Test cases for 2CAA terrestrial endpoint.
    """
    def setUp(self):
        """
        """
        
        ###################################################################
        # Internal parameters and container.
        ###################################################################
        
        # Internal parameters.        
        self._terrestrial_platform_id = 'terrestrial_id'
        self._remote_platform_id = 'remote_id'
        self._resource_id = 'fake_id'
        self._xs_name = 'remote1'
        self._terrestrial_svc_name = 'terrestrial_endpoint'
        self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
        self._remote_svc_name = 'remote_endpoint'
        self._remote_listen_name = self._remote_svc_name + self._xs_name
        self._remote_port = 0
        self._terrestrial_port = 0
        self._te_client = None
        self._re_client = None
        self._remote_pid = None
        self._terrestrial_pid = None

        # Async test results.
        self._no_requests = 10
        self._no_telem_evts = 2
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._telem_evts = []
        self._queue_mod_evts = []
        self._cmd_tx_evts = []
        self._results_recv = {}
        self._requests_sent = {}
        self._done_telem_evt = AsyncResult()
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()

        # Start container.
        log.debug('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message).
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Create a container client.
        log.debug('Creating container client.')
        self._container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        
        ###################################################################
        # Start endpoints, agent.
        ###################################################################

        self._start_terrestrial()
        self._start_remote()
        self._start_agent()
            
        ###################################################################
        # Assign client ports.
        # This is primarily for test purposes as the IP config in
        # deployment will be fixed in advance.
        ###################################################################
        
        self.te_client.set_client_port(self._remote_port)
        check_port = self.te_client.get_client_port()
        log.debug('Terrestrial client port is: %i', check_port)
    
        self.re_client.set_client_port(self._terrestrial_port)
        check_port = self.re_client.get_client_port()
        log.debug('Remote client port is: %i', check_port)
        
        ###################################################################
        # Start the event publisher and subscribers.
        # Used to send fake agent telemetry publications to the endpoints,
        # and to receive endpoint publications.
        ###################################################################
        self._event_publisher = EventPublisher()

        # Start the event subscriber for remote namespace platform events.
        # This event could be changed to RemoteNamespaceEvent.
        self._event_subscriber = EventSubscriber(
            event_type='PlatformEvent',
            callback=self.consume_event,
            origin=self._xs_name)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._event_subscriber.stop)

        # Start the result subscriber for remote resource events.
        self._resource_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=IA_RESOURCE_ID,
            callback=self.consume_event)
        self._resource_result_subscriber.start()
        self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._resource_result_subscriber.stop)

        # Start the result subscriber for remote service events.
        self._service_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin='resource_registry' + 'remote1',
            callback=self.consume_event)
        self._service_result_subscriber.start()
        self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._service_result_subscriber.stop)

        # Start the result subscriber for fake resource results.      
        self._fake_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=self._resource_id,
            callback=self.consume_event)
        self._fake_result_subscriber.start()
        self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._fake_result_subscriber.stop)

    ###################################################################
    # Start/stop helpers.
    ###################################################################

    def _start_agent(self):
        """
        Start an instrument agent and client.
        """
        
        log.info('Creating driver integration test support:')
        log.info('driver module: %s', DRV_MOD)
        log.info('driver class: %s', DRV_CLS)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)        
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }
        self.addCleanup(self._support.stop_pagent)    
                        
        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : {},
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }
    
        # Start instrument agent.
        log.debug("Starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
    
        ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config)
    
        log.info('Agent pid=%s.', str(ia_pid))
    
        # Start a resource agent client to talk with the instrument agent.
    
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))
        
    def _start_terrestrial(self):
        """
        """
        # Create terrestrial config.
        terrestrial_endpoint_config = {
            'other_host' : 'localhost',
            'other_port' : self._remote_port,
            'this_port' : self._terrestrial_port,
            'platform_resource_id' : self._terrestrial_platform_id,
            'xs_name' : self._xs_name,
            'process' : {
                'listen_name' : self._terrestrial_listen_name
            }
        }
        
        # Spawn the terrestrial enpoint process.
        log.debug('Spawning terrestrial endpoint process.')
        self._terrestrial_pid = self._container_client.spawn_process(
            name=self._terrestrial_listen_name,
            module='ion.services.sa.tcaa.terrestrial_endpoint',
            cls='TerrestrialEndpoint',
            config=terrestrial_endpoint_config)
        log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))

        # Create a terrestrial client.
        self.te_client = TerrestrialEndpointClient(
            process=FakeProcess(),
            to_name=self._terrestrial_listen_name)
        log.debug('Got te client %s.', str(self.te_client))
        self._terrestrial_port = self.te_client.get_port()
        log.debug('Terrestrial port is: %i', self._terrestrial_port)
        
    def _start_remote(self):
        """
        """        
        # Create agent config.
        remote_endpoint_config = {
            'other_host' : 'localhost',
            'other_port' : self._terrestrial_port,
            'this_port' : self._remote_port,
            'platform_resource_id' : self._remote_platform_id,
            'xs_name' : self._xs_name,
            'process' : {
                'listen_name' : self._remote_listen_name
            }
        }
        
        # Spawn the remote enpoint process.
        log.debug('Spawning remote endpoint process.')
        self._remote_pid = self._container_client.spawn_process(
            name=self._remote_listen_name,
            module='ion.services.sa.tcaa.remote_endpoint',
            cls='RemoteEndpoint',
            config=remote_endpoint_config)
        log.debug('Remote endpoint pid=%s.', str(self._remote_pid))

        # Create an endpoint client.
        self.re_client = RemoteEndpointClient(
            process=FakeProcess(),
            to_name=self._remote_listen_name)
        log.debug('Got re client %s.', str(self.re_client))
        
        # Remember the remote port.
        self._remote_port = self.re_client.get_port()
        log.debug('The remote port is: %i.', self._remote_port)
        
        
    ###################################################################
    # Telemetry publications to start/top endpoint.
    # (Normally be published by appropriate platform agents.)
    ###################################################################

    def terrestrial_link_up(self):
        """
        Publish telemetry available to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry available event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._terrestrial_platform_id,
                            status = TelemetryStatusType.AVAILABLE)
        
    def terrestrial_link_down(self):
        """
        Publish telemetry unavailable to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry unavailable event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._terrestrial_platform_id,
                            status = TelemetryStatusType.UNAVAILABLE)
    
    def remote_link_up(self):
        """
        Publish telemetry available to the remote endpoint.
        """
        # Publish a link up event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry available event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._remote_platform_id,
                            status = TelemetryStatusType.AVAILABLE)
    
    def remote_link_down(self):
        """
        Publish telemetry unavailable to the remote endpoint.
        """
        # Publish a link down event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry unavailable event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._remote_platform_id,
                            status = TelemetryStatusType.UNAVAILABLE)
    
    def consume_event(self, evt, *args, **kwargs):
        """
        Test callback for events.
        """
        log.debug('Test got event: %s, args: %s, kwargs: %s',
                  str(evt), str(args), str(kwargs))
        
        if evt.type_ == 'PublicPlatformTelemetryEvent':
            self._telem_evts.append(evt)
            if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
                    self._done_telem_evt.set()
                    
        elif evt.type_ == 'RemoteQueueModifiedEvent':
            self._queue_mod_evts.append(evt)
            if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
                    self._done_queue_mod_evt.set()
            
        elif evt.type_ == 'RemoteCommandTransmittedEvent':
            self._cmd_tx_evts.append(evt)
            if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
                    self._done_cmd_tx_evt.set()
        
        elif evt.type_ == 'RemoteCommandResult':
            cmd = evt.command
            self._results_recv[cmd.command_id] = cmd
            if len(self._results_recv) == self._no_requests:
                self._done_cmd_evt.set()
        
    ###################################################################
    # Misc helpers.
    ###################################################################
        
    def make_fake_command(self, no):
        """
        Build a fake command for use in tests.
        """
            
        cmdstr = 'fake_cmd_%i' % no
        cmd = IonObject('RemoteCommand',
                             resource_id='fake_id',
                             command=cmdstr,
                             args=['arg1', 23],
                             kwargs={'kwargs1':'someval'})
        return cmd
        
    ###################################################################
    # Tests.
    ###################################################################

    def test_queued_fake(self):
        """
        test_queued_fake
        Test fake resource commands queued prior to linkup.
        """
                
        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            cmd = self.te_client.enqueue_command(cmd)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()
        
        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    # The following error occurs when queue persistence is enabled.
    # Need to verify correct solution to enable persistent queues.
    """
    2012-11-06 14:27:13,517 INFO     pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8
    Traceback (most recent call last):
      File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run
        result = self._run(*self.args, **self.kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop
        self._callback(cmd_result)
      File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete
        self._client.enqueue(result)
    AttributeError: 'NoneType' object has no attribute 'enqueue'
    <Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError
    
    2012-11-06 14:27:13,586 INFO     pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7
    2012-11-06 14:27:13,665 INFO     pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6
    2012-11-06 14:27:13,739 INFO     pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5
    2012-11-06 14:27:13,807 INFO     pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4
    2012-11-06 14:27:13,874 INFO     pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3
    2012-11-06 14:27:13,941 INFO     pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1
    2012-11-06 14:27:13,945 INFO     pyon.event.event EventSubscriber stopped. Event pattern=#
    2012-11-06 14:27:14,124 INFO     pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984
    2012-11-06 14:27:14,399 INFO     pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB
    
    ======================================================================
    ERROR: test_process_online
    ----------------------------------------------------------------------
    Traceback (most recent call last):
      File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner
        return f(*args, **kw)
      File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online
        cmd = self.te_client.enqueue_command(cmd)
      File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command
        return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request
        return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request
        retval, headers = e.send(msg, headers=headers, timeout=timeout)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send
        result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send
        return self._invite_and_send(to_role, msg, headers, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send
        return self._send(to_role, to_role_name, msg, header, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send
        return self._end_point_unit._message_send(msg, headers, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send
        return ProcessRPCRequestEndpointUnit.send(self, msg, headers,  **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send
        return self._send(_msg, _header, **kwargs)
      File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send
        raise ex
    Conflict: 409 - Object not based on most current version
    
    """

    def test_process_online(self):
        """
        test_process_online
        Test fake resource commands queued while link is up.
        """
                
        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            cmd = self.te_client.enqueue_command(cmd)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mods, command transmissions and results.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    def test_remote_late(self):
        """
        test_remote_late
        Test fake resource commands queued prior to linkup.
        Delay remote side linkup substantially to test terrestrial
        behavior when remote server not initially available.
        """
                
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            cmd = self.te_client.enqueue_command(cmd)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Actually stop the remote process, since the server is
        # available even if it hasn't linked up yet and the test is running
        # in the same container. The test will remember the appropriate
        # remote port numbers.
        self._container_client.terminate_process(self._remote_pid)
        self.terrestrial_link_up()
        gevent.sleep(10)
        
        # Restart remote side and publish remote link up.
        self._start_remote()
        self.remote_link_up()
        
        # Block for transmission and result events.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())
        
    def test_resource_commands(self):
        """
        test_resource_commands
        """

        # Set up to verify the two commands queued.        
        self._no_requests = 2
        self._no_telem_evts = 2
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests

        # Use IA client to verify IA.
        state = self._ia_client.get_agent_state()
        log.debug('Agent state is: %s', state)
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        retval = self._ia_client.ping_agent()
        log.debug('Agent ping is: %s', str(retval))
        self.assertIn('ping from InstrumentAgent', retval)

        # Create and enqueue commands.
        state_cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={})
        state_cmd = self.te_client.enqueue_command(state_cmd)
        self._requests_sent[state_cmd.command_id] = state_cmd

        ping_cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='ping_agent',
                             args=[],
                             kwargs={})
        ping_cmd = self.te_client.enqueue_command(ping_cmd)
        self._requests_sent[ping_cmd.command_id] = ping_cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()

        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())        
        
        self.assertEqual(self._results_recv[state_cmd.command_id].result,
                         ResourceAgentState.UNINITIALIZED)
        self.assertIn('ping from InstrumentAgent',
                      self._results_recv[ping_cmd.command_id].result)
        
    def test_service_command_sequence(self):
        """
        test_service_commands
        """
        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []        
        self._cmd_tx_evts = []
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='create',
                             args=[obj],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns obj_id, obj_rev.
        obj_id, obj_rev = self._results_recv[cmd.command_id].result
        
        # Confirm the results are valid.
        
        #Result is a tuple of strings.
        #{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
        #'1-00a308c45fff459c7cda1db9a7314de6'],
        #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
        
        self.assertIsInstance(obj_id, str)
        self.assertNotEqual(obj_id, '')
        self.assertIsInstance(obj_rev, str)
        self.assertNotEqual(obj_rev, '')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []        
        self._cmd_tx_evts = []
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='read',
                             args=[obj_id],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        
                
        # Returns read_obj.
        read_obj = self._results_recv[cmd.command_id].result
        
        # Confirm the results are valid.
        
        #Result is a user info object with the name set.
        #{'lcstate': 'DEPLOYED_AVAILABLE',
        #'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
        #'description': '',
        #'ts_updated': '1349213207638',
        #'type_': 'UserInfo',
        #'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
        #'_id': '27832d93f4cd4535a75ac75c06e00a7e',
        #'ts_created': '1349213207638',
        #'variables': [{'name': '', 'value': ''}],
        #'name': 'some_name'}
        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_name')

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []
        self._cmd_tx_evts = []        
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        read_obj.name = 'some_other_name'
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='update',
                             args=[read_obj],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        

        # Returns nothing.

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []
        self._cmd_tx_evts = []        
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='read',
                             args=[obj_id],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        
        
        # Returns read_obj.
        read_obj = self._results_recv[cmd.command_id].result
        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_other_name')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()
        self._queue_mod_evts = []
        self._cmd_tx_evts = []        
        self._requests_sent = {}
        self._results_recv = {}

        # Create user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='delete',
                             args=[obj_id],
                             kwargs='')
        cmd = self.te_client.enqueue_command(cmd)

        # Block for the queue to be modified, the command to be transmitted,
        # and the result to be received.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)        

        # Returns nothing.
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
示例#27
0
class TestRemoteClient(IonIntegrationTestCase):
    """
    Test cases for 2CAA remote clients.
    """
    def setUp(self):
        """
        Setup the test parameters.
        Start the container and retrieve client.
        Start the endpoints and resource agent.
        Start publisher and subscribers.
        """
        ###################################################################
        # Internal parameters and container.
        ###################################################################
        
        # Internal parameters.        
        self._terrestrial_platform_id = 'terrestrial_id'
        self._remote_platform_id = 'remote_id'
        self._resource_id = 'fake_id'
        self._xs_name = 'remote1'
        self._terrestrial_svc_name = 'terrestrial_endpoint'
        self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name
        self._remote_svc_name = 'remote_endpoint'
        self._remote_listen_name = self._remote_svc_name + self._xs_name
        self._remote_port = 0
        self._terrestrial_port = 0
        self._te_client = None
        self._re_client = None
        self._remote_pid = None
        self._terrestrial_pid = None

        # Async test results.
        self._no_requests = 10
        self._no_telem_evts = 2
        self._no_cmd_tx_evts = self._no_requests
        self._no_queue_mod_evts = self._no_requests
        self._telem_evts = []
        self._queue_mod_evts = []
        self._cmd_tx_evts = []
        self._results_recv = {}
        self._requests_sent = {}
        self._done_telem_evt = AsyncResult()
        self._done_queue_mod_evt = AsyncResult()
        self._done_cmd_tx_evt = AsyncResult()
        self._done_cmd_evt = AsyncResult()

        # Start container.
        log.debug('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message).
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Create a container client.
        log.debug('Creating container client.')
        self._container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        
        ###################################################################
        # Start endpoints, agent.
        ###################################################################

        self._start_terrestrial()
        self._start_remote()
        self._start_agent()
                        
        ###################################################################
        # Assign client ports.
        # This is primarily for test purposes as the IP config in
        # deployment will be fixed in advance.
        ###################################################################
        
        self.te_client.set_client_port(self._remote_port)
        check_port = self.te_client.get_client_port()
        log.debug('Terrestrial client port is: %i', check_port)
    
        self.re_client.set_client_port(self._terrestrial_port)
        check_port = self.re_client.get_client_port()
        log.debug('Remote client port is: %i', check_port)
        
        ###################################################################
        # Start the event publisher and subscribers.
        # Used to send fake agent telemetry publications to the endpoints,
        # and to receive endpoint publications.
        ###################################################################
        self._event_publisher = EventPublisher()

        # Start the event subscriber for remote namespace platform events.
        # This event could be changed to RemoteNamespaceEvent.
        self._event_subscriber = EventSubscriber(
            event_type='PlatformEvent',
            callback=self.consume_event,
            origin=self._xs_name)
        self._event_subscriber.start()
        self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._event_subscriber.stop)

        # Start the result subscriber for remote resource events.
        self._resource_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=IA_RESOURCE_ID,
            callback=self.consume_event)
        self._resource_result_subscriber.start()
        self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._resource_result_subscriber.stop)

        # Start the result subscriber for remote service events.
        self._service_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin='resource_registry' + 'remote1',
            callback=self.consume_event)
        self._service_result_subscriber.start()
        self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._service_result_subscriber.stop)

        # Start the result subscriber for fake resource results.      
        self._fake_result_subscriber = EventSubscriber(
            event_type='RemoteCommandResult',
            origin=self._resource_id,
            callback=self.consume_event)
        self._fake_result_subscriber.start()
        self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
        self.addCleanup(self._fake_result_subscriber.stop)

    ###################################################################
    # Start/stop helpers.
    ###################################################################

    def _start_agent(self):
        """
        Start an instrument agent and client.
        """
        
        log.info('Creating driver integration test support:')
        log.info('driver module: %s', DRV_MOD)
        log.info('driver class: %s', DRV_CLS)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)        
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port
        }
        self.addCleanup(self._support.stop_pagent)    
                        
        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : {},
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }
    
        # Start instrument agent.
        log.debug("Starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
    
        ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config)
    
        log.info('Agent pid=%s.', str(ia_pid))
    
        # Start a resource agent client to talk with the instrument agent.
    
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))
        
    def _start_terrestrial(self):
        """
        Start up the terrestrial endpoint.
        """
        # Create terrestrial config.
        terrestrial_endpoint_config = {
            'other_host' : 'localhost',
            'other_port' : self._remote_port,
            'this_port' : self._terrestrial_port,
            'platform_resource_id' : self._terrestrial_platform_id,
            'xs_name' : self._xs_name,
            'process' : {
                'listen_name' : self._terrestrial_listen_name
            }
        }
        
        # Spawn the terrestrial enpoint process.
        log.debug('Spawning terrestrial endpoint process.')
        self._terrestrial_pid = self._container_client.spawn_process(
            name=self._terrestrial_listen_name,
            module='ion.services.sa.tcaa.terrestrial_endpoint',
            cls='TerrestrialEndpoint',
            config=terrestrial_endpoint_config)
        log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid))

        # Create a terrestrial client.
        self.te_client = TerrestrialEndpointClient(
            process=FakeProcess(),
            to_name=self._terrestrial_listen_name)
        log.debug('Got te client %s.', str(self.te_client))
        self._terrestrial_port = self.te_client.get_port()
        log.debug('Terrestrial port is: %i', self._terrestrial_port)
        
    def _start_remote(self):
        """
        Start up the remote endpoint.
        """        
        # Create agent config.
        remote_endpoint_config = {
            'other_host' : 'localhost',
            'other_port' : self._terrestrial_port,
            'this_port' : self._remote_port,
            'platform_resource_id' : self._remote_platform_id,
            'xs_name' : self._xs_name,
            'process' : {
                'listen_name' : self._remote_listen_name
            }
        }
        
        # Spawn the remote enpoint process.
        log.debug('Spawning remote endpoint process.')
        self._remote_pid = self._container_client.spawn_process(
            name=self._remote_listen_name,
            module='ion.services.sa.tcaa.remote_endpoint',
            cls='RemoteEndpoint',
            config=remote_endpoint_config)
        log.debug('Remote endpoint pid=%s.', str(self._remote_pid))

        # Create an endpoint client.
        self.re_client = RemoteEndpointClient(
            process=FakeProcess(),
            to_name=self._remote_listen_name)
        log.debug('Got re client %s.', str(self.re_client))
        
        # Remember the remote port.
        self._remote_port = self.re_client.get_port()
        log.debug('The remote port is: %i.', self._remote_port)
        
        
    ###################################################################
    # Telemetry publications to start/top endpoint.
    # (Normally be published by appropriate platform agents.)
    ###################################################################

    def terrestrial_link_up(self):
        """
        Publish telemetry available to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry available event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._terrestrial_platform_id,
                            status = TelemetryStatusType.AVAILABLE)
        
    def terrestrial_link_down(self):
        """
        Publish telemetry unavailable to the terrestrial endpoint.
        """
        # Publish a link up event to be caught by the terrestrial endpoint.
        log.debug('Publishing terrestrial telemetry unavailable event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._terrestrial_platform_id,
                            status = TelemetryStatusType.UNAVAILABLE)
    
    def remote_link_up(self):
        """
        Publish telemetry available to the remote endpoint.
        """
        # Publish a link up event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry available event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._remote_platform_id,
                            status = TelemetryStatusType.AVAILABLE)
    
    def remote_link_down(self):
        """
        Publish telemetry unavailable to the remote endpoint.
        """
        # Publish a link down event to be caught by the remote endpoint.
        log.debug('Publishing remote telemetry unavailable event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._remote_platform_id,
                            status = TelemetryStatusType.UNAVAILABLE)
    
    def consume_event(self, evt, *args, **kwargs):
        """
        Test callback for events.
        """
        log.debug('Test got event: %s, args: %s, kwargs: %s',
                  str(evt), str(args), str(kwargs))
        
        if evt.type_ == 'PublicPlatformTelemetryEvent':
            self._telem_evts.append(evt)
            if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
                    self._done_telem_evt.set()
                    
        elif evt.type_ == 'RemoteQueueModifiedEvent':
            self._queue_mod_evts.append(evt)
            if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
                    self._done_queue_mod_evt.set()
            
        elif evt.type_ == 'RemoteCommandTransmittedEvent':
            self._cmd_tx_evts.append(evt)
            if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
                    self._done_cmd_tx_evt.set()
        
        elif evt.type_ == 'RemoteCommandResult':
            cmd = evt.command
            self._results_recv[cmd.command_id] = cmd
            if len(self._results_recv) == self._no_requests:
                self._done_cmd_evt.set()
        
    ###################################################################
    # Misc helpers.
    ###################################################################
        
    def make_fake_command(self, no):
        """
        Build a fake command for use in tests.
        """
            
        cmdstr = 'fake_cmd_%i' % no
        cmd = IonObject('RemoteCommand',
                             resource_id='fake_id',
                             command=cmdstr,
                             args=['arg1', 23],
                             kwargs={'kwargs1':'someval'})
        return cmd

    ###################################################################
    # Tests.
    ###################################################################

    def test_resource_client_online(self):
        """
        test_recourse_client_online
        Test the client transparently forwards commands to the remote
        resource while link is up.
        """

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()
        
        remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
            resource_id='fake_id', process=FakeProcess())
        
        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent()
            self._requests_sent[cmd.command_id] = cmd
            
        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)                
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())
    
    #@unittest.skip('For some reason this bastard wont run on the builder.')
    def test_resource_client_blocking(self):
        """
        test_resource_client_blocking
        Test the client can block on remote resource command results.
        """

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()
        
        remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
            resource_id=IA_RESOURCE_ID, process=FakeProcess())
        
        # Queue up a series of fake commands to be handled by the remote side.
        """
        {'time_completed': 1350421095.804607, 'resource_id': '123xyz',
        'time_queued': 1350421095.623531, 'args': [], 'type_': 'RemoteCommand',
        'command': 'ping_agent', 'result': 'ping from InstrumentAgent
        (name=Agent007,id=Edwards-MacBook-Pro_local_10126.35,type=agent),
        time: 1350421095757', 'kwargs': {}, 'svc_name': '',
        'command_id': '76be11b4-a22c-49de-89cd-4e019463d7c9'}
        """
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent(remote_timeout=CFG.endpoint.receive.timeout)
            self.assertEqual(cmd.resource_id, IA_RESOURCE_ID)
            self.assertEqual(cmd.command, 'ping_agent')
            self.assertIn('ping from InstrumentAgent', cmd.result)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

    #@unittest.skip('For some reason this bastard wont run on the builder.')
    def test_service_client_blocking(self):
        """
        test_service_client_blocking
        Test the client can command remote services and block on their
        results.
        """

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()

        # Create remote client for the resource registry.
        remote_client = RemoteClient(iface=IResourceRegistryService,
            xs_name=self._xs_name, svc_name='resource_registry',
            process=FakeProcess())

        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        result = remote_client.create(obj,
            remote_timeout=CFG.endpoint.receive.timeout)

        # Returns obj_id, obj_rev.
        obj_id, obj_rev = result.result
        
        # Confirm the results are valid.
        
        #Result is a tuple of strings.
        #{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
        #'1-00a308c45fff459c7cda1db9a7314de6'],
        #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
        
        self.assertIsInstance(obj_id, str)
        self.assertNotEqual(obj_id, '')
        self.assertIsInstance(obj_rev, str)
        self.assertNotEqual(obj_rev, '')

        # Read user object.
        result = remote_client.read(obj_id,
            remote_timeout=CFG.endpoint.receive.timeout)

        # Returns read_obj.
        read_obj = result.result
        
        # Confirm the results are valid.
        
        #Result is a user info object with the name set.
        #{'lcstate': 'DEPLOYED_AVAILABLE',
        #'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
        #'description': '',
        #'ts_updated': '1349213207638',
        #'type_': 'UserInfo',
        #'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
        #'_id': '27832d93f4cd4535a75ac75c06e00a7e',
        #'ts_created': '1349213207638',
        #'variables': [{'name': '', 'value': ''}],
        #'name': 'some_name'}
        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_name')

        # Update user object.
        read_obj.name = 'some_other_name'
        result = remote_client.update(read_obj,
            remote_timeout=CFG.endpoint.receive.timeout)
        
        # Returns nothing.

        # Read user object.
        result = remote_client.read(obj_id,
            remote_timeout=CFG.endpoint.receive.timeout)

        # Returns read_obj.
        read_obj = result.result
        
        # Confirm results are valid.        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_other_name')

        # Delete user object.
        result = remote_client.delete(obj_id,
            remote_timeout=CFG.endpoint.receive.timeout)
        
        # Returns nothing.

        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

    def test_queue_manipulators(self):
        """
        test_queue_manipulators
        Test ability to instpect and manipulate the command queue corresponding
        to this resource or service.
        """

        remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
            resource_id='fake_id', process=FakeProcess())
        
        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent(link=False)
            self._requests_sent[cmd.command_id] = cmd

        # Block on queue mod events.
        self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout)

        queue = remote_client.get_queue()
        self.assertEqual(len(queue), self._no_requests)

        popped = remote_client.clear_queue()
        self.assertEqual(len(popped), self._no_requests)

        self._requests_sent = {}

        # Queue up a series of fake commands to be handled by the remote side.
        for i in range(self._no_requests):
            cmd = remote_client.ping_agent(link=False)
            self._requests_sent[cmd.command_id] = cmd

        # Pop the last three commands.
        cmd_ids = self._requests_sent.keys()[:3]
        poped = []
        for x in cmd_ids:
            poped.append(remote_client.pop_queue(x))
            self._requests_sent.pop(x)

        queue = remote_client.get_queue()
        self.assertEqual(len(queue), self._no_requests - 3)
        self.assertEqual(len(poped), 3)

        self._no_requests = self._no_requests - 3
        self._no_cmd_tx_evts = self._no_requests

        # Publish link up events.
        self.terrestrial_link_up()
        self.remote_link_up()            
        
        # Block on command transmissions and results.
        self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        pending = remote_client.get_pending()
        for x in pending:
            self.assertIn(x.command_id, self._requests_sent.keys())
        self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Publish link down events.
        self.terrestrial_link_down()
        self.remote_link_down()
        
        # Block on terrestrial public telemetry events.
        self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout)

        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    def test_errors(self):
        """
        test_errors
        Test various error conditions.
        """
        
        # Constructed without a xs name.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(iface=IResourceAgent,
                    resource_id=IA_RESOURCE_ID, process=FakeProcess())
        
        # Constructed without an interface.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(xs_name=self._xs_name,
                    resource_id=IA_RESOURCE_ID, process=FakeProcess())

        # Construct with invalid interface.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient('Bogus_interface', xs_name=self._xs_name,
            resource_id=IA_RESOURCE_ID, process=FakeProcess())
        
        # Construct with no resource or service specified.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
                    process=FakeProcess())

        # Construct with both resource and service specified.
        with self.assertRaises(ConfigNotFound):
            remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
                    resource_id=IA_RESOURCE_ID, svc_name='resource_registry', process=FakeProcess())

        # Create a valid resource client.
        remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
            resource_id=IA_RESOURCE_ID, process=FakeProcess())
        
        # Send a command while link is down.
        with self.assertRaises(Conflict):
            result = remote_client.ping_agent()

        # Test port manipulators refused by remote proxy client.
        with self.assertRaises(BadRequest):
            remote_client.get_port()
        with self.assertRaises(BadRequest):
            remote_client.set_client_port()
        with self.assertRaises(BadRequest):
            remote_client.get_client_port()

    def test_interfaces(self):
        """
        test_interfaces
        Test that the client declare the correct interfaces.
        """
        remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name,
            resource_id='fake_id', process=FakeProcess())

        interfaces = providedBy(remote_client)
        self.assertIn(IResourceAgent, interfaces)
        self.assertIn(ITerrestrialEndpoint, interfaces)
示例#28
0
class TestRemoteEndpoint(IonIntegrationTestCase):
    """
    Test cases for 2CAA terrestrial endpoint.
    """
    def setUp(self):
        """
        Start fake terrestrial components and add cleanup.
        Start terrestrial server and retrieve port.
        Set internal variables.
        Start container.
        Start deployment.
        Start container agent.
        Spawn remote endpoint process.
        Create remote endpoint client and retrieve remote server port.
        Create event publisher.
        """
        
        self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close)
        self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close)
        self.addCleanup(self._terrestrial_server.stop)
        self.addCleanup(self._terrestrial_client.stop)
        self._other_port = self._terrestrial_server.start('*', 0)
        log.debug('Terrestrial server binding to *:%i', self._other_port)
        
        self._other_host = 'localhost'
        self._platform_resource_id = 'abc123'
        self._resource_id = 'fake_id'
        self._no_requests = 10
        self._requests_sent = {}
        self._results_recv = {}
        self._no_telem_events = 0
        self._done_evt = AsyncResult()
        self._done_telem_evts = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        
        # Start container.
        log.debug('Staring capability container.')
        self._start_container()
        
        # Bring up services in a deploy file (no need to message).
        log.info('Staring deploy services.')
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        # Create a container client.
        log.debug('Creating container client.')
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)

        # Create agent config.
        endpoint_config = {
            'other_host' : self._other_host,
            'other_port' : self._other_port,
            'this_port' : 0,
            'platform_resource_id' : self._platform_resource_id
        }
        
        # Spawn the remote enpoint process.
        log.debug('Spawning remote endpoint process.')
        re_pid = container_client.spawn_process(
            name='remote_endpoint_1',
            module='ion.services.sa.tcaa.remote_endpoint',
            cls='RemoteEndpoint',
            config=endpoint_config)
        log.debug('Endpoint pid=%s.', str(re_pid))

        # Create an endpoint client.
        self.re_client = RemoteEndpointClient(
            process=FakeProcess(),
            to_name=re_pid)
        log.debug('Got re client %s.', str(self.re_client))
        
        # Remember the remote port.
        self._this_port = self.re_client.get_port()
        log.debug('The remote port is: %i.', self._this_port)
        
        # Start the event publisher.
        self._event_publisher = EventPublisher()
      
    ######################################################################    
    # Helpers.
    ######################################################################    

    def on_link_up(self):
        """
        Called by a test to simulate turning the link on.
        """
        log.debug('Terrestrial client connecting to localhost:%i.',
                 self._this_port)
        self._terrestrial_client.start('localhost', self._this_port)
        # Publish a link up event to be caught by the endpoint.
        log.debug('Publishing telemetry event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._platform_resource_id,
                            status = TelemetryStatusType.AVAILABLE)
    
    def on_link_down(self):
        """
        Called by a test to simulate turning the link off.
        """
        self._terrestrial_client.stop()
        # Publish a link down event to be caught by the endpoint.
        log.debug('Publishing telemetry event.')
        self._event_publisher.publish_event(
                            event_type='PlatformTelemetryEvent',
                            origin=self._platform_resource_id,
                            status = TelemetryStatusType.UNAVAILABLE)    
    
    def consume_req(self, res):
        """
        Consume a terrestrial request setting async event when necessary.
        """
        command_id = res['command_id']
        self._results_recv[command_id] = res
        if len(self._results_recv) == self._no_requests:
            self._done_evt.set()
    
    def consume_ack(self, cmd):
        """
        Consume terrestrial ack setting async event when necessary.
        """
        self._requests_sent[cmd.command_id] = cmd
        if len(self._requests_sent) == self._no_requests:
            self._cmd_tx_evt.set()
        
    def terrestrial_server_close(self):
        """
        Callback when terrestrial server closes.
        """
        pass
    
    def terrestrial_client_close(self):
        """
        Callback when terrestrial client closes.
        """
        pass
    
    def make_fake_command(self, no):
        """
        Build a fake command for use in tests.
        """
            
        cmdstr = 'fake_cmd_%i' % no
        cmd = IonObject('RemoteCommand',
                             resource_id=self._resource_id,
                             command=cmdstr,
                             args=['arg1', 23],
                             kwargs={'worktime':3},
                             command_id = str(uuid.uuid4()))
        return cmd

    def start_agent(self):
        """
        Start an instrument agent and client.
        """
        
        log.info('Creating driver integration test support:')
        log.info('driver module: %s', DRV_MOD)
        log.info('driver class: %s', DRV_CLS)
        log.info('device address: %s', DEV_ADDR)
        log.info('device port: %s', DEV_PORT)
        log.info('log delimiter: %s', DELIM)
        log.info('work dir: %s', WORK_DIR)        
        self._support = DriverIntegrationTestSupport(DRV_MOD,
                                                     DRV_CLS,
                                                     DEV_ADDR,
                                                     DEV_PORT,
                                                     DATA_PORT,
                                                     CMD_PORT,
                                                     PA_BINARY,
                                                     DELIM,
                                                     WORK_DIR)
        
        # Start port agent, add stop to cleanup.
        port = self._support.start_pagent()
        log.info('Port agent started at port %i',port)
        
        # Configure driver to use port agent port number.
        DVR_CONFIG['comms_config'] = {
            'addr' : 'localhost',
            'port' : port,
            'cmd_port' : CMD_PORT
        }
        self.addCleanup(self._support.stop_pagent)    
                        
        # Create agent config.
        agent_config = {
            'driver_config' : DVR_CONFIG,
            'stream_config' : {},
            'agent'         : {'resource_id': IA_RESOURCE_ID},
            'test_mode' : True
        }
    
        # Start instrument agent.
        log.debug("Starting IA.")
        container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
    
        ia_pid = container_client.spawn_process(name=IA_NAME,
            module=IA_MOD,
            cls=IA_CLS,
            config=agent_config)
    
        log.info('Agent pid=%s.', str(ia_pid))
    
        # Start a resource agent client to talk with the instrument agent.
    
        self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
        log.info('Got ia client %s.', str(self._ia_client))                
                
    ######################################################################    
    # Tests.
    ######################################################################    

    def test_process_queued(self):
        """
        test_process_queued
        Test that queued commands are forwarded to and handled by
        remote endpoint when link comes up.
        """        
        
        # Create and enqueue some requests.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            self._terrestrial_client.enqueue(cmd)

        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Wait for all the enqueued commands to be acked.
        # Wait for all the responses to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        # Confirm the results match the commands sent.
        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())
    
    def test_process_online(self):
        """
        test_process_online
        Test commands are forwarded and handled while link is up.
        """        
        
        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Wait for the link to be up.
        # The remote side does not publish public telemetry events
        # so we can't wait for that.
        gevent.sleep(1)

        # Create and enqueue some requests.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            self._terrestrial_client.enqueue(cmd)

        # Wait for all the enqueued commands to be acked.
        # Wait for all the responses to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        # Confirm the results match the commands sent.
        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    def test_terrestrial_late(self):
        """
        test_terrestrial_late
        Test queued commands are forwarded and handled by remote endpoint
        when terrestrial side is late to come up.
        """        
        
        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Wait for the link to be up.
        # The remote side does not publish public telemetry events
        # so we can't wait for that.
        gevent.sleep(1)

        # Manually stop the terrestrial endpoint.
        # This will cause it to be unavailable when commands are queued
        # to simulate stability during asynchronous wake ups.
        self._terrestrial_server.stop()
        self._terrestrial_client.stop()

        # Create and enqueue some requests.
        for i in range(self._no_requests):
            cmd = self.make_fake_command(i)
            self._terrestrial_client.enqueue(cmd)

        # Remote side awaits the terrestrial waking up.
        gevent.sleep(3)

        # Terrestrail endpoint eventually wakes up and starts transmitting.        
        self._terrestrial_client.start('localhost', self._this_port)
        self._terrestrial_server.start('*', self._other_port)
    
        # Wait for all the enqueued commands to be acked.
        # Wait for all the responses to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        # Confirm the results match the commands sent.
        self.assertItemsEqual(self._requests_sent.keys(),
                                  self._results_recv.keys())

    def test_service_commands(self):
        """
        test_service_commands
        Test that real service commands are handled by the remote endpoint.
        """
        
        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}
        
        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='create',
                             args=[obj],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Returns obj_id, obj_rev.
        obj_id, obj_rev = self._results_recv[cmd.command_id]['result']
        
        # Confirm the results are valid.
        """
        Result is a tuple of strings.
        {'result': ['ad183ff26bae4f329ddd85fd69d160a9',
        '1-00a308c45fff459c7cda1db9a7314de6'],
        'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
        """
        self.assertIsInstance(obj_id, str)
        self.assertNotEqual(obj_id, '')
        self.assertIsInstance(obj_rev, str)
        self.assertNotEqual(obj_rev, '')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Read user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='read',
                             args=[obj_id],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)

        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns read_obj.
        read_obj = self._results_recv[cmd.command_id]['result']
        
        # Confirm the results are valid.
        """
        Result is a user info object with the name set.
        {'lcstate': 'DEPLOYED_AVAILABLE',
        '_rev': '1-851f067bac3c34b2238c0188b3340d0f',
        'description': '',
        'ts_updated': '1349213207638',
        'type_': 'UserInfo',
        'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
        '_id': '27832d93f4cd4535a75ac75c06e00a7e',
        'ts_created': '1349213207638',
        'variables': [{'name': '', 'value': ''}],
        'name': 'some_name'}
        """
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_name')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Update user object.
        read_obj.name = 'some_other_name'
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='update',
                             args=[read_obj],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)

        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns nothing.
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Read user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='read',
                             args=[obj_id],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)        

        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns read_obj.
        read_obj = self._results_recv[cmd.command_id]['result']
        
        self.assertIsInstance(read_obj, UserInfo)
        self.assertEquals(read_obj.name, 'some_other_name')
        
        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}
        
        # Delete user object.
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='delete',
                             args=[obj_id],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)        

        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns nothing.
            
        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        gevent.sleep(1)
        
    def test_resource_commands(self):
        """
        test_resource_commands
        Test that real resource commands are handled by the remote endpoint.
        """
        
        # Start the IA and check it's out there and behaving.
        self.start_agent()
        
        state = self._ia_client.get_agent_state()
        log.debug('Agent state is: %s', state)
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        retval = self._ia_client.ping_agent()
        log.debug('Agent ping is: %s', str(retval))
        self.assertIn('ping from InstrumentAgent', retval)

        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Wait for the link to be up.
        # The remote side does not publish public telemetry events
        # so we can't wait for that.
        gevent.sleep(1)

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Get agent state via remote endpoint.        
        cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Returns agent state.
        state = self._results_recv[cmd.command_id]['result']
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Ping agent via remote endpoint. 
        cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='ping_agent',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Returns agent state.
        ping = self._results_recv[cmd.command_id]['result']
        self.assertIn('ping from InstrumentAgent', ping)
        
        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        gevent.sleep(1)

    def test_bad_service_name_resource_id(self):
        """
        test_bad_service_name_resource_id
        Test for proper exception behavior when a bad service name or
        resource id is used in a command forwarded to the remote endpoint.
        """
        
        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Wait for the link to be up.
        # The remote side does not publish public telemetry events
        # so we can't wait for that.
        gevent.sleep(1)

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}
        
        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='bogus_service',
                             command='create',
                             args=[obj],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Returns NotFound.
        result = self._results_recv[cmd.command_id]['result']
        self.assertIsInstance(result, NotFound)

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Get agent state via remote endpoint.        
        cmd = IonObject('RemoteCommand',
                             resource_id='bogus_resource_id',
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns NotFound.
        result = self._results_recv[cmd.command_id]['result']
        self.assertIsInstance(result, NotFound)

        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        gevent.sleep(1)

    def test_bad_commands(self):
        """
        test_bad_commands
        Test for correct exception behavior if a bad command name is forwarded
        to a remote service or resource.
        """
        
        # Start the IA and check it's out there and behaving.
        self.start_agent()
        
        state = self._ia_client.get_agent_state()
        log.debug('Agent state is: %s', state)
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        retval = self._ia_client.ping_agent()
        log.debug('Agent ping is: %s', str(retval))
        self.assertIn('ping from InstrumentAgent', retval)
        
        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()

        # Wait for the link to be up.
        # The remote side does not publish public telemetry events
        # so we can't wait for that.
        gevent.sleep(1)

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}
        
        # Create user object.
        obj = IonObject("UserInfo", name="some_name")
        cmd = IonObject('RemoteCommand',
                             resource_id='',
                             svc_name='resource_registry',
                             command='what_the_flunk',
                             args=[obj],
                             kwargs='',
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
        
        # Returns BadRequest.
        result = self._results_recv[cmd.command_id]['result']
        self.assertIsInstance(result, BadRequest)

        # Send commands one at a time.
        # Reset queues and events.
        self._no_requests = 1
        self._done_evt = AsyncResult()
        self._cmd_tx_evt = AsyncResult()
        self._requests_sent = {}
        self._results_recv = {}

        # Get agent state via remote endpoint.        
        cmd = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='what_the_flunk',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd)
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Returns NotFound.
        result = self._results_recv[cmd.command_id]['result']
        self.assertIsInstance(result, BadRequest)
        
        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        gevent.sleep(1)

    def test_resource_command_sequence(self):
        """
        test_resource_command_sequence
        Test for successful completion of a properly ordered sequence of
        resource commands queued for forwarding to the remote endpoint.
        """
        # Start the IA and check it's out there and behaving.
        self.start_agent()
        
        state = self._ia_client.get_agent_state()
        log.debug('Agent state is: %s', state)
        self.assertEqual(state, ResourceAgentState.UNINITIALIZED)

        retval = self._ia_client.ping_agent()
        log.debug('Agent ping is: %s', str(retval))
        self.assertIn('ping from InstrumentAgent', retval)

        # We execute a sequence of twelve consecutive events.
        self._no_requests = 12

        # Get agent state.
        cmd1 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd1)
        
        # Initialize agent.
        cmd2 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_agent',
                             args=[AgentCommand(command=ResourceAgentEvent.INITIALIZE)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd2)
        
        # Get agent state.
        cmd3 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd3)
        
        # Go active.
        cmd4 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_agent',
                             args=[AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd4)
        
        # Get agent state.
        cmd5 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd5)
        
        # Run.
        cmd6 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_agent',
                             args=[AgentCommand(command=ResourceAgentEvent.RUN)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd6)
        
        # Get agent state.
        cmd7 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd7)
        
        # Acquire sample.
        cmd8 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_resource',
                             args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd8)
        
        # Acquire sample
        cmd9 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_resource',
                             args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd9)
        
        # Acquire sample.
        cmd10 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_resource',
                             args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd10)
        
        # Reset.
        cmd11 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='execute_agent',
                             args=[AgentCommand(command=ResourceAgentEvent.RESET)],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd11)
        
        # Get agent state.
        cmd12 = IonObject('RemoteCommand',
                             resource_id=IA_RESOURCE_ID,
                             svc_name='',
                             command='get_agent_state',
                             args=[],
                             kwargs={},
                             command_id = str(uuid.uuid4()))
        self._terrestrial_client.enqueue(cmd12)

        
        # Publish a telemetry available event.
        # This will cause the endpoint clients to wake up and connect.
        self.on_link_up()
        
        # Wait for command request to be acked.
        # Wait for response to arrive.
        self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
        self._done_evt.get(timeout=CFG.endpoint.receive.timeout)

        # Check results of command sequence.
        """
        0ccf1e10-eeca-400d-aefe-f9d6888ec963   {'result': 'RESOURCE_AGENT_STATE_INACTIVE', 'command_id': '0ccf1e10-eeca-400d-aefe-f9d6888ec963'}
        92531bdf-c2c8-4aa8-817d-5107c7311b37   {'result': <interface.objects.AgentCommandResult object at 0x10d7f11d0>, 'command_id': '92531bdf-c2c8-4aa8-817d-5107c7311b37'}
        509934a1-5038-40d8-8014-591e2d8042b6   {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '509934a1-5038-40d8-8014-591e2d8042b6'}
        88bacbb7-5366-4d27-9ecf-fff2bec34b2c   {'result': <interface.objects.AgentCommandResult object at 0x10d389190>, 'command_id': '88bacbb7-5366-4d27-9ecf-fff2bec34b2c'}
        f8b4d3fa-a249-439b-8bd4-ac212b6100aa   {'result': <interface.objects.AgentCommandResult object at 0x10d3893d0>, 'command_id': 'f8b4d3fa-a249-439b-8bd4-ac212b6100aa'}
        8ae98e39-fdb3-4218-ad8f-584620397d9f   {'result': <interface.objects.AgentCommandResult object at 0x10d739990>, 'command_id': '8ae98e39-fdb3-4218-ad8f-584620397d9f'}
        746364a1-c4c7-400f-96d4-ee36df5dc1a4   {'result': BadRequest('Execute argument "command" not set.',), 'command_id': '746364a1-c4c7-400f-96d4-ee36df5dc1a4'}
        d516d3d9-e4f9-4ea5-80e0-34639a6377b5   {'result': <interface.objects.AgentCommandResult object at 0x10d3b2350>, 'command_id': 'd516d3d9-e4f9-4ea5-80e0-34639a6377b5'}
        c7da03f5-59bc-420a-9e10-0a7794266599   {'result': 'RESOURCE_AGENT_STATE_IDLE', 'command_id': 'c7da03f5-59bc-420a-9e10-0a7794266599'}
        678d870a-bf18-424a-afb0-f80ecf3277e2   {'result': <interface.objects.AgentCommandResult object at 0x10d739590>, 'command_id': '678d870a-bf18-424a-afb0-f80ecf3277e2'}
        750c6a30-56eb-4535-99c2-a81fefab1b1f   {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '750c6a30-56eb-4535-99c2-a81fefab1b1f'}
        c17bd658-3775-4aa3-8844-02df70a0e3c0   {'result': 'RESOURCE_AGENT_STATE_UNINITIALIZED', 'command_id': 'c17bd658-3775-4aa3-8844-02df70a0e3c0'}
        """        
        
        # First result is a state string.
        result1 = self._results_recv[cmd1.command_id]['result']
        self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
        
        # Second result is an empty AgentCommandResult.
        result2 = self._results_recv[cmd2.command_id]['result']

        # Third result is a state string.
        result3 = self._results_recv[cmd3.command_id]['result']
        self.assertEqual(result3, ResourceAgentState.INACTIVE)
        
        # Fourth result is an empty AgentCommandResult.
        result4 = self._results_recv[cmd4.command_id]['result']

        # Fifth result is a state string.
        result5 = self._results_recv[cmd5.command_id]['result']
        self.assertEqual(result5, ResourceAgentState.IDLE)

        # Sixth result is an empty AgentCommandResult.
        result6 = self._results_recv[cmd6.command_id]['result']

        # Seventh result is a state string.
        result7 = self._results_recv[cmd7.command_id]['result']
        self.assertEqual(result7, ResourceAgentState.COMMAND)
        
        """
        {'raw': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
        'stream_name': 'raw', 'pkt_format_id': 'JSON_Data',
        'pkt_version': 1, '
        values': [{'binary': True, 'value_id': 'raw',
        'value': 'NzkuNDM3MywxNy4yMDU2NCwgNzYxLjg4NSwgICA2LjIxOTgsIDE1MDYuMzk3LCAwMSBGZWIgMjAwMSwgMDE6MDE6MDA='}],
        'driver_timestamp': 3558286748.8039923},
        'parsed': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
        'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1,
        'values': [{'value_id': 'temp', 'value': 79.4373},
        {'value_id': 'conductivity', 'value': 17.20564},
        {'value_id': 'pressure', 'value': 761.885}],
        'driver_timestamp': 3558286748.8039923}}
        """
        
        # Eigth result is an AgentCommandResult containing a sample.
        result8 = self._results_recv[cmd8.command_id]['result']
        self.assertTrue('parsed',result8.result )
        
        # Ninth result is an AgentCommandResult containing a sample.
        result9 = self._results_recv[cmd9.command_id]['result']
        self.assertTrue('parsed',result9.result )

        # Tenth result is an AgentCommandResult containing a sample.
        result10 = self._results_recv[cmd10.command_id]['result']
        self.assertTrue('parsed',result10.result )

        # Eleventh result is an empty AgentCommandResult.
        result11 = self._results_recv[cmd11.command_id]['result']

        # Twelth result is a state string.
        result12 = self._results_recv[cmd12.command_id]['result']
        self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
        
        # Publish a telemetry unavailable event.
        # This will cause the endpoint clients to disconnect and go to sleep.
        self.on_link_down()

        gevent.sleep(1)
示例#29
0
class TestSBE37Driver(unittest.TestCase):
    """
    Integration tests for the sbe37 driver. This class tests and shows
    use patterns for the sbe37 driver as a zmq driver process.
    """
    def setUp(self):
        """
        Setup test cases.
        """
        self.device_addr = DEV_ADDR
        self.device_port = DEV_PORT
        self.work_dir = WORK_DIR
        self.delim = DELIM

        self.driver_class = DVR_CLS
        self.driver_module = DVR_MOD
        self._support = DriverIntegrationTestSupport(self.driver_module,
                                                     self.driver_class,
                                                     self.device_addr,
                                                     self.device_port,
                                                     self.delim, self.work_dir)

        # Clear driver event list.
        self._events = []

        # The port agent object. Used to start and stop the port agent.
        self._pagent = None

        # The driver process popen object.
        self._dvr_proc = None

        # The driver client.
        self._dvr_client = None

        # Create and start the port agent.
        mi_logger.info('start')
        COMMS_CONFIG['port'] = self._support.start_pagent()
        self.addCleanup(self._support.stop_pagent)

        # Create and start the driver.
        self._support.start_driver()
        self.addCleanup(self._support.stop_driver)

        # Grab some variables from support that we need
        self._dvr_client = self._support._dvr_client
        self._dvr_proc = self._support._dvr_proc
        self._pagent = self._support._pagent
        self._events = self._support._events

    def assertSampleDict(self, val):
        """
        Verify the value is a sample dictionary for the sbe37.
        """
        #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
        self.assertTrue(isinstance(val, dict))
        self.assertTrue(val.has_key('c'))
        self.assertTrue(val.has_key('t'))
        self.assertTrue(val.has_key('p'))
        self.assertTrue(val.has_key('time'))
        c = val['c'][0]
        t = val['t'][0]
        p = val['p'][0]
        time = val['time'][0]

        self.assertTrue(isinstance(c, float))
        self.assertTrue(isinstance(t, float))
        self.assertTrue(isinstance(p, float))
        self.assertTrue(isinstance(time, float))

    def assertParamDict(self, pd, all_params=False):
        """
        Verify all device parameters exist and are correct type.
        """
        if all_params:
            self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
            #print str(pd)
            #print str(PARAMS)
            for (key, type_val) in PARAMS.iteritems():
                #print key
                self.assertTrue(isinstance(pd[key], type_val))
        else:
            for (key, val) in pd.iteritems():
                self.assertTrue(PARAMS.has_key(key))
                self.assertTrue(isinstance(val, PARAMS[key]))

    def assertParamVals(self, params, correct_params):
        """
        Verify parameters take the correct values.
        """
        self.assertEqual(set(params.keys()), set(correct_params.keys()))
        for (key, val) in params.iteritems():
            correct_val = correct_params[key]
            if isinstance(val, float):
                # Verify to 5% of the larger value.
                max_val = max(abs(val), abs(correct_val))
                self.assertAlmostEqual(val, correct_val, delta=max_val * .01)

            else:
                # int, bool, str, or tuple of same
                self.assertEqual(val, correct_val)

    def test_process(self):
        """
        Test for correct launch of driver process and communications, including
        asynchronous driver events.
        """

        # Verify processes exist.
        self.assertNotEqual(self._dvr_proc, None)
        drv_pid = self._dvr_proc.pid
        self.assertTrue(isinstance(drv_pid, int))

        self.assertNotEqual(self._pagent, None)
        pagent_pid = self._pagent.get_pid()
        self.assertTrue(isinstance(pagent_pid, int))

        # Send a test message to the process interface, confirm result.
        msg = 'I am a ZMQ message going to the process.'
        reply = self._dvr_client.cmd_dvr('process_echo', msg)
        self.assertEqual(reply, 'process_echo: ' + msg)

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Send a test message to the driver interface, confirm result.
        msg = 'I am a ZMQ message going to the driver.'
        reply = self._dvr_client.cmd_dvr('driver_echo', msg)
        self.assertEqual(reply, 'driver_echo: ' + msg)

        # Test the event thread publishes and client side picks up events.
        events = ['I am important event #1!', 'And I am important event #2!']
        reply = self._dvr_client.cmd_dvr('test_events', events=events)
        gevent.sleep(1)

        # Confirm the events received are as expected.
        self.assertEqual(self._events, events)

        # Test the exception mechanism.
        with self.assertRaises(InstrumentException):
            exception_str = 'Oh no, something bad happened!'
            reply = self._dvr_client.cmd_dvr('test_exceptions', exception_str)

        # Verify we received a driver error event.
        gevent.sleep(1)
        error_events = [
            evt for evt in self._events
            if isinstance(evt, dict) and evt['type'] == DriverAsyncEvent.ERROR
        ]
        self.assertTrue(len(error_events) == 1)

    def test_config(self):
        """
        Test to configure the driver process for device comms and transition
        to disconnected state.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver returned state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_connect(self):
        """
        Test configuring and connecting to the device through the port
        agent. Discover device state.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_get_set(self):
        """
        Test device parameter access.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Get all device parameters. Confirm all expected keys are retrived
        # and have correct type.
        reply = self._dvr_client.cmd_dvr('get', SBE37Parameter.ALL)
        self.assertParamDict(reply, True)

        # Remember original configuration.
        orig_config = reply

        # Grab a subset of parameters.
        params = [
            SBE37Parameter.TA0, SBE37Parameter.INTERVAL,
            SBE37Parameter.STORETIME, SBE37Parameter.TCALDATE
        ]
        reply = self._dvr_client.cmd_dvr('get', params)
        self.assertParamDict(reply)

        # Remember the original subset.
        orig_params = reply

        # Construct new parameters to set.
        old_date = orig_params[SBE37Parameter.TCALDATE]
        new_params = {
            SBE37Parameter.TA0: orig_params[SBE37Parameter.TA0] * 1.2,
            SBE37Parameter.INTERVAL: orig_params[SBE37Parameter.INTERVAL] + 1,
            SBE37Parameter.STORETIME:
            not orig_params[SBE37Parameter.STORETIME],
            SBE37Parameter.TCALDATE:
            (old_date[0], old_date[1], old_date[2] + 1)
        }

        # Set parameters and verify.
        reply = self._dvr_client.cmd_dvr('set', new_params)
        reply = self._dvr_client.cmd_dvr('get', params)
        self.assertParamVals(reply, new_params)

        # Restore original parameters and verify.
        reply = self._dvr_client.cmd_dvr('set', orig_params)
        reply = self._dvr_client.cmd_dvr('get', params)
        self.assertParamVals(reply, orig_params)

        # Retrieve the configuration and ensure it matches the original.
        # Remove samplenum as it is switched by autosample and storetime.
        reply = self._dvr_client.cmd_dvr('get', SBE37Parameter.ALL)
        reply.pop('SAMPLENUM')
        orig_config.pop('SAMPLENUM')
        self.assertParamVals(reply, orig_config)

        # Disconnect from the port agent.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is disconnected.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Deconfigure the driver.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_poll(self):
        """
        Test sample polling commands and events.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)

        # Confirm that 3 samples arrived as published events.
        gevent.sleep(1)
        sample_events = [
            evt for evt in self._events
            if evt['type'] == DriverAsyncEvent.SAMPLE
        ]
        self.assertEqual(len(sample_events), 3)

        # Disconnect from the port agent.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Deconfigure the driver.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_autosample(self):
        """
        Test autosample mode.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Make sure the device parameters are set to sample frequently.
        params = {SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5}
        reply = self._dvr_client.cmd_dvr('set', params)

        reply = self._dvr_client.cmd_dvr('execute_start_autosample')

        # Test the driver is in autosample mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.AUTOSAMPLE)

        # Wait for a few samples to roll in.
        gevent.sleep(30)

        # Return to command mode. Catch timeouts and retry if necessary.
        count = 0
        while True:
            try:
                reply = self._dvr_client.cmd_dvr('execute_stop_autosample')

            except InstrumentTimeoutException:
                count += 1
                if count >= 5:
                    self.fail(
                        'Could not wakeup device to leave autosample mode.')

            else:
                break

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Verify we received at least 2 samples.
        sample_events = [
            evt for evt in self._events
            if evt['type'] == DriverAsyncEvent.SAMPLE
        ]
        self.assertTrue(len(sample_events) >= 2)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    @unittest.skip('Not supported by simulator and very long (> 5 min).')
    def test_test(self):
        """
        Test the hardware testing mode.
        """
        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        start_time = time.time()
        reply = self._dvr_client.cmd_dvr('execute_test')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.TEST)

        while state != SBE37ProtocolState.COMMAND:
            gevent.sleep(5)
            elapsed = time.time() - start_time
            mi_logger.info('Device testing %f seconds elapsed.' % elapsed)
            state = self._dvr_client.cmd_dvr('get_current_state')

        # Verify we received the test result and it passed.
        test_results = [
            evt for evt in self._events
            if evt['type'] == DriverAsyncEvent.TEST_RESULT
        ]
        self.assertTrue(len(test_results) == 1)
        self.assertEqual(test_results[0]['value']['success'], 'Passed')

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    def test_errors(self):
        """
        Test response to erroneous commands and parameters.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Assert for an unknown driver command.
        with self.assertRaises(InstrumentCommandException):
            reply = self._dvr_client.cmd_dvr('bogus_command')

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_acquire_sample')

        # Assert we forgot the comms parameter.
        with self.assertRaises(InstrumentParameterException):
            reply = self._dvr_client.cmd_dvr('configure')

        # Assert we send a bad config object (not a dict).
        with self.assertRaises(InstrumentParameterException):
            BOGUS_CONFIG = 'not a config dict'
            reply = self._dvr_client.cmd_dvr('configure', BOGUS_CONFIG)

        # Assert we send a bad config object (missing addr value).
        with self.assertRaises(InstrumentParameterException):
            BOGUS_CONFIG = COMMS_CONFIG.copy()
            BOGUS_CONFIG.pop('addr')
            reply = self._dvr_client.cmd_dvr('configure', BOGUS_CONFIG)

        # Assert we send a bad config object (bad addr value).
        with self.assertRaises(InstrumentParameterException):
            BOGUS_CONFIG = COMMS_CONFIG.copy()
            BOGUS_CONFIG['addr'] = ''
            reply = self._dvr_client.cmd_dvr('configure', BOGUS_CONFIG)

        # Configure for comms.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_acquire_sample')

        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_acquire_sample')

        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Poll for a sample and confirm result.
        reply = self._dvr_client.cmd_dvr('execute_acquire_sample')
        self.assertSampleDict(reply)

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('execute_stop_autosample')

        # Assert for a known command, invalid state.
        with self.assertRaises(InstrumentStateException):
            reply = self._dvr_client.cmd_dvr('connect')

        # Get all device parameters. Confirm all expected keys are retrived
        # and have correct type.
        reply = self._dvr_client.cmd_dvr('get', SBE37Parameter.ALL)
        self.assertParamDict(reply, True)

        # Assert get fails without a parameter.
        with self.assertRaises(InstrumentParameterException):
            reply = self._dvr_client.cmd_dvr('get')

        # Assert get fails without a bad parameter (not ALL or a list).
        with self.assertRaises(InstrumentParameterException):
            bogus_params = 'I am a bogus param list.'
            reply = self._dvr_client.cmd_dvr('get', bogus_params)

        # Assert get fails without a bad parameter (not ALL or a list).
        #with self.assertRaises(InvalidParameterValueError):
        with self.assertRaises(InstrumentParameterException):
            bogus_params = [
                'a bogus parameter name', SBE37Parameter.INTERVAL,
                SBE37Parameter.STORETIME, SBE37Parameter.TCALDATE
            ]
            reply = self._dvr_client.cmd_dvr('get', bogus_params)

        # Assert we cannot set a bogus parameter.
        with self.assertRaises(InstrumentParameterException):
            bogus_params = {'a bogus parameter name': 'bogus value'}
            reply = self._dvr_client.cmd_dvr('set', bogus_params)

        # Assert we cannot set a real parameter to a bogus value.
        with self.assertRaises(InstrumentParameterException):
            bogus_params = {SBE37Parameter.INTERVAL: 'bogus value'}
            reply = self._dvr_client.cmd_dvr('set', bogus_params)

        # Disconnect from the port agent.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Deconfigure the driver.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

    @unittest.skip('Not supported by simulator.')
    def test_discover_autosample(self):
        """
        Test the device can discover autosample mode.
        """

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('discover')

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Make sure the device parameters are set to sample frequently.
        params = {SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5}
        reply = self._dvr_client.cmd_dvr('set', params)

        reply = self._dvr_client.cmd_dvr('execute_start_autosample')

        # Test the driver is in autosample mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.AUTOSAMPLE)

        # Let a sample or two come in.
        gevent.sleep(30)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)

        # Wait briefly before we restart the comms.
        gevent.sleep(10)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('configure', COMMS_CONFIG)

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('connect')

        # Test the driver is in unknown state.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.UNKNOWN)

        # Configure driver for comms and transition to disconnected.
        count = 0
        while True:
            try:
                reply = self._dvr_client.cmd_dvr('discover')

            except InstrumentTimeoutException:
                count += 1
                if count >= 5:
                    self.fail('Could not discover device state.')

            else:
                break

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.AUTOSAMPLE)

        # Let a sample or two come in.
        # This device takes awhile to begin transmitting again after you
        # prompt it in autosample mode.
        gevent.sleep(30)

        # Return to command mode. Catch timeouts and retry if necessary.
        count = 0
        while True:
            try:
                reply = self._dvr_client.cmd_dvr('execute_stop_autosample')

            except InstrumentTimeoutException:
                count += 1
                if count >= 5:
                    self.fail(
                        'Could not wakeup device to leave autosample mode.')

            else:
                break

        # Test the driver is in command mode.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, SBE37ProtocolState.COMMAND)

        # Configure driver for comms and transition to disconnected.
        reply = self._dvr_client.cmd_dvr('disconnect')

        # Test the driver is configured for comms.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.DISCONNECTED)

        # Initialize the driver and transition to unconfigured.
        reply = self._dvr_client.cmd_dvr('initialize')

        # Test the driver is in state unconfigured.
        state = self._dvr_client.cmd_dvr('get_current_state')
        self.assertEqual(state, DriverConnectionState.UNCONFIGURED)