class SingleConnectionInstrumentDriver(InstrumentDriver):
    """
    Base class for instrument drivers with a single device connection.
    Provides connection state logic for single connection drivers. This is
    the base class for the majority of driver implementation classes.
    """
    __metaclass__ = META_LOGGER

    def __init__(self, event_callback, refdes=None):
        """
        Constructor for singly connected instrument drivers.
        @param event_callback Callback to the driver process to send asynchronous
        driver events back to the agent.
        """
        InstrumentDriver.__init__(self, event_callback)

        # The one and only instrument connection.
        # Exists in the connected state.
        self._connection = None

        # The one and only instrument protocol.
        self._protocol = None

        # Consul
        self.consul = consulate.Consul()

        # Reference Designator to the port agent service
        self.refdes = refdes

        # Build connection state machine.
        self._connection_fsm = ThreadSafeFSM(DriverConnectionState,
                                             DriverEvent,
                                             DriverEvent.ENTER,
                                             DriverEvent.EXIT)

        # Add handlers for all events.
        handlers = {
            DriverState.UNCONFIGURED: [
                (DriverEvent.ENTER, self._handler_unconfigured_enter),
                (DriverEvent.EXIT, self._handler_unconfigured_exit),
                (DriverEvent.INITIALIZE, self._handler_unconfigured_initialize),
                (DriverEvent.CONFIGURE, self._handler_unconfigured_configure),
            ],
            DriverConnectionState.DISCONNECTED: [
                (DriverEvent.ENTER, self._handler_disconnected_enter),
                (DriverEvent.EXIT, self._handler_disconnected_exit),
                (DriverEvent.INITIALIZE, self._handler_disconnected_initialize),
                (DriverEvent.CONFIGURE, self._handler_disconnected_configure),
                (DriverEvent.CONNECT, self._handler_disconnected_connect),
            ],
            DriverConnectionState.CONNECTED: [
                (DriverEvent.ENTER, self._handler_connected_enter),
                (DriverEvent.EXIT, self._handler_connected_exit),
                (DriverEvent.DISCONNECT, self._handler_connected_disconnect),
                (DriverEvent.CONNECTION_LOST, self._handler_connected_connection_lost),
                (DriverEvent.DISCOVER, self._handler_connected_protocol_event),
                (DriverEvent.GET, self._handler_connected_protocol_event),
                (DriverEvent.SET, self._handler_connected_protocol_event),
                (DriverEvent.EXECUTE, self._handler_connected_protocol_event),
                (DriverEvent.FORCE_STATE, self._handler_connected_protocol_event),
                (DriverEvent.START_DIRECT, self._handler_connected_start_direct_event),
                (DriverEvent.STOP_DIRECT, self._handler_connected_stop_direct_event),
            ],
        }

        for state in handlers:
            for event, handler in handlers[state]:
                self._connection_fsm.add_handler(state, event, handler)

        self._pre_da_config = {}
        self._startup_config = {}

        # Idempotency flag for lost connections.
        # This set to false when a connection is established to
        # allow for lost callback to become activated.
        self._connection_lost = True

        # Autoconnect flag
        # Set this to false to disable autoconnect
        self._autoconnect = True
        self._reconnect_interval = STARTING_RECONNECT_INTERVAL
        self._max_reconnect_interval = MAXIMUM_RECONNECT_INTERVAL

        # Start state machine.
        self._connection_fsm.start(DriverConnectionState.UNCONFIGURED)

    #############################################################
    # Device connection interface.
    #############################################################

    def initialize(self, *args, **kwargs):
        """
        Initialize driver connection, bringing communications parameters
        into unconfigured state (no connection object).
        @raises InstrumentStateException if command not allowed in current state
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.INITIALIZE, *args, **kwargs)

    def configure(self, *args, **kwargs):
        """
        Configure the driver for communications with the device via
        port agent / logger (valid but unconnected connection object).
        @param arg[0] comms config dict.
        @raises InstrumentStateException if command not allowed in current state
        @throws InstrumentParameterException if missing comms or invalid config dict.
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.CONFIGURE, *args, **kwargs)

    def connect(self, *args, **kwargs):
        """
        Establish communications with the device via port agent / logger
        (connected connection object).
        @raises InstrumentStateException if command not allowed in current state
        @throws InstrumentConnectionException if the connection failed.
        """
        # Forward event and argument to the connection FSM.
        result = self._connection_fsm.on_event(DriverEvent.CONNECT, *args, **kwargs)
        init_config = {}
        if len(args) > 0 and isinstance(args[0], dict):
            init_config = args[0]

        self.set_init_params(init_config)
        return result

    def disconnect(self, *args, **kwargs):
        """
        Disconnect from device via port agent / logger.
        @raises InstrumentStateException if command not allowed in current state
        """
        # Disable autoconnect if manually disconnected
        self._autoconnect = False
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.DISCONNECT, *args, **kwargs)

    #############################################################
    # Configuration logic
    #############################################################
    def get_init_params(self):
        """
        get the driver initialization parameters
        @return driver configuration dictionary
        """
        return self._startup_config

    def set_init_params(self, config):
        """
        Set the initialization parameters down in the protocol and store the
        driver configuration in the driver.

        If the protocol hasn't been setup yet cache the config.  Next time
        this method is called, if you call it with an empty config it will
        read from the cache.

        @param config This default configuration assumes a structure driver
        configuration dict with keys named in DriverConfigKey.
        Stranger parameters can be adjusted by over riding this method.
        @raise InstrumentParameterException If the config cannot be applied
        """
        if not isinstance(config, dict):
            raise InstrumentParameterException("Incompatible initialization parameters")

        if self._protocol:
            param_config = None
            if config:
                param_config = config
            elif self._startup_config:
                param_config = self._startup_config

            if param_config:
                self._protocol.set_init_params(param_config)
                self._protocol.initialize_scheduler()

        if config:
            self._startup_config = config

    def apply_startup_params(self):
        """
        Apply the startup values previously stored in the protocol to
        the running config of the live instrument. The startup values are the
        values that are (1) marked as startup parameters and are (2) the "best"
        value to use at startup. Preference is given to the previously-set init
        value, then the default value, then the currently used value.

        This default implementation simply pushes the logic down into the protocol
        for processing should the action be better accomplished down there.

        The driver writer can decide to overload this method in the derived
        driver class and apply startup parameters in the driver (likely calling
        some get and set methods for the resource). If the driver does not
        implement an apply_startup_params() method in the driver, this method
        will call into the protocol. Deriving protocol classes are expected to
        implement an apply_startup_params() method lest they get the exception
        from the base InstrumentProtocol implementation.
        """
        log.debug("Base driver applying startup params...")
        self._protocol.apply_startup_params()

    def get_cached_config(self):
        """
        Return the configuration object that shows the instrument's
        configuration as cached in the protocol parameter dictionary.
        @retval The running configuration in the instruments config format. By
        default, it is a dictionary of parameter names and values.
        """
        if self._protocol:
            return self._protocol.get_cached_config()

    def get_config_metadata(self):
        """
        Return the configuration metadata object in JSON format
        @retval The description of the parameters, commands, and driver info
        in a JSON string
        @see https://confluence.oceanobservatories.org/display/syseng/
        CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
        """
        log.debug("Getting metadata from driver...")
        protocol = self._protocol

        # Because the config requires information from the protocol param dict
        # we temporarily instantiate a protocol object to get at the static
        # information.
        if not protocol:
            self._build_protocol()

        log.debug("Getting metadata from protocol...")
        return self._protocol.get_config_metadata_dict()

    def restore_direct_access_params(self, config):
        """
        Restore the correct values out of the full config that is given when
        returning from direct access. By default, this takes a simple dict of
        param name and value. Override this class as needed as it makes some
        simple assumptions about how your instrument sets things.

        @param config The configuration that was previously saved (presumably
        to disk somewhere by the driver that is working with this protocol)
        """
        vals = {}
        # for each parameter that is read only, restore
        da_params = self._protocol.get_direct_access_params()
        for param in da_params:
            vals[param] = config[param]

        log.debug("Restore DA Parameters: %r", vals)
        self.set_resource(vals, True)

    #############################################################
    # Command and control interface.
    #############################################################

    def discover_state(self, *args, **kwargs):
        """
        Determine initial state upon establishing communications.
        @param timeout=timeout Optional command timeout.
        @retval Current device state.
        @raises InstrumentTimeoutException if could not wake device.
        @raises InstrumentStateException if command not allowed in current state or if
        device state not recognized.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(
            DriverEvent.DISCOVER, DriverEvent.DISCOVER, *args, **kwargs)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise return all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.
        """

        if self._protocol:
            return self._protocol.get_resource_capabilities(current_state)

        else:
            return [[], []]

    def get_resource_state(self, *args, **kwargs):
        """
        Return the current state of the driver.
        @retval str current driver state.
        @raises NotImplementedException if not implemented by subclass.
        """
        connection_state = self._connection_fsm.get_current_state()
        if connection_state == DriverConnectionState.CONNECTED:
            return self._protocol.get_current_state()
        else:
            return connection_state

    def get_resource(self, *args, **kwargs):
        """
        Retrieve device parameters.
        @param args[0] DriverParameter.ALL or a list of parameters to retrieve.
        @retval parameter : value dict.
        @raises InstrumentParameterException if missing or invalid get parameters.
        @raises InstrumentStateException if command not allowed in current state
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.GET, DriverEvent.GET, *args, **kwargs)

    def set_resource(self, *args, **kwargs):
        """
        Set device parameters.
        @param args[0] parameter : value dict of parameters to set.
        @param timeout=timeout Optional command timeout.
        @raises InstrumentParameterException if missing or invalid set parameters.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if set command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.SET, DriverEvent.SET, *args, **kwargs)

    def execute_resource(self, resource_cmd, *args, **kwargs):
        """
        Poll for a sample.
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.EXECUTE, resource_cmd, *args, **kwargs)

    def start_direct(self, *args, **kwargs):
        """
        start direct access mode
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Need to pass the event as a parameter because the event handler to capture the current
        # pre-da config requires it.
        return self._connection_fsm.on_event(DriverEvent.START_DIRECT, DriverEvent.START_DIRECT)

    def execute_direct(self, *args, **kwargs):
        """
        execute direct access command
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        return self.execute_resource(DriverEvent.EXECUTE_DIRECT, *args, **kwargs)

    def stop_direct(self, *args, **kwargs):
        """
        stop direct access mode
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        return self._connection_fsm.on_event(DriverEvent.STOP_DIRECT, DriverEvent.STOP_DIRECT)

    def test_force_state(self, *args, **kwargs):
        """
        Force driver into a given state for the purposes of unit testing
        @param state=desired_state Required desired state to change to.
        @raises InstrumentParameterException if no state parameter.
        @raises TestModeException if not in test mode
        """

        if not self._test_mode:
            raise TestModeException()

        # Get the required param
        state = kwargs.get('state', None)  # via kwargs
        if state is None:
            raise InstrumentParameterException('Missing state parameter.')

        # We are mucking with internal FSM parameters which may be bad.
        # The alternative was to raise an event to change the state. Don't
        # know which is better.
        self._protocol._protocol_fsm.current_state = state

    ########################################################################
    # Unconfigured handlers.
    ########################################################################

    def _handler_unconfigured_enter(self, *args, **kwargs):
        """
        Enter unconfigured state.
        """
        # Send state change event to agent.
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)
        # attempt to auto-configure from consul
        self._auto_config_with_backoff()

    def _handler_unconfigured_exit(self, *args, **kwargs):
        """
        Exit unconfigured state.
        """
        pass

    def _handler_unconfigured_initialize(self, *args, **kwargs):
        """
        Initialize handler. We are already in unconfigured state, do nothing.
        @retval (next_state, result) tuple, (None, None).
        """
        return None, None

    def _handler_unconfigured_configure(self, *args, **kwargs):
        """
        Configure driver for device comms.
        @param args[0] Communications config dictionary.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None) if successful, (None, None) otherwise.
        @raises InstrumentParameterException if missing or invalid param dict.
        """
        # Verify configuration dict, and update connection if possible.
        try:
            self._connection = self._build_connection(*args, **kwargs)
        except InstrumentException:
            self._auto_config_with_backoff()
            raise

        return DriverConnectionState.DISCONNECTED, None

    ########################################################################
    # Disconnected handlers.
    ########################################################################

    def _handler_disconnected_enter(self, *args, **kwargs):
        """
        Enter disconnected state.
        """
        # Send state change event to agent.
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)

        if self._autoconnect:
            self._async_raise_event(DriverEvent.CONNECT, *args, **kwargs)

    def _handler_disconnected_exit(self, *args, **kwargs):
        """
        Exit disconnected state.
        """
        pass

    def _handler_disconnected_initialize(self, *args, **kwargs):
        """
        Initialize device communications. Causes the connection parameters to
        be reset.
        @retval (next_state, result) tuple, (DriverConnectionState.UNCONFIGURED,
        None).
        """
        self._connection = None
        return DriverConnectionState.UNCONFIGURED, None

    def _handler_disconnected_configure(self, *args, **kwargs):
        """
        Configure driver for device comms.
        @param args[0] Communications config dictionary.
        @retval (next_state, result) tuple, (None, None).
        @raises InstrumentParameterException if missing or invalid param dict.
        """
        # Verify configuration dict, and update connection if possible.
        self._connection = self._build_connection(*args, **kwargs)
        return DriverConnectionState.UNCONFIGURED, None

    def _handler_disconnected_connect(self, *args, **kwargs):
        """
        Establish communications with the device via port agent / logger and
        construct and initialize a protocol FSM for device interaction.
        @retval (next_state, result) tuple, (DriverConnectionState.CONNECTED,
        None) if successful.
        @raises InstrumentConnectionException if the attempt to connect failed.
        """
        result = None
        self._build_protocol()
        try:
            self._connection.init_comms(self._protocol.got_data,
                                        self._protocol.got_raw,
                                        self._got_config,
                                        self._got_exception,
                                        self._lost_connection_callback)
            self._protocol._connection = self._connection
            next_state = DriverConnectionState.CONNECTED
        except InstrumentConnectionException as e:
            log.error("Connection Exception: %s", e)
            log.error("Instrument Driver returning to unconfigured state.")
            next_state = DriverConnectionState.UNCONFIGURED

        return next_state, result

    ########################################################################
    # Connected handlers.
    ########################################################################

    def _handler_connected_enter(self, *args, **kwargs):
        """
        Enter connected state.
        """
        # Send state change event to agent.
        self._connection_lost = False
        # reset the reconnection interval to 1
        self._reconnect_interval = STARTING_RECONNECT_INTERVAL
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)

    def _handler_connected_exit(self, *args, **kwargs):
        """
        Exit connected state.
        """
        pass

    def _handler_connected_disconnect(self, *args, **kwargs):
        """
        Disconnect to the device via port agent / logger and destroy the
        protocol FSM.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None) if successful.
        """
        log.info("_handler_connected_disconnect: invoking stop_comms().")
        self._connection.stop_comms()

        scheduler = self._protocol._scheduler
        if scheduler:
            scheduler._scheduler.shutdown()
        scheduler = None
        self._protocol = None

        return DriverConnectionState.UNCONFIGURED, None

    def _handler_connected_connection_lost(self, *args, **kwargs):
        """
        The device connection was lost. Stop comms, destroy protocol FSM and
        revert to disconnected state.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None).
        """
        log.info("_handler_connected_connection_lost: invoking stop_comms().")
        self._connection.stop_comms()

        scheduler = self._protocol._scheduler
        if scheduler:
            scheduler._scheduler.shutdown()
        scheduler = None
        self._protocol = None

        # Send async agent state change event.
        log.info("_handler_connected_connection_lost: sending LOST_CONNECTION "
                 "event, moving to UNCONFIGURED state.")
        self._driver_event(DriverAsyncEvent.AGENT_EVENT,
                           ResourceAgentEvent.LOST_CONNECTION)

        return DriverConnectionState.UNCONFIGURED, None

    def _handler_connected_protocol_event(self, event, *args, **kwargs):
        """
        Forward a driver command event to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        return next_state, result

    def _handler_connected_start_direct_event(self, event, *args, **kwargs):
        """
        Stash the current config first, then forward a driver command event
        to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None

        # Get the value for all direct access parameters and store them in the protocol
        self._pre_da_config = self.get_resource(self._protocol.get_direct_access_params())
        self._protocol.store_direct_access_config(self._pre_da_config)
        self._protocol.enable_da_initialization()
        log.debug("starting DA.  Storing DA parameters for restore: %s", self._pre_da_config)

        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        return next_state, result

    def _handler_connected_stop_direct_event(self, event, *args, **kwargs):
        """
        Restore previous config first, then forward a driver command event
        to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)

        # Moving the responsibility for applying DA parameters to the
        # protocol.
        # self.restore_direct_access_params(self._pre_da_config)

        return next_state, result

    ########################################################################
    # Helpers.
    ########################################################################

    def _build_connection(self, *args, **kwargs):
        """
        Constructs and returns a Connection object according to the given
        configuration. The connection object is a LoggerClient instance in
        this base class. Subclasses can overwrite this operation as needed.
        The value returned by this operation is assigned to self._connection
        and also to self._protocol._connection upon entering in the
        DriverConnectionState.CONNECTED state.

        @param config configuration dict

        @retval a Connection instance, which will be assigned to
                  self._connection

        @throws InstrumentParameterException Invalid configuration.
        """
        # Get required config param dict.
        config = kwargs.get('config', None)  # via kwargs
        if config is None and len(args) > 0:
            config = args[0]  # via first argument

        if config is None:
            config = self._get_config_from_consul(self.refdes)

        if config is None:
            raise InstrumentParameterException('No port agent config supplied and failed to auto-discover port agent')

        if 'mock_port_agent' in config:
            mock_port_agent = config['mock_port_agent']
            # check for validity here...
            if mock_port_agent is not None:
                return mock_port_agent

        try:
            addr = config['addr']
            port = config['port']
            cmd_port = config.get('cmd_port')

            if isinstance(addr, basestring) and isinstance(port, int) and len(addr) > 0:
                return PortAgentClient(addr, port, cmd_port)
            else:
                raise InstrumentParameterException('Invalid comms config dict.')

        except (TypeError, KeyError):
            raise InstrumentParameterException('Invalid comms config dict.')

    def _get_config_from_consul(self, tag):
        """
        Query consul for the port agent service
        configuration parameters: data port, command port, and address
        This will retry a specified number of times with exponential backoff.
        """
        try:
            data_port = self.consul.health.service('port-agent', passing=True, tag=tag)
            cmd_port = self.consul.health.service('command-port-agent', passing=True, tag=tag)

            if data_port and cmd_port:
                port = data_port[0]['Service']['Port']
                addr = data_port[0]['Node']['Address']
                cmd_port = cmd_port[0]['Service']['Port']
                port_agent_config = {'port': port, 'cmd_port': cmd_port, 'addr': addr}
                return port_agent_config
        except ConnectionError:
            return None

    def _got_exception(self, exception):
        """
        Callback for the client for exception handling with async data.  Exceptions
        are wrapped in an event and sent up to the agent.
        """
        try:
            log.error("ASYNC Data Exception Detected: %s (%s)",
                      exception.__class__.__name__, str(exception))
        finally:
            self._driver_event(DriverAsyncEvent.ERROR, exception)

    def _got_config(self, port_agent_packet):
        data = port_agent_packet.get_data()

        configuration = {}

        for each in data.split('\n'):
            if each == '':
                continue

            key, value = each.split(None, 1)
            try:
                value = int(value)
            except ValueError:
                pass
            configuration[key] = value

        self._driver_event(DriverAsyncEvent.DRIVER_CONFIG, configuration)

    def _lost_connection_callback(self, error_string):
        """
        A callback invoked by the port agent client when it looses
        connectivity to the port agent.
        """

        if not self._connection_lost:
            log.info("_lost_connection_callback: starting thread to send "
                     "CONNECTION_LOST event to instrument driver.")
            self._connection_lost = True
            self._async_raise_event(DriverEvent.CONNECTION_LOST)
        else:
            log.info("_lost_connection_callback: connection_lost flag true.")

    def _build_protocol(self):
        """
        Construct device specific single connection protocol FSM.
        Overridden in device specific subclasses.
        """
        pass

    def _auto_config_with_backoff(self):
        # randomness to prevent all instrument drivers from trying to reconnect at the same exact time.
        self._reconnect_interval = self._reconnect_interval * 2 + random.uniform(-.5, .5)
        interval = min(self._reconnect_interval, self._max_reconnect_interval)
        self._async_raise_event(DriverEvent.CONFIGURE, event_delay=interval)
        log.info('Created delayed CONFIGURE event with %.2f second delay', interval)

    def _async_raise_event(self, event, *args, **kwargs):
        delay = kwargs.pop('event_delay', 0)

        def inner():
            try:
                time.sleep(delay)
                log.info('Async raise event: %r', event)
                self._connection_fsm.on_event(event)
            except Exception as exc:
                log.exception('Exception in asynchronous thread: %r', exc)
                self._driver_event(DriverAsyncEvent.ERROR, exc)
            log.info('_async_raise_fsm_event: event complete. bub bye thread. (%r)', args)

        thread = Thread(target=inner)
        thread.start()
class SingleConnectionInstrumentDriver(InstrumentDriver):
    """
    Base class for instrument drivers with a single device connection.
    Provides connenction state logic for single connection drivers. This is
    the base class for the majority of driver implementation classes.
    """
    
    def __init__(self, event_callback):
        """
        Constructor for singly connected instrument drivers.
        @param event_callback Callback to the driver process to send asynchronous
        driver events back to the agent.
        """
        InstrumentDriver.__init__(self, event_callback)
        
        # The only and only instrument connection.
        # Exists in the connected state.
        self._connection = None

        # The one and only instrument protocol.
        self._protocol = None
        
        # Build connection state machine.
        self._connection_fsm = ThreadSafeFSM(DriverConnectionState,
                                                DriverEvent,
                                                DriverEvent.ENTER,
                                                DriverEvent.EXIT)
        
        # Add handlers for all events.
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.ENTER, self._handler_unconfigured_enter)
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.EXIT, self._handler_unconfigured_exit)
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.INITIALIZE, self._handler_unconfigured_initialize)
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.CONFIGURE, self._handler_unconfigured_configure)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.ENTER, self._handler_disconnected_enter)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.EXIT, self._handler_disconnected_exit)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.INITIALIZE, self._handler_disconnected_initialize)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.CONFIGURE, self._handler_disconnected_configure)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.CONNECT, self._handler_disconnected_connect)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.ENTER, self._handler_connected_enter)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.EXIT, self._handler_connected_exit)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.DISCONNECT, self._handler_connected_disconnect)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.CONNECTION_LOST, self._handler_connected_connection_lost)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.DISCOVER, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.GET, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.SET, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.EXECUTE, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.FORCE_STATE, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.START_DIRECT, self._handler_connected_start_direct_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.STOP_DIRECT, self._handler_connected_stop_direct_event)
        
            
        # Start state machine.
        self._connection_fsm.start(DriverConnectionState.UNCONFIGURED)
        
        self._pre_da_config = {}
        self._startup_config = {}
        
        # Idempotency flag for lost connections.
        # This set to false when a connection is established to
        # allow for lost callback to become activated.
        self._connection_lost = True
        
    #############################################################
    # Device connection interface.
    #############################################################

    def initialize(self, *args, **kwargs):
        """
        Initialize driver connection, bringing communications parameters
        into unconfigured state (no connection object).
        @raises InstrumentStateException if command not allowed in current state        
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.INITIALIZE, *args, **kwargs)
        
    def configure(self, *args, **kwargs):
        """
        Configure the driver for communications with the device via
        port agent / logger (valid but unconnected connection object).
        @param arg[0] comms config dict.
        @raises InstrumentStateException if command not allowed in current state        
        @throws InstrumentParameterException if missing comms or invalid config dict.
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.CONFIGURE, *args, **kwargs)
        
    def connect(self, *args, **kwargs):
        """
        Establish communications with the device via port agent / logger
        (connected connection object).
        @raises InstrumentStateException if command not allowed in current state
        @throws InstrumentConnectionException if the connection failed.
        """
        # Forward event and argument to the connection FSM.
        result = self._connection_fsm.on_event(DriverEvent.CONNECT, *args, **kwargs)
        init_config = {}
        if len(args) > 0 and isinstance(args[0], dict):
            init_config = args[0]

        self.set_init_params(init_config)
        return result
    
    def disconnect(self, *args, **kwargs):
        """
        Disconnect from device via port agent / logger.
        @raises InstrumentStateException if command not allowed in current state
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.DISCONNECT, *args, **kwargs)

    #############################################################
    # Configuration logic
    #############################################################
    def get_init_params(self):
        """
        get the driver initialization parameters
        @return driver configuration dictionary
        """
        return self._startup_config

    def set_init_params(self, config):
        """
        Set the initialization parameters down in the protocol and store the
        driver configuration in the driver.

        If the protocol hasn't been setup yet cache the config.  Next time
        this method is called, if you call it with an empty config it will
        read from the cache.

        @param config This default configuration assumes a structure driver
        configuration dict with keys named in DriverConfigKey.
        Stranger parameters can be adjusted by over riding this method.
        @raise InstrumentParameterException If the config cannot be applied
        """
        if not isinstance(config, dict):
            raise InstrumentParameterException("Incompatible initialization parameters")

        if(self._protocol):
            param_config = None
            if(len(config)):
                param_config = config
            elif(len(self._startup_config)):
                param_config = self._startup_config

            if(param_config):
                self._protocol.set_init_params(param_config)
                self._protocol.initialize_scheduler()
                
        self._startup_config = config
    
    def apply_startup_params(self):
        """
        Apply the startup values previously stored in the protocol to
        the running config of the live instrument. The startup values are the
        values that are (1) marked as startup parameters and are (2) the "best"
        value to use at startup. Preference is given to the previously-set init
        value, then the default value, then the currently used value.

        This default implementation simply pushes the logic down into the protocol
        for processing should the action be better accomplished down there.
        
        The driver writer can decide to overload this method in the derived
        driver class and apply startup parameters in the driver (likely calling
        some get and set methods for the resource). If the driver does not
        implement an apply_startup_params() method in the driver, this method
        will call into the protocol. Deriving protocol classes are expected to
        implement an apply_startup_params() method lest they get the exception
        from the base InstrumentProtocol implementation.
        """
        log.debug("Base driver applying startup params...")
        self._protocol.apply_startup_params()
        
    def get_cached_config(self):
        """
        Return the configuration object that shows the instrument's
        configuration as cached in the protocol parameter dictionary.
        @retval The running configuration in the instruments config format. By
        default, it is a dictionary of parameter names and values.
        """
        if self._protocol:
            return self._protocol.get_cached_config()
                
    def restore_direct_access_params(self, config):
        """
        Restore the correct values out of the full config that is given when
        returning from direct access. By default, this takes a simple dict of
        param name and value. Override this class as needed as it makes some
        simple assumptions about how your instrument sets things.
        
        @param config The configuration that was previously saved (presumably
        to disk somewhere by the driver that is working with this protocol)
        """
        vals = {}
        # for each parameter that is read only, restore
        da_params = self._protocol.get_direct_access_params()        
        for param in da_params:
            vals[param] = config[param]

        log.debug("Restore DA Parameters: %s" % vals)
        self.set_resource(vals, True)
        
    #############################################################
    # Commande and control interface.
    #############################################################

    def discover_state(self, *args, **kwargs):
        """
        Determine initial state upon establishing communications.
        @param timeout=timeout Optional command timeout.        
        @retval Current device state.
        @raises InstrumentTimeoutException if could not wake device.
        @raises InstrumentStateException if command not allowed in current state or if
        device state not recognized.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.DISCOVER, DriverEvent.DISCOVER, *args, **kwargs)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise reutrn all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.        
        """

        if self._protocol:
            return self._protocol.get_resource_capabilities(current_state)
        
        else:
            return [[], []]

                
    def get_resource_state(self, *args, **kwargs):
        """
        Return the current state of the driver.
        @retval str current driver state.
        @raises NotImplementedException if not implemented by subclass.        
        """
        connection_state = self._connection_fsm.get_current_state()
        if connection_state == DriverConnectionState.CONNECTED:
            return self._protocol.get_current_state()
        else:
            return connection_state

    def get_resource(self, *args, **kwargs):
        """
        Retrieve device parameters.
        @param args[0] DriverParameter.ALL or a list of parameters to retrive.
        @retval parameter : value dict.
        @raises InstrumentParameterException if missing or invalid get parameters.
        @raises InstrumentStateException if command not allowed in current state
        @raises NotImplementedException if not implemented by subclass.                        
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.GET, DriverEvent.GET, *args, **kwargs)

    def set_resource(self, *args, **kwargs):
        """
        Set device parameters.
        @param args[0] parameter : value dict of parameters to set.
        @param timeout=timeout Optional command timeout.
        @raises InstrumentParameterException if missing or invalid set parameters.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if set command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.                        
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.SET, DriverEvent.SET, *args, **kwargs)

    def execute_resource(self, resource_cmd, *args, **kwargs):
        """
        Poll for a sample.
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.EXECUTE, resource_cmd, *args, **kwargs)

    def start_direct(self, *args, **kwargs):
        """
        start direct access mode
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Need to pass the event as a parameter because the event handler to capture the current
        # pre-da config requires it.
        return self._connection_fsm.on_event(DriverEvent.START_DIRECT, DriverEvent.START_DIRECT)

    def execute_direct(self, *args, **kwargs):
        """
        execute direct accesscommand
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        return self.execute_resource(DriverEvent.EXECUTE_DIRECT, *args, **kwargs)

    def stop_direct(self, *args, **kwargs):
        """
        stop direct access mode
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        return self._connection_fsm.on_event(DriverEvent.STOP_DIRECT, DriverEvent.STOP_DIRECT)

    def test_force_state(self, *args, **kwargs):
        """
        Force driver into a given state for the purposes of unit testing 
        @param state=desired_state Required desired state to change to.
        @raises InstrumentParameterException if no state parameter.
        @raises TestModeException if not in test mode
        """

        if(not self._test_mode):
            raise TestModeException();

       # Get the required param 
        state = kwargs.get('state', None)  # via kwargs
        if state is None:
            raise InstrumentParameterException('Missing state parameter.')

        # We are mucking with internal FSM parameters which may be bad.
        # The alternative was to raise an event to change the state.  Dont
        # know which is better.
        self._protocol._protocol_fsm.current_state = state

    ########################################################################
    # Unconfigured handlers.
    ########################################################################

    def _handler_unconfigured_enter(self, *args, **kwargs):
        """
        Enter unconfigured state.
        """
        # Send state change event to agent.
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)
    
    def _handler_unconfigured_exit(self, *args, **kwargs):
        """
        Exit unconfigured state.
        """
        pass

    def _handler_unconfigured_initialize(self, *args, **kwargs):
        """
        Initialize handler. We are already in unconfigured state, do nothing.
        @retval (next_state, result) tuple, (None, None).
        """
        next_state = None
        result = None
        
        return (next_state, result)

    def _handler_unconfigured_configure(self, *args, **kwargs):
        """
        Configure driver for device comms.
        @param args[0] Communiations config dictionary.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None) if successful, (None, None) otherwise.
        @raises InstrumentParameterException if missing or invalid param dict.
        """
        next_state = None
        result = None

        # Get the required param dict.
        config = kwargs.get('config', None)  # via kwargs
        # TODO use kwargs as the only mechanism
        if config is None:
            try:
                config = args[0]  # via first argument
            except IndexError:
                pass

        if config is None:
            raise InstrumentParameterException('Missing comms config parameter.')

        # Verify dict and construct connection client.
        self._connection = self._build_connection(config)
        next_state = DriverConnectionState.DISCONNECTED

        return (next_state, result)

    ########################################################################
    # Disconnected handlers.
    ########################################################################

    def _handler_disconnected_enter(self, *args, **kwargs):
        """
        Enter disconnected state.
        """
        # Send state change event to agent.
        self._connection_lost = True
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)

    def _handler_disconnected_exit(self, *args, **kwargs):
        """
        Exit disconnected state.
        """
        pass

    def _handler_disconnected_initialize(self, *args, **kwargs):
        """
        Initialize device communications. Causes the connection parameters to
        be reset.
        @retval (next_state, result) tuple, (DriverConnectionState.UNCONFIGURED,
        None).
        """
        next_state = None
        result = None
        
        self._connection = None
        next_state = DriverConnectionState.UNCONFIGURED
        
        return (next_state, result)

    def _handler_disconnected_configure(self, *args, **kwargs):
        """
        Configure driver for device comms.
        @param args[0] Communiations config dictionary.
        @retval (next_state, result) tuple, (None, None).
        @raises InstrumentParameterException if missing or invalid param dict.
        """
        next_state = None
        result = None

        # Get required config param dict.
        config = kwargs.get('config', None)  # via kwargs
        # TODO use kwargs as the only mechanism
        if config is None:
            try:
                config = args[0]  # via first argument
            except IndexError:
                pass

        if config is None:
            raise InstrumentParameterException('Missing comms config parameter.')

        # Verify configuration dict, and update connection if possible.
        self._connection = self._build_connection(config)

        return (next_state, result)

    def _handler_disconnected_connect(self, *args, **kwargs):
        """
        Establish communications with the device via port agent / logger and
        construct and intialize a protocol FSM for device interaction.
        @retval (next_state, result) tuple, (DriverConnectionState.CONNECTED,
        None) if successful.
        @raises InstrumentConnectionException if the attempt to connect failed.
        """
        next_state = None
        result = None
        
        self._build_protocol()
        self._connection.init_comms(self._protocol.got_data, 
                                    self._protocol.got_raw,
                                    self._lost_connection_callback)
        self._protocol._connection = self._connection
        next_state = DriverConnectionState.CONNECTED
        
        return (next_state, result)

    ########################################################################
    # Connected handlers.
    ########################################################################

    def _handler_connected_enter(self, *args, **kwargs):
        """
        Enter connected state.
        """
        # Send state change event to agent.
        self._connection_lost = False
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)

    def _handler_connected_exit(self, *args, **kwargs):
        """
        Exit connected state.
        """
        pass

    def _handler_connected_disconnect(self, *args, **kwargs):
        """
        Disconnect to the device via port agent / logger and destroy the
        protocol FSM.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None) if successful.
        """
        next_state = None
        result = None
        
        self._connection.stop_comms()
        self._protocol = None
        next_state = DriverConnectionState.DISCONNECTED
        
        return (next_state, result)

    def _handler_connected_connection_lost(self, *args, **kwargs):
        """
        The device connection was lost. Stop comms, destroy protocol FSM and
        revert to disconnected state.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None).
        """
        next_state = None
        result = None
        
        self._connection.stop_comms()
        self._protocol = None
        
        # Send async agent state change event.
        self._driver_event(DriverAsyncEvent.AGENT_EVENT,
                           ResourceAgentEvent.LOST_CONNECTION)
         
        next_state = DriverConnectionState.DISCONNECTED
        
        return (next_state, result)

    def _handler_connected_protocol_event(self, event, *args, **kwargs):
        """
        Forward a driver command event to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        return (next_state, result)

    def _handler_connected_start_direct_event(self, event, *args, **kwargs):
        """
        Stash the current config first, then forward a driver command event
        to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        self._pre_da_config = self.get_resource(DriverParameter.ALL)

        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        return (next_state, result)
    
    def _handler_connected_stop_direct_event(self, event, *args, **kwargs):
        """
        Restore previous config first, then forward a driver command event
        to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        self.restore_direct_access_params(self._pre_da_config)
        return (next_state, result)

    ########################################################################
    # Helpers.
    ########################################################################
    
    def _build_connection(self, config):
        """
        Constructs and returns a Connection object according to the given
        configuration. The connection object is a LoggerClient instance in
        this base class. Subclasses can overwrite this operation as needed.
        The value returned by this operation is assigned to self._connection
        and also to self._protocol._connection upon entering in the
        DriverConnectionState.CONNECTED state.

        @param config configuration dict

        @retval a Connection instance, which will be assigned to
                  self._connection

        @throws InstrumentParameterException Invalid configuration.
        """
        if 'mock_port_agent' in config:
            mock_port_agent = config['mock_port_agent']
            # check for validity here...
            if (mock_port_agent is not None):
                return mock_port_agent
        try:
            addr = config['addr']
            port = config['port']
            cmd_port = config.get('cmd_port')

            if isinstance(addr, str) and isinstance(port, int) and len(addr)>0:
                return PortAgentClient(addr, port, cmd_port)
            else:
                raise InstrumentParameterException('Invalid comms config dict.')

        except (TypeError, KeyError):
            raise InstrumentParameterException('Invalid comms config dict.')

    def _lost_connection_callback(self, error_string):
        """
        A callback invoked by the port agent client when it looses
        connectivity to the port agent.
        """
        
        if not self._connection_lost:
            self._connection_lost = True
            lost_comms_thread = Thread(
                target=self._connection_fsm.on_event,
                args=(DriverEvent.CONNECTION_LOST, ))
            lost_comms_thread.start()
            
    def _build_protocol(self):
        """
        Construct device specific single connection protocol FSM.
        Overridden in device specific subclasses.
        """
        pass
class SingleConnectionInstrumentDriver(InstrumentDriver):
    """
    Base class for instrument drivers with a single device connection.
    Provides connenction state logic for single connection drivers. This is
    the base class for the majority of driver implementation classes.
    """
    
    def __init__(self, event_callback):
        """
        Constructor for singly connected instrument drivers.
        @param event_callback Callback to the driver process to send asynchronous
        driver events back to the agent.
        """
        InstrumentDriver.__init__(self, event_callback)
        
        # The only and only instrument connection.
        # Exists in the connected state.
        self._connection = None

        # The one and only instrument protocol.
        self._protocol = None
        
        # Build connection state machine.
        self._connection_fsm = ThreadSafeFSM(DriverConnectionState,
                                                DriverEvent,
                                                DriverEvent.ENTER,
                                                DriverEvent.EXIT)
        
        # Add handlers for all events.
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.ENTER, self._handler_unconfigured_enter)
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.EXIT, self._handler_unconfigured_exit)
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.INITIALIZE, self._handler_unconfigured_initialize)
        self._connection_fsm.add_handler(DriverConnectionState.UNCONFIGURED, DriverEvent.CONFIGURE, self._handler_unconfigured_configure)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.ENTER, self._handler_disconnected_enter)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.EXIT, self._handler_disconnected_exit)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.INITIALIZE, self._handler_disconnected_initialize)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.CONFIGURE, self._handler_disconnected_configure)
        self._connection_fsm.add_handler(DriverConnectionState.DISCONNECTED, DriverEvent.CONNECT, self._handler_disconnected_connect)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.ENTER, self._handler_connected_enter)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.EXIT, self._handler_connected_exit)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.DISCONNECT, self._handler_connected_disconnect)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.CONNECTION_LOST, self._handler_connected_connection_lost)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.DISCOVER, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.GET, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.SET, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.EXECUTE, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.FORCE_STATE, self._handler_connected_protocol_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.START_DIRECT, self._handler_connected_start_direct_event)
        self._connection_fsm.add_handler(DriverConnectionState.CONNECTED, DriverEvent.STOP_DIRECT, self._handler_connected_stop_direct_event)
        
            
        # Start state machine.
        self._connection_fsm.start(DriverConnectionState.UNCONFIGURED)
        
        self._pre_da_config = {}
        self._startup_config = {}
        
        # Idempotency flag for lost connections.
        # This set to false when a connection is established to
        # allow for lost callback to become activated.
        self._connection_lost = True
        
    #############################################################
    # Device connection interface.
    #############################################################

    def initialize(self, *args, **kwargs):
        """
        Initialize driver connection, bringing communications parameters
        into unconfigured state (no connection object).
        @raises InstrumentStateException if command not allowed in current state        
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.INITIALIZE, *args, **kwargs)
        
    def configure(self, *args, **kwargs):
        """
        Configure the driver for communications with the device via
        port agent / logger (valid but unconnected connection object).
        @param arg[0] comms config dict.
        @raises InstrumentStateException if command not allowed in current state        
        @throws InstrumentParameterException if missing comms or invalid config dict.
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.CONFIGURE, *args, **kwargs)
        
    def connect(self, *args, **kwargs):
        """
        Establish communications with the device via port agent / logger
        (connected connection object).
        @raises InstrumentStateException if command not allowed in current state
        @throws InstrumentConnectionException if the connection failed.
        """
        # Forward event and argument to the connection FSM.
        result = self._connection_fsm.on_event(DriverEvent.CONNECT, *args, **kwargs)
        init_config = {}
        if len(args) > 0 and isinstance(args[0], dict):
            init_config = args[0]

        self.set_init_params(init_config)
        return result
    
    def disconnect(self, *args, **kwargs):
        """
        Disconnect from device via port agent / logger.
        @raises InstrumentStateException if command not allowed in current state
        """
        # Forward event and argument to the connection FSM.
        return self._connection_fsm.on_event(DriverEvent.DISCONNECT, *args, **kwargs)

    #############################################################
    # Configuration logic
    #############################################################
    def get_init_params(self):
        """
        get the driver initialization parameters
        @return driver configuration dictionary
        """
        return self._startup_config

    def set_init_params(self, config):
        """
        Set the initialization parameters down in the protocol and store the
        driver configuration in the driver.

        If the protocol hasn't been setup yet cache the config.  Next time
        this method is called, if you call it with an empty config it will
        read from the cache.

        @param config This default configuration assumes a structure driver
        configuration dict with keys named in DriverConfigKey.
        Stranger parameters can be adjusted by over riding this method.
        @raise InstrumentParameterException If the config cannot be applied
        """
        if not isinstance(config, dict):
            raise InstrumentParameterException("Incompatible initialization parameters")

        if self._protocol:
            param_config = None
            if config:
                param_config = config
            elif self._startup_config:
                param_config = self._startup_config

            if param_config:
                self._protocol.set_init_params(param_config)
                self._protocol.initialize_scheduler()

        if config:
            self._startup_config = config
    
    def apply_startup_params(self):
        """
        Apply the startup values previously stored in the protocol to
        the running config of the live instrument. The startup values are the
        values that are (1) marked as startup parameters and are (2) the "best"
        value to use at startup. Preference is given to the previously-set init
        value, then the default value, then the currently used value.

        This default implementation simply pushes the logic down into the protocol
        for processing should the action be better accomplished down there.
        
        The driver writer can decide to overload this method in the derived
        driver class and apply startup parameters in the driver (likely calling
        some get and set methods for the resource). If the driver does not
        implement an apply_startup_params() method in the driver, this method
        will call into the protocol. Deriving protocol classes are expected to
        implement an apply_startup_params() method lest they get the exception
        from the base InstrumentProtocol implementation.
        """
        log.debug("Base driver applying startup params...")
        self._protocol.apply_startup_params()
        
    def get_cached_config(self):
        """
        Return the configuration object that shows the instrument's
        configuration as cached in the protocol parameter dictionary.
        @retval The running configuration in the instruments config format. By
        default, it is a dictionary of parameter names and values.
        """
        if self._protocol:
            return self._protocol.get_cached_config()
                
    def get_config_metadata(self):
        """
        Return the configuration metadata object in JSON format
        @retval The description of the parameters, commands, and driver info
        in a JSON string
        @see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
        """
        log.debug("Getting metadata from driver...")
        protocol = self._protocol

        # Because the config requires information from the protocol param dict
        # we temporarily instantiate a protocol object to get at the static
        # information.
        if not protocol:
            self._build_protocol()

        log.debug("Getting metadata from protocol...")
        return json.dumps(self._protocol.get_config_metadata_dict(),
                          sort_keys=True)
            
    def restore_direct_access_params(self, config):
        """
        Restore the correct values out of the full config that is given when
        returning from direct access. By default, this takes a simple dict of
        param name and value. Override this class as needed as it makes some
        simple assumptions about how your instrument sets things.
        
        @param config The configuration that was previously saved (presumably
        to disk somewhere by the driver that is working with this protocol)
        """
        vals = {}
        # for each parameter that is read only, restore
        da_params = self._protocol.get_direct_access_params()        
        for param in da_params:
            vals[param] = config[param]

        log.debug("Restore DA Parameters: %s" % vals)
        self.set_resource(vals, True)
        
    #############################################################
    # Commande and control interface.
    #############################################################

    def discover_state(self, *args, **kwargs):
        """
        Determine initial state upon establishing communications.
        @param timeout=timeout Optional command timeout.        
        @retval Current device state.
        @raises InstrumentTimeoutException if could not wake device.
        @raises InstrumentStateException if command not allowed in current state or if
        device state not recognized.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.DISCOVER, DriverEvent.DISCOVER, *args, **kwargs)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise reutrn all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.        
        """

        if self._protocol:
            return self._protocol.get_resource_capabilities(current_state)
        
        else:
            return [['foobb'], ['fooaa']]

                
    def get_resource_state(self, *args, **kwargs):
        """
        Return the current state of the driver.
        @retval str current driver state.
        @raises NotImplementedException if not implemented by subclass.        
        """
        connection_state = self._connection_fsm.get_current_state()
        if connection_state == DriverConnectionState.CONNECTED:
            return self._protocol.get_current_state()
        else:
            return connection_state

    def get_resource(self, *args, **kwargs):
        """
        Retrieve device parameters.
        @param args[0] DriverParameter.ALL or a list of parameters to retrive.
        @retval parameter : value dict.
        @raises InstrumentParameterException if missing or invalid get parameters.
        @raises InstrumentStateException if command not allowed in current state
        @raises NotImplementedException if not implemented by subclass.                        
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.GET, DriverEvent.GET, *args, **kwargs)

    def set_resource(self, *args, **kwargs):
        """
        Set device parameters.
        @param args[0] parameter : value dict of parameters to set.
        @param timeout=timeout Optional command timeout.
        @raises InstrumentParameterException if missing or invalid set parameters.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if set command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.                        
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.SET, DriverEvent.SET, *args, **kwargs)

    def execute_resource(self, resource_cmd, *args, **kwargs):
        """
        Poll for a sample.
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Forward event and argument to the protocol FSM.
        return self._connection_fsm.on_event(DriverEvent.EXECUTE, resource_cmd, *args, **kwargs)

    def start_direct(self, *args, **kwargs):
        """
        start direct access mode
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        # Need to pass the event as a parameter because the event handler to capture the current
        # pre-da config requires it.
        return self._connection_fsm.on_event(DriverEvent.START_DIRECT, DriverEvent.START_DIRECT)

    def execute_direct(self, *args, **kwargs):
        """
        execute direct accesscommand
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        return self.execute_resource(DriverEvent.EXECUTE_DIRECT, *args, **kwargs)

    def stop_direct(self, *args, **kwargs):
        """
        stop direct access mode
        @param timeout=timeout Optional command timeout.
        @ retval Device sample dict.
        @raises InstrumentTimeoutException if could not wake device or no response.
        @raises InstrumentProtocolException if acquire command not recognized.
        @raises InstrumentStateException if command not allowed in current state.
        @raises NotImplementedException if not implemented by subclass.
        """
        return self._connection_fsm.on_event(DriverEvent.STOP_DIRECT, DriverEvent.STOP_DIRECT)

    def test_force_state(self, *args, **kwargs):
        """
        Force driver into a given state for the purposes of unit testing 
        @param state=desired_state Required desired state to change to.
        @raises InstrumentParameterException if no state parameter.
        @raises TestModeException if not in test mode
        """

        if(not self._test_mode):
            raise TestModeException();

       # Get the required param 
        state = kwargs.get('state', None)  # via kwargs
        if state is None:
            raise InstrumentParameterException('Missing state parameter.')

        # We are mucking with internal FSM parameters which may be bad.
        # The alternative was to raise an event to change the state.  Dont
        # know which is better.
        self._protocol._protocol_fsm.current_state = state

    ########################################################################
    # Unconfigured handlers.
    ########################################################################

    def _handler_unconfigured_enter(self, *args, **kwargs):
        """
        Enter unconfigured state.
        """
        # Send state change event to agent.
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)
    
    def _handler_unconfigured_exit(self, *args, **kwargs):
        """
        Exit unconfigured state.
        """
        pass

    def _handler_unconfigured_initialize(self, *args, **kwargs):
        """
        Initialize handler. We are already in unconfigured state, do nothing.
        @retval (next_state, result) tuple, (None, None).
        """
        next_state = None
        result = None
        
        return (next_state, result)

    def _handler_unconfigured_configure(self, *args, **kwargs):
        """
        Configure driver for device comms.
        @param args[0] Communiations config dictionary.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None) if successful, (None, None) otherwise.
        @raises InstrumentParameterException if missing or invalid param dict.
        """
        next_state = None
        result = None

        # Get the required param dict.
        config = kwargs.get('config', None)  # via kwargs
        # TODO use kwargs as the only mechanism
        if config is None:
            try:
                config = args[0]  # via first argument
            except IndexError:
                pass

        if config is None:
            raise InstrumentParameterException('Missing comms config parameter.')

        # Verify dict and construct connection client.
        self._connection = self._build_connection(config)
        next_state = DriverConnectionState.DISCONNECTED

        return (next_state, result)

    ########################################################################
    # Disconnected handlers.
    ########################################################################

    def _handler_disconnected_enter(self, *args, **kwargs):
        """
        Enter disconnected state.
        """
        # Send state change event to agent.
        self._connection_lost = True
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)

    def _handler_disconnected_exit(self, *args, **kwargs):
        """
        Exit disconnected state.
        """
        pass

    def _handler_disconnected_initialize(self, *args, **kwargs):
        """
        Initialize device communications. Causes the connection parameters to
        be reset.
        @retval (next_state, result) tuple, (DriverConnectionState.UNCONFIGURED,
        None).
        """
        next_state = None
        result = None
        
        self._connection = None
        next_state = DriverConnectionState.UNCONFIGURED
        
        return (next_state, result)

    def _handler_disconnected_configure(self, *args, **kwargs):
        """
        Configure driver for device comms.
        @param args[0] Communiations config dictionary.
        @retval (next_state, result) tuple, (None, None).
        @raises InstrumentParameterException if missing or invalid param dict.
        """
        next_state = None
        result = None

        # Get required config param dict.
        config = kwargs.get('config', None)  # via kwargs
        # TODO use kwargs as the only mechanism
        if config is None:
            try:
                config = args[0]  # via first argument
            except IndexError:
                pass

        if config is None:
            raise InstrumentParameterException('Missing comms config parameter.')

        # Verify configuration dict, and update connection if possible.
        self._connection = self._build_connection(config)

        return (next_state, result)

    def _handler_disconnected_connect(self, *args, **kwargs):
        """
        Establish communications with the device via port agent / logger and
        construct and intialize a protocol FSM for device interaction.
        @retval (next_state, result) tuple, (DriverConnectionState.CONNECTED,
        None) if successful.
        @raises InstrumentConnectionException if the attempt to connect failed.
        """
        next_state = None
        result = None
        self._build_protocol()
        try:
            self._connection.init_comms(self._protocol.got_data, 
                                        self._protocol.got_raw,
                                        self._got_exception,
                                        self._lost_connection_callback)
            self._protocol._connection = self._connection
            next_state = DriverConnectionState.CONNECTED
        except InstrumentConnectionException as e:
            log.error("Connection Exception: %s", e)
            log.error("Instrument Driver remaining in disconnected state.")
            # Re-raise the exception
            raise
        
        return (next_state, result)

    ########################################################################
    # Connected handlers.
    ########################################################################

    def _handler_connected_enter(self, *args, **kwargs):
        """
        Enter connected state.
        """
        # Send state change event to agent.
        self._connection_lost = False
        self._driver_event(DriverAsyncEvent.STATE_CHANGE)

    def _handler_connected_exit(self, *args, **kwargs):
        """
        Exit connected state.
        """
        pass

    def _handler_connected_disconnect(self, *args, **kwargs):
        """
        Disconnect to the device via port agent / logger and destroy the
        protocol FSM.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None) if successful.
        """
        next_state = None
        result = None
        
        log.info("_handler_connected_disconnect: invoking stop_comms().")
        self._connection.stop_comms()
        self._protocol = None
        next_state = DriverConnectionState.DISCONNECTED
        
        return (next_state, result)

    def _handler_connected_connection_lost(self, *args, **kwargs):
        """
        The device connection was lost. Stop comms, destroy protocol FSM and
        revert to disconnected state.
        @retval (next_state, result) tuple, (DriverConnectionState.DISCONNECTED,
        None).
        """
        next_state = None
        result = None
        
        log.info("_handler_connected_connection_lost: invoking stop_comms().")
        self._connection.stop_comms()
        self._protocol = None
        
        # Send async agent state change event.
        log.info("_handler_connected_connection_lost: sending LOST_CONNECTION " \
                 "event, moving to DISCONNECTED state.")
        self._driver_event(DriverAsyncEvent.AGENT_EVENT,
                           ResourceAgentEvent.LOST_CONNECTION)
         
        next_state = DriverConnectionState.DISCONNECTED
        
        return (next_state, result)

    def _handler_connected_protocol_event(self, event, *args, **kwargs):
        """
        Forward a driver command event to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        return (next_state, result)

    def _handler_connected_start_direct_event(self, event, *args, **kwargs):
        """
        Stash the current config first, then forward a driver command event
        to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None

        # Get the value for all direct access parameters and store them in the protocol
        self._pre_da_config = self.get_resource(self._protocol.get_direct_access_params())
        self._protocol.store_direct_access_config(self._pre_da_config)
        self._protocol.enable_da_initialization()
        log.debug("starting DA.  Storing DA parameters for restore: %s", self._pre_da_config)

        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)
        return (next_state, result)
    
    def _handler_connected_stop_direct_event(self, event, *args, **kwargs):
        """
        Restore previous config first, then forward a driver command event
        to the protocol FSM.
        @param args positional arguments to pass on.
        @param kwargs keyword arguments to pass on.
        @retval (next_state, result) tuple, (None, protocol result).
        """
        next_state = None
        result = self._protocol._protocol_fsm.on_event(event, *args, **kwargs)

        # Moving the responsibility for applying DA parameters to the
        # protocol.
        #self.restore_direct_access_params(self._pre_da_config)

        return (next_state, result)

    ########################################################################
    # Helpers.
    ########################################################################
    
    def _build_connection(self, config):
        """
        Constructs and returns a Connection object according to the given
        configuration. The connection object is a LoggerClient instance in
        this base class. Subclasses can overwrite this operation as needed.
        The value returned by this operation is assigned to self._connection
        and also to self._protocol._connection upon entering in the
        DriverConnectionState.CONNECTED state.

        @param config configuration dict

        @retval a Connection instance, which will be assigned to
                  self._connection

        @throws InstrumentParameterException Invalid configuration.
        """
        if 'mock_port_agent' in config:
            mock_port_agent = config['mock_port_agent']
            # check for validity here...
            if (mock_port_agent is not None):
                return mock_port_agent
        try:
            addr = config['addr']
            port = config['port']
            cmd_port = config.get('cmd_port')

            if isinstance(addr, str) and isinstance(port, int) and len(addr)>0:
                return PortAgentClient(addr, port, cmd_port)
            else:
                raise InstrumentParameterException('Invalid comms config dict.')

        except (TypeError, KeyError):
            raise InstrumentParameterException('Invalid comms config dict.')

    def _got_exception(self, exception):
        """
        Callback for the client for exception handling with async data.  Exceptions
        are wrapped in an event and sent up to the agent.
        """
        try:
            log.error("ASYNC Data Exception Detected: %s (%s)", exception.__class__.__name__, str(exception))
        finally:
            self._driver_event(DriverAsyncEvent.ERROR, exception)

    def _lost_connection_callback(self, error_string):
        """
        A callback invoked by the port agent client when it looses
        connectivity to the port agent.
        """
        
        if not self._connection_lost:
            log.info("_lost_connection_callback: starting thread to send " \
                     "CONNECTION_LOST event to instrument driver.")
            self._connection_lost = True
            lost_comms_thread = Thread(
                target=self._connection_fsm.on_event,
                args=(DriverEvent.CONNECTION_LOST, ))
            lost_comms_thread.start()
        else:
            log.info("_lost_connection_callback: connection_lost flag true.")
            
            
    def _build_protocol(self):
        """
        Construct device specific single connection protocol FSM.
        Overridden in device specific subclasses.
        """
        pass