Example #1
0
 def __init__(self, sandesh_instance, server, event_handler,
              sandesh_msg_handler):
     self._sandesh_instance = sandesh_instance
     self._logger = sandesh_instance._logger
     self._event_handler = event_handler
     self._reader = SandeshReader(self, sandesh_msg_handler)
     self._writer = SandeshWriter(self)
     self._send_queue = WorkQueue(self._send_sandesh,
                                  self._is_ready_to_send_sandesh)
     TcpSession.__init__(self, server)
Example #2
0
 def init_generator(self, module, source, node_type, instance_id,
                    collectors, client_context,
                    http_port, sandesh_req_uve_pkg_list=None,
                    discovery_client=None, connect_to_collector=True,
                    logger_class=None, logger_config_file=None,
                    host_ip='127.0.0.1', alarm_ack_callback=None):
     self._role = self.SandeshRole.GENERATOR
     self._module = module
     self._source = source
     self._node_type = node_type
     self._instance_id = instance_id
     self._host_ip = host_ip
     self._client_context = client_context
     self._collectors = collectors
     self._connect_to_collector = connect_to_collector
     self._rcv_queue = WorkQueue(self._process_rx_sandesh)
     self._init_logger(module, logger_class=logger_class,
                       logger_config_file=logger_config_file)
     self._logger.info('SANDESH: CONNECT TO COLLECTOR: %s',
                       connect_to_collector)
     self._stats = SandeshStats()
     self._trace = trace.Trace()
     self._sandesh_request_dict = {}
     self._alarm_ack_callback = alarm_ack_callback
     self._uve_type_maps = SandeshUVETypeMaps(self._logger)
     if sandesh_req_uve_pkg_list is None:
         sandesh_req_uve_pkg_list = []
     # Initialize the request handling
     # Import here to break the cyclic import dependency
     import sandesh_req_impl
     sandesh_req_impl = sandesh_req_impl.SandeshReqImpl(self)
     sandesh_req_uve_pkg_list.append('pysandesh.gen_py')
     for pkg_name in sandesh_req_uve_pkg_list:
         self._create_sandesh_request_and_uve_lists(pkg_name)
     self._gev_httpd = None
     if http_port != -1:
         self._http_server = SandeshHttp(
             self, module, http_port, sandesh_req_uve_pkg_list)
         self._gev_httpd = gevent.spawn(self._http_server.start_http_server)
     primary_collector = None
     secondary_collector = None
     if self._collectors is not None:
         if len(self._collectors) > 0:
             primary_collector = self._collectors[0]
         if len(self._collectors) > 1:
             secondary_collector = self._collectors[1]
     if self._connect_to_collector:
         self._client = SandeshClient(
             self, primary_collector, secondary_collector,
             discovery_client)
         self._client.initiate()
Example #3
0
class SandeshSession(TcpSession):
    _KEEPALIVE_IDLE_TIME = 45  # in secs
    _KEEPALIVE_INTERVAL = 3  # in secs
    _KEEPALIVE_PROBES = 5

    def __init__(self, sandesh_instance, server, event_handler,
                 sandesh_msg_handler):
        self._sandesh_instance = sandesh_instance
        self._logger = sandesh_instance._logger
        self._event_handler = event_handler
        self._reader = SandeshReader(self, sandesh_msg_handler)
        self._writer = SandeshWriter(self)
        self._send_queue = WorkQueue(self._send_sandesh,
                                     self._is_ready_to_send_sandesh)
        TcpSession.__init__(self, server)
    # end __init__

    # Public functions

    def sandesh_instance(self):
        return self._sandesh_instance
    # end sandesh_instance

    def is_send_queue_empty(self):
        return self._send_queue.is_queue_empty()
    # end is_send_queue_empty

    def is_connected(self):
        return self._connected
    # end is_connected

    def enqueue_sandesh(self, sandesh):
        self._send_queue.enqueue(sandesh)
    # end enqueue_sandesh

    def send_queue(self):
        return self._send_queue
    # end send_queue

    # Overloaded functions from TcpSession

    def connect(self):
        TcpSession.connect(self, timeout=5)
    # end connect

    def _on_read(self, buf):
        if self._reader.read_msg(buf) < 0:
            self._logger.error('SandeshReader Error. Close Collector session')
            self.close()
    # end _on_read

    def _handle_event(self, event):
        self._event_handler(self, event)
    # end _handle_event

    def _set_socket_options(self):
        self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
        if hasattr(socket, 'TCP_KEEPIDLE'):
            self._socket.setsockopt(
                socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
                self._KEEPALIVE_IDLE_TIME)
        if hasattr(socket, 'TCP_KEEPALIVE'):
            self._socket.setsockopt(
                socket.IPPROTO_TCP, socket.TCP_KEEPALIVE,
                self._KEEPALIVE_IDLE_TIME)
        if hasattr(socket, 'TCP_KEEPINTVL'):
            self._socket.setsockopt(
                socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
                self._KEEPALIVE_INTERVAL)
        if hasattr(socket, 'TCP_KEEPCNT'):
            self._socket.setsockopt(
                socket.IPPROTO_TCP, socket.TCP_KEEPCNT, self._KEEPALIVE_PROBES)
    # end _set_socket_options

    # Private functions

    def _send_sandesh(self, sandesh):
        if self._send_queue.is_queue_empty():
            more = False
        else:
            more = True
        if not self._connected:
            if self._sandesh_instance.is_logging_dropped_allowed(sandesh):
                self._logger.error(
                    "SANDESH: %s: %s" % ("Not connected", sandesh.log()))
            return
        if sandesh.is_logging_allowed(self._sandesh_instance):
            self._logger.log(
                SandeshLogger.get_py_logger_level(sandesh.level()),
                sandesh.log())
        self._writer.send_msg(sandesh, more)
    # end _send_sandesh

    def _is_ready_to_send_sandesh(self):
        return self._sandesh_instance.is_send_queue_enabled()
Example #4
0
class Sandesh(object):
    _DEFAULT_LOG_FILE = sand_logger.SandeshLogger._DEFAULT_LOG_FILE
    _DEFAULT_SYSLOG_FACILITY = (
        sand_logger.SandeshLogger._DEFAULT_SYSLOG_FACILITY)

    class SandeshRole:
        INVALID = 0
        GENERATOR = 1
        COLLECTOR = 2
    # end class SandeshRole

    def __init__(self):
        self._context = ''
        self._scope = ''
        self._module = ''
        self._source = ''
        self._node_type = ''
        self._instance_id = ''
        self._timestamp = 0
        self._versionsig = 0
        self._type = 0
        self._hints = 0
        self._client_context = ''
        self._client = None
        self._role = self.SandeshRole.INVALID
        self._logger = None
        self._level = SandeshLevel.INVALID
        self._category = ''
        self._send_queue_enabled = True
        self._http_server = None
        self._connect_to_collector = True
    # end __init__

    # Public functions

    def init_generator(self, module, source, node_type, instance_id,
                       collectors, client_context,
                       http_port, sandesh_req_uve_pkg_list=None,
                       discovery_client=None, connect_to_collector=True,
                       logger_class=None, logger_config_file=None,
                       host_ip='127.0.0.1', alarm_ack_callback=None):
        self._role = self.SandeshRole.GENERATOR
        self._module = module
        self._source = source
        self._node_type = node_type
        self._instance_id = instance_id
        self._host_ip = host_ip
        self._client_context = client_context
        self._collectors = collectors
        self._connect_to_collector = connect_to_collector
        self._rcv_queue = WorkQueue(self._process_rx_sandesh)
        self._init_logger(module, logger_class=logger_class,
                          logger_config_file=logger_config_file)
        self._logger.info('SANDESH: CONNECT TO COLLECTOR: %s',
                          connect_to_collector)
        self._stats = SandeshStats()
        self._trace = trace.Trace()
        self._sandesh_request_dict = {}
        self._alarm_ack_callback = alarm_ack_callback
        self._uve_type_maps = SandeshUVETypeMaps(self._logger)
        if sandesh_req_uve_pkg_list is None:
            sandesh_req_uve_pkg_list = []
        # Initialize the request handling
        # Import here to break the cyclic import dependency
        import sandesh_req_impl
        sandesh_req_impl = sandesh_req_impl.SandeshReqImpl(self)
        sandesh_req_uve_pkg_list.append('pysandesh.gen_py')
        for pkg_name in sandesh_req_uve_pkg_list:
            self._create_sandesh_request_and_uve_lists(pkg_name)
        self._gev_httpd = None
        if http_port != -1:
            self._http_server = SandeshHttp(
                self, module, http_port, sandesh_req_uve_pkg_list)
            self._gev_httpd = gevent.spawn(self._http_server.start_http_server)
        primary_collector = None
        secondary_collector = None
        if self._collectors is not None:
            if len(self._collectors) > 0:
                primary_collector = self._collectors[0]
            if len(self._collectors) > 1:
                secondary_collector = self._collectors[1]
        if self._connect_to_collector:
            self._client = SandeshClient(
                self, primary_collector, secondary_collector,
                discovery_client)
            self._client.initiate()
    # end init_generator

    def uninit(self):
        self.kill_httpd()

    def kill_httpd(self):
        if self._gev_httpd:
            try:
                self._gev_httpd.kill()
            except Exception as e:
                self._logger.debug(str(e))

    def record_port(self, name, port):
        pipe_name = '/tmp/%s.%d.%s_port' % (self._module, os.getppid(), name)
        try:
            pipeout = os.open(pipe_name, os.O_WRONLY)
        except Exception:
            self._logger.error('Cannot write %s_port %d to %s'
                               % (name, port, pipe_name))
        else:
            self._logger.error('Writing %s_port %d to %s'
                               % (name, port, pipe_name))
            os.write(pipeout, '%d\n' % port)
            os.close(pipeout)

    def logger(self):
        return self._logger
    # end logger

    def sandesh_logger(self):
        return self._sandesh_logger
    # end sandesh_logger

    def set_logging_params(self, enable_local_log=False, category='',
                           level=SandeshLevel.SYS_INFO,
                           file=sand_logger.SandeshLogger._DEFAULT_LOG_FILE,
                           enable_syslog=False,
                           syslog_facility=_DEFAULT_SYSLOG_FACILITY,
                           enable_trace_print=False,
                           enable_flow_log=False):
        self._sandesh_logger.set_logging_params(
            enable_local_log=enable_local_log, category=category,
            level=level, file=file, enable_syslog=enable_syslog,
            syslog_facility=syslog_facility,
            enable_trace_print=enable_trace_print,
            enable_flow_log=enable_flow_log)
    # end set_logging_params

    def set_trace_print(self, enable_trace_print):
        self._sandesh_logger.set_trace_print(enable_trace_print)
    # end set_trace_print

    def set_flow_logging(self, enable_flow_log):
        self._sandesh_logger.set_flow_logging(enable_flow_log)
    # end set_flow_logging

    def set_local_logging(self, enable_local_log):
        self._sandesh_logger.set_local_logging(enable_local_log)
    # end set_local_logging

    def set_logging_level(self, level):
        self._sandesh_logger.set_logging_level(level)
    # end set_logging_level

    def set_logging_category(self, category):
        self._sandesh_logger.set_logging_category(category)
    # end set_logging_category

    def set_logging_file(self, file):
        self._sandesh_logger.set_logging_file(file)
    # end set_logging_file

    def is_logging_dropped_allowed(self, sandesh):
        if sandesh.type() == SandeshType.FLOW:
            return self.is_flow_logging_enabled()
        else:
            return True
    # end is_logging_dropped_allowed

    def is_send_queue_enabled(self):
        return self._send_queue_enabled
    # end is_send_queue_enabled

    def is_connect_to_collector_enabled(self):
        return self._connect_to_collector
    # end is_connect_to_collector_enabled

    def set_send_queue(self, enable):
        if self._send_queue_enabled != enable:
            self._logger.info("SANDESH: CLIENT: SEND QUEUE: %s -> %s",
                              self._send_queue_enabled, enable)
            self._send_queue_enabled = enable
            if enable:
                connection = self._client.connection()
                if connection and connection.session():
                    connection.session().send_queue().may_be_start_runner()
    # end set_send_queue

    def init_collector(self):
        pass
    # end init_collector

    def stats(self):
        return self._stats
    # end stats

    @classmethod
    def next_seqnum(cls):
        if not hasattr(cls, '_lseqnum'):
            cls._lseqnum = 1
        else:
            cls._lseqnum += 1
        return cls._lseqnum
    # end next_seqnum

    @classmethod
    def lseqnum(cls):
        if not hasattr(cls, '_lseqnum'):
            cls._lseqnum = 0
        return cls._lseqnum
    # end lseqnum

    def module(self):
        return self._module
    # end module

    def source_id(self):
        return self._source
    # end source_id

    def node_type(self):
        return self._node_type
    # end node_type

    def instance_id(self):
        return self._instance_id
    # end instance_id

    def host_ip(self):
        return self._host_ip
    # end host_ip

    def scope(self):
        return self._scope
    # end scope

    def context(self):
        return self._context
    # end context

    def seqnum(self):
        return self._seqnum
    # end seqnum

    def timestamp(self):
        return self._timestamp
    # end timestamp

    def versionsig(self):
        return self._versionsig
    # end versionsig

    def type(self):
        return self._type
    # end type

    def hints(self):
        return self._hints
    # end hints

    def client(self):
        return self._client
    # end client

    def level(self):
        return self._level
    # end level

    def category(self):
        return self._category
    # end category

    def validate(self):
        return
    # end validate

    def alarm_ack_callback(self):
        return self._alarm_ack_callback
    # end alarm_ack_callback

    def is_flow_logging_enabled(self):
        return self._sandesh_logger.is_flow_logging_enabled()
    # end is_flow_logging_enabled

    def is_trace_print_enabled(self):
        return self._sandesh_logger.is_trace_print_enabled()
    # end is_trace_print_enabled

    def is_local_logging_enabled(self):
        return self._sandesh_logger.is_local_logging_enabled()
    # end is_local_logging_enabled

    def logging_level(self):
        return self._sandesh_logger.logging_level()
    # end logging_level

    def logging_category(self):
        return self._sandesh_logger.logging_category()
    # end logging_category

    def is_syslog_logging_enabled(self):
        return self._sandesh_logger.is_syslog_logging_enabled()
    # end is_syslog_logging_enabled

    def logging_syslog_facility(self):
        return self._sandesh_logger.logging_syslog_facility()
    # end logging_syslog_facility

    def is_unit_test(self):
        return self._role == self.SandeshRole.INVALID
    # end is_unit_test

    def handle_test(self, sandesh_init):
        if sandesh_init.is_unit_test() or self._is_level_ut():
            if self._is_logging_allowed(sandesh_init):
                sandesh_init._logger.debug(self.log())
                return True
        return False

    def is_logging_allowed(self, sandesh_init):
        if self._type == SandeshType.FLOW:
            return sandesh_init.is_flow_logging_enabled()

        if not sandesh_init.is_local_logging_enabled():
            return False

        logging_level = sandesh_init.logging_level()
        level_allowed = logging_level >= self._level

        logging_category = sandesh_init.logging_category()
        if logging_category is None or len(logging_category) == 0:
            category_allowed = True
        else:
            category_allowed = logging_category == self._category

        return level_allowed and category_allowed
    # end is_logging_allowed

    def enqueue_sandesh_request(self, sandesh):
        self._rcv_queue.enqueue(sandesh)
    # end enqueue_sandesh_request

    def send_sandesh(self, tx_sandesh):
        if self._client:
            self._client.send_sandesh(tx_sandesh)
        else:
            if self._connect_to_collector:
                if self.is_logging_dropped_allowed(tx_sandesh):
                    self._logger.error('SANDESH: No Client: %s',
                                       tx_sandesh.log())
            else:
                self._logger.log(
                    sand_logger.SandeshLogger.get_py_logger_level(
                        tx_sandesh.level()), tx_sandesh.log())
    # end send_sandesh

    def send_generator_info(self):
        from gen_py.sandesh_uve.ttypes import SandeshClientInfo, \
            ModuleClientState, SandeshModuleClientTrace
        client_info = SandeshClientInfo()
        try:
            client_start_time = self._start_time
        except Exception:
            self._start_time = util.UTCTimestampUsec()
        finally:
            client_info.start_time = self._start_time
            client_info.pid = os.getpid()
            if self._http_server is not None:
                client_info.http_port = self._http_server.get_port()
            client_info.collector_name = self._client.connection().collector()
            client_info.status = self._client.connection().state()
            client_info.successful_connections = (
                self._client.connection().statemachine().connect_count())
            client_info.primary = self._client.connection().primary_collector()
            if client_info.primary is None:
                client_info.primary = ''
            client_info.secondary = (
                self._client.connection().secondary_collector())
            if client_info.secondary is None:
                client_info.secondary = ''
            module_state = ModuleClientState(name=self._source + ':' +
                                             self._node_type + ':' +
                                             self._module + ':' +
                                             self._instance_id,
                                             client_info=client_info)
            generator_info = SandeshModuleClientTrace(
                data=module_state, sandesh=self)
            generator_info.send(sandesh=self)
    # end send_generator_info

    def get_sandesh_request_object(self, request):
        try:
            req_module = self._sandesh_request_dict[request]
        except KeyError:
            self._logger.error('Invalid Sandesh Request "%s"' % (request))
            return None
        else:
            if req_module:
                try:
                    imp_module = importlib.import_module(req_module)
                except ImportError:
                    self._logger.error(
                        'Failed to import Module "%s"' % (req_module))
                else:
                    try:
                        sandesh_request = getattr(imp_module, request)()
                        return sandesh_request
                    except AttributeError:
                        self._logger.error(
                            'Failed to create Sandesh Request "%s"' %
                            (request))
                        return None
            else:
                self._logger.error(
                    'Sandesh Request "%s" not implemented' % (request))
                return None
    # end get_sandesh_request_object

    def trace_enable(self):
        self._trace.TraceOn()
    # end trace_enable

    def trace_disable(self):
        self._trace.TraceOff()
    # end trace_disable

    def is_trace_enabled(self):
        return self._trace.IsTraceOn()
    # end is_trace_enabled

    def trace_buffer_create(self, name, size, enable=True):
        self._trace.TraceBufAdd(name, size, enable)
    # end trace_buffer_create

    def trace_buffer_delete(self, name):
        self._trace.TraceBufDelete(name)
    # end trace_buffer_delete

    def trace_buffer_enable(self, name):
        self._trace.TraceBufOn(name)
    # end trace_buffer_enable

    def trace_buffer_disable(self, name):
        self._trace.TraceBufOff(name)
    # end trace_buffer_disable

    def is_trace_buffer_enabled(self, name):
        return self._trace.IsTraceBufOn(name)
    # end is_trace_buffer_enabled

    def trace_buffer_list_get(self):
        return self._trace.TraceBufListGet()
    # end trace_buffer_list_get

    def trace_buffer_size_get(self, name):
        return self._trace.TraceBufSizeGet(name)
    # end trace_buffer_size_get

    def trace_buffer_read(self, name, read_context, count, read_cb):
        self._trace.TraceRead(name, read_context, count, read_cb)
    # end trace_buffer_read

    def trace_buffer_read_done(self, name, context):
        self._trace.TraceReadDone(name, context)
    # end trace_buffer_read_done

    # API to send the trace buffer to the Collector.
    # If trace count is not specified/or zero, then the entire trace buffer
    # is sent to the Collector.
    # [Note] No duplicate trace message sent to the Collector. i.e., If there
    # is no trace message added between two consequent calls to this API, then
    # no trace message is sent to the Collector.
    def send_sandesh_trace_buffer(self, trace_buf, count=0):
        trace_req_runner = SandeshTraceRequestRunner(
            sandesh=self, request_buffer_name=trace_buf, request_context='',
            read_context='Collector', request_count=count)
        trace_req_runner.Run()
    # end send_sandesh_trace_buffer

    # Private functions

    def _is_level_ut(self):
        return (self._level >= SandeshLevel.UT_START and
                self._level <= SandeshLevel.UT_END)
    # end _is_level_ut

    def _create_task(self):
        return gevent.spawn(self._runner.run_for_ever)
    # end _create_task

    def _process_rx_sandesh(self, rx_sandesh):
        handle_request_fn = getattr(rx_sandesh, "handle_request", None)
        if callable(handle_request_fn):
            handle_request_fn(rx_sandesh)
        else:
            self._logger.error('Sandesh Request "%s" not implemented' %
                               (rx_sandesh.__class__.__name__))
    # end _process_rx_sandesh

    def _create_sandesh_request_and_uve_lists(self, package):
        try:
            imp_pkg = __import__(package)
        except ImportError:
            self._logger.error('Failed to import package "%s"' % (package))
        else:
            try:
                pkg_path = imp_pkg.__path__
            except AttributeError:
                self._logger.error(
                    'Failed to get package [%s] path' % (package))
                return
            for importer, mod, ispkg in (
                pkgutil.walk_packages(path=pkg_path,
                                      prefix=imp_pkg.__name__ + '.')):
                if not ispkg:
                    module = mod.rsplit('.', 1)[-1]
                    if 'ttypes' == module:
                        self._logger.debug(
                            'Add Sandesh requests in module "%s"' % (mod))
                        self._add_sandesh_request(mod)
                        self._logger.debug(
                            'Add Sandesh UVEs in module "%s"' % (mod))
                        self._add_sandesh_uve(mod)
                        self._logger.debug(
                            'Add Sandesh Alarms in module "%s"' % (mod))
                        self._add_sandesh_alarm(mod)
    # end _create_sandesh_request_and_uve_lists

    def _add_sandesh_request(self, mod):
        try:
            imp_module = importlib.import_module(mod)
        except ImportError:
            self._logger.error('Failed to import Module "%s"' % (mod))
        else:
            try:
                sandesh_req_list = getattr(imp_module, '_SANDESH_REQUEST_LIST')
            except AttributeError:
                self._logger.error(
                    '"%s" module does not have sandesh request list' % (mod))
            else:
                # Add sandesh requests to the dictionary.
                for req in sandesh_req_list:
                    self._sandesh_request_dict[req] = mod
    # end _add_sandesh_request

    def _get_sandesh_uve_list(self, imp_module):
        try:
            sandesh_uve_list = getattr(imp_module, '_SANDESH_UVE_LIST')
        except AttributeError:
            self._logger.error(
                '"%s" module does not have sandesh UVE list' %
                (imp_module.__name__))
            return None
        else:
            return sandesh_uve_list
    # end _get_sandesh_uve_list

    def _get_sandesh_uve_data_list(self, imp_module):
        try:
            sandesh_uve_data_list = getattr(imp_module,
                                            '_SANDESH_UVE_DATA_LIST')
        except AttributeError:
            self._logger.error(
                '"%s" module does not have sandesh UVE data list' %
                (imp_module.__name__))
            return None
        else:
            return sandesh_uve_data_list
    # end _get_sandesh_uve_data_list

    def _add_sandesh_uve(self, mod):
        try:
            imp_module = importlib.import_module(mod)
        except ImportError:
            self._logger.error('Failed to import Module "%s"' % (mod))
        else:
            sandesh_uve_list = self._get_sandesh_uve_list(imp_module)
            sandesh_uve_data_list = self._get_sandesh_uve_data_list(imp_module)
            if sandesh_uve_list is None or sandesh_uve_data_list is None:
                return
            if len(sandesh_uve_list) != len(sandesh_uve_data_list):
                self._logger.error(
                    '"%s" module sandesh UVE and UVE data list do not match'
                    % (mod))
                return
            sandesh_uve_info_list = zip(sandesh_uve_list,
                                        sandesh_uve_data_list)
            # Register sandesh UVEs
            for uve_type_name, uve_data_type_name in sandesh_uve_info_list:
                SandeshUVEPerTypeMap(self, SandeshType.UVE,
                                     uve_type_name, uve_data_type_name, mod)
    # end _add_sandesh_uve

    def _get_sandesh_alarm_list(self, imp_module):
        try:
            sandesh_alarm_list = getattr(imp_module, '_SANDESH_ALARM_LIST')
        except AttributeError:
            self._logger.error(
                '"%s" module does not have sandesh Alarm list' %
                (imp_module.__name__))
            return None
        else:
            return sandesh_alarm_list
    # end _get_sandesh_alarm_list

    def _get_sandesh_alarm_data_list(self, imp_module):
        try:
            sandesh_alarm_data_list = getattr(imp_module,
                                              '_SANDESH_ALARM_DATA_LIST')
        except AttributeError:
            self._logger.error(
                '"%s" module does not have sandesh Alarm data list' %
                (imp_module.__name__))
            return None
        else:
            return sandesh_alarm_data_list
    # end _get_sandesh_alarm_data_list

    def _add_sandesh_alarm(self, mod):
        try:
            imp_module = importlib.import_module(mod)
        except ImportError:
            self._logger.error('Failed to import Module "%s"' % (mod))
        else:
            sandesh_alarm_list = self._get_sandesh_alarm_list(imp_module)
            sandesh_alarm_data_list = self._get_sandesh_alarm_data_list(
                imp_module)
            if sandesh_alarm_list is None or sandesh_alarm_data_list is None:
                return
            if len(sandesh_alarm_list) != len(sandesh_alarm_data_list):
                self._logger.error(
                    '"%s" module sandesh Alarm and Alarm data list do not '
                    'match' % (mod))
                return
            sandesh_alarm_info_list = zip(sandesh_alarm_list,
                                          sandesh_alarm_data_list)
            # Register sandesh Alarms
            for (alarm_type_name, alarm_data_type_name) in (
                    sandesh_alarm_info_list):
                SandeshUVEPerTypeMap(self, SandeshType.ALARM, alarm_type_name,
                                     alarm_data_type_name, mod)
    # end _add_sandesh_alarm

    def _init_logger(self, module, logger_class=None,
                     logger_config_file=None):
        if not module:
            module = 'sandesh'

        if logger_class:
            self._sandesh_logger = (sand_logger.create_logger(
                module, logger_class,
                logger_config_file=logger_config_file))
        else:
            generator = '%s:%s:%s:%s' % (self._source, module, self._node_type,
                                         self._instance_id)
            self._sandesh_logger = sand_logger.SandeshLogger(
                generator, logger_config_file=logger_config_file)
        self._logger = self._sandesh_logger.logger()
    def __init__(self, connection, logger, primary_collector, 
                 secondary_collector):

        def _update_connection_state(e, status):
            from connection_info import ConnectionState
            from gen_py.process_info.ttypes import ConnectionType
            collector_addr = e.sm._active_collector
            if collector_addr is None:
                collector_addr = ''
            ConnectionState.update(conn_type = ConnectionType.COLLECTOR,
                name = '',
                status = status,
                server_addrs = [collector_addr],
                message = '%s to %s on %s' % (e.src, e.dst, e.event))
        #end _update_connection_state

        def _connection_state_up(e):
            from gen_py.process_info.ttypes import ConnectionStatus
            _update_connection_state(e, ConnectionStatus.UP)
        #end _connection_state_up

        def _connection_state_down(e):
            from gen_py.process_info.ttypes import ConnectionStatus
            _update_connection_state(e, ConnectionStatus.DOWN)
        #end _connection_state_down

        def _connection_state_init(e):
            from gen_py.process_info.ttypes import ConnectionStatus
            _update_connection_state(e, ConnectionStatus.INIT)
        #end _connection_state_init

        def _on_idle(e):
            if e.sm._connect_timer is not None:
                e.sm._cancel_connect_timer()
            # Reset active and backup collector
            self._active_collector = self._connection.primary_collector()
            self._backup_collector = self._connection.secondary_collector()
            # clean up existing connection
            e.sm._delete_session()
            if e.sm._disable != True:
	        e.sm._start_idle_hold_timer()
            # update connection state
            _connection_state_down(e)
        #end _on_idle

        def _on_disconnect(e):
            # update connection state
            _connection_state_down(e)
        #end _on_disconnect

        def _on_connect(e):
            if e.sm._idle_hold_timer is not None:
                e.sm._cancel_idle_hold_timer()
            e.sm._connection.reset_collector()
            # clean up existing connection
            e.sm._delete_session()
            if e.sm._active_collector is not None:
                # update connection state
                _connection_state_init(e)
                e.sm._create_session()
                e.sm._start_connect_timer()
                e.sm._session.connect()
            else:
                e.sm.enqueue_event(Event(event = Event._EV_COLLECTOR_UNKNOWN))
        #end _on_connect

        def _on_connect_to_backup(e):
            if e.sm._connect_timer is not None:
                e.sm._cancel_connect_timer()
            # clean up existing connection
            e.sm._delete_session()
            # try to connect to the backup collector, if known
            if e.sm._backup_collector is not None:
                e.sm._active_collector, e.sm._backup_collector = \
                    e.sm._backup_collector, e.sm._active_collector
                # update connection state
                _connection_state_init(e)
                e.sm._create_session()
                e.sm._start_connect_timer()
                e.sm._session.connect()
            else:
                e.sm.enqueue_event(Event(event = Event._EV_BACKUP_COLLECTOR_UNKNOWN))
        #end _on_connect_to_backup

        def _on_client_init(e):
            e.sm._connects += 1
            gevent.spawn(e.sm._session.read)
            e.sm._connection.handle_initialized(e.sm._connects)
            e.sm._connection.sandesh_instance().send_generator_info()
            # update connection state
            _connection_state_init(e)
        #end _on_client_init
        
        def _on_established(e):
            e.sm._cancel_connect_timer()
            e.sm._connection.set_collector(e.sm_event.source)
            e.sm._connection.handle_sandesh_ctrl_msg(e.sm_event.msg)
            e.sm._connection.sandesh_instance().send_generator_info()
            # update connection state
            _connection_state_up(e)
        #end _on_established

        # FSM - Fysom
        self._fsm = Fysom({
                           'initial': {'state' : State._IDLE,
                                       'event' : Event._EV_START,
                                       'defer' : True
                                      },
                           'events': [
                                      # _IDLE
                                      {'name' : Event._EV_IDLE_HOLD_TIMER_EXPIRED,
                                       'src'  : State._IDLE,
                                       'dst'  : State._CONNECT
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._IDLE,
                                       'dst'  : State._CONNECT
                                      },
                                      {'name' : Event._EV_START,
                                       'src'  : State._IDLE,
                                       'dst'  : State._CONNECT
                                      },

                                      # _DISCONNECT 
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._DISCONNECT,
                                       'dst'  : State._CONNECT
                                      },

                                      # _CONNECT
                                      {'name' : Event._EV_COLLECTOR_UNKNOWN,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._DISCONNECT
                                      },
                                      {'name' : Event._EV_TCP_CONNECT_FAIL,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._CONNECT_TO_BACKUP
                                      },
                                      {'name' : Event._EV_CONNECT_TIMER_EXPIRED,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._CONNECT_TO_BACKUP
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CONNECTED,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._CLIENT_INIT
                                      },

                                      # _CONNECT_TO_BACKUP
                                      {'name' : Event._EV_BACKUP_COLLECTOR_UNKNOWN,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CONNECT_FAIL,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_CONNECT_TIMER_EXPIRED,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CONNECTED,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._CLIENT_INIT
                                      },

                                      # _CLIENT_INIT
                                      {'name' : Event._EV_CONNECT_TIMER_EXPIRED,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CLOSE,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_SANDESH_CTRL_MESSAGE_RECV,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._ESTABLISHED
                                      },

                                      # _ESTABLISHED
                                      {'name' : Event._EV_TCP_CLOSE,
                                       'src'  : State._ESTABLISHED,
                                       'dst'  : State._CONNECT_TO_BACKUP
                                      },
                                      {'name' : Event._EV_STOP,
                                       'src'  : State._ESTABLISHED,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._ESTABLISHED,
                                       'dst'  : State._CONNECT
                                      }
                                     ],
                           'callbacks': {
                                         'on' + State._IDLE : _on_idle,
                                         'on' + State._CONNECT : _on_connect,
                                         'on' + State._CONNECT_TO_BACKUP : _on_connect_to_backup,
                                         'on' + State._CLIENT_INIT : _on_client_init,
                                         'on' + State._ESTABLISHED : _on_established,
                                        }
                          })

        self._connection = connection
        self._session = None
        self._connects = 0
        self._disable = False
        self._idle_hold_timer = None
        self._connect_timer = None
        self._active_collector = primary_collector
        self._backup_collector = secondary_collector
        self._logger = logger
        self._event_queue = WorkQueue(self._dequeue_event,
                                      self._is_ready_to_dequeue_event)
class SandeshStateMachine(object):

    _IDLE_HOLD_TIME = 5 # in seconds
    _CONNECT_TIME = 30 # in seconds

    def __init__(self, connection, logger, primary_collector, 
                 secondary_collector):

        def _update_connection_state(e, status):
            from connection_info import ConnectionState
            from gen_py.process_info.ttypes import ConnectionType
            collector_addr = e.sm._active_collector
            if collector_addr is None:
                collector_addr = ''
            ConnectionState.update(conn_type = ConnectionType.COLLECTOR,
                name = '',
                status = status,
                server_addrs = [collector_addr],
                message = '%s to %s on %s' % (e.src, e.dst, e.event))
        #end _update_connection_state

        def _connection_state_up(e):
            from gen_py.process_info.ttypes import ConnectionStatus
            _update_connection_state(e, ConnectionStatus.UP)
        #end _connection_state_up

        def _connection_state_down(e):
            from gen_py.process_info.ttypes import ConnectionStatus
            _update_connection_state(e, ConnectionStatus.DOWN)
        #end _connection_state_down

        def _connection_state_init(e):
            from gen_py.process_info.ttypes import ConnectionStatus
            _update_connection_state(e, ConnectionStatus.INIT)
        #end _connection_state_init

        def _on_idle(e):
            if e.sm._connect_timer is not None:
                e.sm._cancel_connect_timer()
            # Reset active and backup collector
            self._active_collector = self._connection.primary_collector()
            self._backup_collector = self._connection.secondary_collector()
            # clean up existing connection
            e.sm._delete_session()
            if e.sm._disable != True:
	        e.sm._start_idle_hold_timer()
            # update connection state
            _connection_state_down(e)
        #end _on_idle

        def _on_disconnect(e):
            # update connection state
            _connection_state_down(e)
        #end _on_disconnect

        def _on_connect(e):
            if e.sm._idle_hold_timer is not None:
                e.sm._cancel_idle_hold_timer()
            e.sm._connection.reset_collector()
            # clean up existing connection
            e.sm._delete_session()
            if e.sm._active_collector is not None:
                # update connection state
                _connection_state_init(e)
                e.sm._create_session()
                e.sm._start_connect_timer()
                e.sm._session.connect()
            else:
                e.sm.enqueue_event(Event(event = Event._EV_COLLECTOR_UNKNOWN))
        #end _on_connect

        def _on_connect_to_backup(e):
            if e.sm._connect_timer is not None:
                e.sm._cancel_connect_timer()
            # clean up existing connection
            e.sm._delete_session()
            # try to connect to the backup collector, if known
            if e.sm._backup_collector is not None:
                e.sm._active_collector, e.sm._backup_collector = \
                    e.sm._backup_collector, e.sm._active_collector
                # update connection state
                _connection_state_init(e)
                e.sm._create_session()
                e.sm._start_connect_timer()
                e.sm._session.connect()
            else:
                e.sm.enqueue_event(Event(event = Event._EV_BACKUP_COLLECTOR_UNKNOWN))
        #end _on_connect_to_backup

        def _on_client_init(e):
            e.sm._connects += 1
            gevent.spawn(e.sm._session.read)
            e.sm._connection.handle_initialized(e.sm._connects)
            e.sm._connection.sandesh_instance().send_generator_info()
            # update connection state
            _connection_state_init(e)
        #end _on_client_init
        
        def _on_established(e):
            e.sm._cancel_connect_timer()
            e.sm._connection.set_collector(e.sm_event.source)
            e.sm._connection.handle_sandesh_ctrl_msg(e.sm_event.msg)
            e.sm._connection.sandesh_instance().send_generator_info()
            # update connection state
            _connection_state_up(e)
        #end _on_established

        # FSM - Fysom
        self._fsm = Fysom({
                           'initial': {'state' : State._IDLE,
                                       'event' : Event._EV_START,
                                       'defer' : True
                                      },
                           'events': [
                                      # _IDLE
                                      {'name' : Event._EV_IDLE_HOLD_TIMER_EXPIRED,
                                       'src'  : State._IDLE,
                                       'dst'  : State._CONNECT
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._IDLE,
                                       'dst'  : State._CONNECT
                                      },
                                      {'name' : Event._EV_START,
                                       'src'  : State._IDLE,
                                       'dst'  : State._CONNECT
                                      },

                                      # _DISCONNECT 
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._DISCONNECT,
                                       'dst'  : State._CONNECT
                                      },

                                      # _CONNECT
                                      {'name' : Event._EV_COLLECTOR_UNKNOWN,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._DISCONNECT
                                      },
                                      {'name' : Event._EV_TCP_CONNECT_FAIL,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._CONNECT_TO_BACKUP
                                      },
                                      {'name' : Event._EV_CONNECT_TIMER_EXPIRED,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._CONNECT_TO_BACKUP
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CONNECTED,
                                       'src'  : State._CONNECT,
                                       'dst'  : State._CLIENT_INIT
                                      },

                                      # _CONNECT_TO_BACKUP
                                      {'name' : Event._EV_BACKUP_COLLECTOR_UNKNOWN,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CONNECT_FAIL,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_CONNECT_TIMER_EXPIRED,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CONNECTED,
                                       'src'  : State._CONNECT_TO_BACKUP,
                                       'dst'  : State._CLIENT_INIT
                                      },

                                      # _CLIENT_INIT
                                      {'name' : Event._EV_CONNECT_TIMER_EXPIRED,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_TCP_CLOSE,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_SANDESH_CTRL_MESSAGE_RECV,
                                       'src'  : State._CLIENT_INIT,
                                       'dst'  : State._ESTABLISHED
                                      },

                                      # _ESTABLISHED
                                      {'name' : Event._EV_TCP_CLOSE,
                                       'src'  : State._ESTABLISHED,
                                       'dst'  : State._CONNECT_TO_BACKUP
                                      },
                                      {'name' : Event._EV_STOP,
                                       'src'  : State._ESTABLISHED,
                                       'dst'  : State._IDLE
                                      },
                                      {'name' : Event._EV_COLLECTOR_CHANGE,
                                       'src'  : State._ESTABLISHED,
                                       'dst'  : State._CONNECT
                                      }
                                     ],
                           'callbacks': {
                                         'on' + State._IDLE : _on_idle,
                                         'on' + State._CONNECT : _on_connect,
                                         'on' + State._CONNECT_TO_BACKUP : _on_connect_to_backup,
                                         'on' + State._CLIENT_INIT : _on_client_init,
                                         'on' + State._ESTABLISHED : _on_established,
                                        }
                          })

        self._connection = connection
        self._session = None
        self._connects = 0
        self._disable = False
        self._idle_hold_timer = None
        self._connect_timer = None
        self._active_collector = primary_collector
        self._backup_collector = secondary_collector
        self._logger = logger
        self._event_queue = WorkQueue(self._dequeue_event,
                                      self._is_ready_to_dequeue_event)
    #end __init__

    # Public functions

    def initialize(self):
        self.enqueue_event(Event(event = Event._EV_START))
    #end initialize

    def session(self):
        return self._session 
    #end session 

    def state(self):
        return self._fsm.current
    #end state 

    def shutdown(self):
        self._disable = True
        self.enqueue_event(Event(event = Event._EV_STOP))
    #end shutdown

    def set_admin_state(self, down):
        if down == True:
            self._disable = True
            self.enqueue_event(Event(event = Event._EV_STOP))
        else:
            self._disable = False
            self.enqueue_event(Event(event = Event._EV_START))
    #end set_admin_state

    def connect_count(self):
        return self._connects
    #end connect_count

    def active_collector(self):
        return self._active_collector
    #end active_collector

    def backup_collector(self):
        return self._backup_collector
    #end backup_collector

    def enqueue_event(self, event):
        self._event_queue.enqueue(event)
    #end enqueue_event

    def on_session_event(self, session, event):
        if session is not self._session:
            self._logger.error("Ignore session event [%d] received for old session" % (event))
            return 
        if SandeshSession.SESSION_ESTABLISHED == event:
            self._logger.info("Session Event: TCP Connected")
            self.enqueue_event(Event(event = Event._EV_TCP_CONNECTED,
                                     session = session))
        elif SandeshSession.SESSION_ERROR == event:
            self._logger.error("Session Event: TCP Connect Fail")
            self.enqueue_event(Event(event = Event._EV_TCP_CONNECT_FAIL,
                                     session = session))
        elif SandeshSession.SESSION_CLOSE == event:
            self._logger.error("Session Event: TCP Connection Closed")
            self.enqueue_event(Event(event = Event._EV_TCP_CLOSE,
                                     session = session))
        else:
            self._logger.error("Received unknown session event [%d]" % (event))
    #end on_session_event

    def on_sandesh_ctrl_msg_receive(self, session, sandesh_ctrl, collector):
        if sandesh_ctrl.success == True:
            self.enqueue_event(Event(event = Event._EV_SANDESH_CTRL_MESSAGE_RECV, 
                                     session = session,
                                     msg = sandesh_ctrl,
                                     source = collector))
        else:
            # Negotiation with the Collector failed, reset the 
            # connection and retry after sometime.
            self._logger.error("Negotiation with the Collector %s failed." % (collector))
            self._session.close()
    #end on_sandesh_ctrl_msg_receive

    def on_sandesh_uve_msg_send(self, sandesh_uve):
        self.enqueue_event(Event(event = Event._EV_SANDESH_UVE_SEND,
                                 msg = sandesh_uve))
    #end on_sandesh_uve_msg_send

    # Private functions

    def _create_session(self):
        assert self._session is None
        col_info = self._active_collector.split(':')
        collector = (col_info[0], int(col_info[1]))
        self._session = SandeshSession(self._connection.sandesh_instance(),
                                       collector,
                                       self.on_session_event,
                                       self._connection._receive_sandesh_msg) 
    #end _create_session

    def _delete_session(self):
        if self._session:
            self._session.close()
            self._session = None
            self._connection.reset_collector()
    #end _delete_session 

    def _start_idle_hold_timer(self):
        if self._idle_hold_timer is None:
            if self._IDLE_HOLD_TIME:
                self._idle_hold_timer = gevent.spawn_later(self._IDLE_HOLD_TIME,
                                            self._idle_hold_timer_expiry_handler)
            else:
                self.enqueue_event(Event(event = Event._EV_IDLE_HOLD_TIMER_EXPIRED))
    #end _start_idle_hold_timer

    def _cancel_idle_hold_timer(self):
        if self._idle_hold_timer is not None:
            gevent.kill(self._idle_hold_timer)
            self._idle_hold_timer = None
    #end _cancel_idle_hold_timer

    def _idle_hold_timer_expiry_handler(self):
        self._idle_hold_timer = None
        self.enqueue_event(Event(event = Event._EV_IDLE_HOLD_TIMER_EXPIRED))
    #end _idle_hold_timer_expiry_handler
    
    def _start_connect_timer(self):
        if self._connect_timer is None:
            self._connect_timer = gevent.spawn_later(self._CONNECT_TIME,
                                        self._connect_timer_expiry_handler, 
                                        self._session)
    #end _start_connect_timer

    def _cancel_connect_timer(self):
        if self._connect_timer is not None:
            gevent.kill(self._connect_timer)
            self._connect_timer = None
    #end _cancel_connect_timer

    def _connect_timer_expiry_handler(self, session):
        self._connect_timer = None
        self.enqueue_event(Event(event = Event._EV_CONNECT_TIMER_EXPIRED,
                                 session = session))
    #end _connect_timer_expiry_handler

    def _is_ready_to_dequeue_event(self):
        return True
    #end _is_ready_to_dequeue_event

    def _log_event(self, event):
        if self._fsm.current == State._ESTABLISHED and \
           event.event == Event._EV_SANDESH_UVE_SEND:
           return False
        return True
    #end _log_event

    def _dequeue_event(self, event):
        if self._log_event(event):
            self._logger.info("Processing event[%s] in state[%s]" \
                              % (event.event, self._fsm.current))
        if event.session is not None and event.session is not self._session:
            self._logger.info("Ignore event [%s] received for old session" \
                              % (event.event))
            return
        if event.event == Event._EV_COLLECTOR_CHANGE:
            old_active_collector = self._active_collector
            self._active_collector = event.primary_collector
            self._backup_collector = event.secondary_collector
            if old_active_collector == self._active_collector:
                self._logger.info("No change in active collector. Ignore event [%s]" \
                                  % (event.event))
                return
        if event.event == Event._EV_SANDESH_UVE_SEND:
            if self._fsm.current == State._ESTABLISHED or self._fsm.current == State._CLIENT_INIT:
                self._connection.handle_sandesh_uve_msg(event.msg)
            else:
                self._logger.info("Discarding event[%s] in state[%s]" \
                                  % (event.event, self._fsm.current))
        elif event.event == Event._EV_SANDESH_CTRL_MESSAGE_RECV and \
                self._fsm.current == State._ESTABLISHED:
            self._connection.handle_sandesh_ctrl_msg(event.msg)
        elif self._fsm.cannot(event.event) is True:
            self._logger.info("Unconsumed event[%s] in state[%s]" \
                              % (event.event, self._fsm.current))
        else:
            prev_state = self.state()
            getattr(self._fsm, event.event)(sm = self, sm_event = event)
            # Log state transition
            self._logger.info("Sandesh Client: Event[%s] => State[%s] -> State[%s]" \
                              % (event.event, prev_state, self.state()))