def start(self): ''' Listen to messages and publish them. ''' self._setup_ipc() log.debug('Using the %s listener', self._listener_type) self._setup_listener() self.listener.start() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: try: log_message, log_source = self.listener.receive() except ListenerException as lerr: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(lerr, exc_info=True) raise NapalmLogsExit(lerr) log.debug('Received %s from %s. Queueing to the server.', log_message, log_source) if not log_message: log.info( 'Empty message received from %s. Not queueing to the server.', log_source) continue self.pub.send(umsgpack.packb((log_message, log_source)))
def start(self): ''' Start listening for messages ''' signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True self.skt = self._open_socket(socket.SOCK_DGRAM) try: self.skt.bind((self.address, int(self.port))) except socket.error as msg: error_string = 'Unable to bind to port {} on {}: {}'.format( self.port, self.address, msg) log.error(error_string, exc_info=True) raise ListenerException(error_string) while self.__up: try: msg, addr = self.skt.recvfrom(BUFFER_SIZE) except socket.error as error: if self.__up is False: return else: msg = 'Received listener socket error: {}'.format(error) log.error(msg, exc_info=True) raise NapalmLogsExit(msg) log.debug( '[{2}] Received {0} from {1}. Adding in the queue'.format( msg, addr, time.time())) self.pipe.send((msg, addr[0]))
def start(self): ''' Listen to auth requests and send the AES key. Each client connection starts a new thread. ''' # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True self.socket.listen(AUTH_MAX_CONN) while self.__up: try: (clientsocket, address) = self.socket.accept() wrapped_auth_skt = ssl.wrap_socket(clientsocket, server_side=True, certfile=self.certificate, keyfile=self.keyfile) except ssl.SSLError: log.exception('SSL error', exc_info=True) continue except socket.error as error: if self.__up is False: return else: msg = 'Received auth socket error: {}'.format(error) log.error(msg, exc_info=True) raise NapalmLogsExit(msg) log.info('{0} connected'.format(address)) log.debug('Starting the handshake') client_thread = threading.Thread(target=self._handshake, args=(wrapped_auth_skt, address)) client_thread.start()
def start(self): ''' Start listening for messages ''' signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True self.skt = self._open_socket(socket.SOCK_STREAM) try: self.skt.bind((self.address, int(self.port))) except socket.error as msg: error_string = 'Unable to bind to port {} on {}: {}'.format( self.port, self.address, msg) log.error(error_string, exc_info=True) raise ListenerException(error_string) while self.__up: self.skt.listen(1) try: conn, addr = self.skt.accept() except socket.error as error: if self.__up is False: return else: msg = 'Received listener socket error: {}'.format(error) log.error(msg, exc_info=True) raise NapalmLogsExit(msg) thread = threading.Thread(target=self._tcp_connection, args=( conn, addr, )) thread.start()
def start(self): ''' Listen to messages and publish them. ''' # self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.transport.start() self.__up = True while self.__up: # bin_obj = self.sub.recv() # already serialized try: obj = self.pipe.recv() except IOError as error: if self.__up is False: return else: msg = 'Received IOError on publisher pipe: {}'.format( error) log.error(msg, exc_info=True) raise NapalmLogsExit(msg) log.debug('Publishing the OC object (serialised)') if not self.disable_security: bin_obj = self._prepare(obj) else: bin_obj = umsgpack.packb(obj) self.transport.publish(bin_obj)
def start(self): ''' Listen to messages and publish them. ''' self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.transport.start() self.__up = True while self.__up: # bin_obj = self.sub.recv() # already serialized try: # obj = self.pipe.recv() bin_obj = self.sub.recv() except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error) log.debug('Publishing the OC object (serialised)') if not self.disable_security and self.__transport_encrypt: bin_obj = self._prepare(bin_obj) self.transport.publish(bin_obj)
def start(self): ''' Take the messages from the queue, inspect and identify the operating system, then queue the message correspondingly. ''' # self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) # If were are to log all processed syslogs we need to initiate the class if self.logger.get('syslog'): self._setup_log_syslog_transport() self.__up = True while self.__up: # Take messages from the main queue # bin_obj = self.sub.recv() # msg, address = umsgpack.unpackb(bin_obj, use_list=False) try: msg, address = self.pipe.recv() except IOError as error: if self.__up is False: return else: msg = 'Received IOError from server pipe: {}'.format(error) log.error(msg, exc_info=True) raise NapalmLogsExit(msg) msg = msg.encode('utf8') log.debug('[{2}] Dequeued message from {0}: {1}'.format( address, msg, time.time())) dev_os, msg_dict = self._identify_os(msg) log.debug('Identified OS: {0}'.format(dev_os)) if self.logger.get('syslog'): if dev_os: self._send_log_syslog(dev_os, msg_dict) else: self._send_log_syslog('unknown', {'message': msg}) if not dev_os or not isinstance(msg_dict, dict): # _identify_os should return a string and a dict # providing the info for the device OS # and the decoded message prefix continue # Then send the message in the right queue # obj = (msg_dict, address) # bin_obj = umsgpack.packb(obj) if dev_os in self.os_pipes: log.debug('Queueing message to {0}'.format(dev_os)) # self.pubs[dev_os].send(bin_obj) self.os_pipes[dev_os].send((msg_dict, address)) else: log.info( 'Unable to queue the message to {0}. Is the sub-process started?' .format(dev_os))
def start(self): ''' Start the worker process. ''' # self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),)) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: # bin_obj = self.sub.recv() # msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False) try: msg_dict, address = self.pipe.recv() except IOError as error: if self.__up is False: return else: msg = 'Received IOError on {} device pipe: {}'.format(self._name, error) log.error(msg, exc_info=True) raise NapalmLogsExit(msg) log.debug('{0}: dequeued {1}, received from {2}'.format(self._name, msg_dict, address)) kwargs = self._parse(msg_dict) if not kwargs: continue try: oc_obj = self._emit(**kwargs) except Exception as err: log.exception('Unexpected error when generating the OC object.', exc_info=True) continue log.debug('Generated OC object:') log.debug(oc_obj) error = kwargs.get('error') model_name = kwargs.get('oc_model') host = msg_dict.get('host') timestamp = self._format_time(msg_dict.get('time', ''), msg_dict.get('date', '')) to_publish = { 'error': error, 'host': host, 'ip': address, 'timestamp': timestamp, 'open_config': oc_obj, 'message_details': msg_dict, 'model_name': model_name, 'os': self._name } log.debug('Queueing to be published:') log.debug(to_publish) self.pub_pipe.send(to_publish)
def start(self): ''' Listen to messages and publish them. ''' # counter metrics for messages c_logs_ingested = Counter( 'napalm_logs_listener_logs_ingested', 'Count of ingested log messages', ['listener_type', 'address', 'port'], ) c_messages_published = Counter( 'napalm_logs_listener_messages_published', 'Count of published messages', ['listener_type', 'address', 'port'], ) self._setup_ipc() log.debug('Using the %s listener', self._listener_type) self._setup_listener() self.listener.start() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: try: log_message, log_source = self.listener.receive() except ListenerException as lerr: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(lerr, exc_info=True) raise NapalmLogsExit(lerr) log.debug('Received %s from %s. Queueing to the server.', log_message, log_source) if not log_message: log.info( 'Empty message received from %s. Not queueing to the server.', log_source, ) continue c_logs_ingested.labels(listener_type=self._listener_type, address=self.address, port=self.port).inc() self.pub.send(umsgpack.packb((log_message, log_source))) c_messages_published.labels(listener_type=self._listener_type, address=self.address, port=self.port).inc()
def start(self): ''' Listen to messages and publish them. ''' self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) try: zmq.proxy(self.sub, self.pub) except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error)
def start(self): ''' Listen to messages and publish them. ''' self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.transport.start() self.__up = True while self.__up: try: bin_obj = self.sub.recv() except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error) obj = umsgpack.unpackb(bin_obj) if not napalm_logs.ext.check_whitelist_blacklist( obj['error'], whitelist=self.error_whitelist, blacklist=self.error_blacklist): # Apply the whitelist / blacklist logic # If it doesn't match, jump over. log.debug('This error type is %s. Skipping for %s #%d', obj['error'], self._transport_type, self.pub_id) continue serialized_obj = self._serialize(obj, bin_obj) log.debug('Publishing the OC object') if not self.disable_security and self.__transport_encrypt: # Encrypt only when needed. serialized_obj = self._prepare(serialized_obj) self.transport.publish(serialized_obj)
def start(self): ''' Listen to messages and publish them. ''' # metrics napalm_logs_publisher_received_messages = Counter( 'napalm_logs_publisher_received_messages', "Count of messages received by the publisher", ['publisher_type', 'address', 'port'] ) napalm_logs_publisher_whitelist_blacklist_check_fail = Counter( 'napalm_logs_publisher_whitelist_blacklist_check_fail', "Count of messages which fail the whitelist/blacklist check", ['publisher_type', 'address', 'port'] ) napalm_logs_publisher_messages_published = Counter( 'napalm_logs_publisher_messages_published', "Count of published messages", ['publisher_type', 'address', 'port'] ) self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),)) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.transport.start() self.__up = True while self.__up: try: bin_obj = self.sub.recv() except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error) obj = umsgpack.unpackb(bin_obj) if self._strip_message_details: obj.pop('message_details', None) bin_obj = self.serializer_fun(obj) napalm_logs_publisher_received_messages.labels( publisher_type=self._transport_type, address=self.address, port=self.port ).inc() if not napalm_logs.ext.check_whitelist_blacklist(obj['error'], whitelist=self.error_whitelist, blacklist=self.error_blacklist): # Apply the whitelist / blacklist logic # If it doesn't match, jump over. log.debug('This error type is %s. Skipping for %s #%d', obj['error'], self._transport_type, self.pub_id) napalm_logs_publisher_whitelist_blacklist_check_fail.labels( publisher_type=self._transport_type, address=self.address, port=self.port ).inc() continue serialized_obj = self._serialize(obj, bin_obj) log.debug('Publishing the OC object') if not self.disable_security and self.__transport_encrypt: # Encrypt only when needed. serialized_obj = self._prepare(serialized_obj) self.transport.publish(serialized_obj) napalm_logs_publisher_messages_published.labels( publisher_type=self._transport_type, address=self.address, port=self.port ).inc()
def start(self): ''' Take the messages from the queue, inspect and identify the operating system, then queue the message correspondingly. ''' # metric counters napalm_logs_server_messages_received = Counter( "napalm_logs_server_messages_received", "Count of messages received from listener processes", ) napalm_logs_server_skipped_buffered_messages = Counter( 'napalm_logs_server_skipped_buffered_messages', 'Count of messages skipped as they were already buffered', ['device_os'], ) napalm_logs_server_messages_with_identified_os = Counter( "napalm_logs_server_messages_with_identified_os", "Count of messages with positive os identification", ['device_os'], ) napalm_logs_server_messages_without_identified_os = Counter( "napalm_logs_server_messages_without_identified_os", "Count of messages with negative os identification", ) napalm_logs_server_messages_failed_device_queuing = Counter( "napalm_logs_server_messages_failed_device_queuing", "Count of messages per device os that fail to be queued to a device process", ['device_os'], ) napalm_logs_server_messages_device_queued = Counter( "napalm_logs_server_messages_device_queued", "Count of messages queued to device processes", ['device_os'], ) napalm_logs_server_messages_unknown_queued = Counter( "napalm_logs_server_messages_unknown_queued", "Count of messages queued as unknown", ) if self.opts.get('metrics_include_attributes', True): napalm_logs_server_messages_attrs = Counter( "napalm_logs_server_messages_attrs", "Count of messages from the server process with their details", ['device_os', 'host', 'tag'], ) self._setup_ipc() # Start suicide polling thread cleanup = threading.Thread(target=self._cleanup_buffer) cleanup.start() thread = threading.Thread( target=self._suicide_when_without_parent, args=(os.getppid(),) ) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: # Take messages from the main queue try: bin_obj = self.sub.recv() msg, address = umsgpack.unpackb(bin_obj, use_list=False) except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error) if isinstance(msg, bytes): if six.PY3: msg = str(msg, 'utf-8') else: msg = msg.encode('utf-8') log.debug('[%s] Dequeued message from %s: %s', address, msg, time.time()) napalm_logs_server_messages_received.inc() os_list = self._identify_os(msg) for dev_os, msg_dict in os_list: if dev_os and dev_os in self.started_os_proc: # Identified the OS and the corresponding process is started. # Then send the message in the right queue log.debug('Identified OS: %s', dev_os) log.debug('Queueing message to %s', dev_os) if six.PY3: dev_os = bytes(dev_os, 'utf-8') napalm_logs_server_messages_with_identified_os.labels( device_os=dev_os.decode() ).inc() if self._buffer: message = '{dev_os}/{host}/{msg}'.format( dev_os=dev_os.decode(), host=msg_dict['host'], msg=msg_dict['message'], ) if six.PY3: message_key = base64.b64encode( bytes(message, 'utf-8') ).decode() else: message_key = base64.b64encode(message) if self._buffer[message_key]: log.info( '"%s" seems to be already buffered, skipping', msg_dict['message'], ) napalm_logs_server_skipped_buffered_messages.labels( device_os=dev_os.decode() ).inc() continue log.debug( '"%s" is not buffered yet, added', msg_dict['message'] ) self._buffer[message_key] = 1 self.pub.send_multipart( [dev_os, umsgpack.packb((msg_dict, address))] ) napalm_logs_server_messages_device_queued.labels( device_os=dev_os.decode() ).inc() if self.opts.get('metrics_server_include_attributes', True): napalm_logs_server_messages_attrs.labels( device_os=dev_os.decode(), host=msg_dict['host'], tag=msg_dict['tag'], ).inc() elif dev_os and dev_os not in self.started_os_proc: # Identified the OS, but the corresponding process does not seem to be started. log.info( 'Unable to queue the message to %s. Is the sub-process started?', dev_os, ) napalm_logs_server_messages_with_identified_os.labels( device_os=dev_os.decode() ).inc() napalm_logs_server_messages_failed_device_queuing.labels( device_os=dev_os.decode() ).inc() elif not dev_os and self.opts['_server_send_unknown']: # OS not identified, but the user requested to publish the message as-is log.debug( 'Unable to identify the OS, sending directly to the publishers' ) to_publish = { 'ip': address, 'host': 'unknown', 'timestamp': int(time.time()), 'message_details': msg_dict, 'os': UNKNOWN_DEVICE_NAME, 'error': 'UNKNOWN', 'model_name': 'unknown', } self.publisher_pub.send(umsgpack.packb(to_publish)) napalm_logs_server_messages_unknown_queued.inc() napalm_logs_server_messages_without_identified_os.inc()
def start(self): ''' Start the worker process. ''' self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: # bin_obj = self.sub.recv() # msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False) try: bin_obj = self.sub.recv() msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False) except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown [%s]', self._name) return else: raise NapalmLogsExit(error) log.debug('%s: dequeued %s, received from %s', self._name, msg_dict, address) if self._name == UNKNOWN_DEVICE_NAME: # If running in the sub-process publishing messages for unknown OSs. # This will always send what receives, as-is. log.debug( 'Publishing the message as-is, no further processing possible.' ) to_publish = { 'ip': address, 'host': 'unknown', 'timestamp': int(time.time()), 'message_details': msg_dict, 'os': UNKNOWN_DEVICE_NAME, 'error': 'UNKNOWN', 'model_name': 'unknown' } log.debug('Queueing to be published:') log.debug(to_publish) # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish)) continue # From here on, we're running in a regular OS sub-process. host = msg_dict.get('host') prefix_id = msg_dict.pop('__prefix_id__') if 'timestamp' in msg_dict: timestamp = msg_dict.pop('timestamp') else: timestamp = self._format_time(msg_dict.get('time', ''), msg_dict.get('date', ''), msg_dict.get('timeZone', 'UTC'), prefix_id) facility = msg_dict.get('facility') severity = msg_dict.get('severity') kwargs = self._parse(msg_dict) if not kwargs: # Unable to identify what model to generate for the message in cause. if self.publisher_opts.get('send_raw'): # But publish the message when the user requested to push raw messages. to_publish = { 'ip': address, 'host': host, 'timestamp': timestamp, 'message_details': msg_dict, 'os': self._name, 'error': 'RAW', 'model_name': 'raw', 'facility': facility, 'severity': severity } log.debug('Queueing to be published:') log.debug(to_publish) # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish)) continue try: if '__python_fun__' in kwargs: log.debug( 'Using the Python parser to determine the YANG-equivalent object' ) yang_obj = kwargs['__python_fun__'](msg_dict) else: yang_obj = self._emit(**kwargs) except Exception: log.exception( 'Unexpected error when generating the OC object.', exc_info=True) continue log.debug('Generated OC object:') log.debug(yang_obj) error = kwargs.get('error') model_name = kwargs.get('model') to_publish = { 'error': error, 'host': host, 'ip': address, 'timestamp': timestamp, 'yang_message': yang_obj, 'message_details': msg_dict, 'yang_model': model_name, 'os': self._name, 'facility': facility, 'severity': severity } log.debug('Queueing to be published:') log.debug(to_publish) # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish))
def start(self): ''' Start the worker process. ''' # metrics napalm_logs_device_messages_received = Counter( 'napalm_logs_device_messages_received', "Count of messages received by the device process", ['device_os']) napalm_logs_device_raw_published_messages = Counter( 'napalm_logs_device_raw_published_messages', "Count of raw type published messages", ['device_os']) napalm_logs_device_published_messages = Counter( 'napalm_logs_device_published_messages', "Count of published messages", ['device_os']) napalm_logs_device_oc_object_failed = Counter( 'napalm_logs_device_oc_object_failed', "Counter of failed OpenConfig object generations", ['device_os']) self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: # bin_obj = self.sub.recv() # msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False) try: bin_obj = self.sub.recv() msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False) except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown [%s]', self._name) return else: raise NapalmLogsExit(error) log.debug('%s: dequeued %s, received from %s', self._name, msg_dict, address) napalm_logs_device_messages_received.labels( device_os=self._name).inc() host = msg_dict.get('host') prefix_id = msg_dict.pop('__prefix_id__') if 'timestamp' in msg_dict: timestamp = msg_dict.pop('timestamp') else: timestamp = self._format_time(msg_dict.get('time', ''), msg_dict.get('date', ''), msg_dict.get('timeZone', 'UTC'), prefix_id) facility = msg_dict.get('facility') severity = msg_dict.get('severity') kwargs = self._parse(msg_dict) if not kwargs: # Unable to identify what model to generate for the message in cause. # But publish the message when the user requested to push raw messages. to_publish = { 'ip': address, 'host': host, 'timestamp': timestamp, 'message_details': msg_dict, 'os': self._name, 'error': 'RAW', 'model_name': 'raw', 'facility': facility, 'severity': severity } log.debug('Queueing to be published:') log.debug(to_publish) # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish)) napalm_logs_device_raw_published_messages.labels( device_os=self._name).inc() continue try: if '__python_fun__' in kwargs: log.debug( 'Using the Python parser to determine the YANG-equivalent object' ) yang_obj = kwargs['__python_fun__'](msg_dict) else: yang_obj = self._emit(**kwargs) except Exception: log.exception( 'Unexpected error when generating the OC object.', exc_info=True) napalm_logs_device_oc_object_failed.labels( device_os=self._name).inc() continue log.debug('Generated OC object:') log.debug(yang_obj) error = kwargs.get('error') model_name = kwargs.get('model') to_publish = { 'error': error, 'host': host, 'ip': address, 'timestamp': timestamp, 'yang_message': yang_obj, 'message_details': msg_dict, 'yang_model': model_name, 'os': self._name, 'facility': facility, 'severity': severity } log.debug('Queueing to be published:') log.debug(to_publish) # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish)) # self._publish(to_publish) napalm_logs_device_published_messages.labels( device_os=self._name).inc()
def start(self): ''' Take the messages from the queue, inspect and identify the operating system, then queue the message correspondingly. ''' self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) self.__up = True while self.__up: # Take messages from the main queue try: bin_obj = self.sub.recv() msg, address = umsgpack.unpackb(bin_obj, use_list=False) except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error) if six.PY3: msg = str(msg, 'utf-8') else: msg = msg.encode('utf-8') log.debug('[%s] Dequeued message from %s: %s', address, msg, time.time()) dev_os, msg_dict = self._identify_os(msg) if dev_os and dev_os in self.started_os_proc: # Identified the OS and the corresponding process is started. # Then send the message in the right queue log.debug('Identified OS: %s', dev_os) log.debug('Queueing message to %s', dev_os) if six.PY3: dev_os = bytes(dev_os, 'utf-8') self.pub.send_multipart( [dev_os, umsgpack.packb((msg_dict, address))]) # self.os_pipes[dev_os].send((msg_dict, address)) elif dev_os and dev_os not in self.started_os_proc: # Identified the OS, but the corresponding process does not seem to be started. log.info( 'Unable to queue the message to %s. Is the sub-process started?', dev_os) elif not dev_os and self.opts['_server_send_unknown']: # OS not identified, but the user requested to publish the message as-is log.debug( 'Unable to identify the OS, sending directly to the publishers' ) to_publish = { 'ip': address, 'host': 'unknown', 'timestamp': int(time.time()), 'message_details': msg_dict, 'os': UNKNOWN_DEVICE_NAME, 'error': 'UNKNOWN', 'model_name': 'unknown' } self.publisher_pub.send(umsgpack.packb(to_publish))
def start(self): ''' Take the messages from the queue, inspect and identify the operating system, then queue the message correspondingly. ''' self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(), )) thread.start() signal.signal(signal.SIGTERM, self._exit_gracefully) # If were are to log all processed syslogs we need to initiate the class if self.logger: self._setup_log_syslog_transport() self.__up = True while self.__up: # Take messages from the main queue # bin_obj = self.sub.recv() # msg, address = umsgpack.unpackb(bin_obj, use_list=False) try: bin_obj = self.sub.recv() msg, address = umsgpack.unpackb(bin_obj, use_list=False) except zmq.ZMQError as error: if self.__up is False: log.info('Exiting on process shutdown') return else: log.error(error, exc_info=True) raise NapalmLogsExit(error) if six.PY3: msg = str(msg, 'utf-8') else: msg = msg.encode('utf-8') log.debug('[%s] Dequeued message from %s: %s', address, msg, time.time()) dev_os, msg_dict = self._identify_os(msg) log.debug('Identified OS: %s', dev_os) if self.logger: if self.logger_opts.get('send_raw') and dev_os: self._send_log_syslog(dev_os, msg_dict) elif self.logger_opts.get('send_unknown') and not dev_os: self._send_log_syslog(UNKNOWN_DEVICE_NAME, {'message': msg}) if dev_os and dev_os in self.started_os_proc: # Identified the OS and the corresponding process is started. # Then send the message in the right queue # obj = (msg_dict, address) # bin_obj = umsgpack.packb(obj) log.debug('Queueing message to %s', dev_os) # self.pubs[dev_os].send(bin_obj) if six.PY3: dev_os = bytes(dev_os, 'utf-8') self.pub.send_multipart( [dev_os, umsgpack.packb((msg_dict, address))]) # self.os_pipes[dev_os].send((msg_dict, address)) elif dev_os and dev_os not in self.started_os_proc: # Identified the OS, but the corresponding process does not seem to be started. log.info( 'Unable to queue the message to %s. Is the sub-process started?', dev_os) elif not dev_os and self.publisher_opts.get('send_unknown'): # OS not identified, but the user requested to publish the message as-is log.debug( 'Publishing message, although not identified, as requested' ) if six.PY3: dev_os = bytes(UNKNOWN_DEVICE_NAME, 'utf-8') else: dev_os = UNKNOWN_DEVICE_NAME self.pub.send_multipart( [dev_os, umsgpack.packb(({ 'message': msg }, address))]) # self.os_pipes[UNKNOWN_DEVICE_NAME].send(({'message': msg}, address)) log.info('No action requested. Ignoring.')