def start(self):
'''
Listen to messages and publish them.
'''
self._setup_ipc()
log.debug('Using the %s listener', self._listener_type)
self._setup_listener()
self.listener.start()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
try:
log_message, log_source = self.listener.receive()
except ListenerException as lerr:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(lerr, exc_info=True)
raise NapalmLogsExit(lerr)
log.debug('Received %s from %s. Queueing to the server.',
log_message, log_source)
if not log_message:
log.info(
'Empty message received from %s. Not queueing to the server.',
log_source)
continue
self.pub.send(umsgpack.packb((log_message, log_source)))
def start(self):
'''
Start listening for messages
'''
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
self.skt = self._open_socket(socket.SOCK_DGRAM)
try:
self.skt.bind((self.address, int(self.port)))
except socket.error as msg:
error_string = 'Unable to bind to port {} on {}: {}'.format(
self.port, self.address, msg)
log.error(error_string, exc_info=True)
raise ListenerException(error_string)
while self.__up:
try:
msg, addr = self.skt.recvfrom(BUFFER_SIZE)
except socket.error as error:
if self.__up is False:
return
else:
msg = 'Received listener socket error: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
log.debug(
'[{2}] Received {0} from {1}. Adding in the queue'.format(
msg, addr, time.time()))
self.pipe.send((msg, addr[0]))
def start(self):
'''
Listen to auth requests and send the AES key.
Each client connection starts a new thread.
'''
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
self.socket.listen(AUTH_MAX_CONN)
while self.__up:
try:
(clientsocket, address) = self.socket.accept()
wrapped_auth_skt = ssl.wrap_socket(clientsocket,
server_side=True,
certfile=self.certificate,
keyfile=self.keyfile)
except ssl.SSLError:
log.exception('SSL error', exc_info=True)
continue
except socket.error as error:
if self.__up is False:
return
else:
msg = 'Received auth socket error: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
log.info('{0} connected'.format(address))
log.debug('Starting the handshake')
client_thread = threading.Thread(target=self._handshake,
args=(wrapped_auth_skt, address))
client_thread.start()
def start(self):
'''
Start listening for messages
'''
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
self.skt = self._open_socket(socket.SOCK_STREAM)
try:
self.skt.bind((self.address, int(self.port)))
except socket.error as msg:
error_string = 'Unable to bind to port {} on {}: {}'.format(
self.port, self.address, msg)
log.error(error_string, exc_info=True)
raise ListenerException(error_string)
while self.__up:
self.skt.listen(1)
try:
conn, addr = self.skt.accept()
except socket.error as error:
if self.__up is False:
return
else:
msg = 'Received listener socket error: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
thread = threading.Thread(target=self._tcp_connection,
args=(
conn,
addr,
))
thread.start()
def start(self):
'''
Listen to messages and publish them.
'''
# self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.transport.start()
self.__up = True
while self.__up:
# bin_obj = self.sub.recv() # already serialized
try:
obj = self.pipe.recv()
except IOError as error:
if self.__up is False:
return
else:
msg = 'Received IOError on publisher pipe: {}'.format(
error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
log.debug('Publishing the OC object (serialised)')
if not self.disable_security:
bin_obj = self._prepare(obj)
else:
bin_obj = umsgpack.packb(obj)
self.transport.publish(bin_obj)
def start(self):
'''
Listen to messages and publish them.
'''
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.transport.start()
self.__up = True
while self.__up:
# bin_obj = self.sub.recv() # already serialized
try:
# obj = self.pipe.recv()
bin_obj = self.sub.recv()
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
log.debug('Publishing the OC object (serialised)')
if not self.disable_security and self.__transport_encrypt:
bin_obj = self._prepare(bin_obj)
self.transport.publish(bin_obj)
def start(self):
'''
Take the messages from the queue,
inspect and identify the operating system,
then queue the message correspondingly.
'''
# self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
# If were are to log all processed syslogs we need to initiate the class
if self.logger.get('syslog'):
self._setup_log_syslog_transport()
self.__up = True
while self.__up:
# Take messages from the main queue
# bin_obj = self.sub.recv()
# msg, address = umsgpack.unpackb(bin_obj, use_list=False)
try:
msg, address = self.pipe.recv()
except IOError as error:
if self.__up is False:
return
else:
msg = 'Received IOError from server pipe: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
msg = msg.encode('utf8')
log.debug('[{2}] Dequeued message from {0}: {1}'.format(
address, msg, time.time()))
dev_os, msg_dict = self._identify_os(msg)
log.debug('Identified OS: {0}'.format(dev_os))
if self.logger.get('syslog'):
if dev_os:
self._send_log_syslog(dev_os, msg_dict)
else:
self._send_log_syslog('unknown', {'message': msg})
if not dev_os or not isinstance(msg_dict, dict):
# _identify_os should return a string and a dict
# providing the info for the device OS
# and the decoded message prefix
continue
# Then send the message in the right queue
# obj = (msg_dict, address)
# bin_obj = umsgpack.packb(obj)
if dev_os in self.os_pipes:
log.debug('Queueing message to {0}'.format(dev_os))
# self.pubs[dev_os].send(bin_obj)
self.os_pipes[dev_os].send((msg_dict, address))
else:
log.info(
'Unable to queue the message to {0}. Is the sub-process started?'
.format(dev_os))
def start(self):
'''
Start the worker process.
'''
# self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
# bin_obj = self.sub.recv()
# msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False)
try:
msg_dict, address = self.pipe.recv()
except IOError as error:
if self.__up is False:
return
else:
msg = 'Received IOError on {} device pipe: {}'.format(self._name, error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
log.debug('{0}: dequeued {1}, received from {2}'.format(self._name, msg_dict, address))
kwargs = self._parse(msg_dict)
if not kwargs:
continue
try:
oc_obj = self._emit(**kwargs)
except Exception as err:
log.exception('Unexpected error when generating the OC object.', exc_info=True)
continue
log.debug('Generated OC object:')
log.debug(oc_obj)
error = kwargs.get('error')
model_name = kwargs.get('oc_model')
host = msg_dict.get('host')
timestamp = self._format_time(msg_dict.get('time', ''), msg_dict.get('date', ''))
to_publish = {
'error': error,
'host': host,
'ip': address,
'timestamp': timestamp,
'open_config': oc_obj,
'message_details': msg_dict,
'model_name': model_name,
'os': self._name
}
log.debug('Queueing to be published:')
log.debug(to_publish)
self.pub_pipe.send(to_publish)
def start(self):
'''
Listen to messages and publish them.
'''
# counter metrics for messages
c_logs_ingested = Counter(
'napalm_logs_listener_logs_ingested',
'Count of ingested log messages',
['listener_type', 'address', 'port'],
)
c_messages_published = Counter(
'napalm_logs_listener_messages_published',
'Count of published messages',
['listener_type', 'address', 'port'],
)
self._setup_ipc()
log.debug('Using the %s listener', self._listener_type)
self._setup_listener()
self.listener.start()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
try:
log_message, log_source = self.listener.receive()
except ListenerException as lerr:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(lerr, exc_info=True)
raise NapalmLogsExit(lerr)
log.debug('Received %s from %s. Queueing to the server.',
log_message, log_source)
if not log_message:
log.info(
'Empty message received from %s. Not queueing to the server.',
log_source,
)
continue
c_logs_ingested.labels(listener_type=self._listener_type,
address=self.address,
port=self.port).inc()
self.pub.send(umsgpack.packb((log_message, log_source)))
c_messages_published.labels(listener_type=self._listener_type,
address=self.address,
port=self.port).inc()
def start(self):
'''
Listen to messages and publish them.
'''
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
try:
zmq.proxy(self.sub, self.pub)
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
def start(self):
'''
Listen to messages and publish them.
'''
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.transport.start()
self.__up = True
while self.__up:
try:
bin_obj = self.sub.recv()
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
obj = umsgpack.unpackb(bin_obj)
if not napalm_logs.ext.check_whitelist_blacklist(
obj['error'],
whitelist=self.error_whitelist,
blacklist=self.error_blacklist):
# Apply the whitelist / blacklist logic
# If it doesn't match, jump over.
log.debug('This error type is %s. Skipping for %s #%d',
obj['error'], self._transport_type, self.pub_id)
continue
serialized_obj = self._serialize(obj, bin_obj)
log.debug('Publishing the OC object')
if not self.disable_security and self.__transport_encrypt:
# Encrypt only when needed.
serialized_obj = self._prepare(serialized_obj)
self.transport.publish(serialized_obj)
def start(self):
'''
Listen to messages and publish them.
'''
# metrics
napalm_logs_publisher_received_messages = Counter(
'napalm_logs_publisher_received_messages',
"Count of messages received by the publisher",
['publisher_type', 'address', 'port']
)
napalm_logs_publisher_whitelist_blacklist_check_fail = Counter(
'napalm_logs_publisher_whitelist_blacklist_check_fail',
"Count of messages which fail the whitelist/blacklist check",
['publisher_type', 'address', 'port']
)
napalm_logs_publisher_messages_published = Counter(
'napalm_logs_publisher_messages_published',
"Count of published messages",
['publisher_type', 'address', 'port']
)
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.transport.start()
self.__up = True
while self.__up:
try:
bin_obj = self.sub.recv()
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
obj = umsgpack.unpackb(bin_obj)
if self._strip_message_details:
obj.pop('message_details', None)
bin_obj = self.serializer_fun(obj)
napalm_logs_publisher_received_messages.labels(
publisher_type=self._transport_type,
address=self.address,
port=self.port
).inc()
if not napalm_logs.ext.check_whitelist_blacklist(obj['error'],
whitelist=self.error_whitelist,
blacklist=self.error_blacklist):
# Apply the whitelist / blacklist logic
# If it doesn't match, jump over.
log.debug('This error type is %s. Skipping for %s #%d',
obj['error'],
self._transport_type,
self.pub_id)
napalm_logs_publisher_whitelist_blacklist_check_fail.labels(
publisher_type=self._transport_type,
address=self.address,
port=self.port
).inc()
continue
serialized_obj = self._serialize(obj, bin_obj)
log.debug('Publishing the OC object')
if not self.disable_security and self.__transport_encrypt:
# Encrypt only when needed.
serialized_obj = self._prepare(serialized_obj)
self.transport.publish(serialized_obj)
napalm_logs_publisher_messages_published.labels(
publisher_type=self._transport_type,
address=self.address,
port=self.port
).inc()
def start(self):
'''
Take the messages from the queue,
inspect and identify the operating system,
then queue the message correspondingly.
'''
# metric counters
napalm_logs_server_messages_received = Counter(
"napalm_logs_server_messages_received",
"Count of messages received from listener processes",
)
napalm_logs_server_skipped_buffered_messages = Counter(
'napalm_logs_server_skipped_buffered_messages',
'Count of messages skipped as they were already buffered',
['device_os'],
)
napalm_logs_server_messages_with_identified_os = Counter(
"napalm_logs_server_messages_with_identified_os",
"Count of messages with positive os identification",
['device_os'],
)
napalm_logs_server_messages_without_identified_os = Counter(
"napalm_logs_server_messages_without_identified_os",
"Count of messages with negative os identification",
)
napalm_logs_server_messages_failed_device_queuing = Counter(
"napalm_logs_server_messages_failed_device_queuing",
"Count of messages per device os that fail to be queued to a device process",
['device_os'],
)
napalm_logs_server_messages_device_queued = Counter(
"napalm_logs_server_messages_device_queued",
"Count of messages queued to device processes",
['device_os'],
)
napalm_logs_server_messages_unknown_queued = Counter(
"napalm_logs_server_messages_unknown_queued",
"Count of messages queued as unknown",
)
if self.opts.get('metrics_include_attributes', True):
napalm_logs_server_messages_attrs = Counter(
"napalm_logs_server_messages_attrs",
"Count of messages from the server process with their details",
['device_os', 'host', 'tag'],
)
self._setup_ipc()
# Start suicide polling thread
cleanup = threading.Thread(target=self._cleanup_buffer)
cleanup.start()
thread = threading.Thread(
target=self._suicide_when_without_parent, args=(os.getppid(),)
)
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
# Take messages from the main queue
try:
bin_obj = self.sub.recv()
msg, address = umsgpack.unpackb(bin_obj, use_list=False)
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
if isinstance(msg, bytes):
if six.PY3:
msg = str(msg, 'utf-8')
else:
msg = msg.encode('utf-8')
log.debug('[%s] Dequeued message from %s: %s', address, msg, time.time())
napalm_logs_server_messages_received.inc()
os_list = self._identify_os(msg)
for dev_os, msg_dict in os_list:
if dev_os and dev_os in self.started_os_proc:
# Identified the OS and the corresponding process is started.
# Then send the message in the right queue
log.debug('Identified OS: %s', dev_os)
log.debug('Queueing message to %s', dev_os)
if six.PY3:
dev_os = bytes(dev_os, 'utf-8')
napalm_logs_server_messages_with_identified_os.labels(
device_os=dev_os.decode()
).inc()
if self._buffer:
message = '{dev_os}/{host}/{msg}'.format(
dev_os=dev_os.decode(),
host=msg_dict['host'],
msg=msg_dict['message'],
)
if six.PY3:
message_key = base64.b64encode(
bytes(message, 'utf-8')
).decode()
else:
message_key = base64.b64encode(message)
if self._buffer[message_key]:
log.info(
'"%s" seems to be already buffered, skipping',
msg_dict['message'],
)
napalm_logs_server_skipped_buffered_messages.labels(
device_os=dev_os.decode()
).inc()
continue
log.debug(
'"%s" is not buffered yet, added', msg_dict['message']
)
self._buffer[message_key] = 1
self.pub.send_multipart(
[dev_os, umsgpack.packb((msg_dict, address))]
)
napalm_logs_server_messages_device_queued.labels(
device_os=dev_os.decode()
).inc()
if self.opts.get('metrics_server_include_attributes', True):
napalm_logs_server_messages_attrs.labels(
device_os=dev_os.decode(),
host=msg_dict['host'],
tag=msg_dict['tag'],
).inc()
elif dev_os and dev_os not in self.started_os_proc:
# Identified the OS, but the corresponding process does not seem to be started.
log.info(
'Unable to queue the message to %s. Is the sub-process started?',
dev_os,
)
napalm_logs_server_messages_with_identified_os.labels(
device_os=dev_os.decode()
).inc()
napalm_logs_server_messages_failed_device_queuing.labels(
device_os=dev_os.decode()
).inc()
elif not dev_os and self.opts['_server_send_unknown']:
# OS not identified, but the user requested to publish the message as-is
log.debug(
'Unable to identify the OS, sending directly to the publishers'
)
to_publish = {
'ip': address,
'host': 'unknown',
'timestamp': int(time.time()),
'message_details': msg_dict,
'os': UNKNOWN_DEVICE_NAME,
'error': 'UNKNOWN',
'model_name': 'unknown',
}
self.publisher_pub.send(umsgpack.packb(to_publish))
napalm_logs_server_messages_unknown_queued.inc()
napalm_logs_server_messages_without_identified_os.inc()
def start(self):
'''
Start the worker process.
'''
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
# bin_obj = self.sub.recv()
# msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False)
try:
bin_obj = self.sub.recv()
msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False)
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown [%s]', self._name)
return
else:
raise NapalmLogsExit(error)
log.debug('%s: dequeued %s, received from %s', self._name,
msg_dict, address)
if self._name == UNKNOWN_DEVICE_NAME:
# If running in the sub-process publishing messages for unknown OSs.
# This will always send what receives, as-is.
log.debug(
'Publishing the message as-is, no further processing possible.'
)
to_publish = {
'ip': address,
'host': 'unknown',
'timestamp': int(time.time()),
'message_details': msg_dict,
'os': UNKNOWN_DEVICE_NAME,
'error': 'UNKNOWN',
'model_name': 'unknown'
}
log.debug('Queueing to be published:')
log.debug(to_publish)
# self.pub_pipe.send(to_publish)
self.pub.send(umsgpack.packb(to_publish))
continue
# From here on, we're running in a regular OS sub-process.
host = msg_dict.get('host')
prefix_id = msg_dict.pop('__prefix_id__')
if 'timestamp' in msg_dict:
timestamp = msg_dict.pop('timestamp')
else:
timestamp = self._format_time(msg_dict.get('time', ''),
msg_dict.get('date', ''),
msg_dict.get('timeZone', 'UTC'),
prefix_id)
facility = msg_dict.get('facility')
severity = msg_dict.get('severity')
kwargs = self._parse(msg_dict)
if not kwargs:
# Unable to identify what model to generate for the message in cause.
if self.publisher_opts.get('send_raw'):
# But publish the message when the user requested to push raw messages.
to_publish = {
'ip': address,
'host': host,
'timestamp': timestamp,
'message_details': msg_dict,
'os': self._name,
'error': 'RAW',
'model_name': 'raw',
'facility': facility,
'severity': severity
}
log.debug('Queueing to be published:')
log.debug(to_publish)
# self.pub_pipe.send(to_publish)
self.pub.send(umsgpack.packb(to_publish))
continue
try:
if '__python_fun__' in kwargs:
log.debug(
'Using the Python parser to determine the YANG-equivalent object'
)
yang_obj = kwargs['__python_fun__'](msg_dict)
else:
yang_obj = self._emit(**kwargs)
except Exception:
log.exception(
'Unexpected error when generating the OC object.',
exc_info=True)
continue
log.debug('Generated OC object:')
log.debug(yang_obj)
error = kwargs.get('error')
model_name = kwargs.get('model')
to_publish = {
'error': error,
'host': host,
'ip': address,
'timestamp': timestamp,
'yang_message': yang_obj,
'message_details': msg_dict,
'yang_model': model_name,
'os': self._name,
'facility': facility,
'severity': severity
}
log.debug('Queueing to be published:')
log.debug(to_publish)
# self.pub_pipe.send(to_publish)
self.pub.send(umsgpack.packb(to_publish))
def start(self):
'''
Start the worker process.
'''
# metrics
napalm_logs_device_messages_received = Counter(
'napalm_logs_device_messages_received',
"Count of messages received by the device process", ['device_os'])
napalm_logs_device_raw_published_messages = Counter(
'napalm_logs_device_raw_published_messages',
"Count of raw type published messages", ['device_os'])
napalm_logs_device_published_messages = Counter(
'napalm_logs_device_published_messages',
"Count of published messages", ['device_os'])
napalm_logs_device_oc_object_failed = Counter(
'napalm_logs_device_oc_object_failed',
"Counter of failed OpenConfig object generations", ['device_os'])
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
# bin_obj = self.sub.recv()
# msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False)
try:
bin_obj = self.sub.recv()
msg_dict, address = umsgpack.unpackb(bin_obj, use_list=False)
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown [%s]', self._name)
return
else:
raise NapalmLogsExit(error)
log.debug('%s: dequeued %s, received from %s', self._name,
msg_dict, address)
napalm_logs_device_messages_received.labels(
device_os=self._name).inc()
host = msg_dict.get('host')
prefix_id = msg_dict.pop('__prefix_id__')
if 'timestamp' in msg_dict:
timestamp = msg_dict.pop('timestamp')
else:
timestamp = self._format_time(msg_dict.get('time', ''),
msg_dict.get('date', ''),
msg_dict.get('timeZone', 'UTC'),
prefix_id)
facility = msg_dict.get('facility')
severity = msg_dict.get('severity')
kwargs = self._parse(msg_dict)
if not kwargs:
# Unable to identify what model to generate for the message in cause.
# But publish the message when the user requested to push raw messages.
to_publish = {
'ip': address,
'host': host,
'timestamp': timestamp,
'message_details': msg_dict,
'os': self._name,
'error': 'RAW',
'model_name': 'raw',
'facility': facility,
'severity': severity
}
log.debug('Queueing to be published:')
log.debug(to_publish)
# self.pub_pipe.send(to_publish)
self.pub.send(umsgpack.packb(to_publish))
napalm_logs_device_raw_published_messages.labels(
device_os=self._name).inc()
continue
try:
if '__python_fun__' in kwargs:
log.debug(
'Using the Python parser to determine the YANG-equivalent object'
)
yang_obj = kwargs['__python_fun__'](msg_dict)
else:
yang_obj = self._emit(**kwargs)
except Exception:
log.exception(
'Unexpected error when generating the OC object.',
exc_info=True)
napalm_logs_device_oc_object_failed.labels(
device_os=self._name).inc()
continue
log.debug('Generated OC object:')
log.debug(yang_obj)
error = kwargs.get('error')
model_name = kwargs.get('model')
to_publish = {
'error': error,
'host': host,
'ip': address,
'timestamp': timestamp,
'yang_message': yang_obj,
'message_details': msg_dict,
'yang_model': model_name,
'os': self._name,
'facility': facility,
'severity': severity
}
log.debug('Queueing to be published:')
log.debug(to_publish)
# self.pub_pipe.send(to_publish)
self.pub.send(umsgpack.packb(to_publish))
# self._publish(to_publish)
napalm_logs_device_published_messages.labels(
device_os=self._name).inc()
def start(self):
'''
Take the messages from the queue,
inspect and identify the operating system,
then queue the message correspondingly.
'''
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
while self.__up:
# Take messages from the main queue
try:
bin_obj = self.sub.recv()
msg, address = umsgpack.unpackb(bin_obj, use_list=False)
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
if six.PY3:
msg = str(msg, 'utf-8')
else:
msg = msg.encode('utf-8')
log.debug('[%s] Dequeued message from %s: %s', address, msg,
time.time())
dev_os, msg_dict = self._identify_os(msg)
if dev_os and dev_os in self.started_os_proc:
# Identified the OS and the corresponding process is started.
# Then send the message in the right queue
log.debug('Identified OS: %s', dev_os)
log.debug('Queueing message to %s', dev_os)
if six.PY3:
dev_os = bytes(dev_os, 'utf-8')
self.pub.send_multipart(
[dev_os, umsgpack.packb((msg_dict, address))])
# self.os_pipes[dev_os].send((msg_dict, address))
elif dev_os and dev_os not in self.started_os_proc:
# Identified the OS, but the corresponding process does not seem to be started.
log.info(
'Unable to queue the message to %s. Is the sub-process started?',
dev_os)
elif not dev_os and self.opts['_server_send_unknown']:
# OS not identified, but the user requested to publish the message as-is
log.debug(
'Unable to identify the OS, sending directly to the publishers'
)
to_publish = {
'ip': address,
'host': 'unknown',
'timestamp': int(time.time()),
'message_details': msg_dict,
'os': UNKNOWN_DEVICE_NAME,
'error': 'UNKNOWN',
'model_name': 'unknown'
}
self.publisher_pub.send(umsgpack.packb(to_publish))
def start(self):
'''
Take the messages from the queue,
inspect and identify the operating system,
then queue the message correspondingly.
'''
self._setup_ipc()
# Start suicide polling thread
thread = threading.Thread(target=self._suicide_when_without_parent,
args=(os.getppid(), ))
thread.start()
signal.signal(signal.SIGTERM, self._exit_gracefully)
# If were are to log all processed syslogs we need to initiate the class
if self.logger:
self._setup_log_syslog_transport()
self.__up = True
while self.__up:
# Take messages from the main queue
# bin_obj = self.sub.recv()
# msg, address = umsgpack.unpackb(bin_obj, use_list=False)
try:
bin_obj = self.sub.recv()
msg, address = umsgpack.unpackb(bin_obj, use_list=False)
except zmq.ZMQError as error:
if self.__up is False:
log.info('Exiting on process shutdown')
return
else:
log.error(error, exc_info=True)
raise NapalmLogsExit(error)
if six.PY3:
msg = str(msg, 'utf-8')
else:
msg = msg.encode('utf-8')
log.debug('[%s] Dequeued message from %s: %s', address, msg,
time.time())
dev_os, msg_dict = self._identify_os(msg)
log.debug('Identified OS: %s', dev_os)
if self.logger:
if self.logger_opts.get('send_raw') and dev_os:
self._send_log_syslog(dev_os, msg_dict)
elif self.logger_opts.get('send_unknown') and not dev_os:
self._send_log_syslog(UNKNOWN_DEVICE_NAME,
{'message': msg})
if dev_os and dev_os in self.started_os_proc:
# Identified the OS and the corresponding process is started.
# Then send the message in the right queue
# obj = (msg_dict, address)
# bin_obj = umsgpack.packb(obj)
log.debug('Queueing message to %s', dev_os)
# self.pubs[dev_os].send(bin_obj)
if six.PY3:
dev_os = bytes(dev_os, 'utf-8')
self.pub.send_multipart(
[dev_os, umsgpack.packb((msg_dict, address))])
# self.os_pipes[dev_os].send((msg_dict, address))
elif dev_os and dev_os not in self.started_os_proc:
# Identified the OS, but the corresponding process does not seem to be started.
log.info(
'Unable to queue the message to %s. Is the sub-process started?',
dev_os)
elif not dev_os and self.publisher_opts.get('send_unknown'):
# OS not identified, but the user requested to publish the message as-is
log.debug(
'Publishing message, although not identified, as requested'
)
if six.PY3:
dev_os = bytes(UNKNOWN_DEVICE_NAME, 'utf-8')
else:
dev_os = UNKNOWN_DEVICE_NAME
self.pub.send_multipart(
[dev_os,
umsgpack.packb(({
'message': msg
}, address))])
# self.os_pipes[UNKNOWN_DEVICE_NAME].send(({'message': msg}, address))
log.info('No action requested. Ignoring.')
