def start(self):
'''
Start listening for messages
'''
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
self.skt = self._open_socket(socket.SOCK_DGRAM)
try:
self.skt.bind((self.address, int(self.port)))
except socket.error as msg:
error_string = 'Unable to bind to port {} on {}: {}'.format(
self.port, self.address, msg)
log.error(error_string, exc_info=True)
raise ListenerException(error_string)
while self.__up:
try:
msg, addr = self.skt.recvfrom(BUFFER_SIZE)
except socket.error as error:
if self.__up is False:
return
else:
msg = 'Received listener socket error: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
log.debug(
'[{2}] Received {0} from {1}. Adding in the queue'.format(
msg, addr, time.time()))
self.pipe.send((msg, addr[0]))
def start(self):
'''
Start listening for messages
'''
# Start suicide polling thread
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
try:
self.consumer = kafka.KafkaConsumer(bootstrap_servers=self.bootstrap_servers,
group_id='napalm-logs')
except kafka.errors.NoBrokersAvailable as err:
log.error(err, exc_info=True)
raise ListenerException(err)
self.consumer.subscribe(topics=[self.kafka_topic])
while self.__up:
try:
msg = next(self.consumer)
except ValueError as error:
if self.__up is False:
return
else:
msg = 'Received kafka error: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
log_source = msg.key
decoded = json.loads(msg.value)
log_message = decoded.get('message')
log.debug('[{2}] Received {0} from {1}. Adding in the queue'.format(log_message, log_source, time.time()))
self.pipe.send((log_message, log_source))
def start(self):
'''
Start listening for messages
'''
signal.signal(signal.SIGTERM, self._exit_gracefully)
self.__up = True
self.skt = self._open_socket(socket.SOCK_STREAM)
try:
self.skt.bind((self.address, int(self.port)))
except socket.error as msg:
error_string = 'Unable to bind to port {} on {}: {}'.format(
self.port, self.address, msg)
log.error(error_string, exc_info=True)
raise ListenerException(error_string)
while self.__up:
self.skt.listen(1)
try:
conn, addr = self.skt.accept()
except socket.error as error:
if self.__up is False:
return
else:
msg = 'Received listener socket error: {}'.format(error)
log.error(msg, exc_info=True)
raise NapalmLogsExit(msg)
thread = threading.Thread(target=self._tcp_connection,
args=(
conn,
addr,
))
thread.start()
def receive(self):
'''
Return the message received and the address.
'''
try:
msg, addr = self.skt.recvfrom(self.buffer_size)
except socket.error as error:
log.error('Received listener socket error: %s', error, exc_info=True)
raise ListenerException(error)
log.debug('[%s] Received %s from %s', msg, addr, time.time())
return msg, addr[0]
def receive(self):
'''
Return the message received.
..note::
In ZMQ we are unable to get the address where we got the message from.
'''
try:
msg = self.sub.recv()
except zmq.Again as error:
log.error('Unable to receive messages: %s', error, exc_info=True)
raise ListenerException(error)
log.debug('[%s] Received %s', time.time(), msg)
return msg, ''
def start(self):
'''
Startup the kafka consumer.
'''
log.debug(
'Creating the consumer using the bootstrap servers: %s and the group ID: %s',
self.bootstrap_servers, self.group_id)
try:
self.consumer = kafka.KafkaConsumer(
bootstrap_servers=self.bootstrap_servers,
group_id=self.group_id)
except kafka.errors.NoBrokersAvailable as err:
log.error(err, exc_info=True)
raise ListenerException(err)
log.debug('Subscribing to the %s topic', self.topic)
self.consumer.subscribe(topics=[self.topic])
def receive(self):
'''
Return the message received and the address.
'''
try:
msg = next(self.consumer)
except ValueError as error:
log.error('Received kafka error: %s', error, exc_info=True)
raise ListenerException(error)
log_source = msg.key
try:
decoded = json.loads(msg.value.decode('utf-8'))
except ValueError:
log.error('Not in json format: %s', msg.value.decode('utf-8'))
return '', ''
log_message = decoded.get('message')
log.debug('[%s] Received %s from %s', log_message, log_source,
time.time())
return log_message, log_source
def _serve_clients(self):
'''
Accept cients and serve, one separate thread per client.
'''
self.__up = True
while self.__up:
log.debug('Waiting for a client to connect')
try:
conn, addr = self.skt.accept()
log.debug('Received connection from %s:%d', addr[0], addr[1])
except socket.error as error:
if not self.__up:
return
msg = 'Received listener socket error: {}'.format(error)
log.error(msg, exc_info=True)
raise ListenerException(msg)
client_thread = threading.Thread(
target=self._client_connection, args=(conn, addr,)
)
client_thread.start()
def _client_connection(self, conn, addr):
'''
Handle the connecition with one client.
'''
log.debug('Established connection with %s:%d', addr[0], addr[1])
conn.settimeout(self.socket_timeout)
try:
while self.__up:
msg = conn.recv(self.buffer_size)
if not msg:
# log.debug('Received empty message from %s', addr)
# disabled ^ as it was too noisy
continue
log.debug('[%s] Received %s from %s. Adding in the queue',
time.time(), msg, addr)
self.buffer.put((msg, '{}:{}'.format(addr[0], addr[1])))
except socket.timeout:
if not self.__up:
return
log.debug('Connection %s:%d timed out', addr[1], addr[0])
raise ListenerException('Connection %s:%d timed out' % addr)
finally:
log.debug('Closing connection with %s', addr)
conn.close()