def test_builtin_kafka(cls):
cls.beaver_config.set('transport', 'kafka')
cls.beaver_config.set('logstash_version', 1)
cls.beaver_config.set('kafka_hosts', cls.server.host + ":" + str(cls.server.port))
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.kafka_transport.KafkaTransport)
data = {}
lines = []
n=100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
transport.callback("test.log", **data)
messages = cls._consume_messages(cls.server.host, cls.server.port)
cls.assertEqual(n, messages.__len__())
for message in messages:
cls.assertIn('"file": "test.log", "message": "log', message.message.value);
print(message)
print('\n')
transport.interrupt()
def test_builtin_rabbitmq(self):
beaver_config = self._get_config(transport='rabbitmq')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(
transport,
beaver.transports.rabbitmq_transport.RabbitmqTransport)
transport.interrupt()
def test_kinesis_send_stream(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_stream', 'stream1')
self.beaver_config.set('kinesis_aws_profile_name', None)
self.beaver_config.set('kinesis_aws_access_key', None)
self.beaver_config.set('kinesis_aws_secret_key', None)
self.beaver_config.set('kinesis_bulk_lines', False)
transport = create_transport(self.beaver_config, logger=self.logger)
mock_send_batch = mock.Mock()
transport._send_message_batch = mock_send_batch
self.assertIsInstance(
transport, beaver.transports.kinesis_transport.KinesisTransport)
data = {}
lines = []
n = 500
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
self.assertTrue(transport.callback("test.log", **data))
self.assertEqual(1, mock_send_batch.call_count)
def test_sqs_send_multi_queue_bulklines(cls):
cls._create_queues()
cls.beaver_config.set('sqs_aws_queue', 'queue1,queue2')
cls.beaver_config.set('sqs_aws_profile_name', None)
cls.beaver_config.set('sqs_aws_access_key', None)
cls.beaver_config.set('sqs_aws_secret_key', None)
cls.beaver_config.set('sqs_bulk_lines', True)
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport)
data = {}
lines = []
n=100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
transport.callback("test.log", **data)
def test_builtin_kafka(cls):
cls.beaver_config.set('transport', 'kafka')
cls.beaver_config.set('logstash_version', 1)
cls.beaver_config.set('kafka_hosts',
cls.server.host + ":" + str(cls.server.port))
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport,
beaver.transports.kafka_transport.KafkaTransport)
data = {}
lines = []
n = 100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
transport.callback("test.log", **data)
messages = cls._consume_messages(cls.server.host, cls.server.port)
cls.assertEqual(n, messages.__len__())
for message in messages:
cls.assertIn('"file": "test.log", "message": "log',
message.message.value)
print(message)
print('\n')
transport.interrupt()
def test_kinesis_send_stream(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_stream', 'stream1')
self.beaver_config.set('kinesis_aws_profile_name', None)
self.beaver_config.set('kinesis_aws_access_key', None)
self.beaver_config.set('kinesis_aws_secret_key', None)
self.beaver_config.set('kinesis_bulk_lines', False)
transport = create_transport(self.beaver_config, logger=self.logger)
mock_send_batch = mock.Mock()
transport._send_message_batch = mock_send_batch
self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport)
data = {}
lines = []
n=500
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
self.assertTrue(transport.callback("test.log", **data))
self.assertEqual(1, mock_send_batch.call_count)
def test_kinesis_auth_profile(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_profile_name', 'beaver_stream')
self.beaver_config.set('kinesis_aws_access_key', None)
self.beaver_config.set('kinesis_aws_secret_key', None)
self.beaver_config.set('kinesis_aws_stream', 'stream1')
transport = create_transport(self.beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport)
def test_sqs_auth_profile(cls):
cls._create_queues()
cls.beaver_config.set('sqs_aws_profile_name', 'beaver_queue')
cls.beaver_config.set('sqs_aws_access_key', None)
cls.beaver_config.set('sqs_aws_secret_key', None)
cls.beaver_config.set('sqs_aws_queue', 'queue1')
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport)
def test_kinesis_auth_profile(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_profile_name', 'beaver_stream')
self.beaver_config.set('kinesis_aws_access_key', None)
self.beaver_config.set('kinesis_aws_secret_key', None)
self.beaver_config.set('kinesis_aws_stream', 'stream1')
transport = create_transport(self.beaver_config, logger=self.logger)
self.assertIsInstance(
transport, beaver.transports.kinesis_transport.KinesisTransport)
def test_sqs_single_queue(cls):
cls._create_queues()
cls.beaver_config.set('sqs_aws_queue', 'queue1')
cls.beaver_config.set('sqs_aws_profile_name', None)
cls.beaver_config.set('sqs_aws_access_key', None)
cls.beaver_config.set('sqs_aws_secret_key', None)
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver_sqs.Transport)
transport.interrupt()
def test_kinesis_auth_account_id(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_stream_owner_acct_id', 'abc123')
self.beaver_config.set('kinesis_aws_profile_name', None)
self.beaver_config.set('kinesis_aws_access_key', 'beaver_test_key')
self.beaver_config.set('kinesis_aws_secret_key', 'beaver_test_secret')
self.beaver_config.set('kinesis_aws_stream', 'stream1')
transport = create_transport(self.beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport)
transport.interrupt()
def test_sqs_auth_account_id(cls):
cls._create_queues()
cls.beaver_config.set('sqs_aws_queue_owner_acct_id', 'abc123')
cls.beaver_config.set('sqs_aws_profile_name', None)
cls.beaver_config.set('sqs_aws_access_key', 'beaver_test_key')
cls.beaver_config.set('sqs_aws_secret_key', 'beaver_test_secret')
cls.beaver_config.set('sqs_aws_queue', 'queue1')
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport)
transport.interrupt()
def test_sqs_multi_queue_bulklines(cls):
cls._create_queues()
cls.beaver_config.set('sqs_aws_queue', 'queue1,queue2')
cls.beaver_config.set('sqs_aws_profile_name', None)
cls.beaver_config.set('sqs_aws_access_key', None)
cls.beaver_config.set('sqs_aws_secret_key', None)
cls.beaver_config.set('sqs_bulk_lines', True)
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport)
transport.interrupt()
def test_kinesis_auth_account_id(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_stream_owner_acct_id', 'abc123')
self.beaver_config.set('kinesis_aws_profile_name', None)
self.beaver_config.set('kinesis_aws_access_key', 'beaver_test_key')
self.beaver_config.set('kinesis_aws_secret_key', 'beaver_test_secret')
self.beaver_config.set('kinesis_aws_stream', 'stream1')
transport = create_transport(self.beaver_config, logger=self.logger)
self.assertIsInstance(
transport, beaver.transports.kinesis_transport.KinesisTransport)
transport.interrupt()
def test_kinesis_send_stream_with_retries(self):
self._create_streams()
transport = create_transport(self.beaver_config, logger=self.logger)
mock_client = mock.Mock()
transport._connection = mock_client
mock_client.put_records.side_effect = [{'FailedRecordCount': 1}, {}]
transport._send_message_batch("doesntmatter")
self.assertEqual(2, mock_client.put_records.call_count)
mock_client.reset_mock()
mock_client.put_records.side_effect = [
Exception('ProvisionedThroughputExceededException'),
Exception('ThrottlingException'), {}
]
transport._send_message_batch("doesntmatter")
self.assertEqual(3, mock_client.put_records.call_count)
def run_queue(queue, beaver_config, logger=None):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
last_update_time = int(time.time())
queue_timeout = beaver_config.get('queue_timeout')
wait_timeout = beaver_config.get('wait_timeout')
transport = None
try:
logger.debug('Logging using the {0} transport'.format(
beaver_config.get('transport')))
transport = create_transport(beaver_config, logger=logger)
failure_count = 0
while True:
if not transport.valid():
logger.info('Transport connection issues, stopping queue')
break
if int(time.time()) - last_update_time > queue_timeout:
logger.info(
'Queue timeout of "{0}" seconds exceeded, stopping queue'.
format(queue_timeout))
break
try:
command, data = queue.get(block=True, timeout=wait_timeout)
last_update_time = int(time.time())
logger.debug(
'Last update time now {0}'.format(last_update_time))
except Queue.Empty:
logger.debug('No data')
continue
if command == 'callback':
try:
if data.get('ignore_empty', False):
logger.debug('removing empty lines')
lines = data['lines']
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
if len(data['lines']) == 0:
logger.debug('0 active lines sent from worker')
continue
transport.callback(**data)
except TransportException:
failure_count = failure_count + 1
if failure_count > beaver_config.get('max_failure'):
failure_count = beaver_config.get('max_failure')
sleep_time = int(
beaver_config.get('respawn_delay'))**failure_count
logger.info(
'Caught transport exception, respawning in %d seconds'
% sleep_time)
try:
time.sleep(sleep_time)
transport.reconnect()
except KeyboardInterrupt:
logger.info('User cancelled respawn.')
transport.interrupt()
sys.exit(0)
elif command == 'addglob':
beaver_config.addglob(*data)
transport.addglob(*data)
elif command == 'exit':
break
except KeyboardInterrupt:
logger.debug('Queue Interruped')
if transport is not None:
transport.interrupt()
logger.debug('Queue Shutdown')
def test_builtin_stdout(self):
beaver_config = self._get_config(transport='stdout')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.stdout_transport.StdoutTransport)
def test_builtin_redis(self):
beaver_config = self._get_config(transport='redis')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.redis_transport.RedisTransport)
def test_monascalog(cls, token_mock):
# dynamic callback to verify the log messages sent by the transport
def request_callback(request, uri, headers):
cls.log_requests.append(request.parsed_body)
return (204, headers, "created")
# fake the first get call that is used for checking connection
httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed")
httpretty.register_uri(httpretty.POST, LOG_URL, status=204, body="Created")
cls.beaver_config.set('transport', 'monascalog')
cls.beaver_config.set('logstash_version', 1)
#cls.beaver_config.set('monascalog_hosts', "{}:{}".format(cls.server_host, cls.server_port))
cls.beaver_config.set('monascalog_max_retries', 3)
cls.beaver_config.set('monascalog_auth_url', cls.keystone_auth_url)
cls.beaver_config.set('monascalog_user_name', cls.keystone_user)
cls.beaver_config.set('monascalog_password', cls.keystone_password)
cls.beaver_config.set('monascalog_project_name', cls.keystone_project_name)
cls.beaver_config.set('monascalog_domain_name', cls.keystone_domain_name)
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.monascalog_transport.MonascalogTransport)
cls.assertEqual(transport.valid(), True)
data = {}
lines = []
n=100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
cls.assertEqual(transport.callback("test.log", **data), True)
# Fake a log api failure
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, status=500, body="Internal Server Error")
cls.assertRaises(TransportException, transport.callback, "test.log", **data)
# simulate a single failure followed by success, to test if retry works
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, responses=[
httpretty.Response(status=503, body="Service Unavailable"),
httpretty.Response(status=204, body="Created")
])
cls.assertEqual(transport.callback("test.log", **data), True)
# next, test if the logs made it to the server
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback)
# clear logs from previous tests
del cls.log_requests[:]
cls.assertEqual(transport.callback("test.log", **data), True)
cls._consume_messages(n)
# repeat same test, but with batching turned off
cls.beaver_config.set('monascalog_enable_batching', False)
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback)
# fake the first get call that is used for checking connection
httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed")
transport = create_transport(cls.beaver_config, logger=cls.logger)
# clear logs from previous tests
del cls.log_requests[:]
cls.assertEqual(transport.callback("test.log", **data), True)
cls._consume_messages(n, batching=False)
transport.interrupt()
def run_queue(queue, beaver_config, logger_name=None):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
if os.name != 'nt':
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
last_update_time = int(time.time())
queue_timeout = beaver_config.get('queue_timeout')
wait_timeout = beaver_config.get('wait_timeout')
count = 0
transport = None
try:
logger = setup_custom_logger(logger_name)
logger.debug('Logging using the {0} transport'.format(
beaver_config.get('transport')))
transport = create_transport(beaver_config, logger=logger)
failure_count = 0
while True:
if not transport.valid():
logger.info('Transport connection issues, stopping queue')
break
command = None
try:
if queue.full():
logger.error("Queue is full")
else:
if count == 1000:
logger.debug("Main consumer queue Size is: " +
str(queue.qsize()))
count = 0
command, data = queue.get(block=True, timeout=wait_timeout)
if command == "callback":
last_update_time = int(time.time())
logger.debug(
'Last update time now {0}'.format(last_update_time))
except Queue.Empty:
if not queue.empty():
logger.error(
'Recieved timeout from main consumer queue - stopping queue'
)
break
else:
logger.debug('No data')
if int(time.time()) - last_update_time > queue_timeout:
logger.info(
'Queue timeout of "{0}" seconds exceeded, stopping queue'.
format(queue_timeout))
break
if command == 'callback':
if data.get('ignore_empty', False):
logger.debug('removing empty lines')
lines = data['lines']
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
if len(data['lines']) == 0:
logger.debug('0 active lines sent from worker')
continue
while True:
try:
transport.callback(**data)
count += 1
logger.debug("Number of transports: " + str(count))
break
except TransportException as e:
failure_count = failure_count + 1
if failure_count > beaver_config.get('max_failure'):
failure_count = beaver_config.get('max_failure')
sleep_time = beaver_config.get(
'respawn_delay')**failure_count
logger.info('Caught transport exception: %s', e)
logger.info('Reconnecting in %d seconds' % sleep_time)
try:
transport.invalidate()
time.sleep(sleep_time)
transport.reconnect()
if transport.valid():
failure_count = 0
logger.info('Reconnected successfully')
except KeyboardInterrupt:
logger.info('User cancelled respawn.')
transport.interrupt()
sys.exit(0)
elif command == 'addglob':
beaver_config.addglob(*data)
transport.addglob(*data)
elif command == 'exit':
break
except KeyboardInterrupt:
logger.debug('Queue Interruped')
if transport is not None:
transport.interrupt()
logger.debug('Queue Shutdown')
def test_custom_transport(self):
beaver_config = self._get_config(transport='beaver.tests.test_transport_config.DummyTransport')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, DummyTransport)
def run_queue(queue, beaver_config, logger=None):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
last_update_time = int(time.time())
queue_timeout = beaver_config.get('queue_timeout')
wait_timeout = beaver_config.get('wait_timeout')
count = 0
transport = None
try:
logger.debug('Logging using the {0} transport'.format(beaver_config.get('transport')))
transport = create_transport(beaver_config, logger=logger)
failure_count = 0
while True:
if not transport.valid():
logger.info('Transport connection issues, stopping queue')
break
if int(time.time()) - last_update_time > queue_timeout:
logger.info('Queue timeout of "{0}" seconds exceeded, stopping queue'.format(queue_timeout))
break
command = None
try:
if queue.full():
logger.error("Queue is full")
else:
if count == 1000:
logger.debug("Main consumer queue Size is: " + str(queue.qsize()))
count = 0
command, data = queue.get(block=True, timeout=wait_timeout)
if command == "callback":
last_update_time = int(time.time())
logger.debug('Last update time now {0}'.format(last_update_time))
except Queue.Empty:
if not queue.empty():
logger.error('Recieved timeout from main consumer queue - stopping queue')
break
else:
logger.debug('No data')
if command == 'callback':
if data.get('ignore_empty', False):
logger.debug('removing empty lines')
lines = data['lines']
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
if len(data['lines']) == 0:
logger.debug('0 active lines sent from worker')
continue
while True:
try:
transport.callback(**data)
count += 1
logger.debug("Number of transports: " + str(count))
break
except TransportException:
failure_count = failure_count + 1
if failure_count > beaver_config.get('max_failure'):
failure_count = beaver_config.get('max_failure')
sleep_time = beaver_config.get('respawn_delay') ** failure_count
logger.info('Caught transport exception, reconnecting in %d seconds' % sleep_time)
try:
transport.invalidate()
time.sleep(sleep_time)
transport.reconnect()
if transport.valid():
failure_count = 0
logger.info('Reconnected successfully')
except KeyboardInterrupt:
logger.info('User cancelled respawn.')
transport.interrupt()
sys.exit(0)
elif command == 'addglob':
beaver_config.addglob(*data)
transport.addglob(*data)
elif command == 'exit':
break
except KeyboardInterrupt:
logger.debug('Queue Interruped')
if transport is not None:
transport.interrupt()
logger.debug('Queue Shutdown')
def test_builtin_udp(self):
beaver_config = self._get_config(transport='udp')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.udp_transport.UdpTransport)
def test_builtin_zmq(self):
beaver_config = self._get_config(transport='zmq')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, ZmqTransport)
def test_builtin_redis(self):
beaver_config = self._get_config(transport='redis')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.redis_transport.RedisTransport)
def test_builtin_stdout(self):
beaver_config = self._get_config(transport='stdout')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.stdout_transport.StdoutTransport)
def test_builtin_rabbitmq(self):
beaver_config = self._get_config(transport='rabbitmq')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.rabbitmq_transport.RabbitmqTransport)
transport.interrupt()
def test_builtin_udp(self):
beaver_config = self._get_config(transport='udp')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.udp_transport.UdpTransport)
def test_builtin_zmq(self):
beaver_config = self._get_config(transport='zmq')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, ZmqTransport)
def test_custom_transport(self):
beaver_config = self._get_config(transport='tests.test_transport_config.DummyTransport')
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, DummyTransport)
def test_builtin_zmq(self):
beaver_config = self._get_config(transport="zmq")
transport = create_transport(beaver_config, logger=self.logger)
self.assertIsInstance(transport, beaver.transports.zmq_transport.ZmqTransport)
