def test_kinesis_send_stream(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_stream', 'stream1')
self.beaver_config.set('kinesis_aws_profile_name', None)
self.beaver_config.set('kinesis_aws_access_key', None)
self.beaver_config.set('kinesis_aws_secret_key', None)
self.beaver_config.set('kinesis_bulk_lines', False)
transport = create_transport(self.beaver_config, logger=self.logger)
mock_send_batch = mock.Mock()
transport._send_message_batch = mock_send_batch
self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport)
data = {}
lines = []
n=500
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
self.assertTrue(transport.callback("test.log", **data))
self.assertEqual(1, mock_send_batch.call_count)
def test_kinesis_send_stream(self):
self._create_streams()
self.beaver_config.set('kinesis_aws_stream', 'stream1')
self.beaver_config.set('kinesis_aws_profile_name', None)
self.beaver_config.set('kinesis_aws_access_key', None)
self.beaver_config.set('kinesis_aws_secret_key', None)
self.beaver_config.set('kinesis_bulk_lines', False)
transport = create_transport(self.beaver_config, logger=self.logger)
mock_send_batch = mock.Mock()
transport._send_message_batch = mock_send_batch
self.assertIsInstance(
transport, beaver.transports.kinesis_transport.KinesisTransport)
data = {}
lines = []
n = 500
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
self.assertTrue(transport.callback("test.log", **data))
self.assertEqual(1, mock_send_batch.call_count)
def test_builtin_kafka(cls):
cls.beaver_config.set('transport', 'kafka')
cls.beaver_config.set('logstash_version', 1)
cls.beaver_config.set('kafka_hosts',
cls.server.host + ":" + str(cls.server.port))
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport,
beaver.transports.kafka_transport.KafkaTransport)
data = {}
lines = []
n = 100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
transport.callback("test.log", **data)
messages = cls._consume_messages(cls.server.host, cls.server.port)
cls.assertEqual(n, messages.__len__())
for message in messages:
cls.assertIn('"file": "test.log", "message": "log',
message.message.value)
print(message)
print('\n')
transport.interrupt()
def test_builtin_kafka(cls):
cls.beaver_config.set('transport', 'kafka')
cls.beaver_config.set('logstash_version', 1)
cls.beaver_config.set('kafka_hosts', cls.server.host + ":" + str(cls.server.port))
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.kafka_transport.KafkaTransport)
data = {}
lines = []
n=100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
transport.callback("test.log", **data)
messages = cls._consume_messages(cls.server.host, cls.server.port)
cls.assertEqual(n, messages.__len__())
for message in messages:
cls.assertIn('"file": "test.log", "message": "log', message.message.value);
print(message)
print('\n')
transport.interrupt()
def test_sqs_send_multi_queue_bulklines(cls):
cls._create_queues()
cls.beaver_config.set('sqs_aws_queue', 'queue1,queue2')
cls.beaver_config.set('sqs_aws_profile_name', None)
cls.beaver_config.set('sqs_aws_access_key', None)
cls.beaver_config.set('sqs_aws_secret_key', None)
cls.beaver_config.set('sqs_bulk_lines', True)
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport)
data = {}
lines = []
n=100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
transport.callback("test.log", **data)
def test_monascalog(cls, token_mock):
# dynamic callback to verify the log messages sent by the transport
def request_callback(request, uri, headers):
cls.log_requests.append(request.parsed_body)
return (204, headers, "created")
# fake the first get call that is used for checking connection
httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed")
httpretty.register_uri(httpretty.POST, LOG_URL, status=204, body="Created")
cls.beaver_config.set('transport', 'monascalog')
cls.beaver_config.set('logstash_version', 1)
#cls.beaver_config.set('monascalog_hosts', "{}:{}".format(cls.server_host, cls.server_port))
cls.beaver_config.set('monascalog_max_retries', 3)
cls.beaver_config.set('monascalog_auth_url', cls.keystone_auth_url)
cls.beaver_config.set('monascalog_user_name', cls.keystone_user)
cls.beaver_config.set('monascalog_password', cls.keystone_password)
cls.beaver_config.set('monascalog_project_name', cls.keystone_project_name)
cls.beaver_config.set('monascalog_domain_name', cls.keystone_domain_name)
transport = create_transport(cls.beaver_config, logger=cls.logger)
cls.assertIsInstance(transport, beaver.transports.monascalog_transport.MonascalogTransport)
cls.assertEqual(transport.valid(), True)
data = {}
lines = []
n=100
for i in range(n):
lines.append('log' + str(i) + '\n')
new_lines = []
for line in lines:
message = unicode_dammit(line)
if len(message) == 0:
continue
new_lines.append(message)
data['lines'] = new_lines
data['fields'] = []
cls.assertEqual(transport.callback("test.log", **data), True)
# Fake a log api failure
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, status=500, body="Internal Server Error")
cls.assertRaises(TransportException, transport.callback, "test.log", **data)
# simulate a single failure followed by success, to test if retry works
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, responses=[
httpretty.Response(status=503, body="Service Unavailable"),
httpretty.Response(status=204, body="Created")
])
cls.assertEqual(transport.callback("test.log", **data), True)
# next, test if the logs made it to the server
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback)
# clear logs from previous tests
del cls.log_requests[:]
cls.assertEqual(transport.callback("test.log", **data), True)
cls._consume_messages(n)
# repeat same test, but with batching turned off
cls.beaver_config.set('monascalog_enable_batching', False)
httpretty.reset()
httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback)
# fake the first get call that is used for checking connection
httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed")
transport = create_transport(cls.beaver_config, logger=cls.logger)
# clear logs from previous tests
del cls.log_requests[:]
cls.assertEqual(transport.callback("test.log", **data), True)
cls._consume_messages(n, batching=False)
transport.interrupt()
