def test_kinesis_send_stream(self): self._create_streams() self.beaver_config.set('kinesis_aws_stream', 'stream1') self.beaver_config.set('kinesis_aws_profile_name', None) self.beaver_config.set('kinesis_aws_access_key', None) self.beaver_config.set('kinesis_aws_secret_key', None) self.beaver_config.set('kinesis_bulk_lines', False) transport = create_transport(self.beaver_config, logger=self.logger) mock_send_batch = mock.Mock() transport._send_message_batch = mock_send_batch self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) data = {} lines = [] n=500 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] self.assertTrue(transport.callback("test.log", **data)) self.assertEqual(1, mock_send_batch.call_count)
def test_kinesis_send_stream(self): self._create_streams() self.beaver_config.set('kinesis_aws_stream', 'stream1') self.beaver_config.set('kinesis_aws_profile_name', None) self.beaver_config.set('kinesis_aws_access_key', None) self.beaver_config.set('kinesis_aws_secret_key', None) self.beaver_config.set('kinesis_bulk_lines', False) transport = create_transport(self.beaver_config, logger=self.logger) mock_send_batch = mock.Mock() transport._send_message_batch = mock_send_batch self.assertIsInstance( transport, beaver.transports.kinesis_transport.KinesisTransport) data = {} lines = [] n = 500 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] self.assertTrue(transport.callback("test.log", **data)) self.assertEqual(1, mock_send_batch.call_count)
def test_builtin_kafka(cls): cls.beaver_config.set('transport', 'kafka') cls.beaver_config.set('logstash_version', 1) cls.beaver_config.set('kafka_hosts', cls.server.host + ":" + str(cls.server.port)) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.kafka_transport.KafkaTransport) data = {} lines = [] n = 100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] transport.callback("test.log", **data) messages = cls._consume_messages(cls.server.host, cls.server.port) cls.assertEqual(n, messages.__len__()) for message in messages: cls.assertIn('"file": "test.log", "message": "log', message.message.value) print(message) print('\n') transport.interrupt()
def test_builtin_kafka(cls): cls.beaver_config.set('transport', 'kafka') cls.beaver_config.set('logstash_version', 1) cls.beaver_config.set('kafka_hosts', cls.server.host + ":" + str(cls.server.port)) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.kafka_transport.KafkaTransport) data = {} lines = [] n=100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] transport.callback("test.log", **data) messages = cls._consume_messages(cls.server.host, cls.server.port) cls.assertEqual(n, messages.__len__()) for message in messages: cls.assertIn('"file": "test.log", "message": "log', message.message.value); print(message) print('\n') transport.interrupt()
def test_sqs_send_multi_queue_bulklines(cls): cls._create_queues() cls.beaver_config.set('sqs_aws_queue', 'queue1,queue2') cls.beaver_config.set('sqs_aws_profile_name', None) cls.beaver_config.set('sqs_aws_access_key', None) cls.beaver_config.set('sqs_aws_secret_key', None) cls.beaver_config.set('sqs_bulk_lines', True) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport) data = {} lines = [] n=100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] transport.callback("test.log", **data)
def test_monascalog(cls, token_mock): # dynamic callback to verify the log messages sent by the transport def request_callback(request, uri, headers): cls.log_requests.append(request.parsed_body) return (204, headers, "created") # fake the first get call that is used for checking connection httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed") httpretty.register_uri(httpretty.POST, LOG_URL, status=204, body="Created") cls.beaver_config.set('transport', 'monascalog') cls.beaver_config.set('logstash_version', 1) #cls.beaver_config.set('monascalog_hosts', "{}:{}".format(cls.server_host, cls.server_port)) cls.beaver_config.set('monascalog_max_retries', 3) cls.beaver_config.set('monascalog_auth_url', cls.keystone_auth_url) cls.beaver_config.set('monascalog_user_name', cls.keystone_user) cls.beaver_config.set('monascalog_password', cls.keystone_password) cls.beaver_config.set('monascalog_project_name', cls.keystone_project_name) cls.beaver_config.set('monascalog_domain_name', cls.keystone_domain_name) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.monascalog_transport.MonascalogTransport) cls.assertEqual(transport.valid(), True) data = {} lines = [] n=100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] cls.assertEqual(transport.callback("test.log", **data), True) # Fake a log api failure httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, status=500, body="Internal Server Error") cls.assertRaises(TransportException, transport.callback, "test.log", **data) # simulate a single failure followed by success, to test if retry works httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, responses=[ httpretty.Response(status=503, body="Service Unavailable"), httpretty.Response(status=204, body="Created") ]) cls.assertEqual(transport.callback("test.log", **data), True) # next, test if the logs made it to the server httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback) # clear logs from previous tests del cls.log_requests[:] cls.assertEqual(transport.callback("test.log", **data), True) cls._consume_messages(n) # repeat same test, but with batching turned off cls.beaver_config.set('monascalog_enable_batching', False) httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback) # fake the first get call that is used for checking connection httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed") transport = create_transport(cls.beaver_config, logger=cls.logger) # clear logs from previous tests del cls.log_requests[:] cls.assertEqual(transport.callback("test.log", **data), True) cls._consume_messages(n, batching=False) transport.interrupt()