def test_builtin_kafka(cls): cls.beaver_config.set('transport', 'kafka') cls.beaver_config.set('logstash_version', 1) cls.beaver_config.set('kafka_hosts', cls.server.host + ":" + str(cls.server.port)) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.kafka_transport.KafkaTransport) data = {} lines = [] n=100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] transport.callback("test.log", **data) messages = cls._consume_messages(cls.server.host, cls.server.port) cls.assertEqual(n, messages.__len__()) for message in messages: cls.assertIn('"file": "test.log", "message": "log', message.message.value); print(message) print('\n') transport.interrupt()
def test_builtin_rabbitmq(self): beaver_config = self._get_config(transport='rabbitmq') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance( transport, beaver.transports.rabbitmq_transport.RabbitmqTransport) transport.interrupt()
def test_kinesis_send_stream(self): self._create_streams() self.beaver_config.set('kinesis_aws_stream', 'stream1') self.beaver_config.set('kinesis_aws_profile_name', None) self.beaver_config.set('kinesis_aws_access_key', None) self.beaver_config.set('kinesis_aws_secret_key', None) self.beaver_config.set('kinesis_bulk_lines', False) transport = create_transport(self.beaver_config, logger=self.logger) mock_send_batch = mock.Mock() transport._send_message_batch = mock_send_batch self.assertIsInstance( transport, beaver.transports.kinesis_transport.KinesisTransport) data = {} lines = [] n = 500 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] self.assertTrue(transport.callback("test.log", **data)) self.assertEqual(1, mock_send_batch.call_count)
def test_sqs_send_multi_queue_bulklines(cls): cls._create_queues() cls.beaver_config.set('sqs_aws_queue', 'queue1,queue2') cls.beaver_config.set('sqs_aws_profile_name', None) cls.beaver_config.set('sqs_aws_access_key', None) cls.beaver_config.set('sqs_aws_secret_key', None) cls.beaver_config.set('sqs_bulk_lines', True) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport) data = {} lines = [] n=100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] transport.callback("test.log", **data)
def test_builtin_kafka(cls): cls.beaver_config.set('transport', 'kafka') cls.beaver_config.set('logstash_version', 1) cls.beaver_config.set('kafka_hosts', cls.server.host + ":" + str(cls.server.port)) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.kafka_transport.KafkaTransport) data = {} lines = [] n = 100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] transport.callback("test.log", **data) messages = cls._consume_messages(cls.server.host, cls.server.port) cls.assertEqual(n, messages.__len__()) for message in messages: cls.assertIn('"file": "test.log", "message": "log', message.message.value) print(message) print('\n') transport.interrupt()
def test_kinesis_send_stream(self): self._create_streams() self.beaver_config.set('kinesis_aws_stream', 'stream1') self.beaver_config.set('kinesis_aws_profile_name', None) self.beaver_config.set('kinesis_aws_access_key', None) self.beaver_config.set('kinesis_aws_secret_key', None) self.beaver_config.set('kinesis_bulk_lines', False) transport = create_transport(self.beaver_config, logger=self.logger) mock_send_batch = mock.Mock() transport._send_message_batch = mock_send_batch self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) data = {} lines = [] n=500 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] self.assertTrue(transport.callback("test.log", **data)) self.assertEqual(1, mock_send_batch.call_count)
def test_kinesis_auth_profile(self): self._create_streams() self.beaver_config.set('kinesis_aws_profile_name', 'beaver_stream') self.beaver_config.set('kinesis_aws_access_key', None) self.beaver_config.set('kinesis_aws_secret_key', None) self.beaver_config.set('kinesis_aws_stream', 'stream1') transport = create_transport(self.beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport)
def test_sqs_auth_profile(cls): cls._create_queues() cls.beaver_config.set('sqs_aws_profile_name', 'beaver_queue') cls.beaver_config.set('sqs_aws_access_key', None) cls.beaver_config.set('sqs_aws_secret_key', None) cls.beaver_config.set('sqs_aws_queue', 'queue1') transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport)
def test_kinesis_auth_profile(self): self._create_streams() self.beaver_config.set('kinesis_aws_profile_name', 'beaver_stream') self.beaver_config.set('kinesis_aws_access_key', None) self.beaver_config.set('kinesis_aws_secret_key', None) self.beaver_config.set('kinesis_aws_stream', 'stream1') transport = create_transport(self.beaver_config, logger=self.logger) self.assertIsInstance( transport, beaver.transports.kinesis_transport.KinesisTransport)
def test_sqs_single_queue(cls): cls._create_queues() cls.beaver_config.set('sqs_aws_queue', 'queue1') cls.beaver_config.set('sqs_aws_profile_name', None) cls.beaver_config.set('sqs_aws_access_key', None) cls.beaver_config.set('sqs_aws_secret_key', None) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver_sqs.Transport) transport.interrupt()
def test_kinesis_auth_account_id(self): self._create_streams() self.beaver_config.set('kinesis_aws_stream_owner_acct_id', 'abc123') self.beaver_config.set('kinesis_aws_profile_name', None) self.beaver_config.set('kinesis_aws_access_key', 'beaver_test_key') self.beaver_config.set('kinesis_aws_secret_key', 'beaver_test_secret') self.beaver_config.set('kinesis_aws_stream', 'stream1') transport = create_transport(self.beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.kinesis_transport.KinesisTransport) transport.interrupt()
def test_sqs_auth_account_id(cls): cls._create_queues() cls.beaver_config.set('sqs_aws_queue_owner_acct_id', 'abc123') cls.beaver_config.set('sqs_aws_profile_name', None) cls.beaver_config.set('sqs_aws_access_key', 'beaver_test_key') cls.beaver_config.set('sqs_aws_secret_key', 'beaver_test_secret') cls.beaver_config.set('sqs_aws_queue', 'queue1') transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport) transport.interrupt()
def test_sqs_multi_queue_bulklines(cls): cls._create_queues() cls.beaver_config.set('sqs_aws_queue', 'queue1,queue2') cls.beaver_config.set('sqs_aws_profile_name', None) cls.beaver_config.set('sqs_aws_access_key', None) cls.beaver_config.set('sqs_aws_secret_key', None) cls.beaver_config.set('sqs_bulk_lines', True) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.sqs_transport.SqsTransport) transport.interrupt()
def test_kinesis_auth_account_id(self): self._create_streams() self.beaver_config.set('kinesis_aws_stream_owner_acct_id', 'abc123') self.beaver_config.set('kinesis_aws_profile_name', None) self.beaver_config.set('kinesis_aws_access_key', 'beaver_test_key') self.beaver_config.set('kinesis_aws_secret_key', 'beaver_test_secret') self.beaver_config.set('kinesis_aws_stream', 'stream1') transport = create_transport(self.beaver_config, logger=self.logger) self.assertIsInstance( transport, beaver.transports.kinesis_transport.KinesisTransport) transport.interrupt()
def test_kinesis_send_stream_with_retries(self): self._create_streams() transport = create_transport(self.beaver_config, logger=self.logger) mock_client = mock.Mock() transport._connection = mock_client mock_client.put_records.side_effect = [{'FailedRecordCount': 1}, {}] transport._send_message_batch("doesntmatter") self.assertEqual(2, mock_client.put_records.call_count) mock_client.reset_mock() mock_client.put_records.side_effect = [ Exception('ProvisionedThroughputExceededException'), Exception('ThrottlingException'), {} ] transport._send_message_batch("doesntmatter") self.assertEqual(3, mock_client.put_records.call_count)
def run_queue(queue, beaver_config, logger=None): signal.signal(signal.SIGTERM, signal.SIG_DFL) signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGQUIT, signal.SIG_DFL) last_update_time = int(time.time()) queue_timeout = beaver_config.get('queue_timeout') wait_timeout = beaver_config.get('wait_timeout') transport = None try: logger.debug('Logging using the {0} transport'.format( beaver_config.get('transport'))) transport = create_transport(beaver_config, logger=logger) failure_count = 0 while True: if not transport.valid(): logger.info('Transport connection issues, stopping queue') break if int(time.time()) - last_update_time > queue_timeout: logger.info( 'Queue timeout of "{0}" seconds exceeded, stopping queue'. format(queue_timeout)) break try: command, data = queue.get(block=True, timeout=wait_timeout) last_update_time = int(time.time()) logger.debug( 'Last update time now {0}'.format(last_update_time)) except Queue.Empty: logger.debug('No data') continue if command == 'callback': try: if data.get('ignore_empty', False): logger.debug('removing empty lines') lines = data['lines'] new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines if len(data['lines']) == 0: logger.debug('0 active lines sent from worker') continue transport.callback(**data) except TransportException: failure_count = failure_count + 1 if failure_count > beaver_config.get('max_failure'): failure_count = beaver_config.get('max_failure') sleep_time = int( beaver_config.get('respawn_delay'))**failure_count logger.info( 'Caught transport exception, respawning in %d seconds' % sleep_time) try: time.sleep(sleep_time) transport.reconnect() except KeyboardInterrupt: logger.info('User cancelled respawn.') transport.interrupt() sys.exit(0) elif command == 'addglob': beaver_config.addglob(*data) transport.addglob(*data) elif command == 'exit': break except KeyboardInterrupt: logger.debug('Queue Interruped') if transport is not None: transport.interrupt() logger.debug('Queue Shutdown')
def test_builtin_stdout(self): beaver_config = self._get_config(transport='stdout') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.stdout_transport.StdoutTransport)
def test_builtin_redis(self): beaver_config = self._get_config(transport='redis') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.redis_transport.RedisTransport)
def test_monascalog(cls, token_mock): # dynamic callback to verify the log messages sent by the transport def request_callback(request, uri, headers): cls.log_requests.append(request.parsed_body) return (204, headers, "created") # fake the first get call that is used for checking connection httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed") httpretty.register_uri(httpretty.POST, LOG_URL, status=204, body="Created") cls.beaver_config.set('transport', 'monascalog') cls.beaver_config.set('logstash_version', 1) #cls.beaver_config.set('monascalog_hosts', "{}:{}".format(cls.server_host, cls.server_port)) cls.beaver_config.set('monascalog_max_retries', 3) cls.beaver_config.set('monascalog_auth_url', cls.keystone_auth_url) cls.beaver_config.set('monascalog_user_name', cls.keystone_user) cls.beaver_config.set('monascalog_password', cls.keystone_password) cls.beaver_config.set('monascalog_project_name', cls.keystone_project_name) cls.beaver_config.set('monascalog_domain_name', cls.keystone_domain_name) transport = create_transport(cls.beaver_config, logger=cls.logger) cls.assertIsInstance(transport, beaver.transports.monascalog_transport.MonascalogTransport) cls.assertEqual(transport.valid(), True) data = {} lines = [] n=100 for i in range(n): lines.append('log' + str(i) + '\n') new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines data['fields'] = [] cls.assertEqual(transport.callback("test.log", **data), True) # Fake a log api failure httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, status=500, body="Internal Server Error") cls.assertRaises(TransportException, transport.callback, "test.log", **data) # simulate a single failure followed by success, to test if retry works httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, responses=[ httpretty.Response(status=503, body="Service Unavailable"), httpretty.Response(status=204, body="Created") ]) cls.assertEqual(transport.callback("test.log", **data), True) # next, test if the logs made it to the server httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback) # clear logs from previous tests del cls.log_requests[:] cls.assertEqual(transport.callback("test.log", **data), True) cls._consume_messages(n) # repeat same test, but with batching turned off cls.beaver_config.set('monascalog_enable_batching', False) httpretty.reset() httpretty.register_uri(httpretty.POST, LOG_URL, body=request_callback) # fake the first get call that is used for checking connection httpretty.register_uri(httpretty.GET, LOG_URL, status=405, body="Method Not Allowed") transport = create_transport(cls.beaver_config, logger=cls.logger) # clear logs from previous tests del cls.log_requests[:] cls.assertEqual(transport.callback("test.log", **data), True) cls._consume_messages(n, batching=False) transport.interrupt()
def run_queue(queue, beaver_config, logger_name=None): signal.signal(signal.SIGTERM, signal.SIG_DFL) signal.signal(signal.SIGINT, signal.SIG_DFL) if os.name != 'nt': signal.signal(signal.SIGQUIT, signal.SIG_DFL) last_update_time = int(time.time()) queue_timeout = beaver_config.get('queue_timeout') wait_timeout = beaver_config.get('wait_timeout') count = 0 transport = None try: logger = setup_custom_logger(logger_name) logger.debug('Logging using the {0} transport'.format( beaver_config.get('transport'))) transport = create_transport(beaver_config, logger=logger) failure_count = 0 while True: if not transport.valid(): logger.info('Transport connection issues, stopping queue') break command = None try: if queue.full(): logger.error("Queue is full") else: if count == 1000: logger.debug("Main consumer queue Size is: " + str(queue.qsize())) count = 0 command, data = queue.get(block=True, timeout=wait_timeout) if command == "callback": last_update_time = int(time.time()) logger.debug( 'Last update time now {0}'.format(last_update_time)) except Queue.Empty: if not queue.empty(): logger.error( 'Recieved timeout from main consumer queue - stopping queue' ) break else: logger.debug('No data') if int(time.time()) - last_update_time > queue_timeout: logger.info( 'Queue timeout of "{0}" seconds exceeded, stopping queue'. format(queue_timeout)) break if command == 'callback': if data.get('ignore_empty', False): logger.debug('removing empty lines') lines = data['lines'] new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines if len(data['lines']) == 0: logger.debug('0 active lines sent from worker') continue while True: try: transport.callback(**data) count += 1 logger.debug("Number of transports: " + str(count)) break except TransportException as e: failure_count = failure_count + 1 if failure_count > beaver_config.get('max_failure'): failure_count = beaver_config.get('max_failure') sleep_time = beaver_config.get( 'respawn_delay')**failure_count logger.info('Caught transport exception: %s', e) logger.info('Reconnecting in %d seconds' % sleep_time) try: transport.invalidate() time.sleep(sleep_time) transport.reconnect() if transport.valid(): failure_count = 0 logger.info('Reconnected successfully') except KeyboardInterrupt: logger.info('User cancelled respawn.') transport.interrupt() sys.exit(0) elif command == 'addglob': beaver_config.addglob(*data) transport.addglob(*data) elif command == 'exit': break except KeyboardInterrupt: logger.debug('Queue Interruped') if transport is not None: transport.interrupt() logger.debug('Queue Shutdown')
def test_custom_transport(self): beaver_config = self._get_config(transport='beaver.tests.test_transport_config.DummyTransport') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, DummyTransport)
def run_queue(queue, beaver_config, logger=None): signal.signal(signal.SIGTERM, signal.SIG_DFL) signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGQUIT, signal.SIG_DFL) last_update_time = int(time.time()) queue_timeout = beaver_config.get('queue_timeout') wait_timeout = beaver_config.get('wait_timeout') count = 0 transport = None try: logger.debug('Logging using the {0} transport'.format(beaver_config.get('transport'))) transport = create_transport(beaver_config, logger=logger) failure_count = 0 while True: if not transport.valid(): logger.info('Transport connection issues, stopping queue') break if int(time.time()) - last_update_time > queue_timeout: logger.info('Queue timeout of "{0}" seconds exceeded, stopping queue'.format(queue_timeout)) break command = None try: if queue.full(): logger.error("Queue is full") else: if count == 1000: logger.debug("Main consumer queue Size is: " + str(queue.qsize())) count = 0 command, data = queue.get(block=True, timeout=wait_timeout) if command == "callback": last_update_time = int(time.time()) logger.debug('Last update time now {0}'.format(last_update_time)) except Queue.Empty: if not queue.empty(): logger.error('Recieved timeout from main consumer queue - stopping queue') break else: logger.debug('No data') if command == 'callback': if data.get('ignore_empty', False): logger.debug('removing empty lines') lines = data['lines'] new_lines = [] for line in lines: message = unicode_dammit(line) if len(message) == 0: continue new_lines.append(message) data['lines'] = new_lines if len(data['lines']) == 0: logger.debug('0 active lines sent from worker') continue while True: try: transport.callback(**data) count += 1 logger.debug("Number of transports: " + str(count)) break except TransportException: failure_count = failure_count + 1 if failure_count > beaver_config.get('max_failure'): failure_count = beaver_config.get('max_failure') sleep_time = beaver_config.get('respawn_delay') ** failure_count logger.info('Caught transport exception, reconnecting in %d seconds' % sleep_time) try: transport.invalidate() time.sleep(sleep_time) transport.reconnect() if transport.valid(): failure_count = 0 logger.info('Reconnected successfully') except KeyboardInterrupt: logger.info('User cancelled respawn.') transport.interrupt() sys.exit(0) elif command == 'addglob': beaver_config.addglob(*data) transport.addglob(*data) elif command == 'exit': break except KeyboardInterrupt: logger.debug('Queue Interruped') if transport is not None: transport.interrupt() logger.debug('Queue Shutdown')
def test_builtin_udp(self): beaver_config = self._get_config(transport='udp') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.udp_transport.UdpTransport)
def test_builtin_zmq(self): beaver_config = self._get_config(transport='zmq') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, ZmqTransport)
def test_builtin_redis(self): beaver_config = self._get_config(transport='redis') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.redis_transport.RedisTransport)
def test_builtin_stdout(self): beaver_config = self._get_config(transport='stdout') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.stdout_transport.StdoutTransport)
def test_builtin_rabbitmq(self): beaver_config = self._get_config(transport='rabbitmq') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.rabbitmq_transport.RabbitmqTransport) transport.interrupt()
def test_builtin_udp(self): beaver_config = self._get_config(transport='udp') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.udp_transport.UdpTransport)
def test_builtin_zmq(self): beaver_config = self._get_config(transport='zmq') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, ZmqTransport)
def test_custom_transport(self): beaver_config = self._get_config(transport='tests.test_transport_config.DummyTransport') transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, DummyTransport)
def test_builtin_zmq(self): beaver_config = self._get_config(transport="zmq") transport = create_transport(beaver_config, logger=self.logger) self.assertIsInstance(transport, beaver.transports.zmq_transport.ZmqTransport)