def setUp(self): self.dispatcher = Dispatcher() self.producer = FakeKafkaProducer() kafka._make_producer = lambda _: self.producer config = { 'kafka-topic': 'foo' } self.publisher = kafka.KafkaPublisher(self.dispatcher, config)
def test_services(self): """ The right type and number of services are created. """ config = {'redis-hosts': set(['10.0.0.2', '10.0.0.3']), 'redis-port': 6379, 'redis-key': 'udplog'} dispatcher = Dispatcher() multiService = redis.makeService(config, dispatcher) services = list(multiService) self.assertEqual(3, len(services)) for service in services[:-1]: self.assertIsInstance(service, TCPClient) self.assertIsInstance(services[-1], redis.RedisPublisher)
class KafkaPublisherServiceTest(unittest.TestCase): def setUp(self): self.dispatcher = Dispatcher() self.producer = FakeKafkaProducer() kafka._make_producer = lambda _: self.producer config = { 'kafka-topic': 'foo' } self.publisher = kafka.KafkaPublisher(self.dispatcher, config) @defer.inlineCallbacks def test_startService(self): """ The publisher registers itself with the dispatcher. """ event = {'message': 'test'} self.dispatcher.eventReceived(event) self.assertEqual(0, len(self.producer.produced)) # When yield self.publisher.startService() # Then self.dispatcher.eventReceived(event) self.assertEqual(1, len(self.producer.produced)) @defer.inlineCallbacks def test_stopService(self): """ The publisher registers itself with the dispatcher. """ event = {'message': 'test'} yield self.publisher.startService() self.dispatcher.eventReceived(event) # When self.publisher.stopService() # Then self.dispatcher.eventReceived(event) self.assertEqual(1, len(self.producer.produced)) @defer.inlineCallbacks def test_sendEvent(self): """ An event is pushed as a JSON string. """ event = {'category': u'test', 'message': u'test', 'timestamp': 1340634165} yield self.publisher.startService() # When self.dispatcher.eventReceived(event) # Then output = self.producer.produced[-1] self.assertEqual('foo', output[0]) eventDict = simplejson.loads(output[1]) self.assertEqual(event, eventDict) @defer.inlineCallbacks def test_sendEventUnserializable(self): """ An event that cannot be serialized is dropped and an error logged. """ class Object(object): pass event = {'category': u'test', 'message': Object(), 'timestamp': 1340634165} yield self.publisher.startService() # When self.dispatcher.eventReceived(event) # Then self.assertEqual(0, len(self.producer.produced)) self.assertEqual(1, len(self.flushLoggedErrors(TypeError)))
def makeService(config): s = service.MultiService() # Set up event dispatcher dispatcher = Dispatcher() # Set up UDPLog server. udplogProtocol = UDPLogProtocol(dispatcher.eventReceived) udplogServer = internet.UDPServer(port=config['udplog-port'], protocol=udplogProtocol, interface=config['udplog-interface'], maxPacketSize=65536) udplogServer.setServiceParent(s) # Set up syslog server if (config.get('syslog-port') is not None or config.get('syslog-unix-socket') is not None): hostname = socket.gethostname() hostnames = { hostname.split('.')[0]: hostname, '': hostname } syslogProtocol = syslog.SyslogDatagramProtocol( dispatcher.eventReceived, hostnames=hostnames) if config.get('syslog-unix-socket') is not None: syslogServer = internet.UNIXDatagramServer( address=config['syslog-unix-socket'], protocol=syslogProtocol, maxPacketSize=65536) syslogServer.setServiceParent(s) if config.get('syslog-port') is not None: syslogServer = internet.UDPServer( port=config['syslog-port'], protocol=syslogProtocol, interface=config.get('syslog-interface', ''), maxPacketSize=65536) syslogServer.setServiceParent(s) # Set up Thrift/Scribe client. if config['scribe-host']: from udplog import scribe factory = UDPLogClientFactory(scribe.ScribeProtocol, dispatcher) scribeClient = internet.TCPClient(config['scribe-host'], config['scribe-port'], factory) scribeClient.setServiceParent(s) # Set up RabbitMQ client. if config['rabbitmq-host']: from udplog import rabbitmq factory = UDPLogClientFactory( rabbitmq.RabbitMQPublisher, dispatcher, vhost=config['rabbitmq-vhost'], exchange=config['rabbitmq-exchange'], queueSize=config['rabbitmq-queue-size']) rabbitmqClient = internet.TCPClient(config['rabbitmq-host'], config['rabbitmq-port'], factory) rabbitmqClient.setServiceParent(s) # Set up Redis client. if config['redis-hosts']: from udplog import redis redisService = redis.makeService(config, dispatcher) redisService.setServiceParent(s) # Set up Kafka client. if config['kafka-brokers']: from udplog import kafka kafkaService = kafka.makeService(config, dispatcher) kafkaService.setServiceParent(s) if config['verbose']: UDPLogToTwistedLog(dispatcher) return s
def setUp(self): self.dispatcher = Dispatcher() self.client = FakeRedisClient() self.publisher = redis.RedisPublisher(self.dispatcher, self.client, "test_list")
class RedisPublisherServiceTest(unittest.TestCase): def setUp(self): self.dispatcher = Dispatcher() self.client = FakeRedisClient() self.publisher = redis.RedisPublisher(self.dispatcher, self.client, "test_list") def test_startService(self): """ The publisher registers itself with the dispatcher. """ event = {'message': 'test'} self.dispatcher.eventReceived(event) self.assertEqual(0, len(self.client.pushes)) self.publisher.startService() self.dispatcher.eventReceived(event) self.assertEqual(1, len(self.client.pushes)) def test_stopService(self): """ The publisher registers itself with the dispatcher. """ event = {'message': 'test'} self.publisher.startService() self.dispatcher.eventReceived(event) self.publisher.stopService() self.dispatcher.eventReceived(event) self.assertEqual(1, len(self.client.pushes)) def test_sendEvent(self): """ An event is pushed as a JSON string. """ event = {'category': u'test', 'message': u'test', 'timestamp': 1340634165} self.publisher.sendEvent(event) output = self.client.pushes[-1] self.assertEqual('test_list', output[0]) eventDict = simplejson.loads(output[1][0]) self.assertEqual(u'test', eventDict['message']) def test_sendEventUnserializable(self): """ An event that cannot be serialized is dropped and an error logged. """ class Object(object): pass event = {'category': u'test', 'message': Object(), 'timestamp': 1340634165} self.publisher.sendEvent(event) self.assertEqual(0, len(self.client.pushes)) self.assertEqual(1, len(self.flushLoggedErrors(TypeError))) def test_sendEventNoClient(self): """ An event that cannot be serialized is dropped and an error logged. """ event = {'category': u'test', 'message': u'test', 'timestamp': 1340634165} def lpush(key, *args, **kwargs): return defer.fail(redis.NoClientError()) self.patch(self.client, "lpush", lpush) self.publisher.sendEvent(event) self.assertEqual(0, len(self.client.pushes)) self.assertEqual(0, len(self.flushLoggedErrors()), "Unexpected error logged")
def setUp(self): self.dispatcher = Dispatcher() self.producer = FakeKafkaProducer() kafka._make_producer = lambda _: self.producer config = {'kafka-topic': 'foo'} self.publisher = kafka.KafkaPublisher(self.dispatcher, config)
class KafkaPublisherServiceTest(unittest.TestCase): def setUp(self): self.dispatcher = Dispatcher() self.producer = FakeKafkaProducer() kafka._make_producer = lambda _: self.producer config = {'kafka-topic': 'foo'} self.publisher = kafka.KafkaPublisher(self.dispatcher, config) @defer.inlineCallbacks def test_startService(self): """ The publisher registers itself with the dispatcher. """ event = {'message': 'test'} self.dispatcher.eventReceived(event) self.assertEqual(0, len(self.producer.produced)) # When yield self.publisher.startService() # Then self.dispatcher.eventReceived(event) self.assertEqual(1, len(self.producer.produced)) @defer.inlineCallbacks def test_stopService(self): """ The publisher registers itself with the dispatcher. """ event = {'message': 'test'} yield self.publisher.startService() self.dispatcher.eventReceived(event) # When self.publisher.stopService() # Then self.dispatcher.eventReceived(event) self.assertEqual(1, len(self.producer.produced)) @defer.inlineCallbacks def test_sendEvent(self): """ An event is pushed as a JSON string. """ event = { 'category': u'test', 'message': u'test', 'timestamp': 1340634165 } yield self.publisher.startService() # When self.dispatcher.eventReceived(event) # Then output = self.producer.produced[-1] self.assertEqual('foo', output[0]) eventDict = simplejson.loads(output[1]) self.assertEqual(event, eventDict) @defer.inlineCallbacks def test_sendEventUnserializable(self): """ An event that cannot be serialized is dropped and an error logged. """ class Object(object): pass event = { 'category': u'test', 'message': Object(), 'timestamp': 1340634165 } yield self.publisher.startService() # When self.dispatcher.eventReceived(event) # Then self.assertEqual(0, len(self.producer.produced)) self.assertEqual(1, len(self.flushLoggedErrors(TypeError)))