def setUp(self):
self.dispatcher = Dispatcher()
self.producer = FakeKafkaProducer()
kafka._make_producer = lambda _: self.producer
config = {
'kafka-topic': 'foo'
}
self.publisher = kafka.KafkaPublisher(self.dispatcher, config)
def test_services(self):
"""
The right type and number of services are created.
"""
config = {'redis-hosts': set(['10.0.0.2', '10.0.0.3']),
'redis-port': 6379,
'redis-key': 'udplog'}
dispatcher = Dispatcher()
multiService = redis.makeService(config, dispatcher)
services = list(multiService)
self.assertEqual(3, len(services))
for service in services[:-1]:
self.assertIsInstance(service, TCPClient)
self.assertIsInstance(services[-1], redis.RedisPublisher)
class KafkaPublisherServiceTest(unittest.TestCase):
def setUp(self):
self.dispatcher = Dispatcher()
self.producer = FakeKafkaProducer()
kafka._make_producer = lambda _: self.producer
config = {
'kafka-topic': 'foo'
}
self.publisher = kafka.KafkaPublisher(self.dispatcher, config)
@defer.inlineCallbacks
def test_startService(self):
"""
The publisher registers itself with the dispatcher.
"""
event = {'message': 'test'}
self.dispatcher.eventReceived(event)
self.assertEqual(0, len(self.producer.produced))
# When
yield self.publisher.startService()
# Then
self.dispatcher.eventReceived(event)
self.assertEqual(1, len(self.producer.produced))
@defer.inlineCallbacks
def test_stopService(self):
"""
The publisher registers itself with the dispatcher.
"""
event = {'message': 'test'}
yield self.publisher.startService()
self.dispatcher.eventReceived(event)
# When
self.publisher.stopService()
# Then
self.dispatcher.eventReceived(event)
self.assertEqual(1, len(self.producer.produced))
@defer.inlineCallbacks
def test_sendEvent(self):
"""
An event is pushed as a JSON string.
"""
event = {'category': u'test',
'message': u'test',
'timestamp': 1340634165}
yield self.publisher.startService()
# When
self.dispatcher.eventReceived(event)
# Then
output = self.producer.produced[-1]
self.assertEqual('foo', output[0])
eventDict = simplejson.loads(output[1])
self.assertEqual(event, eventDict)
@defer.inlineCallbacks
def test_sendEventUnserializable(self):
"""
An event that cannot be serialized is dropped and an error logged.
"""
class Object(object):
pass
event = {'category': u'test',
'message': Object(),
'timestamp': 1340634165}
yield self.publisher.startService()
# When
self.dispatcher.eventReceived(event)
# Then
self.assertEqual(0, len(self.producer.produced))
self.assertEqual(1, len(self.flushLoggedErrors(TypeError)))
def makeService(config):
s = service.MultiService()
# Set up event dispatcher
dispatcher = Dispatcher()
# Set up UDPLog server.
udplogProtocol = UDPLogProtocol(dispatcher.eventReceived)
udplogServer = internet.UDPServer(port=config['udplog-port'],
protocol=udplogProtocol,
interface=config['udplog-interface'],
maxPacketSize=65536)
udplogServer.setServiceParent(s)
# Set up syslog server
if (config.get('syslog-port') is not None or
config.get('syslog-unix-socket') is not None):
hostname = socket.gethostname()
hostnames = {
hostname.split('.')[0]: hostname,
'': hostname
}
syslogProtocol = syslog.SyslogDatagramProtocol(
dispatcher.eventReceived, hostnames=hostnames)
if config.get('syslog-unix-socket') is not None:
syslogServer = internet.UNIXDatagramServer(
address=config['syslog-unix-socket'],
protocol=syslogProtocol,
maxPacketSize=65536)
syslogServer.setServiceParent(s)
if config.get('syslog-port') is not None:
syslogServer = internet.UDPServer(
port=config['syslog-port'],
protocol=syslogProtocol,
interface=config.get('syslog-interface', ''),
maxPacketSize=65536)
syslogServer.setServiceParent(s)
# Set up Thrift/Scribe client.
if config['scribe-host']:
from udplog import scribe
factory = UDPLogClientFactory(scribe.ScribeProtocol,
dispatcher)
scribeClient = internet.TCPClient(config['scribe-host'],
config['scribe-port'],
factory)
scribeClient.setServiceParent(s)
# Set up RabbitMQ client.
if config['rabbitmq-host']:
from udplog import rabbitmq
factory = UDPLogClientFactory(
rabbitmq.RabbitMQPublisher, dispatcher,
vhost=config['rabbitmq-vhost'],
exchange=config['rabbitmq-exchange'],
queueSize=config['rabbitmq-queue-size'])
rabbitmqClient = internet.TCPClient(config['rabbitmq-host'],
config['rabbitmq-port'],
factory)
rabbitmqClient.setServiceParent(s)
# Set up Redis client.
if config['redis-hosts']:
from udplog import redis
redisService = redis.makeService(config, dispatcher)
redisService.setServiceParent(s)
# Set up Kafka client.
if config['kafka-brokers']:
from udplog import kafka
kafkaService = kafka.makeService(config, dispatcher)
kafkaService.setServiceParent(s)
if config['verbose']:
UDPLogToTwistedLog(dispatcher)
return s
def setUp(self):
self.dispatcher = Dispatcher()
self.client = FakeRedisClient()
self.publisher = redis.RedisPublisher(self.dispatcher,
self.client,
"test_list")
class RedisPublisherServiceTest(unittest.TestCase):
def setUp(self):
self.dispatcher = Dispatcher()
self.client = FakeRedisClient()
self.publisher = redis.RedisPublisher(self.dispatcher,
self.client,
"test_list")
def test_startService(self):
"""
The publisher registers itself with the dispatcher.
"""
event = {'message': 'test'}
self.dispatcher.eventReceived(event)
self.assertEqual(0, len(self.client.pushes))
self.publisher.startService()
self.dispatcher.eventReceived(event)
self.assertEqual(1, len(self.client.pushes))
def test_stopService(self):
"""
The publisher registers itself with the dispatcher.
"""
event = {'message': 'test'}
self.publisher.startService()
self.dispatcher.eventReceived(event)
self.publisher.stopService()
self.dispatcher.eventReceived(event)
self.assertEqual(1, len(self.client.pushes))
def test_sendEvent(self):
"""
An event is pushed as a JSON string.
"""
event = {'category': u'test',
'message': u'test',
'timestamp': 1340634165}
self.publisher.sendEvent(event)
output = self.client.pushes[-1]
self.assertEqual('test_list', output[0])
eventDict = simplejson.loads(output[1][0])
self.assertEqual(u'test', eventDict['message'])
def test_sendEventUnserializable(self):
"""
An event that cannot be serialized is dropped and an error logged.
"""
class Object(object):
pass
event = {'category': u'test',
'message': Object(),
'timestamp': 1340634165}
self.publisher.sendEvent(event)
self.assertEqual(0, len(self.client.pushes))
self.assertEqual(1, len(self.flushLoggedErrors(TypeError)))
def test_sendEventNoClient(self):
"""
An event that cannot be serialized is dropped and an error logged.
"""
event = {'category': u'test',
'message': u'test',
'timestamp': 1340634165}
def lpush(key, *args, **kwargs):
return defer.fail(redis.NoClientError())
self.patch(self.client, "lpush", lpush)
self.publisher.sendEvent(event)
self.assertEqual(0, len(self.client.pushes))
self.assertEqual(0, len(self.flushLoggedErrors()),
"Unexpected error logged")
def setUp(self):
self.dispatcher = Dispatcher()
self.producer = FakeKafkaProducer()
kafka._make_producer = lambda _: self.producer
config = {'kafka-topic': 'foo'}
self.publisher = kafka.KafkaPublisher(self.dispatcher, config)
class KafkaPublisherServiceTest(unittest.TestCase):
def setUp(self):
self.dispatcher = Dispatcher()
self.producer = FakeKafkaProducer()
kafka._make_producer = lambda _: self.producer
config = {'kafka-topic': 'foo'}
self.publisher = kafka.KafkaPublisher(self.dispatcher, config)
@defer.inlineCallbacks
def test_startService(self):
"""
The publisher registers itself with the dispatcher.
"""
event = {'message': 'test'}
self.dispatcher.eventReceived(event)
self.assertEqual(0, len(self.producer.produced))
# When
yield self.publisher.startService()
# Then
self.dispatcher.eventReceived(event)
self.assertEqual(1, len(self.producer.produced))
@defer.inlineCallbacks
def test_stopService(self):
"""
The publisher registers itself with the dispatcher.
"""
event = {'message': 'test'}
yield self.publisher.startService()
self.dispatcher.eventReceived(event)
# When
self.publisher.stopService()
# Then
self.dispatcher.eventReceived(event)
self.assertEqual(1, len(self.producer.produced))
@defer.inlineCallbacks
def test_sendEvent(self):
"""
An event is pushed as a JSON string.
"""
event = {
'category': u'test',
'message': u'test',
'timestamp': 1340634165
}
yield self.publisher.startService()
# When
self.dispatcher.eventReceived(event)
# Then
output = self.producer.produced[-1]
self.assertEqual('foo', output[0])
eventDict = simplejson.loads(output[1])
self.assertEqual(event, eventDict)
@defer.inlineCallbacks
def test_sendEventUnserializable(self):
"""
An event that cannot be serialized is dropped and an error logged.
"""
class Object(object):
pass
event = {
'category': u'test',
'message': Object(),
'timestamp': 1340634165
}
yield self.publisher.startService()
# When
self.dispatcher.eventReceived(event)
# Then
self.assertEqual(0, len(self.producer.produced))
self.assertEqual(1, len(self.flushLoggedErrors(TypeError)))
