class EventBusPublisher(object):
def __init__(self, kafka_proxy, config):
self.kafka_proxy = kafka_proxy
self.config = config
self.topic_mappings = config.get('topic_mappings', {})
self.event_bus = EventBusClient()
self.subscriptions = None
def start(self):
log.debug('starting')
self.subscriptions = list()
self._setup_subscriptions(self.topic_mappings)
log.info('started')
return self
def stop(self):
try:
log.debug('stopping-event-bus')
if self.subscriptions:
for subscription in self.subscriptions:
self.event_bus.unsubscribe(subscription)
log.info('stopped-event-bus')
except Exception as e:
log.exception('failed-stopping-event-bus', e=e)
return
def _setup_subscriptions(self, mappings):
for event_bus_topic, mapping in mappings.items():
kafka_topic = mapping.get('kafka_topic', None)
if kafka_topic is None:
log.error('no-kafka-topic-in-config',
event_bus_topic=event_bus_topic,
mapping=mapping)
continue
self.subscriptions.append(self.event_bus.subscribe(
event_bus_topic,
# to avoid Python late-binding to the last registered
# kafka_topic, we force instant binding with the default arg
lambda _, m, k=kafka_topic: self.forward(k, m)))
log.info('event-to-kafka', kafka_topic=kafka_topic,
event_bus_topic=event_bus_topic)
def forward(self, kafka_topic, msg):
try:
# convert to JSON string if msg is a protobuf msg
if isinstance(msg, Message):
msg = dumps(MessageToDict(msg, True, True))
log.debug('forward-event-bus-publisher')
self.kafka_proxy.send_message(kafka_topic, msg)
except Exception as e:
log.exception('failed-forward-event-bus-publisher', e=e)
def test_topic_filtering(self):
ebc = EventBusClient(EventBus())
mock = Mock()
ebc.subscribe('news', mock)
ebc.publish('news', 'msg1')
ebc.publish('alerts', 'msg2')
ebc.publish('logs', 'msg3')
self.assertEqual(mock.call_count, 1)
mock.assert_called_with('news', 'msg1')
def test_unsubscribe(self):
ebc = EventBusClient(EventBus())
sub = ebc.subscribe('news', lambda msg, topic: None)
ebc.unsubscribe(sub)
self.assertEqual(ebc.list_subscribers(), [])
self.assertEqual(ebc.list_subscribers('news'), [])
def test_subscribe(self):
ebc = EventBusClient()
sub = ebc.subscribe('news', lambda msg, topic: None)
self.assertEqual(len(ebc.list_subscribers()), 1)
self.assertEqual(len(ebc.list_subscribers('news')), 1)
self.assertEqual(len(ebc.list_subscribers('other')), 0)
def test_subscribers_that_unsubscribe_when_called(self):
# VOL-943 bug fix check
ebc = EventBusClient(EventBus())
class UnsubscribeWhenCalled(object):
def __init__(self):
self.subscription = ebc.subscribe('news', self.unsubscribe)
self.called = False
def unsubscribe(self, _topic, _msg):
self.called = True
ebc.unsubscribe(self.subscription)
ebc1 = UnsubscribeWhenCalled()
ebc2 = UnsubscribeWhenCalled()
ebc3 = UnsubscribeWhenCalled()
ebc.publish('news', 'msg1')
self.assertTrue(ebc1.called)
self.assertTrue(ebc2.called)
self.assertTrue(ebc3.called)
def test_simple_publish(self):
ebc = EventBusClient(EventBus())
mock = Mock()
ebc.subscribe('news', mock)
ebc.publish('news', 'message')
self.assertEqual(mock.call_count, 1)
mock.assert_called_with('news', 'message')
def test_multiple_subscribers(self):
ebc = EventBusClient(EventBus())
mock1 = Mock()
ebc.subscribe('news', mock1)
mock2 = Mock()
ebc.subscribe('alerts', mock2)
mock3 = Mock()
ebc.subscribe('logs', mock3)
mock4 = Mock()
ebc.subscribe('logs', mock4)
ebc.publish('news', 'msg1')
ebc.publish('alerts', 'msg2')
ebc.publish('logs', 'msg3')
self.assertEqual(mock1.call_count, 1)
mock1.assert_called_with('news', 'msg1')
self.assertEqual(mock2.call_count, 1)
mock2.assert_called_with('alerts', 'msg2')
self.assertEqual(mock3.call_count, 1)
mock3.assert_called_with('logs', 'msg3')
self.assertEqual(mock4.call_count, 1)
mock4.assert_called_with('logs', 'msg3')
def test_wildcard_topic(self):
ebc = EventBusClient(EventBus())
subs = []
wildcard_sub = Mock()
subs.append(ebc.subscribe(re.compile(r'.*'), wildcard_sub))
prefix_sub = Mock()
subs.append(ebc.subscribe(re.compile(r'ham.*'), prefix_sub))
contains_sub = Mock()
subs.append(ebc.subscribe(re.compile(r'.*burg.*'), contains_sub))
ebc.publish('news', 1)
ebc.publish('hamsters', 2)
ebc.publish('hamburgers', 3)
ebc.publish('nonsense', 4)
c = call
self.assertEqual(wildcard_sub.call_count, 4)
wildcard_sub.assert_has_calls([
c('news', 1),
c('hamsters', 2),
c('hamburgers', 3),
c('nonsense', 4)])
self.assertEqual(prefix_sub.call_count, 2)
prefix_sub.assert_has_calls([
c('hamsters', 2),
c('hamburgers', 3)])
self.assertEqual(contains_sub.call_count, 1)
contains_sub.assert_has_calls([c('hamburgers', 3)])
for sub in subs:
ebc.unsubscribe(sub)
self.assertEqual(ebc.list_subscribers(), [])
def test_predicates(self):
ebc = EventBusClient(EventBus())
get_foos = Mock()
ebc.subscribe('', get_foos, lambda msg: msg.startswith('foo'))
get_bars = Mock()
ebc.subscribe('', get_bars, lambda msg: msg.endswith('bar'))
get_all = Mock()
ebc.subscribe('', get_all)
get_none = Mock()
ebc.subscribe('', get_none, lambda msg: msg.find('zoo') >= 0)
errored = Mock()
ebc.subscribe('', errored, lambda msg: 1/0)
ebc.publish('', 'foo')
ebc.publish('', 'foobar')
ebc.publish('', 'bar')
c = call
self.assertEqual(get_foos.call_count, 2)
get_foos.assert_has_calls([c('', 'foo'), c('', 'foobar')])
self.assertEqual(get_bars.call_count, 2)
get_bars.assert_has_calls([c('', 'foobar'), c('', 'bar')])
self.assertEqual(get_all.call_count, 3)
get_all.assert_has_calls([c('', 'foo'), c('', 'foobar'), c('', 'bar')])
get_none.assert_not_called()
errored.assert_not_called()
def __init__(self, kafka_proxy, config):
self.kafka_proxy = kafka_proxy
self.config = config
self.topic_mappings = config.get('topic_mappings', {})
self.event_bus = EventBusClient()
self.subscriptions = None