def relay(self,
consumer_group,
commit_log_topic,
synchronize_commit_group,
commit_batch_size=100,
initial_offset_reset='latest'):
consumer = SynchronizedConsumer(
bootstrap_servers=self.producer_configuration['bootstrap.servers'],
consumer_group=consumer_group,
commit_log_topic=commit_log_topic,
synchronize_commit_group=synchronize_commit_group,
initial_offset_reset=initial_offset_reset,
)
consumer.subscribe([self.publish_topic])
offsets = {}
def commit_offsets():
consumer.commit(offsets=[
TopicPartition(topic, partition, offset)
for (topic, partition), offset in offsets.items()
],
asynchronous=False)
try:
i = 0
while True:
message = consumer.poll(0.1)
if message is None:
continue
error = message.error()
if error is not None:
raise Exception(error)
i = i + 1
offsets[(message.topic(),
message.partition())] = message.offset() + 1
payload = parse_event_message(message.value())
if payload is not None:
post_process_group.delay(**payload)
if i % commit_batch_size == 0:
commit_offsets()
except KeyboardInterrupt:
pass
logger.info('Committing offsets and closing consumer...')
if offsets:
commit_offsets()
consumer.close()
def test_parse_event_message_version_1():
event_data = {
'project_id': 1,
'group_id': 2,
'event_id': '00000000000010008080808080808080',
'message': 'message',
'platform': 'python',
'datetime': '2018-07-20T21:04:27.600640Z',
'data': {},
'extra': {},
'primary_hash': '49f68a5c8493ec2c0bf489821c21fc3b',
}
task_state = {
'is_new': True,
'is_sample': False,
'is_regression': False,
'is_new_group_environment': True,
}
kwargs = parse_event_message(
json.dumps([1, 'insert', event_data, task_state]))
event = kwargs.pop('event')
assert event.project_id == 1
assert event.group_id == 2
assert event.event_id == '00000000000010008080808080808080'
assert event.message == 'message'
assert event.platform == 'python'
assert event.datetime == datetime(2018,
7,
20,
21,
4,
27,
600640,
tzinfo=pytz.utc)
assert dict(event.data) == {}
assert kwargs.pop('primary_hash') == '49f68a5c8493ec2c0bf489821c21fc3b'
assert kwargs.pop('is_new') is True
assert kwargs.pop('is_sample') is False
assert kwargs.pop('is_regression') is False
assert kwargs.pop('is_new_group_environment') is True
assert not kwargs, 'unexpected values remaining: {!r}'.format(kwargs)
def test_parse_event_message_version_1_unexpected_operation():
with pytest.raises(UnexpectedOperation):
parse_event_message(json.dumps([1, 'invalid', {}, {}]))
def test_parse_event_message_version_1_unsupported_operation():
assert parse_event_message(json.dumps([1, 'delete', {}])) is None
def test_parse_event_message_invalid_version():
with pytest.raises(InvalidVersion):
parse_event_message(json.dumps([0, 'insert', {}]))
def test_parse_event_message_invalid_payload():
with pytest.raises(InvalidPayload):
parse_event_message('{"format": "invalid"}')