def _create_file_download_event(timestamp,
bucket_id='B0000000000000000000000000000001',
file_id='F0000000000000000000000000000001',
file_key='test.pdf',
visitor_id=100):
"""Create a file_download event content."""
doc = dict(
timestamp=datetime.datetime(*timestamp).isoformat(),
# What:
bucket_id=str(bucket_id),
file_id=str(file_id),
file_key=file_key,
visitor_id=100)
return build_file_unique_id(doc)
def _create_file_download_event(timestamp,
bucket_id='B0000000000000000000000000000001',
file_id='F0000000000000000000000000000001',
file_key='test.pdf', visitor_id=100):
"""Create a file_download event content."""
doc = dict(
timestamp=datetime.datetime(*timestamp).isoformat(),
# What:
bucket_id=str(bucket_id),
file_id=str(file_id),
file_key=file_key,
visitor_id=100
)
return build_file_unique_id(doc)
def mock_event_queue(app, mock_datetime, request_headers, objects,
mock_user_ctx):
"""Create a mock queue containing a few file download events."""
mock_queue = Mock()
mock_queue.routing_key = 'stats-file-download'
with patch('datetime.datetime', mock_datetime), \
app.test_request_context(headers=request_headers['user']):
events = [
build_file_unique_id(
file_download_event_builder({}, app, objects[0])
) for idx in range(100)
]
mock_queue.consume.return_value = iter(events)
# Save the queued events for later tests
mock_queue.queued_events = deepcopy(events)
return mock_queue
def test_events_indexer_preprocessors(app, mock_event_queue):
"""Check that EventsIndexer calls properly the preprocessors."""
def test_preprocessor1(event):
event['test1'] = 42
event['visitor_id'] = 'testuser1'
return event
def test_preprocessor2(event):
event['test2'] = 21
return event
indexer = EventsIndexer(mock_event_queue,
preprocessors=[
build_file_unique_id, test_preprocessor1,
test_preprocessor2
])
# Generate the events
received_docs = []
def bulk(client, generator, *args, **kwargs):
received_docs.extend(generator)
with patch('elasticsearch.helpers.bulk', side_effect=bulk):
indexer.run()
# Process the events as we expect them to be
expected_docs = []
for event in mock_event_queue.queued_events:
event = build_file_unique_id(event)
event = test_preprocessor1(event)
event = test_preprocessor2(event)
_id = hash_id('2017-01-01T00:00:00', event)
expected_docs.append(
dict(
_id=_id,
_op_type='index',
_index='events-stats-file-download-2017-01-01',
_type=get_doctype('stats-file-download'),
_source=event,
))
assert received_docs == expected_docs