def test_process_events(app, es, event_queues):
"""Test process event."""
current_stats.publish('file-download', [
dict(timestamp='2017-01-01T00:00:00',
visitor_id='testuser1',
unique_id='2017-01-01T00:00:00-hash',
data='val')
])
process_events.delay(['file-download'])
def _patch_stats_publish():
original_publish = current_stats.publish
event_batches = defaultdict(list)
def _patched_publish(self, event_type, events):
events[0].update(event_data)
event_batches[event_type].append(events[0])
current_stats.publish = MethodType(_patched_publish, current_stats)
yield
current_stats.publish = original_publish
for event_type, events in event_batches.items():
current_stats.publish(event_type, events)
def _patch_stats_publish():
original_publish = current_stats.publish
event_batches = defaultdict(list)
def _patched_publish(self, event_type, events):
events[0].update(event_data)
event_batches[event_type].append(events[0])
current_stats.publish = MethodType(_patched_publish, current_stats)
yield
current_stats.publish = original_publish
for event_type, events in event_batches.items():
current_stats.publish(event_type, events)
def test_metric_aggregations(app, event_queues, es_with_templates):
"""Test aggregation metrics."""
es = es_with_templates
current_stats.publish('file-download', [
_create_file_download_event(date, user_id='1')
for date in [(2018, 1, 1, 12,
10), (2018, 1, 1, 12,
20), (2018, 1, 1, 12,
30), (2018, 1, 1, 13,
10), (2018, 1, 1, 13,
20), (2018, 1, 1, 13,
30), (2018, 1, 1, 14, 10),
(2018, 1, 1, 14,
20), (2018, 1, 1, 14,
30), (2018, 1, 1, 15,
10), (2018, 1, 1, 15,
20), (2018, 1, 1, 15, 30)]
])
process_events(['file-download'])
es.indices.refresh(index='*')
StatAggregator(name='file-download-agg',
client=current_search_client,
event='file-download',
aggregation_field='file_id',
metric_aggregation_fields={
'unique_count': ('cardinality', 'unique_session_id', {
'precision_threshold': 1000
}),
'volume': ('sum', 'size', {})
},
aggregation_interval='day').run()
es.indices.refresh(index='*')
query = Search(using=current_search_client,
index='stats-file-download',
doc_type='file-download-day-aggregation')
results = query.execute()
assert len(results) == 1
assert results[0].count == 12 # 3 views over 4 differnet hour slices
assert results[0].unique_count == 4 # 4 different hour slices accessed
assert results[0].volume == 9000 * 12
def test_events_process(script_info, event_queues, es_with_templates):
"""Test "events process" CLI command."""
es = es_with_templates
search = Search(using=es)
runner = CliRunner()
# Invalid argument
result = runner.invoke(
stats, ['events', 'process', 'invalid-event-type', '--eager'],
obj=script_info)
assert result.exit_code == 2
assert 'Invalid event type(s):' in result.output
current_stats.publish('file-download', [
_create_file_download_event(date)
for date in [(2018, 1, 1, 10), (2018, 1, 1, 12), (2018, 1, 1, 14)]
])
current_stats.publish('record-view', [
_create_record_view_event(date)
for date in [(2018, 1, 1, 10), (2018, 1, 1, 12), (2018, 1, 1, 14)]
])
result = runner.invoke(stats,
['events', 'process', 'file-download', '--eager'],
obj=script_info)
assert result.exit_code == 0
current_search.flush_and_refresh(index='*')
assert search.index('events-stats-file-download-2018-01-01').count() == 3
assert search.index('events-stats-file-download').count() == 3
assert not es.indices.exists('events-stats-record-view-2018-01-01')
assert not es.indices.exists_alias(name='events-stats-record-view')
result = runner.invoke(stats,
['events', 'process', 'record-view', '--eager'],
obj=script_info)
assert result.exit_code == 0
current_search.flush_and_refresh(index='*')
assert search.index('events-stats-file-download-2018-01-01').count() == 3
assert search.index('events-stats-file-download').count() == 3
assert search.index('events-stats-record-view-2018-01-01').count() == 3
assert search.index('events-stats-record-view').count() == 3
# Create some more events
current_stats.publish('file-download',
[_create_file_download_event((2018, 2, 1, 12))])
current_stats.publish('record-view',
[_create_record_view_event((2018, 2, 1, 10))])
# Process all event types via a celery task
result = runner.invoke(stats, ['events', 'process'], obj=script_info)
assert result.exit_code == 0
current_search.flush_and_refresh(index='*')
assert search.index('events-stats-file-download-2018-01-01').count() == 3
assert search.index('events-stats-file-download-2018-02-01').count() == 1
assert search.index('events-stats-file-download').count() == 4
assert search.index('events-stats-record-view-2018-01-01').count() == 3
assert search.index('events-stats-record-view-2018-02-01').count() == 1
assert search.index('events-stats-record-view').count() == 4
def test_process_events(app, es, event_queues):
"""Test process event."""
current_stats.publish('file-download', [dict(data='val')])
process_events.delay(['file-download'])
def test_process_events(app, es, event_queues):
"""Test process event."""
current_stats.publish('file-download', [dict(data='val')])
process_events.delay(['file-download'])