def test_piwik_exporter(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True, do_aggregate_events=False, do_update_record_statistics=False ) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:00:00' PiwikExporter().run() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:30:00'
def test_basic_stats(app, db, es, locations, event_queues, minimal_record): """Test basic statistics results.""" search = Search(using=es) records = create_stats_fixtures( # (10 * 2) -> 20 records and (10 * 2 * 3) -> 60 files metadata=minimal_record, n_records=10, n_versions=2, n_files=3, event_data={'user_id': '1'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30)) # Events indices prefix = app.config['SEARCH_INDEX_PREFIX'] # 2 versions * 10 records * 3 files * 4 events -> 240 assert search.index(prefix + 'events-stats-file-download').count() == 240 # 2 versions * 10 records * 4 events -> 80 assert search.index(prefix + 'events-stats-record-view').count() == 80 # Aggregations indices # (2 versions + 1 concept) * 10 records -> 30 documents + 2 bookmarks # 30d assert search.index(prefix + 'stats-file-download').count() == 30 # 30d assert search.index(prefix + 'stats-record-view').count() == 30 # 2bm + 2bm assert search.index(prefix + 'stats-bookmarks').count() == 4 # Records index for _, record, _ in records: doc = \ current_search_client.get( index=build_alias_name('records'), id=str(record.id), params={'_source_includes': '_stats'} ) assert doc['_source']['_stats'] == { # 4 view events 'views': 4.0, 'version_views': 8.0, # 4 view events over 2 different hours 'unique_views': 2.0, 'version_unique_views': 2.0, # 4 download events * 3 files 'downloads': 12.0, 'version_downloads': 24.0, # 4 download events * 3 files over 2 different hours 'unique_downloads': 2.0, 'version_unique_downloads': 2.0, # 4 download events * 3 files * 10 bytes 'volume': 120.0, 'version_volume': 240.0, }
def test_piwik_exporter(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True, do_aggregate_events=False, do_update_record_statistics=False ) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:00:00' PiwikExporter().run() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:30:00'
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={ 'user_id': '1', 'country': 'CH' }, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked: PiwikExporter().run() mocked.assert_not_called() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={ 'user_id': '1', 'country': 'CH' }, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) with pytest.raises(PiwikExportRequestError): PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_large_stats(app, db, es, locations, event_queues, minimal_record): """Test a larger number of events, aggregations, and results.""" search = Search(using=es) records = create_stats_fixtures( # (3 * 4) -> 12 records and (3 * 4 * 2) -> 24 files metadata=minimal_record, n_records=3, n_versions=4, n_files=2, event_data={'user_id': '1'}, # (31 + 30) * 2 -> 122 event timestamps (61 days and 2 events/day) start_date=datetime(2018, 3, 1), end_date=datetime(2018, 5, 1), interval=timedelta(hours=12)) # Events indices prefix = app.config['SEARCH_INDEX_PREFIX'] # 4 versions * 3 records * 2 files * 122 events -> 2928 assert search.index(prefix + 'events-stats-file-download').count() == 2928 # 4 versions * 3 records * 122 events -> 1464 assert search.index(prefix + 'events-stats-record-view').count() == 1464 # Aggregations indices # (4 versions + 1 concept) * 3 records -> 15 documents + 2 bookmarks q = search.index(prefix + 'stats-file-download') q = q.doc_type('file-download-day-aggregation') assert q.count() == 915 # 61 days * 15 records q = search.index(prefix + 'stats-record-view') q = q.doc_type('record-view-day-aggregation') assert q.count() == 915 # 61 days * 15 records # Records index for _, record, _ in records: doc = \ current_search_client.get( index=build_alias_name('records'), id=str(record.id), params={'_source_includes': '_stats'} ) assert doc['_source']['_stats'] == { # 4 view events 'views': 122.0, 'version_views': 488.0, # 4 view events over 2 different hours 'unique_views': 122.0, 'version_unique_views': 122.0, # 4 download events * 3 files 'downloads': 244.0, 'version_downloads': 976.0, # 4 download events * 3 files over 2 different hours 'unique_downloads': 122.0, 'version_unique_downloads': 122.0, # 4 download events * 3 files * 10 bytes 'volume': 2440.0, 'version_volume': 9760.0, }
def test_large_stats(app, db, es, locations, event_queues, minimal_record): """Test record page view event import.""" search = Search(using=es) records = create_stats_fixtures( # (3 * 4) -> 12 records and (3 * 4 * 2) -> 24 files metadata=minimal_record, n_records=3, n_versions=4, n_files=2, event_data={'user_id': '1'}, # (31 + 30) * 2 -> 122 event timestamps (61 days and 2 events/day) start_date=datetime(2018, 3, 1), end_date=datetime(2018, 5, 1), interval=timedelta(hours=12)) # Events indices # 4 versions * 3 records * 2 files * 122 events -> 2928 assert search.index('events-stats-file-download').count() == 2928 # 4 versions * 3 records * 122 events -> 1464 assert search.index('events-stats-record-view').count() == 1464 # Aggregations indices # (4 versions + 1 concept) * 3 records -> 15 documents + 2 bookmarks q = search.index('stats-file-download') q = q.doc_type('file-download-day-aggregation') assert q.count() == 915 # 61 days * 15 records q = search.index('stats-record-view') q = q.doc_type('record-view-day-aggregation') assert q.count() == 915 # 61 days * 15 records # Reords index for _, record, _ in records: doc = (RecordsSearch().get_record( record.id).source(include='_stats').execute()[0]) assert doc['_stats'] == { # 4 view events 'views': 122.0, 'version_views': 488.0, # 4 view events over 2 different hours 'unique_views': 122.0, 'version_unique_views': 122.0, # 4 download events * 3 files 'downloads': 244.0, 'version_downloads': 976.0, # 4 download events * 3 files over 2 different hours 'unique_downloads': 122.0, 'version_unique_downloads': 122.0, # 4 download events * 3 files * 10 bytes 'volume': 2440.0, 'version_volume': 9760.0, }
def test_basic_stats(app, db, es, locations, event_queues, minimal_record): """Test basic statistics results.""" search = Search(using=es) records = create_stats_fixtures( # (10 * 2) -> 20 records and (10 * 2 * 3) -> 60 files metadata=minimal_record, n_records=10, n_versions=2, n_files=3, event_data={'user_id': '1'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30)) # Events indices # 2 versions * 10 records * 3 files * 4 events -> 240 assert search.index('events-stats-file-download').count() == 240 # 2 versions * 10 records * 4 events -> 80 assert search.index('events-stats-record-view').count() == 80 # Aggregations indices # (2 versions + 1 concept) * 10 records -> 30 documents + 2 bookmarks assert search.index('stats-file-download').count() == 32 # 2bm + 30d assert search.index('stats-record-view').count() == 32 # 2bm + 30d # Reords index for _, record, _ in records: doc = (RecordsSearch().get_record( record.id).source(include='_stats').execute()[0]) assert doc['_stats'] == { # 4 view events 'views': 4.0, 'version_views': 8.0, # 4 view events over 2 different hours 'unique_views': 2.0, 'version_unique_views': 2.0, # 4 download events * 3 files 'downloads': 12.0, 'version_downloads': 24.0, # 4 download events * 3 files over 2 different hours 'unique_downloads': 2.0, 'version_unique_downloads': 2.0, # 4 download events * 3 files * 10 bytes 'volume': 120.0, 'version_volume': 240.0, }
def test_large_stats(app, db, es, locations, event_queues, minimal_record): """Test record page view event import.""" search = Search(using=es) records = create_stats_fixtures( # (3 * 4) -> 12 records and (3 * 4 * 2) -> 24 files metadata=minimal_record, n_records=3, n_versions=4, n_files=2, event_data={'user_id': '1'}, # (31 + 30) * 2 -> 122 event timestamps (61 days and 2 events/day) start_date=datetime(2018, 3, 1), end_date=datetime(2018, 5, 1), interval=timedelta(hours=12)) # Events indices # 4 versions * 3 records * 2 files * 122 events -> 2928 assert search.index('events-stats-file-download').count() == 2928 # 4 versions * 3 records * 122 events -> 1464 assert search.index('events-stats-record-view').count() == 1464 # Aggregations indices # (4 versions + 1 concept) * 3 records -> 15 documents + 2 bookmarks q = search.index('stats-file-download') q = q.doc_type('file-download-day-aggregation') assert q.count() == 915 # 61 days * 15 records q = search.index('stats-record-view') q = q.doc_type('record-view-day-aggregation') assert q.count() == 915 # 61 days * 15 records # Reords index for _, record, _ in records: doc = ( RecordsSearch().get_record(record.id) .source(include='_stats').execute()[0]) assert doc['_stats'] == { # 4 view events 'views': 122.0, 'version_views': 488.0, # 4 view events over 2 different hours 'unique_views': 122.0, 'version_unique_views': 122.0, # 4 download events * 3 files 'downloads': 244.0, 'version_downloads': 976.0, # 4 download events * 3 files over 2 different hours 'unique_downloads': 122.0, 'version_unique_downloads': 122.0, # 4 download events * 3 files * 10 bytes 'volume': 2440.0, 'version_volume': 9760.0, }
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked: PiwikExporter().run() mocked.assert_not_called() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_basic_stats(app, db, es, locations, event_queues, minimal_record): """Test basic statistics results.""" search = Search(using=es) records = create_stats_fixtures( # (10 * 2) -> 20 records and (10 * 2 * 3) -> 60 files metadata=minimal_record, n_records=10, n_versions=2, n_files=3, event_data={'user_id': '1'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30)) # Events indices # 2 versions * 10 records * 3 files * 4 events -> 240 assert search.index('events-stats-file-download').count() == 240 # 2 versions * 10 records * 4 events -> 80 assert search.index('events-stats-record-view').count() == 80 # Aggregations indices # (2 versions + 1 concept) * 10 records -> 30 documents + 2 bookmarks assert search.index('stats-file-download').count() == 32 # 2bm + 30d assert search.index('stats-record-view').count() == 32 # 2bm + 30d # Reords index for _, record, _ in records: doc = ( RecordsSearch().get_record(record.id) .source(include='_stats').execute()[0]) assert doc['_stats'] == { # 4 view events 'views': 4.0, 'version_views': 8.0, # 4 view events over 2 different hours 'unique_views': 2.0, 'version_unique_views': 2.0, # 4 download events * 3 files 'downloads': 12.0, 'version_downloads': 24.0, # 4 download events * 3 files over 2 different hours 'unique_downloads': 2.0, 'version_unique_downloads': 2.0, # 4 download events * 3 files * 10 bytes 'volume': 120.0, 'version_volume': 240.0, }
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) with pytest.raises(PiwikExportRequestError): PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_update_record_statistics(app, db, es, locations, event_queues, minimal_record): """Test record statistics update task.""" records = create_stats_fixtures( metadata=minimal_record, n_records=1, n_versions=5, n_files=3, event_data={'user_id': '1'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), # This also runs the task we want to test and indexes records. do_update_record_statistics=True) expected_stats = { 'views': 4.0, 'version_views': 20.0, 'unique_views': 2.0, 'version_unique_views': 2.0, 'downloads': 12.0, 'version_downloads': 60.0, 'unique_downloads': 2.0, 'version_unique_downloads': 2.0, 'volume': 120.0, 'version_volume': 600.0, } # Check current stats for all records for recid, _, _ in records: stats = get_record_stats(recid.object_uuid) assert stats == expected_stats # Perform a view and all-files download today on the first version recid_v1, record_v1, file_objects_v1 = records[0] with app.test_client() as client: for f in file_objects_v1: file_url = url_for('invenio_records_ui.recid_files', pid_value=recid_v1.pid_value, filename=f.key) assert client.get(file_url).status_code == 200 record_url = url_for( 'invenio_records_ui.recid', pid_value=recid_v1.pid_value) assert client.get(record_url).status_code == 200 process_events(['record-view', 'file-download']) current_search.flush_and_refresh(index='events-stats-*') aggregate_events( ['record-view-agg', 'record-view-all-versions-agg', 'record-download-agg', 'record-download-all-versions-agg']) current_search.flush_and_refresh(index='stats-*') update_record_statistics() RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index='records') # Check current stats for all records stats = get_record_stats(recid_v1.object_uuid) assert stats == { 'views': 5.0, 'version_views': 21.0, 'unique_views': 3.0, 'version_unique_views': 3.0, 'downloads': 15.0, 'version_downloads': 63.0, 'unique_downloads': 3.0, 'version_unique_downloads': 3.0, 'volume': 150.0, 'version_volume': 630.0, } # Other versions will have only their `version_*` statistics updated expected_stats['version_views'] += 1 expected_stats['version_unique_views'] += 1 expected_stats['version_downloads'] += 3 expected_stats['version_unique_downloads'] += 1 expected_stats['version_volume'] += 30 for recid, _, _ in records[1:]: stats = get_record_stats(recid.object_uuid) assert stats == expected_stats
def test_update_record_statistics(app, db, es, locations, event_queues, minimal_record): """Test record statistics update task.""" records = create_stats_fixtures( metadata=minimal_record, n_records=1, n_versions=5, n_files=3, event_data={'user_id': '1'}, # 4 event timestamps (half-hours between 13:00-15:00) start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), # This also runs the task we want to test and indexes records. do_update_record_statistics=True) expected_stats = { 'views': 4.0, 'version_views': 20.0, 'unique_views': 2.0, 'version_unique_views': 2.0, 'downloads': 12.0, 'version_downloads': 60.0, 'unique_downloads': 2.0, 'version_unique_downloads': 2.0, 'volume': 120.0, 'version_volume': 600.0, } # Check current stats for all records for recid, _, _ in records: stats = get_record_stats(recid.object_uuid) assert stats == expected_stats # Perform a view and all-files download today on the first version recid_v1, record_v1, file_objects_v1 = records[0] with app.test_client() as client: for f in file_objects_v1: file_url = url_for('invenio_records_ui.recid_files', pid_value=recid_v1.pid_value, filename=f.key) assert client.get(file_url).status_code == 200 record_url = url_for('invenio_records_ui.recid', pid_value=recid_v1.pid_value) assert client.get(record_url).status_code == 200 process_events(['record-view', 'file-download']) current_search.flush_and_refresh(index='events-stats-*') aggregate_events([ 'record-view-agg', 'record-view-all-versions-agg', 'record-download-agg', 'record-download-all-versions-agg' ]) current_search.flush_and_refresh(index='stats-*') update_record_statistics() RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index='records') # Check current stats for all records stats = get_record_stats(recid_v1.object_uuid) assert stats == { 'views': 5.0, 'version_views': 21.0, 'unique_views': 3.0, 'version_unique_views': 3.0, 'downloads': 15.0, 'version_downloads': 63.0, 'unique_downloads': 3.0, 'version_unique_downloads': 3.0, 'volume': 150.0, 'version_volume': 630.0, } # Other versions will have only their `version_*` statistics updated expected_stats['version_views'] += 1 expected_stats['version_unique_views'] += 1 expected_stats['version_downloads'] += 3 expected_stats['version_unique_downloads'] += 1 expected_stats['version_volume'] += 30 for recid, _, _ in records[1:]: stats = get_record_stats(recid.object_uuid) assert stats == expected_stats