def test_piwik_exporter(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True, do_aggregate_events=False, do_update_record_statistics=False ) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:00:00' PiwikExporter().run() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:30:00'
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={ 'user_id': '1', 'country': 'CH' }, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) with pytest.raises(PiwikExportRequestError): PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_get_user_by_email_forces_to_refresh_cache_when_nothing_found_first_time( mock_query_rt, inspire_app): mock_query_rt.side_effect = [ ["id: user/1", "Name: user1", "EmailAddress: [email protected]"], [ "id: user/1", "Name: user1", "EmailAddress: [email protected]", "", "--", "", "id: user/2", "Name: user2", "EmailAddress: [email protected]", ], ] expected_user = { "id": "user/2", "Name": "user2", "EmailAddress": "*****@*****.**" } current_cache.delete("rt_users_with_emails") user = get_rt_user_by_email("*****@*****.**") assert user == expected_user assert mock_query_rt.call_count == 2
def openaire_direct_index(record_uuid): """Send record for direct indexing at OpenAIRE. :param record_uuid: Record Metadata UUID. :type record_uuid: str """ try: record = ZenodoRecord.get_record(record_uuid) # Bail out if not an OpenAIRE record. if not (is_openaire_publication(record) or is_openaire_dataset(record)): return data = openaire_json_v1.serialize(record.pid, record) url = '{}/api/results/feedObject'.format( current_app.config['OPENAIRE_API_URL']) req = _openaire_request_factory() res = req.post(url, data=data) if not res.ok: raise OpenAIRERequestError(res.text) else: recid = record.get('recid') current_cache.delete('openaire_direct_index:{}'.format(recid)) except Exception as exc: recid = record.get('recid') current_cache.set('openaire_direct_index:{}'.format(recid), datetime.now(), timeout=-1) openaire_direct_index.retry(exc=exc)
def test_cache_ui(): with_update = request.args.get('update', '0') with_delete = request.args.get('delete', '0') with_update = True if int(with_update) else False with_delete = True if int(with_delete) else False if with_update: if with_delete: current_cache.delete('test_key') current_cache.set('test_key', ObjDict(**{ 'device_id': 12345678, 'cache_time': datetime.today() }), timeout=60) test_value = current_cache.get('test_key') if not test_value: cache_rtn = current_cache.set('test_key', ObjDict(**{ 'device_id': 12345678, 'cache_time': datetime.today() }), timeout=60) if not cache_rtn: return jsonify({ 'code': -1, 'msg': 'cache set error' }) test_value = current_cache.get('test_key') return jsonify({ 'code': 0, 'msg': 'success', 'data': { 'dev_id': test_value.device_id, 'store_time': test_value.cache_time.strftime('%Y-%m-%d %H:%M:%S') } })
def test_get_from_cache_when_forced(inspire_app): def _test_generator(): return [1, 2, 3] def _second_generator_which_should_run(): return [4, 5, 6] expected = [1, 2, 3] current_cache.delete("test") # Cache empty, so run generator and return data result = _get_from_cache("test", _test_generator) assert result == expected assert expected == current_cache.get("test") # Forcing so generator should run again expected = [4, 5, 6] result = _get_from_cache("test", _second_generator_which_should_run, force_update=True) assert result == expected assert expected == current_cache.get("test") # Cache is set and not forcing so cached data should be returned result = _get_from_cache("test", _test_generator) assert result == expected assert expected == current_cache.get("test")
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={ 'user_id': '1', 'country': 'CH' }, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked: PiwikExporter().run() mocked.assert_not_called() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_all_collections_are_cached(app, app_client): """Test that collection info gets cached.""" # Remove collection cache key current_cache.delete('restricted_collections') app_client.get("/old-literature/1497201") # Check that cache key has been correctly filled assert current_cache.get('restricted_collections') == set(['HERMES Internal Notes'])
def test_all_collections_are_cached(app, app_client): """Test that collection info gets cached.""" # Remove collection cache key current_cache.delete('restricted_collections') app_client.get("/old-literature/1497201") # Check that cache key has been correctly filled assert current_cache.get('restricted_collections') == set( ['HERMES Internal Notes'])
def test_all_collections_are_cached(app, app_client): """Test that collection info gets cached.""" # Remove collection cache key current_cache.delete('restricted_collections') app_client.get("/literature/111") # Check that cache key has been correctly filled assert current_cache.get('restricted_collections') == \ set([u'Another Restricted Collection', u'Restricted Collection', u'Role only collection'])
def test_rt_queues_are_cached(mock_query_rt, inspire_app): mock_query_rt.return_value = [ "35: Admin", "63: Admin-curator", "60: Admin-Dev" ] current_cache.delete("rt_queues") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/queues") assert current_cache.get("rt_queues") == orjson.loads(response.data)
def test_get_user_by_email_forces_to_refresh_only_once_per_call( mock_query_rt, inspire_app): mock_query_rt.return_value = [ "id: user/1", "Name: user1", "EmailAddress: [email protected]", ] current_cache.delete("rt_users_with_emails") with pytest.raises(NoUsersFound): get_rt_user_by_email("*****@*****.**") assert mock_query_rt.call_count == 2
def test_rt_users_are_cached(mock_get_all_of, inspire_app): mock_get_all_of.return_value = [ {"id": "10309", "name": "atkinson"}, {"id": "1125438", "name": "bhecker"}, {"id": "460354", "name": "Catherine"}, ] current_cache.delete("rt_users") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/users") assert current_cache.get("rt_users") == json.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, inspire_app): mock_get_all_of.return_value = [ {"id": "35", "name": "Admin"}, {"id": "63", "name": "Admin-curator"}, {"id": "60", "name": "Admin-Dev"}, ] current_cache.delete("rt_queues") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/queues") assert current_cache.get("rt_queues") == json.loads(response.data)
def test_rt_users_are_cached(mock_query_rt, inspire_app): mock_query_rt.return_value = [ "10309: atkinson", "1125438: bhecker", "460354: Catherine", ] current_cache.delete("rt_users") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/users") assert current_cache.get("rt_users") == orjson.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, log_in_as_cataloger, api_client): mock_get_all_of.return_value = [{ "id": "35", "name": "Admin" }, { "id": "63", "name": "Admin-curator" }, { "id": "60", "name": "Admin-Dev" }] current_cache.delete('rt_queues') response = api_client.get('/editor/rt/queues') assert current_cache.get('rt_queues') == json.loads(response.data)
def restricted_record(app): restricted_collection = Collection( name='Restricted Collection', dbquery='_collections:"Restricted Collection"', ) another_restricted_collection = Collection( name='Another Restricted Collection', dbquery='_collections:"Another Restricted Collection"', ) db.session.add_all([restricted_collection, another_restricted_collection]) db.session.commit() record = { '$schema': 'http://localhost:5000/schemas/records/hep.json', '_collections': [ 'Literature', 'Restricted Collection', 'Another Restricted Collection', ], 'control_number': 222, 'document_type': [ 'article', ], 'titles': [{ 'title': 'restricted' }], } record = _create_and_index_record(record) yield record collection = Collection.query.filter_by(name='Restricted Collection').one() another_collection = Collection.query.filter_by( name='Another Restricted Collection').one() pid = PersistentIdentifier.get('lit', '222') db.session.delete(collection) db.session.delete(another_collection) db.session.delete(pid) record._delete(force=True) current_app.extensions[ 'invenio-db'].versioning_manager.transaction_cls.query.delete() db.session.commit() current_cache.delete('restricted_collections')
def test_get_user_by_email(mock_query_rt, inspire_app): mock_query_rt.return_value = [ "id: user/1", "Name: user1", "EmailAddress: [email protected]", "", "--", "", "id: user/2", "Name: user2", "EmailAddress: [email protected]", ] expected_user = {"Name": "user2", "EmailAddress": "*****@*****.**"} current_cache.delete("rt_users_with_emails") user = get_rt_user_by_email("*****@*****.**") del user["id"] assert user == expected_user
def test_rt_users_are_cached(mock_get_all_of, log_in_as_cataloger, api_client): mock_get_all_of.return_value = [ { "id": "10309", "name": "atkinson" }, { "id": "1125438", "name": "bhecker" }, { "id": "460354", "name": "Catherine" }, ] current_cache.delete('rt_users') response = api_client.get('/editor/rt/users') assert current_cache.get('rt_users') == json.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, log_in_as_cataloger, api_client): mock_get_all_of.return_value = [ { "id": "35", "name": "Admin" }, { "id": "63", "name": "Admin-curator" }, { "id": "60", "name": "Admin-Dev" } ] current_cache.delete('rt_queues') response = api_client.get('/editor/rt/queues') assert current_cache.get('rt_queues') == json.loads(response.data)
def delete(self, key): """Delete the key. Args: key (str): a key name. Returns: bool: if the key is deleted. """ return current_cache.delete(self._prefix(key))
def test_get_from_cache(inspire_app): def _test_generator(): return [1, 2, 3] def _second_generator_which_should_not_run(): raise AssertionError expected = [1, 2, 3] current_cache.delete("test") # Cache empty, so run generator and return data result = _get_from_cache("test", _test_generator) assert result == expected assert expected == current_cache.get("test") # Cache set so generator should not run result = _get_from_cache("test", _second_generator_which_should_not_run) assert result == expected assert expected == current_cache.get("test")
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked: PiwikExporter().run() mocked.assert_not_called() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def openaire_direct_index(record_uuid): """Send record for direct indexing at OpenAIRE. :param record_uuid: Record Metadata UUID. :type record_uuid: str """ try: record = ZenodoRecord.get_record(record_uuid) # Bail out if not an OpenAIRE record. if not (is_openaire_publication(record) or is_openaire_dataset(record) or is_openaire_software(record) or is_openaire_other(record)): return data = openaire_json_v1.serialize(record.pid, record) url = '{}/feedObject'.format( current_app.config['OPENAIRE_API_URL']) req = _openaire_request_factory() res = req.post(url, data=data) if not res.ok: raise OpenAIRERequestError(res.text) res_beta = None if current_app.config['OPENAIRE_API_URL_BETA']: url_beta = '{}/feedObject'.format( current_app.config['OPENAIRE_API_URL_BETA']) res_beta = req.post(url_beta, data=data) if res_beta and not res_beta.ok: raise OpenAIRERequestError(res_beta.text) else: recid = record.get('recid') current_cache.delete('openaire_direct_index:{}'.format(recid)) except Exception as exc: recid = record.get('recid') current_cache.set('openaire_direct_index:{}'.format(recid), datetime.now(), timeout=-1) openaire_direct_index.retry(exc=exc)
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) with pytest.raises(PiwikExportRequestError): PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def clear_cache(self): """Clear the sitemap cache.""" for key in self.cache_keys: current_cache.delete(key) self.cache_keys = set()
def del_test_cache_ui(): current_cache.delete('test_key') return jsonify({ 'code': 0, 'msg': 'success' })
def clear_cache_pages(self): """Clear all cached pages.""" page_keys = current_cache.get(self.cached_pages_set_key) or set() for page in page_keys: current_cache.delete(page) current_cache.delete(self.cached_pages_set_key)