def test_piwik_exporter(app, db, es, locations, event_queues, full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True,
do_aggregate_events=False,
do_update_record_statistics=False
)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:00:00'
PiwikExporter().run()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:30:00'
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record,
n_records=1,
n_versions=1,
n_files=1,
event_data={
'user_id': '1',
'country': 'CH'
},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
with pytest.raises(PiwikExportRequestError):
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def test_piwik_exporter(app, db, es, locations, event_queues, full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True,
do_aggregate_events=False,
do_update_record_statistics=False
)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:00:00'
PiwikExporter().run()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:30:00'
def test_get_user_by_email_forces_to_refresh_cache_when_nothing_found_first_time(
mock_query_rt, inspire_app):
mock_query_rt.side_effect = [
["id: user/1", "Name: user1", "EmailAddress: [email protected]"],
[
"id: user/1",
"Name: user1",
"EmailAddress: [email protected]",
"",
"--",
"",
"id: user/2",
"Name: user2",
"EmailAddress: [email protected]",
],
]
expected_user = {
"id": "user/2",
"Name": "user2",
"EmailAddress": "*****@*****.**"
}
current_cache.delete("rt_users_with_emails")
user = get_rt_user_by_email("*****@*****.**")
assert user == expected_user
assert mock_query_rt.call_count == 2
def openaire_direct_index(record_uuid):
"""Send record for direct indexing at OpenAIRE.
:param record_uuid: Record Metadata UUID.
:type record_uuid: str
"""
try:
record = ZenodoRecord.get_record(record_uuid)
# Bail out if not an OpenAIRE record.
if not (is_openaire_publication(record)
or is_openaire_dataset(record)):
return
data = openaire_json_v1.serialize(record.pid, record)
url = '{}/api/results/feedObject'.format(
current_app.config['OPENAIRE_API_URL'])
req = _openaire_request_factory()
res = req.post(url, data=data)
if not res.ok:
raise OpenAIRERequestError(res.text)
else:
recid = record.get('recid')
current_cache.delete('openaire_direct_index:{}'.format(recid))
except Exception as exc:
recid = record.get('recid')
current_cache.set('openaire_direct_index:{}'.format(recid),
datetime.now(),
timeout=-1)
openaire_direct_index.retry(exc=exc)
def test_cache_ui():
with_update = request.args.get('update', '0')
with_delete = request.args.get('delete', '0')
with_update = True if int(with_update) else False
with_delete = True if int(with_delete) else False
if with_update:
if with_delete:
current_cache.delete('test_key')
current_cache.set('test_key', ObjDict(**{
'device_id': 12345678,
'cache_time': datetime.today()
}), timeout=60)
test_value = current_cache.get('test_key')
if not test_value:
cache_rtn = current_cache.set('test_key', ObjDict(**{
'device_id': 12345678,
'cache_time': datetime.today()
}), timeout=60)
if not cache_rtn:
return jsonify({
'code': -1,
'msg': 'cache set error'
})
test_value = current_cache.get('test_key')
return jsonify({
'code': 0,
'msg': 'success',
'data': {
'dev_id': test_value.device_id,
'store_time': test_value.cache_time.strftime('%Y-%m-%d %H:%M:%S')
}
})
def test_get_from_cache_when_forced(inspire_app):
def _test_generator():
return [1, 2, 3]
def _second_generator_which_should_run():
return [4, 5, 6]
expected = [1, 2, 3]
current_cache.delete("test")
# Cache empty, so run generator and return data
result = _get_from_cache("test", _test_generator)
assert result == expected
assert expected == current_cache.get("test")
# Forcing so generator should run again
expected = [4, 5, 6]
result = _get_from_cache("test",
_second_generator_which_should_run,
force_update=True)
assert result == expected
assert expected == current_cache.get("test")
# Cache is set and not forcing so cached data should be returned
result = _get_from_cache("test", _test_generator)
assert result == expected
assert expected == current_cache.get("test")
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record,
n_records=1,
n_versions=1,
n_files=1,
event_data={
'user_id': '1',
'country': 'CH'
},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked:
PiwikExporter().run()
mocked.assert_not_called()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def test_all_collections_are_cached(app, app_client):
"""Test that collection info gets cached."""
# Remove collection cache key
current_cache.delete('restricted_collections')
app_client.get("/old-literature/1497201")
# Check that cache key has been correctly filled
assert current_cache.get('restricted_collections') == set(['HERMES Internal Notes'])
def test_all_collections_are_cached(app, app_client):
"""Test that collection info gets cached."""
# Remove collection cache key
current_cache.delete('restricted_collections')
app_client.get("/old-literature/1497201")
# Check that cache key has been correctly filled
assert current_cache.get('restricted_collections') == set(
['HERMES Internal Notes'])
def test_all_collections_are_cached(app, app_client):
"""Test that collection info gets cached."""
# Remove collection cache key
current_cache.delete('restricted_collections')
app_client.get("/literature/111")
# Check that cache key has been correctly filled
assert current_cache.get('restricted_collections') == \
set([u'Another Restricted Collection', u'Restricted Collection', u'Role only collection'])
def test_rt_queues_are_cached(mock_query_rt, inspire_app):
mock_query_rt.return_value = [
"35: Admin", "63: Admin-curator", "60: Admin-Dev"
]
current_cache.delete("rt_queues")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/queues")
assert current_cache.get("rt_queues") == orjson.loads(response.data)
def test_get_user_by_email_forces_to_refresh_only_once_per_call(
mock_query_rt, inspire_app):
mock_query_rt.return_value = [
"id: user/1",
"Name: user1",
"EmailAddress: [email protected]",
]
current_cache.delete("rt_users_with_emails")
with pytest.raises(NoUsersFound):
get_rt_user_by_email("*****@*****.**")
assert mock_query_rt.call_count == 2
def test_rt_users_are_cached(mock_get_all_of, inspire_app):
mock_get_all_of.return_value = [
{"id": "10309", "name": "atkinson"},
{"id": "1125438", "name": "bhecker"},
{"id": "460354", "name": "Catherine"},
]
current_cache.delete("rt_users")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/users")
assert current_cache.get("rt_users") == json.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, inspire_app):
mock_get_all_of.return_value = [
{"id": "35", "name": "Admin"},
{"id": "63", "name": "Admin-curator"},
{"id": "60", "name": "Admin-Dev"},
]
current_cache.delete("rt_queues")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/queues")
assert current_cache.get("rt_queues") == json.loads(response.data)
def test_rt_users_are_cached(mock_query_rt, inspire_app):
mock_query_rt.return_value = [
"10309: atkinson",
"1125438: bhecker",
"460354: Catherine",
]
current_cache.delete("rt_users")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/users")
assert current_cache.get("rt_users") == orjson.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, log_in_as_cataloger,
api_client):
mock_get_all_of.return_value = [{
"id": "35",
"name": "Admin"
}, {
"id": "63",
"name": "Admin-curator"
}, {
"id": "60",
"name": "Admin-Dev"
}]
current_cache.delete('rt_queues')
response = api_client.get('/editor/rt/queues')
assert current_cache.get('rt_queues') == json.loads(response.data)
def restricted_record(app):
restricted_collection = Collection(
name='Restricted Collection',
dbquery='_collections:"Restricted Collection"',
)
another_restricted_collection = Collection(
name='Another Restricted Collection',
dbquery='_collections:"Another Restricted Collection"',
)
db.session.add_all([restricted_collection, another_restricted_collection])
db.session.commit()
record = {
'$schema':
'http://localhost:5000/schemas/records/hep.json',
'_collections': [
'Literature',
'Restricted Collection',
'Another Restricted Collection',
],
'control_number':
222,
'document_type': [
'article',
],
'titles': [{
'title': 'restricted'
}],
}
record = _create_and_index_record(record)
yield record
collection = Collection.query.filter_by(name='Restricted Collection').one()
another_collection = Collection.query.filter_by(
name='Another Restricted Collection').one()
pid = PersistentIdentifier.get('lit', '222')
db.session.delete(collection)
db.session.delete(another_collection)
db.session.delete(pid)
record._delete(force=True)
current_app.extensions[
'invenio-db'].versioning_manager.transaction_cls.query.delete()
db.session.commit()
current_cache.delete('restricted_collections')
def test_get_user_by_email(mock_query_rt, inspire_app):
mock_query_rt.return_value = [
"id: user/1",
"Name: user1",
"EmailAddress: [email protected]",
"",
"--",
"",
"id: user/2",
"Name: user2",
"EmailAddress: [email protected]",
]
expected_user = {"Name": "user2", "EmailAddress": "*****@*****.**"}
current_cache.delete("rt_users_with_emails")
user = get_rt_user_by_email("*****@*****.**")
del user["id"]
assert user == expected_user
def test_rt_users_are_cached(mock_get_all_of, log_in_as_cataloger, api_client):
mock_get_all_of.return_value = [
{
"id": "10309",
"name": "atkinson"
},
{
"id": "1125438",
"name": "bhecker"
},
{
"id": "460354",
"name": "Catherine"
},
]
current_cache.delete('rt_users')
response = api_client.get('/editor/rt/users')
assert current_cache.get('rt_users') == json.loads(response.data)
def test_rt_users_are_cached(mock_get_all_of, log_in_as_cataloger, api_client):
mock_get_all_of.return_value = [
{
"id": "10309",
"name": "atkinson"
},
{
"id": "1125438",
"name": "bhecker"
},
{
"id": "460354",
"name": "Catherine"
},
]
current_cache.delete('rt_users')
response = api_client.get('/editor/rt/users')
assert current_cache.get('rt_users') == json.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, log_in_as_cataloger, api_client):
mock_get_all_of.return_value = [
{
"id": "35",
"name": "Admin"
},
{
"id": "63",
"name": "Admin-curator"
},
{
"id": "60",
"name": "Admin-Dev"
}
]
current_cache.delete('rt_queues')
response = api_client.get('/editor/rt/queues')
assert current_cache.get('rt_queues') == json.loads(response.data)
def delete(self, key):
"""Delete the key.
Args:
key (str): a key name.
Returns:
bool: if the key is deleted.
"""
return current_cache.delete(self._prefix(key))
def delete(self, key):
"""Delete the key.
Args:
key (str): a key name.
Returns:
bool: if the key is deleted.
"""
return current_cache.delete(self._prefix(key))
def test_get_from_cache(inspire_app):
def _test_generator():
return [1, 2, 3]
def _second_generator_which_should_not_run():
raise AssertionError
expected = [1, 2, 3]
current_cache.delete("test")
# Cache empty, so run generator and return data
result = _get_from_cache("test", _test_generator)
assert result == expected
assert expected == current_cache.get("test")
# Cache set so generator should not run
result = _get_from_cache("test", _second_generator_which_should_not_run)
assert result == expected
assert expected == current_cache.get("test")
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked:
PiwikExporter().run()
mocked.assert_not_called()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def openaire_direct_index(record_uuid):
"""Send record for direct indexing at OpenAIRE.
:param record_uuid: Record Metadata UUID.
:type record_uuid: str
"""
try:
record = ZenodoRecord.get_record(record_uuid)
# Bail out if not an OpenAIRE record.
if not (is_openaire_publication(record) or
is_openaire_dataset(record) or
is_openaire_software(record) or
is_openaire_other(record)):
return
data = openaire_json_v1.serialize(record.pid, record)
url = '{}/feedObject'.format(
current_app.config['OPENAIRE_API_URL'])
req = _openaire_request_factory()
res = req.post(url, data=data)
if not res.ok:
raise OpenAIRERequestError(res.text)
res_beta = None
if current_app.config['OPENAIRE_API_URL_BETA']:
url_beta = '{}/feedObject'.format(
current_app.config['OPENAIRE_API_URL_BETA'])
res_beta = req.post(url_beta, data=data)
if res_beta and not res_beta.ok:
raise OpenAIRERequestError(res_beta.text)
else:
recid = record.get('recid')
current_cache.delete('openaire_direct_index:{}'.format(recid))
except Exception as exc:
recid = record.get('recid')
current_cache.set('openaire_direct_index:{}'.format(recid),
datetime.now(), timeout=-1)
openaire_direct_index.retry(exc=exc)
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
with pytest.raises(PiwikExportRequestError):
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def clear_cache(self):
"""Clear the sitemap cache."""
for key in self.cache_keys:
current_cache.delete(key)
self.cache_keys = set()
def clear_cache(self):
"""Clear the sitemap cache."""
for key in self.cache_keys:
current_cache.delete(key)
self.cache_keys = set()
def del_test_cache_ui():
current_cache.delete('test_key')
return jsonify({
'code': 0,
'msg': 'success'
})
def clear_cache_pages(self):
"""Clear all cached pages."""
page_keys = current_cache.get(self.cached_pages_set_key) or set()
for page in page_keys:
current_cache.delete(page)
current_cache.delete(self.cached_pages_set_key)
