def test_piwik_exporter(app, db, es, locations, event_queues, full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True,
do_aggregate_events=False,
do_update_record_statistics=False
)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:00:00'
PiwikExporter().run()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:30:00'
def test_piwik_exporter(app, db, es, locations, event_queues, full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True,
do_aggregate_events=False,
do_update_record_statistics=False
)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:00:00'
PiwikExporter().run()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark == u'2018-01-01T14:30:00'
def test_cache_ui():
with_update = request.args.get('update', '0')
with_delete = request.args.get('delete', '0')
with_update = True if int(with_update) else False
with_delete = True if int(with_delete) else False
if with_update:
if with_delete:
current_cache.delete('test_key')
current_cache.set('test_key', ObjDict(**{
'device_id': 12345678,
'cache_time': datetime.today()
}), timeout=60)
test_value = current_cache.get('test_key')
if not test_value:
cache_rtn = current_cache.set('test_key', ObjDict(**{
'device_id': 12345678,
'cache_time': datetime.today()
}), timeout=60)
if not cache_rtn:
return jsonify({
'code': -1,
'msg': 'cache set error'
})
test_value = current_cache.get('test_key')
return jsonify({
'code': 0,
'msg': 'success',
'data': {
'dev_id': test_value.device_id,
'store_time': test_value.cache_time.strftime('%Y-%m-%d %H:%M:%S')
}
})
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record,
n_records=1,
n_versions=1,
n_files=1,
event_data={
'user_id': '1',
'country': 'CH'
},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
with pytest.raises(PiwikExportRequestError):
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record,
n_records=1,
n_versions=1,
n_files=1,
event_data={
'user_id': '1',
'country': 'CH'
},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked:
PiwikExporter().run()
mocked.assert_not_called()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def test_get_from_cache_when_forced(inspire_app):
def _test_generator():
return [1, 2, 3]
def _second_generator_which_should_run():
return [4, 5, 6]
expected = [1, 2, 3]
current_cache.delete("test")
# Cache empty, so run generator and return data
result = _get_from_cache("test", _test_generator)
assert result == expected
assert expected == current_cache.get("test")
# Forcing so generator should run again
expected = [4, 5, 6]
result = _get_from_cache("test",
_second_generator_which_should_run,
force_update=True)
assert result == expected
assert expected == current_cache.get("test")
# Cache is set and not forcing so cached data should be returned
result = _get_from_cache("test", _test_generator)
assert result == expected
assert expected == current_cache.get("test")
def get_widget_design_setting(repository_id, current_language, page_id=None):
"""Get widget design setting.
@param repository_id: The repository identifier
@param current_language: The current language
@param page_id: The Page identifier.
@return:
"""
def validate_response():
"""Check the response data can compress with gzip.
@return: True if the response data can compress with gzip
"""
is_valid = True
accept_encoding = request.headers.get('Accept-Encoding', '')
response = jsonify({})
if not config.WEKO_GRIDLAYOUT_IS_COMPRESS_WIDGET or \
response.direct_passthrough or \
'gzip' not in accept_encoding.lower() or \
'Content-Encoding' in response.headers:
is_valid = False
return is_valid
def get_widget_response(_page_id):
"""Get widget setting response.
:param _page_id: The Page identifier.
@return: The widget setting response
"""
lang_code = current_language or get_default_language()
if _page_id:
from .services import WidgetDesignPageServices
widget_setting_data = WidgetDesignPageServices\
.get_widget_design_setting(_page_id, lang_code)
else:
from .services import WidgetDesignServices
widget_setting_data = WidgetDesignServices\
.get_widget_design_setting(
repository_id, lang_code)
return jsonify(widget_setting_data)
if validate_response() and current_language:
if page_id:
key = (config.WEKO_GRIDLAYOUT_WIDGET_PAGE_CACHE_KEY +
repository_id + "_" + page_id + "_" + current_language)
else:
key = (config.WEKO_GRIDLAYOUT_WIDGET_CACHE_KEY + repository_id +
"_" + current_language)
if current_cache.get(key) is None:
data = compress_widget_response(get_widget_response(page_id))
current_cache.set(key, data)
return data
else:
return current_cache.get(key)
else:
return get_widget_response(page_id)
def _load_cache_pages(self):
"""Get pages from cache instead of re-creating them."""
kwargs = dict(
_external=True,
_scheme=current_app.config.get('WEKO_SITEMAP_URL_SCHEME')
)
kwargs['page'] = 0
page_keys = current_cache.get(self.cached_pages_set_key) or set()
for page_number in page_keys:
kwargs['page'] += 1
page = current_cache.get(page_number)
if page:
yield {'loc': url_for('flask_sitemap.page', **kwargs),
'lastmod': page['lastmod']}
def wrapper(*args, **kwargs):
key = key_prefix
for value in args:
key += str(value)
cache_fun = current_cache.cached(
timeout=timeout,
key_prefix=key,
forced_update=is_update_cache,
)
if current_cache.get(key) is None:
data = cache_fun(f)(*args, **kwargs)
current_cache.set(key, data)
return data
else:
return current_cache.get(key)
def get_organization_relationships(uuid):
"""Get a source by any PID received as a argument, including UUID"""
try:
rtype = request.args.get('type') if request.args.get('type') else None
cache = current_cache.get(
"get_organization_relationships:{0}{1}".format(uuid, rtype)) or {}
if "date" not in cache:
cache["date"] = datetime.datetime.now()
if datetime.datetime.now() - cache["date"] < datetime.timedelta(
days=1) and "stats" in cache:
result = cache["stats"]
return jsonify(result)
else:
result = _get_organization_relationships(uuid, rtype)
cache["date"] = datetime.datetime.now()
cache["stats"] = result
current_cache.set("get_organization_relationships:{0}{1}".format(
uuid, rtype),
cache,
timeout=-1)
return jsonify(result)
except Exception as e:
return jsonify({
'ERROR': str(e),
})
def load_restricted_collections():
restricted_collections = current_cache.get('restricted_collections')
if restricted_collections:
return restricted_collections
else:
restricted_collections = set(
[
a.argument for a in ActionUsers.query.filter_by(
action='view-restricted-collection').all()
]
)
restricted_collections = restricted_collections | set(
[
a.argument for a in ActionRoles.query.filter_by(
action='view-restricted-collection').all()
]
)
if restricted_collections:
current_cache.set(
'restricted_collections',
restricted_collections,
timeout=current_app.config.get(
'INSPIRE_COLLECTIONS_RESTRICTED_CACHE_TIMEOUT', 120)
)
return restricted_collections
def schedule_update_datacite_metadata(max_count):
"""Schedule the update of DataCite metadata."""
task_details = current_cache.get('update_datacite:task_details')
if task_details is None or 'from_date' not in task_details or 'until_date' not in task_details:
return
pids = find_registered_doi_pids(task_details['from_date'],
task_details['until_date'],
current_app.config['ZENODO_LOCAL_DOI_PREFIXES'])
pids_count = pids.count()
task_details['left_pids'] = pids_count
task_details['last_update'] = datetime.utcnow()
current_cache.set('update_datacite:task_details', task_details)
if pids_count == 0:
if 'finish_date' not in task_details:
task_details['finish_date'] = datetime.utcnow()
current_cache.set('update_datacite:task_details', task_details)
return
scheduled_pids_count = max_count if max_count < pids_count else pids_count
scheduled_pids = pids.limit(scheduled_pids_count)
for pid in scheduled_pids:
update_datacite_metadata.delay(pid.pid_value, pid.object_uuid, task_details['job_id'])
def decorated(*args, **kwargs):
key = f'{prefix}:{key_func(f, *args, **kwargs)}'
res = current_cache.get(key)
if not res:
res = f(*args, **kwargs)
current_cache.set(key, res, timeout=timeout)
return res
def page(self, page):
"""Override to get sitemap page from cache if it exists."""
sitemap_page = current_cache.get('sitemap_' + str(page).zfill(4))
if sitemap_page:
return self.gzip_response(sitemap_page['page'])
# return sitemap_page['page']
return flask_sitemap.render_page(urlset=[None])
def read_all(self, identity, fields, cache=True, **kwargs):
"""Search for records matching the querystring."""
cache_key = "-".join(fields)
results = current_cache.get(cache_key)
es_query = Q("match_all")
if not results:
results = self._read_many(
identity, es_query, fields, **kwargs)
if cache:
# ES DSL Response is not pickable.
# If saved in cache serialization wont work with to_dict()
current_cache.set(cache_key, results.to_dict())
else:
search = self.create_search(
identity=identity,
record_cls=self.record_cls,
search_opts=self.config.search,
permission_action='search',
).query(es_query)
results = Response(search, results)
return self.result_list(self, identity, results)
def query_cuor_by_uuid(cls, uuid):
""""""
try:
cache = current_cache.get(
"query_cuor_by_pid:{0}".format(uuid)) or {}
if "date" not in cache:
cache["date"] = datetime.now()
if datetime.now() - cache["date"] < timedelta(
days=1) and "org" in cache:
return cache["org"]
api_endpoint = current_app.config['CUOR_API_ENDPOINT']
session = requests.Session()
url = api_endpoint + '/' + uuid
response = session.get(url, verify=False)
result = json.loads(response.text)
cache["org"] = result
cache["date"] = datetime.now()
current_cache.set("query_cuor_by_pid:{0}".format(uuid),
cache,
timeout=-1)
return result
except Exception:
print(traceback.format_exc())
return None
def load_restricted_collections():
restricted_collections = current_cache.get('restricted_collections')
if restricted_collections:
return restricted_collections
else:
restricted_collections = set(
[
a.argument for a in ActionUsers.query.filter_by(
action='view-restricted-collection').all()
]
)
restricted_collections = restricted_collections | set(
[
a.argument for a in ActionRoles.query.filter_by(
action='view-restricted-collection').all()
]
)
if restricted_collections:
current_cache.set(
'restricted_collections',
restricted_collections,
timeout=current_app.config.get(
'INSPIRE_COLLECTIONS_RESTRICTED_CACHE_TIMEOUT', 120)
)
return restricted_collections
def query_cuor_by_pid(cls, pid):
"""Request an Organization by Persistent Identifier
not the CUOR UUID
"""
try:
cache = current_cache.get(
"query_cuor_by_pid:{0}".format(pid)) or {}
if "date" not in cache:
cache["date"] = datetime.now()
if datetime.now() - cache["date"] < timedelta(
days=1) and "org" in cache:
print("USING CACHE ORGANIZATION")
if 'status' in cache["org"] and cache["org"]['status'] == '404':
cache["org"] = None
return None
return cache["org"]
api_endpoint = current_app.config['CUOR_API_ENDPOINT']
session = requests.Session()
url = api_endpoint + '/pid?value=' + pid
response = session.get(url, verify=False)
result = json.loads(response.text)
if 'status' in result and result['status'] == '404':
return None
cache["org"] = result
cache["date"] = datetime.now()
current_cache.set("query_cuor_by_pid:{0}".format(pid),
cache,
timeout=-1)
return result
except Exception:
return None
def _get_from_cache(cache_key, generator, force_update=False):
"""Retrieve specified key from cache.
If key is missing or is empty or if `force_update` is set to `True`
update cache with generator and return cached data.
Args:
cache_key: name of the key where cache is stored
generator: partial to generate data for specified key if key is empty
or when `force_update` is `True`.
force_update: when `True` updates cache key then return cached data.
Returns:
Whatever was under `cache_key` key in cache.
"""
data = current_cache.get(cache_key)
if not data or force_update:
new_data = generator()
current_cache.set(
cache_key,
new_data,
timeout=current_app.config.get("RT_USERS_CACHE_TIMEOUT", 86400),
)
return new_data
return data
def read_all(self, identity, fields, type, cache=True, **kwargs):
"""Search for records matching the querystring."""
cache_key = type + "_" + "-".join(fields)
results = current_cache.get(cache_key)
es_query = Q("match_all")
if not results:
# If not found, NoResultFound is raised (caught by the resource).
vocabulary_type = VocabularyType.query.filter_by(id=type).one()
vocab_id_filter = Q('term', type__id=vocabulary_type.id)
results = self._read_many(identity,
es_query,
fields,
extra_filter=vocab_id_filter,
**kwargs)
if cache:
# ES DSL Response is not pickable.
# If saved in cache serialization wont work with to_dict()
current_cache.set(cache_key, results.to_dict())
else:
search = self.create_search(
identity=identity,
record_cls=self.record_cls,
search_opts=self.config.search,
permission_action='search',
).query(es_query)
results = Response(search, results)
return self.result_list(self, identity, results)
def schedule_update_datacite_metadata(max_count):
"""Schedule the update of DataCite metadata."""
task_details = current_cache.get('update_datacite:task_details')
if task_details is None or 'from_date' not in task_details or 'until_date' not in task_details:
return
doi_pids = find_registered_doi_pids(task_details['from_date'],
task_details['until_date'],
current_app.config['ZENODO_LOCAL_DOI_PREFIXES'])
dois_count = doi_pids.count()
task_details['left_pids'] = dois_count
task_details['last_update'] = datetime.utcnow()
current_cache.set('update_datacite:task_details', task_details, timeout=-1)
if dois_count == 0:
if 'finish_date' not in task_details:
task_details['finish_date'] = datetime.utcnow()
current_cache.set('update_datacite:task_details', task_details, timeout=-1)
return
scheduled_dois_count = max_count if max_count < dois_count else dois_count
scheduled_dois_pids = doi_pids.limit(scheduled_dois_count)
for doi_pid in scheduled_dois_pids:
update_datacite_metadata.delay(doi_pid.pid_value,
str(doi_pid.object_uuid),
task_details['job_id'])
def run(self, start_date=None, end_date=None, update_bookmark=True):
"""Run export job."""
if start_date is None:
bookmark = current_cache.get('piwik_export:bookmark')
start_date = dateutil_parse(bookmark) if bookmark else None
time_range = {}
if start_date is not None:
time_range['gte'] = start_date.replace(microsecond=0).isoformat()
if end_date is not None:
time_range['lte'] = end_date.replace(microsecond=0).isoformat()
events = Search(using=current_search_client,
index='events-stats-*').filter(
'range', timestamp=time_range).sort({
'timestamp': {
'order': 'asc'
}
}).params(preserve_order=True).scan()
url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get(
'url', None)
token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \
.get('token_auth', None)
chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\
.get('chunk_size', 0)
for event_chunk in chunkify(events, chunk_size):
query_strings = []
for event in event_chunk:
query_string = self._build_query_string(event)
query_strings.append(query_string)
payload = {'requests': query_strings, 'token_auth': token_auth}
res = requests.post(url, json=payload)
# Failure: not 200 or not "success"
content = res.json() if res.ok else None
if res.status_code == 200 and content.get('status') == 'success':
if content.get('invalid') != 0:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
'invalid_events': content.get('invalid')
}
current_app.logger.warning(msg, extra=info)
elif update_bookmark is True:
current_cache.set('piwik_export:bookmark',
event_chunk[-1].timestamp,
timeout=-1)
else:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
}
raise PiwikExportRequestError(msg, export_info=info)
def set_cache_page(self, key, value):
"""Set the page into the cache."""
# Check if there is a set of keys, if not create one_or_none
current_key_set = current_cache.get(self.cached_pages_set_key) or set()
current_key_set.add(key)
current_cache.set(self.cached_pages_set_key, current_key_set)
current_cache.set(
key, value,
timeout=current_app.config['WEKO_SITEMAP_CACHE_TIMEOUT'])
def update_datacite(self):
"""."""
form = UpdateDataciteForm()
cancel_or_new_task_form = FlaskForm()
is_task_running = False
time = 0
task_details = current_cache.get('update_datacite:task_details')
if task_details:
is_task_running = True
if cancel_or_new_task_form.validate_on_submit():
current_cache.set('update_datacite:task_details', None)
return redirect(url_for('updatedataciteview.update_datacite'))
else:
if form.validate_on_submit():
from_date = request.form['from_date']
until_date = request.form['until_date']
action = request.form['action']
if action == 'SubmitDates':
if from_date > until_date:
flash("Error: the 'From' date should precede the 'Until' date.")
else:
pids_count = find_registered_doi_pids(from_date,
until_date,
current_app.config['ZENODO_LOCAL_DOI_PREFIXES']).count()
task_details = dict(
total_pids=pids_count
)
time = pids_count/current_app.config['DATACITE_UPDATING_RATE_PER_HOUR']
elif action == 'Confirm':
pids_count = find_registered_doi_pids(from_date,
until_date,
current_app.config['ZENODO_LOCAL_DOI_PREFIXES']).count()
task_details = dict(
start_date=datetime.utcnow(),
job_id=str(uuid.uuid4()),
from_date=from_date,
until_date=until_date,
total_pids=pids_count,
left_pids=pids_count,
last_update=datetime.utcnow()
)
current_cache.set('update_datacite:task_details',
task_details, timeout=-1)
return redirect(url_for('updatedataciteview.update_datacite'))
elif action == 'Cancel':
return redirect(url_for('updatedataciteview.update_datacite'))
return self.render('zenodo_records/update_datacite.html',
form=form,
cancel_or_new_task_form=cancel_or_new_task_form,
details=task_details,
is_task_running=is_task_running, time=time)
def test_read_all_cache(lang_type, lang_data_many, service, identity, cache):
""" read_all method should return all languages created in this scope. """
items = service.read_all(identity,
fields=['id'],
type='languages',
cache=True)
assert set(lang_data_many).issubset(set([i["id"] for i in items]))
cached = current_cache.get("languages__id")
assert cached is not None
def get_anonymization_salt(ts):
"""Get the anonymization salt based on the event timestamp's day."""
salt_key = 'stats:salt:{}'.format(ts.date().isoformat())
salt = current_cache.get(salt_key)
if not salt:
salt_bytes = os.urandom(32)
salt = b64encode(salt_bytes).decode('utf-8')
current_cache.set(salt_key, salt, timeout=60 * 60 * 24)
return salt
def get(self, key):
"""Get the key value.
Args:
key (str): a key name.
Returns:
str: the value of the given key.
"""
return current_cache.get(self._prefix(key))
def test_get_from_cache(inspire_app):
def _test_generator():
return [1, 2, 3]
def _second_generator_which_should_not_run():
raise AssertionError
expected = [1, 2, 3]
current_cache.delete("test")
# Cache empty, so run generator and return data
result = _get_from_cache("test", _test_generator)
assert result == expected
assert expected == current_cache.get("test")
# Cache set so generator should not run
result = _get_from_cache("test", _second_generator_which_should_not_run)
assert result == expected
assert expected == current_cache.get("test")
def test_all_collections_are_cached(app, app_client):
"""Test that collection info gets cached."""
# Remove collection cache key
current_cache.delete('restricted_collections')
app_client.get("/old-literature/1497201")
# Check that cache key has been correctly filled
assert current_cache.get('restricted_collections') == set(['HERMES Internal Notes'])
def get(self, key):
"""Get the key value.
Args:
key (str): a key name.
Returns:
str: the value of the given key.
"""
return current_cache.get(self._prefix(key))
def test_rt_queues_are_cached(mock_query_rt, inspire_app):
mock_query_rt.return_value = [
"35: Admin", "63: Admin-curator", "60: Admin-Dev"
]
current_cache.delete("rt_queues")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/queues")
assert current_cache.get("rt_queues") == orjson.loads(response.data)
def test_all_collections_are_cached(app, app_client):
"""Test that collection info gets cached."""
# Remove collection cache key
current_cache.delete('restricted_collections')
app_client.get("/old-literature/1497201")
# Check that cache key has been correctly filled
assert current_cache.get('restricted_collections') == set(
['HERMES Internal Notes'])
def test_all_collections_are_cached(app, app_client):
"""Test that collection info gets cached."""
# Remove collection cache key
current_cache.delete('restricted_collections')
app_client.get("/literature/111")
# Check that cache key has been correctly filled
assert current_cache.get('restricted_collections') == \
set([u'Another Restricted Collection', u'Restricted Collection', u'Role only collection'])
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked:
PiwikExporter().run()
mocked.assert_not_called()
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def test_read_all_no_cache(
app, es_clear, service, cache, identity_simple, input_data
):
# Create an items
item_one = service.create(identity_simple, input_data)
item_two = service.create(identity_simple, input_data)
Record.index.refresh()
_assert_read_all(service, identity_simple, cache=False)
cached = current_cache.get("metadata.title")
assert not cached
def test_rt_users_are_cached(mock_query_rt, inspire_app):
mock_query_rt.return_value = [
"10309: atkinson",
"1125438: bhecker",
"460354: Catherine",
]
current_cache.delete("rt_users")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/users")
assert current_cache.get("rt_users") == orjson.loads(response.data)
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues,
full_record):
records = create_stats_fixtures(
metadata=full_record, n_records=1, n_versions=1, n_files=1,
event_data={'user_id': '1', 'country': 'CH'},
# 4 event timestamps
start_date=datetime(2018, 1, 1, 13),
end_date=datetime(2018, 1, 1, 15),
interval=timedelta(minutes=30),
do_process_events=True)
current_cache.delete('piwik_export:bookmark')
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
start_date = datetime(2018, 1, 1, 12)
end_date = datetime(2018, 1, 1, 14)
with pytest.raises(PiwikExportRequestError):
PiwikExporter().run(start_date=start_date, end_date=end_date)
bookmark = current_cache.get('piwik_export:bookmark')
assert bookmark is None
def test_rt_queues_are_cached(mock_get_all_of, inspire_app):
mock_get_all_of.return_value = [
{"id": "35", "name": "Admin"},
{"id": "63", "name": "Admin-curator"},
{"id": "60", "name": "Admin-Dev"},
]
current_cache.delete("rt_queues")
user = create_user(role=Roles.cataloger.value)
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.get("api/editor/rt/queues")
assert current_cache.get("rt_queues") == json.loads(response.data)
def get_users():
"""Returns list of all users as {id, name} dict
:rtype: dict - with ``name (string)``, ``id (integer)`` properties
"""
queues = current_cache.get('rt_users')
if queues:
return queues
else:
queues = _get_all_of("user")
if queues:
current_cache.set('rt_users', queues, timeout=current_app.config.get(
'RT_USERS_CACHE_TIMEOUT', 86400))
return queues
def test_rt_users_are_cached(mock_get_all_of, log_in_as_cataloger, api_client):
mock_get_all_of.return_value = [
{
"id": "10309",
"name": "atkinson"
},
{
"id": "1125438",
"name": "bhecker"
},
{
"id": "460354",
"name": "Catherine"
},
]
current_cache.delete('rt_users')
response = api_client.get('/editor/rt/users')
assert current_cache.get('rt_users') == json.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, log_in_as_cataloger, api_client):
mock_get_all_of.return_value = [
{
"id": "35",
"name": "Admin"
},
{
"id": "63",
"name": "Admin-curator"
},
{
"id": "60",
"name": "Admin-Dev"
}
]
current_cache.delete('rt_queues')
response = api_client.get('/editor/rt/queues')
assert current_cache.get('rt_queues') == json.loads(response.data)
def update_datacite_metadata(doi, object_uuid, job_id):
"""Update DataCite metadata of a single PersistentIdentifier.
:param doi: Value of doi PID, with pid_type='doi'. It could be a normal
DOI or a concept DOI.
:type doi: str
:param object_uuid: Record Metadata UUID.
:type object_uuid: str
:param job_id: id of the job to which this task belongs.
:type job_id: str
"""
task_details = current_cache.get('update_datacite:task_details')
if task_details is None or job_id != task_details['job_id']:
return
record = Record.get_record(object_uuid)
dcp = DataCiteProvider.get(doi)
if dcp.pid.status != PIDStatus.REGISTERED:
return
doc = datacite_v41.serialize(dcp.pid, record)
for validator in xsd41():
validator.assertValid(etree.XML(doc.encode('utf8')))
url = None
if doi == record.get('doi'):
url = current_app.config['ZENODO_RECORDS_UI_LINKS_FORMAT'].format(
recid=str(record['recid']))
elif doi == record.get('conceptdoi'):
url = current_app.config['ZENODO_RECORDS_UI_LINKS_FORMAT'].format(
recid=str(record['conceptrecid']))
result = dcp.update(url, doc)
if result is True:
dcp.pid.updated = datetime.utcnow()
db.session.commit()
def get_cache(key):
"""Get the sitemap cache."""
current_cache.get(key)
def _get_cached_or_404(page):
data = current_cache.get('sitemap:' + str(page))
if data:
return current_app.response_class(data, mimetype='text/xml')
else:
abort(404)
def run(self, start_date=None, end_date=None, update_bookmark=True):
"""Run export job."""
if start_date is None:
bookmark = current_cache.get('piwik_export:bookmark')
if bookmark is None:
msg = 'Bookmark not found, and no start date specified.'
current_app.logger.warning(msg)
return
start_date = dateutil_parse(bookmark) if bookmark else None
time_range = {}
if start_date is not None:
time_range['gte'] = start_date.replace(microsecond=0).isoformat()
if end_date is not None:
time_range['lte'] = end_date.replace(microsecond=0).isoformat()
events = Search(
using=current_search_client,
index='events-stats-*'
).filter(
'range', timestamp=time_range
).sort(
{'timestamp': {'order': 'asc'}}
).params(preserve_order=True).scan()
url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get('url', None)
token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \
.get('token_auth', None)
chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\
.get('chunk_size', 0)
for event_chunk in chunkify(events, chunk_size):
query_strings = []
for event in event_chunk:
if 'recid' not in event:
continue
try:
query_string = self._build_query_string(event)
query_strings.append(query_string)
except PIDDeletedError:
pass
payload = {
'requests': query_strings,
'token_auth': token_auth
}
res = requests.post(url, json=payload)
# Failure: not 200 or not "success"
content = res.json() if res.ok else None
if res.status_code == 200 and content.get('status') == 'success':
if content.get('invalid') != 0:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
'invalid_events': content.get('invalid')
}
current_app.logger.warning(msg, extra=info)
elif update_bookmark is True:
current_cache.set('piwik_export:bookmark',
event_chunk[-1].timestamp,
timeout=-1)
else:
msg = 'Invalid events in Piwik export request.'
info = {
'begin_event_timestamp': event_chunk[0].timestamp,
'end_event_timestamp': event_chunk[-1].timestamp,
}
raise PiwikExportRequestError(msg, export_info=info)
