def test_piwik_exporter(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True, do_aggregate_events=False, do_update_record_statistics=False ) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:00:00' PiwikExporter().run() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark == u'2018-01-01T14:30:00'
def test_cache_ui(): with_update = request.args.get('update', '0') with_delete = request.args.get('delete', '0') with_update = True if int(with_update) else False with_delete = True if int(with_delete) else False if with_update: if with_delete: current_cache.delete('test_key') current_cache.set('test_key', ObjDict(**{ 'device_id': 12345678, 'cache_time': datetime.today() }), timeout=60) test_value = current_cache.get('test_key') if not test_value: cache_rtn = current_cache.set('test_key', ObjDict(**{ 'device_id': 12345678, 'cache_time': datetime.today() }), timeout=60) if not cache_rtn: return jsonify({ 'code': -1, 'msg': 'cache set error' }) test_value = current_cache.get('test_key') return jsonify({ 'code': 0, 'msg': 'success', 'data': { 'dev_id': test_value.device_id, 'store_time': test_value.cache_time.strftime('%Y-%m-%d %H:%M:%S') } })
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={ 'user_id': '1', 'country': 'CH' }, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) with pytest.raises(PiwikExportRequestError): PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={ 'user_id': '1', 'country': 'CH' }, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked: PiwikExporter().run() mocked.assert_not_called() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_get_from_cache_when_forced(inspire_app): def _test_generator(): return [1, 2, 3] def _second_generator_which_should_run(): return [4, 5, 6] expected = [1, 2, 3] current_cache.delete("test") # Cache empty, so run generator and return data result = _get_from_cache("test", _test_generator) assert result == expected assert expected == current_cache.get("test") # Forcing so generator should run again expected = [4, 5, 6] result = _get_from_cache("test", _second_generator_which_should_run, force_update=True) assert result == expected assert expected == current_cache.get("test") # Cache is set and not forcing so cached data should be returned result = _get_from_cache("test", _test_generator) assert result == expected assert expected == current_cache.get("test")
def get_widget_design_setting(repository_id, current_language, page_id=None): """Get widget design setting. @param repository_id: The repository identifier @param current_language: The current language @param page_id: The Page identifier. @return: """ def validate_response(): """Check the response data can compress with gzip. @return: True if the response data can compress with gzip """ is_valid = True accept_encoding = request.headers.get('Accept-Encoding', '') response = jsonify({}) if not config.WEKO_GRIDLAYOUT_IS_COMPRESS_WIDGET or \ response.direct_passthrough or \ 'gzip' not in accept_encoding.lower() or \ 'Content-Encoding' in response.headers: is_valid = False return is_valid def get_widget_response(_page_id): """Get widget setting response. :param _page_id: The Page identifier. @return: The widget setting response """ lang_code = current_language or get_default_language() if _page_id: from .services import WidgetDesignPageServices widget_setting_data = WidgetDesignPageServices\ .get_widget_design_setting(_page_id, lang_code) else: from .services import WidgetDesignServices widget_setting_data = WidgetDesignServices\ .get_widget_design_setting( repository_id, lang_code) return jsonify(widget_setting_data) if validate_response() and current_language: if page_id: key = (config.WEKO_GRIDLAYOUT_WIDGET_PAGE_CACHE_KEY + repository_id + "_" + page_id + "_" + current_language) else: key = (config.WEKO_GRIDLAYOUT_WIDGET_CACHE_KEY + repository_id + "_" + current_language) if current_cache.get(key) is None: data = compress_widget_response(get_widget_response(page_id)) current_cache.set(key, data) return data else: return current_cache.get(key) else: return get_widget_response(page_id)
def _load_cache_pages(self): """Get pages from cache instead of re-creating them.""" kwargs = dict( _external=True, _scheme=current_app.config.get('WEKO_SITEMAP_URL_SCHEME') ) kwargs['page'] = 0 page_keys = current_cache.get(self.cached_pages_set_key) or set() for page_number in page_keys: kwargs['page'] += 1 page = current_cache.get(page_number) if page: yield {'loc': url_for('flask_sitemap.page', **kwargs), 'lastmod': page['lastmod']}
def wrapper(*args, **kwargs): key = key_prefix for value in args: key += str(value) cache_fun = current_cache.cached( timeout=timeout, key_prefix=key, forced_update=is_update_cache, ) if current_cache.get(key) is None: data = cache_fun(f)(*args, **kwargs) current_cache.set(key, data) return data else: return current_cache.get(key)
def get_organization_relationships(uuid): """Get a source by any PID received as a argument, including UUID""" try: rtype = request.args.get('type') if request.args.get('type') else None cache = current_cache.get( "get_organization_relationships:{0}{1}".format(uuid, rtype)) or {} if "date" not in cache: cache["date"] = datetime.datetime.now() if datetime.datetime.now() - cache["date"] < datetime.timedelta( days=1) and "stats" in cache: result = cache["stats"] return jsonify(result) else: result = _get_organization_relationships(uuid, rtype) cache["date"] = datetime.datetime.now() cache["stats"] = result current_cache.set("get_organization_relationships:{0}{1}".format( uuid, rtype), cache, timeout=-1) return jsonify(result) except Exception as e: return jsonify({ 'ERROR': str(e), })
def load_restricted_collections(): restricted_collections = current_cache.get('restricted_collections') if restricted_collections: return restricted_collections else: restricted_collections = set( [ a.argument for a in ActionUsers.query.filter_by( action='view-restricted-collection').all() ] ) restricted_collections = restricted_collections | set( [ a.argument for a in ActionRoles.query.filter_by( action='view-restricted-collection').all() ] ) if restricted_collections: current_cache.set( 'restricted_collections', restricted_collections, timeout=current_app.config.get( 'INSPIRE_COLLECTIONS_RESTRICTED_CACHE_TIMEOUT', 120) ) return restricted_collections
def schedule_update_datacite_metadata(max_count): """Schedule the update of DataCite metadata.""" task_details = current_cache.get('update_datacite:task_details') if task_details is None or 'from_date' not in task_details or 'until_date' not in task_details: return pids = find_registered_doi_pids(task_details['from_date'], task_details['until_date'], current_app.config['ZENODO_LOCAL_DOI_PREFIXES']) pids_count = pids.count() task_details['left_pids'] = pids_count task_details['last_update'] = datetime.utcnow() current_cache.set('update_datacite:task_details', task_details) if pids_count == 0: if 'finish_date' not in task_details: task_details['finish_date'] = datetime.utcnow() current_cache.set('update_datacite:task_details', task_details) return scheduled_pids_count = max_count if max_count < pids_count else pids_count scheduled_pids = pids.limit(scheduled_pids_count) for pid in scheduled_pids: update_datacite_metadata.delay(pid.pid_value, pid.object_uuid, task_details['job_id'])
def decorated(*args, **kwargs): key = f'{prefix}:{key_func(f, *args, **kwargs)}' res = current_cache.get(key) if not res: res = f(*args, **kwargs) current_cache.set(key, res, timeout=timeout) return res
def page(self, page): """Override to get sitemap page from cache if it exists.""" sitemap_page = current_cache.get('sitemap_' + str(page).zfill(4)) if sitemap_page: return self.gzip_response(sitemap_page['page']) # return sitemap_page['page'] return flask_sitemap.render_page(urlset=[None])
def read_all(self, identity, fields, cache=True, **kwargs): """Search for records matching the querystring.""" cache_key = "-".join(fields) results = current_cache.get(cache_key) es_query = Q("match_all") if not results: results = self._read_many( identity, es_query, fields, **kwargs) if cache: # ES DSL Response is not pickable. # If saved in cache serialization wont work with to_dict() current_cache.set(cache_key, results.to_dict()) else: search = self.create_search( identity=identity, record_cls=self.record_cls, search_opts=self.config.search, permission_action='search', ).query(es_query) results = Response(search, results) return self.result_list(self, identity, results)
def query_cuor_by_uuid(cls, uuid): """""" try: cache = current_cache.get( "query_cuor_by_pid:{0}".format(uuid)) or {} if "date" not in cache: cache["date"] = datetime.now() if datetime.now() - cache["date"] < timedelta( days=1) and "org" in cache: return cache["org"] api_endpoint = current_app.config['CUOR_API_ENDPOINT'] session = requests.Session() url = api_endpoint + '/' + uuid response = session.get(url, verify=False) result = json.loads(response.text) cache["org"] = result cache["date"] = datetime.now() current_cache.set("query_cuor_by_pid:{0}".format(uuid), cache, timeout=-1) return result except Exception: print(traceback.format_exc()) return None
def query_cuor_by_pid(cls, pid): """Request an Organization by Persistent Identifier not the CUOR UUID """ try: cache = current_cache.get( "query_cuor_by_pid:{0}".format(pid)) or {} if "date" not in cache: cache["date"] = datetime.now() if datetime.now() - cache["date"] < timedelta( days=1) and "org" in cache: print("USING CACHE ORGANIZATION") if 'status' in cache["org"] and cache["org"]['status'] == '404': cache["org"] = None return None return cache["org"] api_endpoint = current_app.config['CUOR_API_ENDPOINT'] session = requests.Session() url = api_endpoint + '/pid?value=' + pid response = session.get(url, verify=False) result = json.loads(response.text) if 'status' in result and result['status'] == '404': return None cache["org"] = result cache["date"] = datetime.now() current_cache.set("query_cuor_by_pid:{0}".format(pid), cache, timeout=-1) return result except Exception: return None
def _get_from_cache(cache_key, generator, force_update=False): """Retrieve specified key from cache. If key is missing or is empty or if `force_update` is set to `True` update cache with generator and return cached data. Args: cache_key: name of the key where cache is stored generator: partial to generate data for specified key if key is empty or when `force_update` is `True`. force_update: when `True` updates cache key then return cached data. Returns: Whatever was under `cache_key` key in cache. """ data = current_cache.get(cache_key) if not data or force_update: new_data = generator() current_cache.set( cache_key, new_data, timeout=current_app.config.get("RT_USERS_CACHE_TIMEOUT", 86400), ) return new_data return data
def read_all(self, identity, fields, type, cache=True, **kwargs): """Search for records matching the querystring.""" cache_key = type + "_" + "-".join(fields) results = current_cache.get(cache_key) es_query = Q("match_all") if not results: # If not found, NoResultFound is raised (caught by the resource). vocabulary_type = VocabularyType.query.filter_by(id=type).one() vocab_id_filter = Q('term', type__id=vocabulary_type.id) results = self._read_many(identity, es_query, fields, extra_filter=vocab_id_filter, **kwargs) if cache: # ES DSL Response is not pickable. # If saved in cache serialization wont work with to_dict() current_cache.set(cache_key, results.to_dict()) else: search = self.create_search( identity=identity, record_cls=self.record_cls, search_opts=self.config.search, permission_action='search', ).query(es_query) results = Response(search, results) return self.result_list(self, identity, results)
def schedule_update_datacite_metadata(max_count): """Schedule the update of DataCite metadata.""" task_details = current_cache.get('update_datacite:task_details') if task_details is None or 'from_date' not in task_details or 'until_date' not in task_details: return doi_pids = find_registered_doi_pids(task_details['from_date'], task_details['until_date'], current_app.config['ZENODO_LOCAL_DOI_PREFIXES']) dois_count = doi_pids.count() task_details['left_pids'] = dois_count task_details['last_update'] = datetime.utcnow() current_cache.set('update_datacite:task_details', task_details, timeout=-1) if dois_count == 0: if 'finish_date' not in task_details: task_details['finish_date'] = datetime.utcnow() current_cache.set('update_datacite:task_details', task_details, timeout=-1) return scheduled_dois_count = max_count if max_count < dois_count else dois_count scheduled_dois_pids = doi_pids.limit(scheduled_dois_count) for doi_pid in scheduled_dois_pids: update_datacite_metadata.delay(doi_pid.pid_value, str(doi_pid.object_uuid), task_details['job_id'])
def run(self, start_date=None, end_date=None, update_bookmark=True): """Run export job.""" if start_date is None: bookmark = current_cache.get('piwik_export:bookmark') start_date = dateutil_parse(bookmark) if bookmark else None time_range = {} if start_date is not None: time_range['gte'] = start_date.replace(microsecond=0).isoformat() if end_date is not None: time_range['lte'] = end_date.replace(microsecond=0).isoformat() events = Search(using=current_search_client, index='events-stats-*').filter( 'range', timestamp=time_range).sort({ 'timestamp': { 'order': 'asc' } }).params(preserve_order=True).scan() url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get( 'url', None) token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \ .get('token_auth', None) chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\ .get('chunk_size', 0) for event_chunk in chunkify(events, chunk_size): query_strings = [] for event in event_chunk: query_string = self._build_query_string(event) query_strings.append(query_string) payload = {'requests': query_strings, 'token_auth': token_auth} res = requests.post(url, json=payload) # Failure: not 200 or not "success" content = res.json() if res.ok else None if res.status_code == 200 and content.get('status') == 'success': if content.get('invalid') != 0: msg = 'Invalid events in Piwik export request.' info = { 'begin_event_timestamp': event_chunk[0].timestamp, 'end_event_timestamp': event_chunk[-1].timestamp, 'invalid_events': content.get('invalid') } current_app.logger.warning(msg, extra=info) elif update_bookmark is True: current_cache.set('piwik_export:bookmark', event_chunk[-1].timestamp, timeout=-1) else: msg = 'Invalid events in Piwik export request.' info = { 'begin_event_timestamp': event_chunk[0].timestamp, 'end_event_timestamp': event_chunk[-1].timestamp, } raise PiwikExportRequestError(msg, export_info=info)
def set_cache_page(self, key, value): """Set the page into the cache.""" # Check if there is a set of keys, if not create one_or_none current_key_set = current_cache.get(self.cached_pages_set_key) or set() current_key_set.add(key) current_cache.set(self.cached_pages_set_key, current_key_set) current_cache.set( key, value, timeout=current_app.config['WEKO_SITEMAP_CACHE_TIMEOUT'])
def update_datacite(self): """.""" form = UpdateDataciteForm() cancel_or_new_task_form = FlaskForm() is_task_running = False time = 0 task_details = current_cache.get('update_datacite:task_details') if task_details: is_task_running = True if cancel_or_new_task_form.validate_on_submit(): current_cache.set('update_datacite:task_details', None) return redirect(url_for('updatedataciteview.update_datacite')) else: if form.validate_on_submit(): from_date = request.form['from_date'] until_date = request.form['until_date'] action = request.form['action'] if action == 'SubmitDates': if from_date > until_date: flash("Error: the 'From' date should precede the 'Until' date.") else: pids_count = find_registered_doi_pids(from_date, until_date, current_app.config['ZENODO_LOCAL_DOI_PREFIXES']).count() task_details = dict( total_pids=pids_count ) time = pids_count/current_app.config['DATACITE_UPDATING_RATE_PER_HOUR'] elif action == 'Confirm': pids_count = find_registered_doi_pids(from_date, until_date, current_app.config['ZENODO_LOCAL_DOI_PREFIXES']).count() task_details = dict( start_date=datetime.utcnow(), job_id=str(uuid.uuid4()), from_date=from_date, until_date=until_date, total_pids=pids_count, left_pids=pids_count, last_update=datetime.utcnow() ) current_cache.set('update_datacite:task_details', task_details, timeout=-1) return redirect(url_for('updatedataciteview.update_datacite')) elif action == 'Cancel': return redirect(url_for('updatedataciteview.update_datacite')) return self.render('zenodo_records/update_datacite.html', form=form, cancel_or_new_task_form=cancel_or_new_task_form, details=task_details, is_task_running=is_task_running, time=time)
def test_read_all_cache(lang_type, lang_data_many, service, identity, cache): """ read_all method should return all languages created in this scope. """ items = service.read_all(identity, fields=['id'], type='languages', cache=True) assert set(lang_data_many).issubset(set([i["id"] for i in items])) cached = current_cache.get("languages__id") assert cached is not None
def get_anonymization_salt(ts): """Get the anonymization salt based on the event timestamp's day.""" salt_key = 'stats:salt:{}'.format(ts.date().isoformat()) salt = current_cache.get(salt_key) if not salt: salt_bytes = os.urandom(32) salt = b64encode(salt_bytes).decode('utf-8') current_cache.set(salt_key, salt, timeout=60 * 60 * 24) return salt
def get(self, key): """Get the key value. Args: key (str): a key name. Returns: str: the value of the given key. """ return current_cache.get(self._prefix(key))
def test_get_from_cache(inspire_app): def _test_generator(): return [1, 2, 3] def _second_generator_which_should_not_run(): raise AssertionError expected = [1, 2, 3] current_cache.delete("test") # Cache empty, so run generator and return data result = _get_from_cache("test", _test_generator) assert result == expected assert expected == current_cache.get("test") # Cache set so generator should not run result = _get_from_cache("test", _second_generator_which_should_not_run) assert result == expected assert expected == current_cache.get("test")
def test_all_collections_are_cached(app, app_client): """Test that collection info gets cached.""" # Remove collection cache key current_cache.delete('restricted_collections') app_client.get("/old-literature/1497201") # Check that cache key has been correctly filled assert current_cache.get('restricted_collections') == set(['HERMES Internal Notes'])
def test_rt_queues_are_cached(mock_query_rt, inspire_app): mock_query_rt.return_value = [ "35: Admin", "63: Admin-curator", "60: Admin-Dev" ] current_cache.delete("rt_queues") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/queues") assert current_cache.get("rt_queues") == orjson.loads(response.data)
def test_all_collections_are_cached(app, app_client): """Test that collection info gets cached.""" # Remove collection cache key current_cache.delete('restricted_collections') app_client.get("/old-literature/1497201") # Check that cache key has been correctly filled assert current_cache.get('restricted_collections') == set( ['HERMES Internal Notes'])
def test_all_collections_are_cached(app, app_client): """Test that collection info gets cached.""" # Remove collection cache key current_cache.delete('restricted_collections') app_client.get("/literature/111") # Check that cache key has been correctly filled assert current_cache.get('restricted_collections') == \ set([u'Another Restricted Collection', u'Restricted Collection', u'Role only collection'])
def test_piwik_exporter_no_bookmark(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None with mock.patch('zenodo.modules.stats.exporters.requests.post') as mocked: PiwikExporter().run() mocked.assert_not_called() bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_read_all_no_cache( app, es_clear, service, cache, identity_simple, input_data ): # Create an items item_one = service.create(identity_simple, input_data) item_two = service.create(identity_simple, input_data) Record.index.refresh() _assert_read_all(service, identity_simple, cache=False) cached = current_cache.get("metadata.title") assert not cached
def test_rt_users_are_cached(mock_query_rt, inspire_app): mock_query_rt.return_value = [ "10309: atkinson", "1125438: bhecker", "460354: Catherine", ] current_cache.delete("rt_users") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/users") assert current_cache.get("rt_users") == orjson.loads(response.data)
def test_piwik_exporter_request_fail(app, db, es, locations, event_queues, full_record): records = create_stats_fixtures( metadata=full_record, n_records=1, n_versions=1, n_files=1, event_data={'user_id': '1', 'country': 'CH'}, # 4 event timestamps start_date=datetime(2018, 1, 1, 13), end_date=datetime(2018, 1, 1, 15), interval=timedelta(minutes=30), do_process_events=True) current_cache.delete('piwik_export:bookmark') bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None start_date = datetime(2018, 1, 1, 12) end_date = datetime(2018, 1, 1, 14) with pytest.raises(PiwikExportRequestError): PiwikExporter().run(start_date=start_date, end_date=end_date) bookmark = current_cache.get('piwik_export:bookmark') assert bookmark is None
def test_rt_queues_are_cached(mock_get_all_of, inspire_app): mock_get_all_of.return_value = [ {"id": "35", "name": "Admin"}, {"id": "63", "name": "Admin-curator"}, {"id": "60", "name": "Admin-Dev"}, ] current_cache.delete("rt_queues") user = create_user(role=Roles.cataloger.value) with inspire_app.test_client() as client: login_user_via_session(client, email=user.email) response = client.get("api/editor/rt/queues") assert current_cache.get("rt_queues") == json.loads(response.data)
def get_users(): """Returns list of all users as {id, name} dict :rtype: dict - with ``name (string)``, ``id (integer)`` properties """ queues = current_cache.get('rt_users') if queues: return queues else: queues = _get_all_of("user") if queues: current_cache.set('rt_users', queues, timeout=current_app.config.get( 'RT_USERS_CACHE_TIMEOUT', 86400)) return queues
def test_rt_users_are_cached(mock_get_all_of, log_in_as_cataloger, api_client): mock_get_all_of.return_value = [ { "id": "10309", "name": "atkinson" }, { "id": "1125438", "name": "bhecker" }, { "id": "460354", "name": "Catherine" }, ] current_cache.delete('rt_users') response = api_client.get('/editor/rt/users') assert current_cache.get('rt_users') == json.loads(response.data)
def test_rt_queues_are_cached(mock_get_all_of, log_in_as_cataloger, api_client): mock_get_all_of.return_value = [ { "id": "35", "name": "Admin" }, { "id": "63", "name": "Admin-curator" }, { "id": "60", "name": "Admin-Dev" } ] current_cache.delete('rt_queues') response = api_client.get('/editor/rt/queues') assert current_cache.get('rt_queues') == json.loads(response.data)
def update_datacite_metadata(doi, object_uuid, job_id): """Update DataCite metadata of a single PersistentIdentifier. :param doi: Value of doi PID, with pid_type='doi'. It could be a normal DOI or a concept DOI. :type doi: str :param object_uuid: Record Metadata UUID. :type object_uuid: str :param job_id: id of the job to which this task belongs. :type job_id: str """ task_details = current_cache.get('update_datacite:task_details') if task_details is None or job_id != task_details['job_id']: return record = Record.get_record(object_uuid) dcp = DataCiteProvider.get(doi) if dcp.pid.status != PIDStatus.REGISTERED: return doc = datacite_v41.serialize(dcp.pid, record) for validator in xsd41(): validator.assertValid(etree.XML(doc.encode('utf8'))) url = None if doi == record.get('doi'): url = current_app.config['ZENODO_RECORDS_UI_LINKS_FORMAT'].format( recid=str(record['recid'])) elif doi == record.get('conceptdoi'): url = current_app.config['ZENODO_RECORDS_UI_LINKS_FORMAT'].format( recid=str(record['conceptrecid'])) result = dcp.update(url, doc) if result is True: dcp.pid.updated = datetime.utcnow() db.session.commit()
def get_cache(key): """Get the sitemap cache.""" current_cache.get(key)
def _get_cached_or_404(page): data = current_cache.get('sitemap:' + str(page)) if data: return current_app.response_class(data, mimetype='text/xml') else: abort(404)
def run(self, start_date=None, end_date=None, update_bookmark=True): """Run export job.""" if start_date is None: bookmark = current_cache.get('piwik_export:bookmark') if bookmark is None: msg = 'Bookmark not found, and no start date specified.' current_app.logger.warning(msg) return start_date = dateutil_parse(bookmark) if bookmark else None time_range = {} if start_date is not None: time_range['gte'] = start_date.replace(microsecond=0).isoformat() if end_date is not None: time_range['lte'] = end_date.replace(microsecond=0).isoformat() events = Search( using=current_search_client, index='events-stats-*' ).filter( 'range', timestamp=time_range ).sort( {'timestamp': {'order': 'asc'}} ).params(preserve_order=True).scan() url = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'].get('url', None) token_auth = current_app.config['ZENODO_STATS_PIWIK_EXPORTER'] \ .get('token_auth', None) chunk_size = current_app.config['ZENODO_STATS_PIWIK_EXPORTER']\ .get('chunk_size', 0) for event_chunk in chunkify(events, chunk_size): query_strings = [] for event in event_chunk: if 'recid' not in event: continue try: query_string = self._build_query_string(event) query_strings.append(query_string) except PIDDeletedError: pass payload = { 'requests': query_strings, 'token_auth': token_auth } res = requests.post(url, json=payload) # Failure: not 200 or not "success" content = res.json() if res.ok else None if res.status_code == 200 and content.get('status') == 'success': if content.get('invalid') != 0: msg = 'Invalid events in Piwik export request.' info = { 'begin_event_timestamp': event_chunk[0].timestamp, 'end_event_timestamp': event_chunk[-1].timestamp, 'invalid_events': content.get('invalid') } current_app.logger.warning(msg, extra=info) elif update_bookmark is True: current_cache.set('piwik_export:bookmark', event_chunk[-1].timestamp, timeout=-1) else: msg = 'Invalid events in Piwik export request.' info = { 'begin_event_timestamp': event_chunk[0].timestamp, 'end_event_timestamp': event_chunk[-1].timestamp, } raise PiwikExportRequestError(msg, export_info=info)