def search_articles(search_query): # get the values for the 2 other bits of search info: the page number and the page size page = request.values.get("page", 1) psize = request.values.get("pageSize", 10) sort = request.values.get("sort") # check the page is an integer try: page = int(page) except: raise Api400Error("Page number was not an integer") # check the page size is an integer try: psize = int(psize) except: raise Api400Error("Page size was not an integer") results = None try: results = DiscoveryApi.search('article', None, search_query, page, psize, sort) except DiscoveryException as e: raise Api400Error(e.message) return jsonify_models(results)
def search_articles(search_query): # get the values for the 2 other bits of search info: the page number and the page size page = request.values.get("page", 1) psize = request.values.get("pageSize", 10) sort = request.values.get("sort") # check the page is an integer try: page = int(page) except: raise Api400Error("Page number was not an integer") # check the page size is an integer try: psize = int(psize) except: raise Api400Error("Page size was not an integer") results = None try: results = DiscoveryApi.search('article', None, search_query, page, psize, sort) except DiscoveryException as e: raise Api400Error(str(e)) return jsonify_models(results)
def test_06_deep_paging_limit(self): # populate the index with some journals jids = [] for i in range(10): j = models.Journal() j.set_in_doaj(True) bj = j.bibjson() bj.title = "Test Journal {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_url("http://homepage.com/{x}".format(x=i), "homepage") j.save() jids.append((j.id, j.last_updated)) self.app_test.config["DISCOVERY_MAX_RECORDS_SIZE"] = 5 # block until all the records are saved for jid, lu in jids: models.Journal.block(jid, lu, sleep=0.05) # now run some queries with self.app_test.test_request_context(): # check that the first page still works res = DiscoveryApi.search("journal", None, "*", 1, 5) assert res.data.get("total") == 10 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 # but that the second page fails with self.assertRaises(DiscoveryException): try: res = DiscoveryApi.search("journal", None, "*", 2, 5) except DiscoveryException as e: data_dump_url = url_for("doaj.public_data_dump") oai_article_url = url_for("oaipmh.oaipmh", specified="article") oai_journal_url = url_for("oaipmh.oaipmh") assert data_dump_url in e.message assert oai_article_url in e.message assert oai_journal_url in e.message raise
def test_06_deep_paging_limit(self): # populate the index with some journals jids = [] for i in range(10): j = models.Journal() j.set_in_doaj(True) bj = j.bibjson() bj.title = "Test Journal {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_url("http://homepage.com/{x}".format(x=i), "homepage") j.save() jids.append((j.id, j.last_updated)) self.app_test.config["DISCOVERY_MAX_RECORDS_SIZE"] = 5 # block until all the records are saved for jid, lu in jids: models.Journal.block(jid, lu, sleep=0.05) # now run some queries with self.app_test.test_request_context(): # check that the first page still works res = DiscoveryApi.search("journal", None, "*", 1, 5) assert res.data.get("total") == 10 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 # but that the second page fails with self.assertRaises(DiscoveryException): try: res = DiscoveryApi.search("journal", None, "*", 2, 5) except DiscoveryException as e: data_dump_url = url_for("doaj.public_data_dump") oai_article_url = url_for("oaipmh.oaipmh", specified="article") oai_journal_url = url_for("oaipmh.oaipmh") assert data_dump_url in e.message assert oai_article_url in e.message assert oai_journal_url in e.message raise
def run(self): """ Execute the task as specified by the background_job :return: """ job = self.background_job params = job.params clean = self.get_param(params, 'clean') prune = self.get_param(params, 'prune') types = self.get_param(params, 'types') tmpStore = StoreFactory.tmp() mainStore = StoreFactory.get("public_data_dump") container = app.config.get("STORE_PUBLIC_DATA_DUMP_CONTAINER") if clean: mainStore.delete_container(container) # create dir with today's date day_at_start = dates.today() # Do the search and save it page_size = app.config.get("DISCOVERY_BULK_PAGE_SIZE", 1000) records_per_file = app.config.get('DISCOVERY_RECORDS_PER_FILE', 100000) if types == 'all': types = ['article', 'journal'] else: types = [types] urls = {"article" : None, "journal" : None} sizes = {"article" : None, "journal" : None} # Scroll for article and/or journal for typ in types: job.add_audit_message(dates.now() + u": Starting export of " + typ) out_dir = tmpStore.path(container, "doaj_" + typ + "_data_" + day_at_start, create_container=True, must_exist=False) out_name = os.path.basename(out_dir) zipped_name = out_name + ".tar.gz" zip_dir = os.path.dirname(out_dir) zipped_path = os.path.join(zip_dir, zipped_name) tarball = tarfile.open(zipped_path, "w:gz") file_num = 1 out_file, path, filename = self._start_new_file(tmpStore, container, typ, day_at_start, file_num) first_in_file = True count = 0 for result in DiscoveryApi.scroll(typ, None, None, page_size, scan=True): if not first_in_file: out_file.write(",\n") else: first_in_file = False out_file.write(json.dumps(result)) count += 1 if count >= records_per_file: file_num += 1 self._finish_file(tmpStore, container, filename, path, out_file, tarball) out_file, path, filename = self._start_new_file(tmpStore, container, typ, day_at_start, file_num) first_in_file = True count = 0 if count > 0: self._finish_file(tmpStore, container, filename, path, out_file, tarball) tarball.close() # Copy the source directory to main store try: filesize = self._copy_on_complete(mainStore, tmpStore, container, zipped_path) except Exception as e: tmpStore.delete_container(container) raise BackgroundException("Error copying {0} data on complete {1}\n".format(typ, e.message)) store_url = mainStore.url(container, zipped_name) urls[typ] = store_url sizes[typ] = filesize if prune: self._prune_container(mainStore, container, day_at_start, types) self.background_job.add_audit_message(u"Removing temp store container {x}".format(x=container)) tmpStore.delete_container(container) # finally update the cache cache.Cache.cache_public_data_dump(urls["article"], sizes["article"], urls["journal"], sizes["journal"]) job.add_audit_message(dates.now() + u": done")
def run(self): """ Execute the task as specified by the background_job :return: """ job = self.background_job params = job.params clean = self.get_param(params, 'clean') prune = self.get_param(params, 'prune') types = self.get_param(params, 'types') tmpStore = StoreFactory.tmp() mainStore = StoreFactory.get("public_data_dump") container = app.config.get("STORE_PUBLIC_DATA_DUMP_CONTAINER") if clean: mainStore.delete_container(container) job.add_audit_message("Deleted existing data dump files") job.save() # create dir with today's date day_at_start = dates.today() # Do the search and save it page_size = app.config.get("DISCOVERY_BULK_PAGE_SIZE", 1000) records_per_file = app.config.get('DISCOVERY_RECORDS_PER_FILE', 100000) if types == 'all': types = ['article', 'journal'] else: types = [types] urls = {"article": None, "journal": None} sizes = {"article": None, "journal": None} # Scroll for article and/or journal for typ in types: job.add_audit_message(dates.now() + ": Starting export of " + typ) job.save() out_dir = tmpStore.path(container, "doaj_" + typ + "_data_" + day_at_start, create_container=True, must_exist=False) out_name = os.path.basename(out_dir) zipped_name = out_name + ".tar.gz" zip_dir = os.path.dirname(out_dir) zipped_path = os.path.join(zip_dir, zipped_name) tarball = tarfile.open(zipped_path, "w:gz") file_num = 1 out_file, path, filename = self._start_new_file( tmpStore, container, typ, day_at_start, file_num) first_in_file = True count = 0 for result in DiscoveryApi.scroll(typ, None, None, page_size, scan=True): if not first_in_file: out_file.write(",\n") else: first_in_file = False out_file.write(json.dumps(result)) count += 1 if count >= records_per_file: file_num += 1 self._finish_file(tmpStore, container, filename, path, out_file, tarball) job.save() out_file, path, filename = self._start_new_file( tmpStore, container, typ, day_at_start, file_num) first_in_file = True count = 0 if count > 0: self._finish_file(tmpStore, container, filename, path, out_file, tarball) job.save() tarball.close() # Copy the source directory to main store try: filesize = self._copy_on_complete(mainStore, tmpStore, container, zipped_path) job.save() except Exception as e: tmpStore.delete_container(container) raise BackgroundException( "Error copying {0} data on complete {1}\n".format( typ, str(e))) store_url = mainStore.url(container, zipped_name) urls[typ] = store_url sizes[typ] = filesize if prune: self._prune_container(mainStore, container, day_at_start, types) job.save() self.background_job.add_audit_message( "Removing temp store container {x}".format(x=container)) tmpStore.delete_container(container) # finally update the cache cache.Cache.cache_public_data_dump(urls["article"], sizes["article"], urls["journal"], sizes["journal"]) job.add_audit_message(dates.now() + ": done")
def test_04_paging_for_link_headers(self): # calc_pagination takes total, page_size, requested_page # and returns page_count, previous_page, next_page, last_page # request 1 of 1 pages assert DiscoveryApi._calc_pagination(0, 10, 1) == (1, None, None, 1) # 0 results still means page 1 assert DiscoveryApi._calc_pagination(1, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(2, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(3, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(9, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(10, 10, 1) == (1, None, None, 1) # request 1st of 2 pages assert DiscoveryApi._calc_pagination(11, 10, 1) == (2, None, 2, 2) assert DiscoveryApi._calc_pagination(12, 10, 1) == (2, None, 2, 2) assert DiscoveryApi._calc_pagination(19, 10, 1) == (2, None, 2, 2) assert DiscoveryApi._calc_pagination(20, 10, 1) == (2, None, 2, 2) # request 2nd of 2 pages assert DiscoveryApi._calc_pagination(11, 10, 2) == (2, 1, None, 2) assert DiscoveryApi._calc_pagination(12, 10, 2) == (2, 1, None, 2) assert DiscoveryApi._calc_pagination(19, 10, 2) == (2, 1, None, 2) assert DiscoveryApi._calc_pagination(20, 10, 2) == (2, 1, None, 2) # various requests for 10s of 1000s of results assert DiscoveryApi._calc_pagination(9900 , 100, 1) == (99, None, 2, 99) assert DiscoveryApi._calc_pagination(9900 , 100, 99) == (99, 98, None, 99) assert DiscoveryApi._calc_pagination(9901 , 100, 1) == (100, None, 2, 100) assert DiscoveryApi._calc_pagination(9902 , 100, 1) == (100, None, 2, 100) assert DiscoveryApi._calc_pagination(10000, 100, 1) == (100, None, 2, 100) assert DiscoveryApi._calc_pagination(10000, 100, 2) == (100, 1, 3, 100) assert DiscoveryApi._calc_pagination(10000, 100, 98) == (100, 97, 99, 100) assert DiscoveryApi._calc_pagination(10000, 100, 99) == (100, 98, 100, 100) assert DiscoveryApi._calc_pagination(10000, 100, 100) == (100, 99, None, 100)
def test_03_applications(self): # create an account that will own the suggestions acc = models.Account() acc.set_id("owner") acc.save() # populate the index with some suggestions owned by this owner for i in range(5): a = models.Suggestion() a.set_owner("owner") bj = a.bibjson() bj.title = "Test Suggestion {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_url("http://homepage.com/{x}".format(x=i), "homepage") a.save() # make sure the last updated dates are suitably different time.sleep(1) # populte the index with some which are not owned by this owner for i in range(5): a = models.Suggestion() a.set_owner("stranger") bj = a.bibjson() bj.title = "Test Suggestion {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) a.save() # make sure the last updated dates are suitably different time.sleep(1) time.sleep(1) # now run some queries with self._make_and_push_test_context(acc=acc): # 1. a general query that should hit everything res = DiscoveryApi.search("application", acc, "Test", 1, 2) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 2 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 2 assert res.data.get("query") == "Test" # 2. a specific field query that should hit just one res = DiscoveryApi.search("application", acc, "title:\"Test Suggestion 2\"", 1, 5) assert res.data.get("total") == 1 assert len(res.data.get("results")) == 1 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 assert res.data.get("query") == "title:\"Test Suggestion 2\"" # 3.paging out of range of results res = DiscoveryApi.search("application", acc, "Test", 2, 10) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 0 assert res.data.get("page") == 2 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 4. paging outside the allowed bounds (lower) res = DiscoveryApi.search("application", acc, "Test", 0, 0) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 5. page size above upper limit res = DiscoveryApi.search("application", acc, "Test", 1, 100000) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 100 assert res.data.get("query") == "Test" # 6. Failed attempt at wildcard search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "Te*t", 1, 10) # 7. Failed attempt at fuzzy search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "title:Test~0.8", 1, 10) # 8. sort on a specific field, expect a default to "asc" res = DiscoveryApi.search("application", acc, "Test", 1, 10, "created_date") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") < res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date" # 9. sort on a specific field in a specified direction res = DiscoveryApi.search("application", acc, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") > res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date:desc" # 10. Malformed sort direction with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "Test", 1, 10, "created_date:whatever") # 11. non-existant sort field with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "Test", 1, 10, "some.missing.field:asc") # 12. with a forward slash, with and without escaping (note that we have to escape the : as it has meaning for lucene) res = DiscoveryApi.search("application", acc, '"http\://homepage.com/1"', 1, 10) assert res.data.get("total") == 1 res = DiscoveryApi.search("application", acc, '"http\:\/\/homepage.com\/1"', 1, 10) assert res.data.get("total") == 1 # 13. A search with an account that isn't either of the ones in the dataset other = models.Account() other.set_id("other") with self._make_and_push_test_context(acc=other): res = DiscoveryApi.search("application", other, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 0
def test_01_journals(self): # populate the index with some journals saved_journals = [] for i in range(5): j = models.Journal() j.set_in_doaj(True) bj = j.bibjson() bj.title = "Test Journal {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_url("http://homepage.com/{x}".format(x=i), "homepage") j.save() saved_journals.append((j.id, j.last_updated)) # make sure the last updated dates are suitably different time.sleep(1) # add one that's not in DOAJ, which shouldn't turn up in our results j = models.Journal() j.set_in_doaj(False) bj = j.bibjson() bj.title = "Test Journal {x}".format(x=6) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=6)) bj.publisher = "Test Publisher {x}".format(x=6) bj.add_url("http://homepage.com/{x}".format(x=6), "homepage") j.save() saved_journals.append((j.id, j.last_updated)) models.Journal.blockall(saved_journals) # now run some queries with self.app_test.test_request_context(): # 1. a general query that should hit everything (except number 6) res = DiscoveryApi.search("journal", None, "Test", 1, 2) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 2 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 2 assert res.data.get("query") == "Test" # 2. a specific field query that should hit just one res = DiscoveryApi.search("journal", None, "title:\"Test Journal 2\"", 1, 5) assert res.data.get("total") == 1 assert len(res.data.get("results")) == 1 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 assert res.data.get("query") == "title:\"Test Journal 2\"" # 3.paging out of range of results res = DiscoveryApi.search("journal", None, "Test", 2, 10) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 0 assert res.data.get("page") == 2 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 4. paging outside the allowed bounds (lower) res = DiscoveryApi.search("journal", None, "Test", 0, 0) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 5. page size above upper limit res = DiscoveryApi.search("journal", None, "Test", 1, 100000) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 100 assert res.data.get("query") == "Test" # 6. Failed attempt at wildcard search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "Te*t", 1, 10) # 7. Failed attempt at fuzzy search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "title:Test~0.8", 1, 10) # 8. sort on a specific field, expect a default to "asc" res = DiscoveryApi.search("journal", None, "Test", 1, 10, "created_date") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") < res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date" # 9. sort on a specific field in a specified direction res = DiscoveryApi.search("journal", None, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") > res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date:desc" # 10. Malformed sort direction with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "Test", 1, 10, "created_date:whatever") # 11. non-existant sort field with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "Test", 1, 10, "some.missing.field:asc") # 12. with a forward slash, with and without escaping (note that we have to escape the : as it has meaning for lucene) res = DiscoveryApi.search("journal", None, '"http\://homepage.com/1"', 1, 10) assert res.data.get("total") == 1 res = DiscoveryApi.search("journal", None, '"http\:\/\/homepage.com\/1"', 1, 10) assert res.data.get("total") == 1
# Handle wayward paths by raising an API404Error @blueprint.route("/<path:invalid_path>", methods=["POST", "GET", "PUT", "DELETE", "PATCH", "HEAD"]) # leaving out methods should mean all, but tests haven't shown that behaviour. def missing_resource(invalid_path): docs_url = app.config.get("BASE_URL", "") + url_for('.docs') spec_url = app.config.get("BASE_URL", "") + url_for('.api_spec') raise Api404Error("No endpoint at {0}. See {1} for valid paths or read the documentation at {2}.".format(invalid_path, spec_url, docs_url)) @blueprint.route('/docs') def docs(): return render_template('api/v1/api_docs.html') @swag(swag_summary='Search your applications <span class="red">[Authenticated, not public]</span>', swag_spec=DiscoveryApi.get_application_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes. @blueprint.route("/search/applications/<path:search_query>") @api_key_required def search_applications(search_query): # get the values for the 2 other bits of search info: the page number and the page size page = request.values.get("page", 1) psize = request.values.get("pageSize", 10) sort = request.values.get("sort") # check the page is an integer try: page = int(page) except: raise Api400Error("Page number was not an integer") # check the page size is an integer
def test_04_paging_for_link_headers(self): # calc_pagination takes total, page_size, requested_page # and returns page_count, previous_page, next_page, last_page # request 1 of 1 pages assert DiscoveryApi._calc_pagination(0, 10, 1) == (1, None, None, 1) # 0 results still means page 1 assert DiscoveryApi._calc_pagination(1, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(2, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(3, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(9, 10, 1) == (1, None, None, 1) assert DiscoveryApi._calc_pagination(10, 10, 1) == (1, None, None, 1) # request 1st of 2 pages assert DiscoveryApi._calc_pagination(11, 10, 1) == (2, None, 2, 2) assert DiscoveryApi._calc_pagination(12, 10, 1) == (2, None, 2, 2) assert DiscoveryApi._calc_pagination(19, 10, 1) == (2, None, 2, 2) assert DiscoveryApi._calc_pagination(20, 10, 1) == (2, None, 2, 2) # request 2nd of 2 pages assert DiscoveryApi._calc_pagination(11, 10, 2) == (2, 1, None, 2) assert DiscoveryApi._calc_pagination(12, 10, 2) == (2, 1, None, 2) assert DiscoveryApi._calc_pagination(19, 10, 2) == (2, 1, None, 2) assert DiscoveryApi._calc_pagination(20, 10, 2) == (2, 1, None, 2) # various requests for 10s of 1000s of results assert DiscoveryApi._calc_pagination(9900 , 100, 1) == (99, None, 2, 99) assert DiscoveryApi._calc_pagination(9900 , 100, 99) == (99, 98, None, 99) assert DiscoveryApi._calc_pagination(9901 , 100, 1) == (100, None, 2, 100) assert DiscoveryApi._calc_pagination(9902 , 100, 1) == (100, None, 2, 100) assert DiscoveryApi._calc_pagination(10000, 100, 1) == (100, None, 2, 100) assert DiscoveryApi._calc_pagination(10000, 100, 2) == (100, 1, 3, 100) assert DiscoveryApi._calc_pagination(10000, 100, 98) == (100, 97, 99, 100) assert DiscoveryApi._calc_pagination(10000, 100, 99) == (100, 98, 100, 100) assert DiscoveryApi._calc_pagination(10000, 100, 100) == (100, 99, None, 100)
def test_03_applications(self): # create an account that will own the suggestions acc = models.Account() acc.set_id("owner") acc.save() # populate the index with some suggestions owned by this owner for i in range(5): a = models.Suggestion() a.set_owner("owner") bj = a.bibjson() bj.title = "Test Suggestion {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_url("http://homepage.com/{x}".format(x=i), "homepage") a.save() # make sure the last updated dates are suitably different time.sleep(1) # populte the index with some which are not owned by this owner for i in range(5): a = models.Suggestion() a.set_owner("stranger") bj = a.bibjson() bj.title = "Test Suggestion {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) a.save() # make sure the last updated dates are suitably different time.sleep(1) time.sleep(1) # now run some queries with self._make_and_push_test_context(acc=acc): # 1. a general query that should hit everything res = DiscoveryApi.search("application", acc, "Test", 1, 2) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 2 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 2 assert res.data.get("query") == "Test" # 2. a specific field query that should hit just one res = DiscoveryApi.search("application", acc, "title:\"Test Suggestion 2\"", 1, 5) assert res.data.get("total") == 1 assert len(res.data.get("results")) == 1 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 assert res.data.get("query") == "title:\"Test Suggestion 2\"" # 3.paging out of range of results res = DiscoveryApi.search("application", acc, "Test", 2, 10) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 0 assert res.data.get("page") == 2 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 4. paging outside the allowed bounds (lower) res = DiscoveryApi.search("application", acc, "Test", 0, 0) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 5. page size above upper limit res = DiscoveryApi.search("application", acc, "Test", 1, 100000) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 100 assert res.data.get("query") == "Test" # 6. Failed attempt at wildcard search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "Te*t", 1, 10) # 7. Failed attempt at fuzzy search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "title:Test~0.8", 1, 10) # 8. sort on a specific field, expect a default to "asc" res = DiscoveryApi.search("application", acc, "Test", 1, 10, "created_date") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") < res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date" # 9. sort on a specific field in a specified direction res = DiscoveryApi.search("application", acc, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") > res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date:desc" # 10. Malformed sort direction with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "Test", 1, 10, "created_date:whatever") # 11. non-existant sort field with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("application", acc, "Test", 1, 10, "some.missing.field:asc") # 12. with a forward slash, with and without escaping (note that we have to escape the : as it has meaning for lucene) res = DiscoveryApi.search("application", acc, '"http\://homepage.com/1"', 1, 10) assert res.data.get("total") == 1 res = DiscoveryApi.search("application", acc, '"http\:\/\/homepage.com\/1"', 1, 10) assert res.data.get("total") == 1 # 13. A search with an account that isn't either of the ones in the dataset other = models.Account() other.set_id("other") with self._make_and_push_test_context(acc=other): res = DiscoveryApi.search("application", other, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 0
def test_01_journals(self): # populate the index with some journals saved_journals = [] for i in range(5): j = models.Journal() j.set_in_doaj(True) bj = j.bibjson() bj.title = "Test Journal {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_url("http://homepage.com/{x}".format(x=i), "homepage") j.save() saved_journals.append((j.id, j.last_updated)) # make sure the last updated dates are suitably different time.sleep(1) # add one that's not in DOAJ, which shouldn't turn up in our results j = models.Journal() j.set_in_doaj(False) bj = j.bibjson() bj.title = "Test Journal {x}".format(x=6) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=6)) bj.publisher = "Test Publisher {x}".format(x=6) bj.add_url("http://homepage.com/{x}".format(x=6), "homepage") j.save() saved_journals.append((j.id, j.last_updated)) models.Journal.blockall(saved_journals) # now run some queries with self.app_test.test_request_context(): # 1. a general query that should hit everything (except number 6) res = DiscoveryApi.search("journal", None, "Test", 1, 2) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 2 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 2 assert res.data.get("query") == "Test" # 2. a specific field query that should hit just one res = DiscoveryApi.search("journal", None, "title:\"Test Journal 2\"", 1, 5) assert res.data.get("total") == 1 assert len(res.data.get("results")) == 1 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 assert res.data.get("query") == "title:\"Test Journal 2\"" # 3.paging out of range of results res = DiscoveryApi.search("journal", None, "Test", 2, 10) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 0 assert res.data.get("page") == 2 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 4. paging outside the allowed bounds (lower) res = DiscoveryApi.search("journal", None, "Test", 0, 0) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 5. page size above upper limit res = DiscoveryApi.search("journal", None, "Test", 1, 100000) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 100 assert res.data.get("query") == "Test" # 6. Failed attempt at wildcard search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "Te*t", 1, 10) # 7. Failed attempt at fuzzy search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "title:Test~0.8", 1, 10) # 8. sort on a specific field, expect a default to "asc" res = DiscoveryApi.search("journal", None, "Test", 1, 10, "created_date") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") < res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date" # 9. sort on a specific field in a specified direction res = DiscoveryApi.search("journal", None, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") > res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date:desc" # 10. Malformed sort direction with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "Test", 1, 10, "created_date:whatever") # 11. non-existant sort field with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("journal", None, "Test", 1, 10, "some.missing.field:asc") # 12. with a forward slash, with and without escaping (note that we have to escape the : as it has meaning for lucene) res = DiscoveryApi.search("journal", None, '"http\://homepage.com/1"', 1, 10) assert res.data.get("total") == 1 res = DiscoveryApi.search("journal", None, '"http\:\/\/homepage.com\/1"', 1, 10) assert res.data.get("total") == 1
def test_02_articles(self): # populate the index with some articles for i in range(5): a = models.Article() a.set_in_doaj(True) bj = a.bibjson() bj.title = "Test Article {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.add_identifier(bj.DOI, "10.test/{x}".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) bj.add_author("Agnieszka Domanska", "CL University", "https://orcid.org/0000-0001-1234-1234") a.save() # make sure the last updated dates are suitably different time.sleep(1) time.sleep(1) # now run some queries with self.app_test.test_request_context(): # 1. a general query that should hit everything res = DiscoveryApi.search("article", None, "Test", 1, 2) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 2 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 2 assert res.data.get("query") == "Test" # 2. a specific field query that should hit just one res = DiscoveryApi.search("article", None, "title:\"Test Article 2\"", 1, 5) assert res.data.get("total") == 1 assert len(res.data.get("results")) == 1 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 assert res.data.get("query") == "title:\"Test Article 2\"" # 3.paging out of range of results res = DiscoveryApi.search("article", None, "Test", 2, 10) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 0 assert res.data.get("page") == 2 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 4. paging outside the allowed bounds (lower) res = DiscoveryApi.search("article", None, "Test", 0, 0) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 5. page size above upper limit res = DiscoveryApi.search("article", None, "Test", 1, 100000) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 100 assert res.data.get("query") == "Test" # 6. Failed attempt at wildcard search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "Te*t", 1, 10) # 7. Failed attempt at fuzzy search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "title:Test~0.8", 1, 10) # 8. sort on a specific field, expect a default to "asc" res = DiscoveryApi.search("article", None, "Test", 1, 10, "created_date") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") < res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date" # 9. sort on a specific field in a specified direction res = DiscoveryApi.search("article", None, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") > res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date:desc" # 10. Malformed sort direction with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "Test", 1, 10, "created_date:whatever") # 11. non-existant sort field with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "Test", 1, 10, "some.missing.field:asc") # 12. with a forward slash, with and without escaping res = DiscoveryApi.search("article", None, '"10.test/1"', 1, 10) assert res.data.get("total") == 1 res = DiscoveryApi.search("article", None, '"10.test\/1"', 1, 10) assert res.data.get("total") == 1
# Handle wayward paths by raising an API404Error @blueprint.route("/<path:invalid_path>", methods=["POST", "GET", "PUT", "DELETE", "PATCH", "HEAD"]) # leaving out methods should mean all, but tests haven't shown that behaviour. def missing_resource(invalid_path): docs_url = app.config.get("BASE_URL", "") + url_for('.docs') spec_url = app.config.get("BASE_URL", "") + url_for('.api_spec') raise Api404Error("No endpoint at {0}. See {1} for valid paths or read the documentation at {2}.".format(invalid_path, spec_url, docs_url)) @blueprint.route('/docs') def docs(): return render_template('api/v1/api_docs.html') @swag(swag_summary='Search your applications <span class="red">[Authenticated, not public]</span>', swag_spec=DiscoveryApi.get_application_swag()) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes. @blueprint.route("/search/applications/<path:search_query>") @api_key_required @analytics.sends_ga_event(GA_CATEGORY, GA_ACTIONS.get('search_applications', 'Search applications'), record_value_of_which_arg='search_query') def search_applications(search_query): # get the values for the 2 other bits of search info: the page number and the page size page = request.values.get("page", 1) psize = request.values.get("pageSize", 10) sort = request.values.get("sort") # check the page is an integer try: page = int(page) except: raise Api400Error("Page number was not an integer")
def test_02_articles(self): # populate the index with some articles for i in range(5): a = models.Article() a.set_in_doaj(True) bj = a.bibjson() bj.title = "Test Article {x}".format(x=i) bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i)) bj.add_identifier(bj.DOI, "10.test/{x}".format(x=i)) bj.publisher = "Test Publisher {x}".format(x=i) a.save() # make sure the last updated dates are suitably different time.sleep(1) time.sleep(1) # now run some queries with self.app_test.test_request_context(): # 1. a general query that should hit everything res = DiscoveryApi.search("article", None, "Test", 1, 2) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 2 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 2 assert res.data.get("query") == "Test" # 2. a specific field query that should hit just one res = DiscoveryApi.search("article", None, "title:\"Test Article 2\"", 1, 5) assert res.data.get("total") == 1 assert len(res.data.get("results")) == 1 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 5 assert res.data.get("query") == "title:\"Test Article 2\"" # 3.paging out of range of results res = DiscoveryApi.search("article", None, "Test", 2, 10) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 0 assert res.data.get("page") == 2 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 4. paging outside the allowed bounds (lower) res = DiscoveryApi.search("article", None, "Test", 0, 0) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" # 5. page size above upper limit res = DiscoveryApi.search("article", None, "Test", 1, 100000) assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 100 assert res.data.get("query") == "Test" # 6. Failed attempt at wildcard search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "Te*t", 1, 10) # 7. Failed attempt at fuzzy search with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "title:Test~0.8", 1, 10) # 8. sort on a specific field, expect a default to "asc" res = DiscoveryApi.search("article", None, "Test", 1, 10, "created_date") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") < res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date" # 9. sort on a specific field in a specified direction res = DiscoveryApi.search("article", None, "Test", 1, 10, "created_date:desc") assert res.data.get("total") == 5 assert len(res.data.get("results")) == 5 assert res.data.get("page") == 1 assert res.data.get("pageSize") == 10 assert res.data.get("query") == "Test" assert res.data.get("results")[0].get("created_date") > res.data.get("results")[1].get("created_date") assert res.data.get("sort") == "created_date:desc" # 10. Malformed sort direction with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "Test", 1, 10, "created_date:whatever") # 11. non-existant sort field with self.assertRaises(DiscoveryException): res = DiscoveryApi.search("article", None, "Test", 1, 10, "some.missing.field:asc") # 12. with a forward slash, with and without escaping res = DiscoveryApi.search("article", None, '"10.test/1"', 1, 10) assert res.data.get("total") == 1 res = DiscoveryApi.search("article", None, '"10.test\/1"', 1, 10) assert res.data.get("total") == 1
docs_url = app.config.get("BASE_URL", "") + url_for('.docs') spec_url = app.config.get("BASE_URL", "") + url_for('.api_spec') raise Api404Error( "No endpoint at {0}. See {1} for valid paths or read the documentation at {2}." .format(invalid_path, spec_url, docs_url)) @blueprint.route('/docs') def docs(): return render_template('api/v1/api_docs.html') @swag( swag_summary= 'Search your applications <span class="red">[Authenticated, not public]</span>', swag_spec=DiscoveryApi.get_application_swag() ) # must be applied after @api_key_(optional|required) decorators. They don't preserve func attributes. @blueprint.route("/search/applications/<path:search_query>") @api_key_required def search_applications(search_query): # get the values for the 2 other bits of search info: the page number and the page size page = request.values.get("page", 1) psize = request.values.get("pageSize", 10) sort = request.values.get("sort") # check the page is an integer try: page = int(page) except: raise Api400Error("Page number was not an integer")