def setUp(self): super(TestApi, self).setUp() self.port = get_first_free_port() self.test_server = TestServer( port=self.port, index=app.config['ELASTIC_SEARCH_TEST_INDEX'], python_app_module_path=os.path.abspath(web.__file__)) self.test_server.spawn() self.appurl = self.test_server.get_server_url() self.apibase = self.appurl + '/api'
def setUp(self): super(TestSearch, self).setUp() self.config = { "PORT" : get_first_free_port(), "ELASTIC_SEARCH_INDEX" : app.config['ELASTIC_SEARCH_INDEX'], "THREADED" : True, "FUNCTIONAL_TEST_MODE" : True } self.cfg_file = paths.rel2abs(__file__, "..", "resources", "test-server.cfg") make_config(self.config, self.cfg_file) self.test_server = TestServer(port=None, index=None, python_app_module_path=os.path.abspath(web.__file__), cfg_file=self.cfg_file) self.test_server.spawn_with_config() self.appurl = "http://localhost:{x}".format(x=self.config["PORT"]) self.api_base = self.appurl + "/search/v1/"
class TestApi(testindex.ESTestCase): def setUp(self): super(TestApi, self).setUp() self.port = get_first_free_port() self.test_server = TestServer( port=self.port, index=app.config['ELASTIC_SEARCH_TEST_INDEX'], python_app_module_path=os.path.abspath(web.__file__)) self.test_server.spawn() self.appurl = self.test_server.get_server_url() self.apibase = self.appurl + '/api' def tearDown(self): super(TestApi, self).tearDown() self.test_server.terminate() def test_01_get_progress(self): pass # Fails on assert r.status_code == requests.codes.ok, r.status_code with a 404 despite the job being created. # It seems that the separate self.test_server is reading off the configured development index (e.g. "lantern") # rather than the configured test index (usually "test"). So the job is being created via workflow.make_spreadsheet_job # in "test" but the progress of the job is being read in from "lantern" - and the job is not found in "lantern" of course. # For some reason the test index does not seem to be working by passing index=app.config['ELASTIC_SEARCH_TEST_INDEX'] on # line 19 above. # job = workflow.make_spreadsheet_job('test task ' + uuid1().hex, "*****@*****.**") # job.save(blocking=True) # expected_results = { # "progress_url": app.config['SERVICE_BASE_URL'] + '/api/compliancejob/progress/{0}'.format(job.id), # "pc": 0.0, # "queue": 1, # "results_url": app.config['SERVICE_BASE_URL'] + '/download_progress/{0}'.format(job.id), # "status": "submitted" # } # # r = requests.get(self.apibase + '/compliancejob/progress/{0}'.format(job.id)) # assert r.status_code == requests.codes.ok, r.status_code # try: # results = r.json() # except JSONDecodeError: # self.fail("The API did not return a JSON response as expected.") # # print results # # assert expected_results == results, diff_dicts(expected_results, results, d1_label="Expected results", d2_label="Actual results") def test_02_create_job(self): obj = { "webhook_callback": "http://your_url.com", "articles": [{ "doi": "10.1/doi", "pmid": "123456", "pmcid": "PMC123456", "title": "Article Title 1" }, { "doi": "10.2/doi" }] } r = requests.post(self.apibase + '/compliancejob', data=json.dumps(obj)) assert r.status_code == requests.codes.ok, r.status_code assert '/api/compliancejob/progress/' in r.json()['progress_url'] assert r.json()['pc'] == 0.0 assert r.json()['queue'] == 1, r.json()['queue'] assert '/download_progress/' in r.json()['results_url'], r.json( )['results_url'] assert r.json()['status'] == 'submitted'
class TestCRUD(ESTestCase): def setUp(self): super(TestCRUD, self).setUp() self.config = { "PORT" : get_first_free_port(), "ELASTIC_SEARCH_INDEX" : app.config['ELASTIC_SEARCH_INDEX'], "THREADED" : True, "FUNCTIONAL_TEST_MODE" : True, "DEBUG" : True, "SSL" : False } self.cfg_file = paths.rel2abs(__file__, "..", "resources", "test-server.cfg") make_config(self.config, self.cfg_file) self.test_server = TestServer(port=None, index=None, python_app_module_path=os.path.abspath(web.__file__), cfg_file=self.cfg_file) self.test_server.spawn_with_config() self.appurl = "http://*****:*****@context"] = app.config.get("API_JSON_LD_CONTEXT") create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 201 j = resp.json() assert j.get("status") == "created" assert j.get("public_id") == "10.1234/me" assert j.get("request_id") is not None time.sleep(2) # now it has been created, and while it still sits in the request space, attempt to retrieve # it - this should fail retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key resp2 = requests.get(retrieve_url) assert resp2.status_code == 404 def test_02_create_error(self): # Create a record and receive an error # create an account with the correct rights acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save() # and a third account without the correct role acc3 = models.MonitorUKAccount() acc3.generate_api_key() acc3.save(blocking=True) record = fixtures.RequestFixtureFactory.record() # try a create without an API key create_url = self.api_base + "apc" resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 401 # try a create with the wrong role create_url = self.api_base + "apc?api_key=" + acc3.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 403 # try a create with the wrong format data create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps({"random" : "junk"}), headers={"Content-Type" : "application/json"}) assert resp.status_code == 400 def test_03_update_request(self): # Create a record and then successfully update it # this is basically 2 consecutive creates for the same item - things get more interesting in the next # test when we publish the item in between ... # create an account with the correct rights acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save(blocking=True) # first, create a record via the API record = fixtures.RequestFixtureFactory.record() record["@context"] = app.config.get("API_JSON_LD_CONTEXT") create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 201 time.sleep(2) # now send an update to the base APC url - which is the only thing we can do at the moment, as the public_id # is not yet active record2 = fixtures.RequestFixtureFactory.record() record2["dc:title"] = "Updated" j = resp.json() update_url = self.api_base + "apc?api_key=" + acc.api_key resp2 = requests.post(update_url, data=json.dumps(record2), headers={"Content-Type" : "application/json"}) assert resp2.status_code == 201 j = resp2.json() assert j.get("status") == "created" assert j.get("public_id") == "10.1234/me" assert j.get("request_id") is not None # no point trying to retrieve it, as it still sits in the public space, where we can't access it def test_04_retrieve_update_public(self): # Create a record, publish it, and then send an update pub_dao = models.PublicAPC() # create an account with the correct rights acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save() # and another account without the correct role acc3 = models.MonitorUKAccount() acc3.generate_api_key() acc3.save(blocking=True) # first, create a record via the API record = fixtures.RequestFixtureFactory.record() create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) j = resp.json() assert resp.status_code == 201 # execute the back-end workflow, which should publish the request time.sleep(2) api.WorkflowApi.process_requests() time.sleep(2) # check that publication took place pubs = pub_dao.find_by_doi(j.get("public_id")) assert len(pubs) == 1 # attempt to retrieve the record from the public space retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key crecord = deepcopy(record) crecord["@context"] = app.config.get("API_JSON_LD_CONTEXT") resp2 = requests.get(retrieve_url) assert resp2.status_code == 200 assert resp2.json() == crecord # now issue an update against that record update_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key record2 = fixtures.RequestFixtureFactory.record() record2["dc:title"] = "Updated" resp3 = requests.put(update_url, data=json.dumps(record2), headers={"Content-Type" : "application/json"}) assert resp3.status_code == 200 j = resp3.json() assert j.get("status") == "updated" assert j.get("public_id") == "10.1234/me" assert j.get("request_id") is not None # execute the back-end workflow, which should publish the update time.sleep(2) api.WorkflowApi.process_requests() time.sleep(2) retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key resp4 = requests.get(retrieve_url) assert resp4.status_code == 200 r4j = resp4.json() del r4j["@context"] assert r4j == record2 # finally, a quick security check - that a user without the correct role can't access retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc3.api_key resp4 = requests.get(retrieve_url) assert resp4.status_code == 403 def test_05_update_error(self): # Create and publish a record, then send an update which contains an error # create an account with the correct rights acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save() # and a third account without the correct role acc3 = models.MonitorUKAccount() acc3.generate_api_key() acc3.save(blocking=True) # first, create a record via the API record = fixtures.RequestFixtureFactory.record() create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 201 j = resp.json() # execute the back-end workflow, which should publish the update time.sleep(2) api.WorkflowApi.process_requests() time.sleep(2) location = self.api_base + "apc/" + j.get("public_id") # try an update without an API key update_url = location resp = requests.put(update_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 401 # try an update with the wrong role update_url = location + "?api_key=" + acc3.api_key resp = requests.put(update_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 403 # try an update with the wrong format data update_url = location + "?api_key=" + acc.api_key resp = requests.put(update_url, data=json.dumps({"random" : "junk"}), headers={"Content-Type" : "application/json"}) assert resp.status_code == 400 def test_06_delete_request(self): # Create and publish a record, then issue a delete request pub_dao = models.PublicAPC() # create an account with the correct rights acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save(blocking=True) # first, create a record via the API record = fixtures.RequestFixtureFactory.record() create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 201 j = resp.json() # execute the back-end workflow, which should publish the update time.sleep(2) api.WorkflowApi.process_requests() time.sleep(2) # check that publication took place pubs = pub_dao.find_by_doi(j.get("public_id")) assert len(pubs) == 1 # now send a delete request delete_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key resp2 = requests.delete(delete_url) assert resp2.status_code == 200 j = resp2.json() assert j.get("status") == "deleted" assert j.get("public_id") == "10.1234/me" assert j.get("request_id") is not None # execute the back-end workflow, which should delete the public record time.sleep(2) api.WorkflowApi.process_requests() time.sleep(2) # request the record, and discover that it is no longer there (even though, secretly, it's still on # the request queue) retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key resp3 = requests.get(retrieve_url) assert resp3.status_code == 404 def test_07_delete_error(self): # Create and publish a record, then issue a delete request with an error # create an account with the correct rights acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save() # and a third account without the correct role acc2 = models.MonitorUKAccount() acc2.generate_api_key() acc.add_role("write_apc") acc2.save(blocking=True) # and a third account without the correct role acc3 = models.MonitorUKAccount() acc3.generate_api_key() acc3.save(blocking=True) # first, create a record via the API record = fixtures.RequestFixtureFactory.record() create_url = self.api_base + "apc?api_key=" + acc.api_key resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"}) assert resp.status_code == 201 j = resp.json() # execute the back-end workflow, which should publish the update time.sleep(2) api.WorkflowApi.process_requests() time.sleep(2) location = self.api_base + "apc/" + j.get("public_id") # try a delete without an API key delete_url = location resp = requests.delete(delete_url) assert resp.status_code == 401 # try a delete with the wrong role delete_url = location + "?api_key=" + acc3.api_key resp = requests.delete(delete_url) assert resp.status_code == 403 # try a delete with an account which has no stake in this record delete_url = location + "?api_key=" + acc2.api_key resp = requests.delete(delete_url) assert resp.status_code == 403
class TestSearch(ESTestCase): def setUp(self): super(TestSearch, self).setUp() self.config = { "PORT" : get_first_free_port(), "ELASTIC_SEARCH_INDEX" : app.config['ELASTIC_SEARCH_INDEX'], "THREADED" : True, "FUNCTIONAL_TEST_MODE" : True } self.cfg_file = paths.rel2abs(__file__, "..", "resources", "test-server.cfg") make_config(self.config, self.cfg_file) self.test_server = TestServer(port=None, index=None, python_app_module_path=os.path.abspath(web.__file__), cfg_file=self.cfg_file) self.test_server.spawn_with_config() self.appurl = "http://*****:*****@context" in r assert r.get("dcterms:dateSubmitted") == "2001-01-01T00:00:00Z" assert r.get("dc:title") == "Title A" assert r.get("dcterms:dateAccepted") == "2002-01-01T00:00:00Z" # 2. more advanced query, with sorting query = "dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-02T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=desc") assert resp.status_code == 200 j = resp.json() assert j.get("total") == 2 assert j.get("page") == 1 assert j.get("pageSize") == 10 assert len(j.get("results")) == 2 r1 = j.get("results")[0] assert r1.get("dc:title") == "Title B", r1.get("dc:title") r2 = j.get("results")[1] assert r2.get("dc:title") == "Title A" # 3. advanced query, with sorting and paging query = "dcterms\\:dateSubmitted:2001-01-03T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-04T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=asc&page=2&pageSize=1") assert resp.status_code == 200 j = resp.json() assert j.get("total") == 2 assert j.get("page") == 2 assert j.get("pageSize") == 1 assert len(j.get("results")) == 1 r1 = j.get("results")[0] assert r1.get("dc:title") == "Title D", r1.get("dc:title") def test_02_public_error(self): # query against the public search api which results in an error # make a user account which can only read generically from public acc2 = models.MonitorUKAccount() acc2.generate_api_key() acc2.save() acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("read_apc") acc.save(blocking=True) # 1. query with incorrect page parameter query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&page=fourteen") assert resp.status_code == 400, resp.status_code # 2. query with incorrect page size query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=huge") assert resp.status_code == 400 # 3. query with incorrect sort_dir query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=10&sortBy=dc:title&sortDir=up") assert resp.status_code == 400 # 4. wildcard query attempt query = "record.dc\\:title:Ti*le" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query) assert resp.status_code == 400 # 5. Proximity query attempt query = "record.dc\\:title:Title~5" resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query) assert resp.status_code == 400 # 6. No API key query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?q=" + query) assert resp.status_code == 401 # 7. Incorrect user role query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "public?api_key=" + acc2.api_key + "&q=" + query) assert resp.status_code == 403 def test_03_private_success(self): # Successful query against the private search API # make user accounts with the correct usernames to query stuff we're about to create then search on acc = models.MonitorUKAccount() acc.id = "test1" acc.generate_api_key() acc.add_role("write_apc") acc.save() acc2 = models.MonitorUKAccount() acc2.id = "test2" acc2.generate_api_key() acc2.add_role("write_apc") acc2.save() # first make a small dataset for us to search over source1 = fixtures.PublicAPCFixtureFactory.make_record("test1", "Title A", "2001-01-01T00:00:00Z", "2002-01-01T00:00:00Z") pub1 = models.PublicAPC(source1) pub1.save() source2 = fixtures.PublicAPCFixtureFactory.make_record("test1", "Title B", "2001-01-02T00:00:00Z", "2002-01-02T00:00:00Z") pub2 = models.PublicAPC(source2) pub2.save() source3 = fixtures.PublicAPCFixtureFactory.make_record("test2", "Title C", "2001-01-03T00:00:00Z", "2002-01-03T00:00:00Z") pub3 = models.PublicAPC(source3) pub3.save() source4 = fixtures.PublicAPCFixtureFactory.make_record("test2", "Title D", "2001-01-04T00:00:00Z", "2002-01-04T00:00:00Z") pub4 = models.PublicAPC(source4) pub4.save(blocking=True) # 1. simple query for a field, with no sorting and no paging, done by the correct user account query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query) assert resp.status_code == 200 j = resp.json() assert j.get("total") == 1 assert j.get("page") == 1 assert j.get("pageSize") == 10 assert len(j.get("results")) == 1 r = j.get("results")[0] assert r.get("dc:title") == "Title A" # 2. more advanced query, with sorting, which would match 2 if it weren't for the user account query = "dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-03T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=desc") assert resp.status_code == 200 j = resp.json() assert j.get("total") == 1 assert j.get("page") == 1 assert j.get("pageSize") == 10 assert len(j.get("results")) == 1 r1 = j.get("results")[0] assert r1.get("dc:title") == "Title A", r1.get("dc:title") # 3. now do the same query as above with a different account, and get the opposite result query = "dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-03T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc2.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=desc") assert resp.status_code == 200 j = resp.json() assert j.get("total") == 1 assert j.get("page") == 1 assert j.get("pageSize") == 10 assert len(j.get("results")) == 1 r1 = j.get("results")[0] assert r1.get("dc:title") == "Title C", r1.get("dc:title") def test_04_private_error(self): # Query against the private search API that results in an error # make a user account which can only read generically from public acc2 = models.MonitorUKAccount() acc2.generate_api_key() acc2.add_role("read_apc") acc2.save() acc = models.MonitorUKAccount() acc.generate_api_key() acc.add_role("write_apc") acc.save(blocking=True) # 1. query with incorrect page parameter query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&page=fourteen") assert resp.status_code == 400, resp.status_code # 2. query with incorrect page size query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=huge") assert resp.status_code == 400 # 3. query with incorrect sort_dir query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=10&sortBy=dc:title&sortDir=up") assert resp.status_code == 400 # 4. wildcard query attempt query = "record.dc\\:title:Ti*le" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query) assert resp.status_code == 400 # 5. Proximity query attempt query = "record.dc\\:title:Title~5" resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query) assert resp.status_code == 400 # 6. No API key query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?q=" + query) assert resp.status_code == 401 # 7. Incorrect user role query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z" resp = requests.get(self.api_base + "private?api_key=" + acc2.api_key + "&q=" + query) assert resp.status_code == 403