def setUp(self):
super(TestApi, self).setUp()
self.port = get_first_free_port()
self.test_server = TestServer(
port=self.port,
index=app.config['ELASTIC_SEARCH_TEST_INDEX'],
python_app_module_path=os.path.abspath(web.__file__))
self.test_server.spawn()
self.appurl = self.test_server.get_server_url()
self.apibase = self.appurl + '/api'
def setUp(self):
super(TestSearch, self).setUp()
self.config = {
"PORT" : get_first_free_port(),
"ELASTIC_SEARCH_INDEX" : app.config['ELASTIC_SEARCH_INDEX'],
"THREADED" : True,
"FUNCTIONAL_TEST_MODE" : True
}
self.cfg_file = paths.rel2abs(__file__, "..", "resources", "test-server.cfg")
make_config(self.config, self.cfg_file)
self.test_server = TestServer(port=None, index=None, python_app_module_path=os.path.abspath(web.__file__), cfg_file=self.cfg_file)
self.test_server.spawn_with_config()
self.appurl = "http://localhost:{x}".format(x=self.config["PORT"])
self.api_base = self.appurl + "/search/v1/"
class TestApi(testindex.ESTestCase):
def setUp(self):
super(TestApi, self).setUp()
self.port = get_first_free_port()
self.test_server = TestServer(
port=self.port,
index=app.config['ELASTIC_SEARCH_TEST_INDEX'],
python_app_module_path=os.path.abspath(web.__file__))
self.test_server.spawn()
self.appurl = self.test_server.get_server_url()
self.apibase = self.appurl + '/api'
def tearDown(self):
super(TestApi, self).tearDown()
self.test_server.terminate()
def test_01_get_progress(self):
pass
# Fails on assert r.status_code == requests.codes.ok, r.status_code with a 404 despite the job being created.
# It seems that the separate self.test_server is reading off the configured development index (e.g. "lantern")
# rather than the configured test index (usually "test"). So the job is being created via workflow.make_spreadsheet_job
# in "test" but the progress of the job is being read in from "lantern" - and the job is not found in "lantern" of course.
# For some reason the test index does not seem to be working by passing index=app.config['ELASTIC_SEARCH_TEST_INDEX'] on
# line 19 above.
# job = workflow.make_spreadsheet_job('test task ' + uuid1().hex, "*****@*****.**")
# job.save(blocking=True)
# expected_results = {
# "progress_url": app.config['SERVICE_BASE_URL'] + '/api/compliancejob/progress/{0}'.format(job.id),
# "pc": 0.0,
# "queue": 1,
# "results_url": app.config['SERVICE_BASE_URL'] + '/download_progress/{0}'.format(job.id),
# "status": "submitted"
# }
#
# r = requests.get(self.apibase + '/compliancejob/progress/{0}'.format(job.id))
# assert r.status_code == requests.codes.ok, r.status_code
# try:
# results = r.json()
# except JSONDecodeError:
# self.fail("The API did not return a JSON response as expected.")
#
# print results
#
# assert expected_results == results, diff_dicts(expected_results, results, d1_label="Expected results", d2_label="Actual results")
def test_02_create_job(self):
obj = {
"webhook_callback":
"http://your_url.com",
"articles": [{
"doi": "10.1/doi",
"pmid": "123456",
"pmcid": "PMC123456",
"title": "Article Title 1"
}, {
"doi": "10.2/doi"
}]
}
r = requests.post(self.apibase + '/compliancejob',
data=json.dumps(obj))
assert r.status_code == requests.codes.ok, r.status_code
assert '/api/compliancejob/progress/' in r.json()['progress_url']
assert r.json()['pc'] == 0.0
assert r.json()['queue'] == 1, r.json()['queue']
assert '/download_progress/' in r.json()['results_url'], r.json(
)['results_url']
assert r.json()['status'] == 'submitted'
class TestCRUD(ESTestCase):
def setUp(self):
super(TestCRUD, self).setUp()
self.config = {
"PORT" : get_first_free_port(),
"ELASTIC_SEARCH_INDEX" : app.config['ELASTIC_SEARCH_INDEX'],
"THREADED" : True,
"FUNCTIONAL_TEST_MODE" : True,
"DEBUG" : True,
"SSL" : False
}
self.cfg_file = paths.rel2abs(__file__, "..", "resources", "test-server.cfg")
make_config(self.config, self.cfg_file)
self.test_server = TestServer(port=None, index=None, python_app_module_path=os.path.abspath(web.__file__), cfg_file=self.cfg_file)
self.test_server.spawn_with_config()
self.appurl = "http://*****:*****@context"] = app.config.get("API_JSON_LD_CONTEXT")
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 201
j = resp.json()
assert j.get("status") == "created"
assert j.get("public_id") == "10.1234/me"
assert j.get("request_id") is not None
time.sleep(2)
# now it has been created, and while it still sits in the request space, attempt to retrieve
# it - this should fail
retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key
resp2 = requests.get(retrieve_url)
assert resp2.status_code == 404
def test_02_create_error(self):
# Create a record and receive an error
# create an account with the correct rights
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save()
# and a third account without the correct role
acc3 = models.MonitorUKAccount()
acc3.generate_api_key()
acc3.save(blocking=True)
record = fixtures.RequestFixtureFactory.record()
# try a create without an API key
create_url = self.api_base + "apc"
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 401
# try a create with the wrong role
create_url = self.api_base + "apc?api_key=" + acc3.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 403
# try a create with the wrong format data
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps({"random" : "junk"}), headers={"Content-Type" : "application/json"})
assert resp.status_code == 400
def test_03_update_request(self):
# Create a record and then successfully update it
# this is basically 2 consecutive creates for the same item - things get more interesting in the next
# test when we publish the item in between ...
# create an account with the correct rights
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save(blocking=True)
# first, create a record via the API
record = fixtures.RequestFixtureFactory.record()
record["@context"] = app.config.get("API_JSON_LD_CONTEXT")
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 201
time.sleep(2)
# now send an update to the base APC url - which is the only thing we can do at the moment, as the public_id
# is not yet active
record2 = fixtures.RequestFixtureFactory.record()
record2["dc:title"] = "Updated"
j = resp.json()
update_url = self.api_base + "apc?api_key=" + acc.api_key
resp2 = requests.post(update_url, data=json.dumps(record2), headers={"Content-Type" : "application/json"})
assert resp2.status_code == 201
j = resp2.json()
assert j.get("status") == "created"
assert j.get("public_id") == "10.1234/me"
assert j.get("request_id") is not None
# no point trying to retrieve it, as it still sits in the public space, where we can't access it
def test_04_retrieve_update_public(self):
# Create a record, publish it, and then send an update
pub_dao = models.PublicAPC()
# create an account with the correct rights
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save()
# and another account without the correct role
acc3 = models.MonitorUKAccount()
acc3.generate_api_key()
acc3.save(blocking=True)
# first, create a record via the API
record = fixtures.RequestFixtureFactory.record()
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
j = resp.json()
assert resp.status_code == 201
# execute the back-end workflow, which should publish the request
time.sleep(2)
api.WorkflowApi.process_requests()
time.sleep(2)
# check that publication took place
pubs = pub_dao.find_by_doi(j.get("public_id"))
assert len(pubs) == 1
# attempt to retrieve the record from the public space
retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key
crecord = deepcopy(record)
crecord["@context"] = app.config.get("API_JSON_LD_CONTEXT")
resp2 = requests.get(retrieve_url)
assert resp2.status_code == 200
assert resp2.json() == crecord
# now issue an update against that record
update_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key
record2 = fixtures.RequestFixtureFactory.record()
record2["dc:title"] = "Updated"
resp3 = requests.put(update_url, data=json.dumps(record2), headers={"Content-Type" : "application/json"})
assert resp3.status_code == 200
j = resp3.json()
assert j.get("status") == "updated"
assert j.get("public_id") == "10.1234/me"
assert j.get("request_id") is not None
# execute the back-end workflow, which should publish the update
time.sleep(2)
api.WorkflowApi.process_requests()
time.sleep(2)
retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key
resp4 = requests.get(retrieve_url)
assert resp4.status_code == 200
r4j = resp4.json()
del r4j["@context"]
assert r4j == record2
# finally, a quick security check - that a user without the correct role can't access
retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc3.api_key
resp4 = requests.get(retrieve_url)
assert resp4.status_code == 403
def test_05_update_error(self):
# Create and publish a record, then send an update which contains an error
# create an account with the correct rights
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save()
# and a third account without the correct role
acc3 = models.MonitorUKAccount()
acc3.generate_api_key()
acc3.save(blocking=True)
# first, create a record via the API
record = fixtures.RequestFixtureFactory.record()
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 201
j = resp.json()
# execute the back-end workflow, which should publish the update
time.sleep(2)
api.WorkflowApi.process_requests()
time.sleep(2)
location = self.api_base + "apc/" + j.get("public_id")
# try an update without an API key
update_url = location
resp = requests.put(update_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 401
# try an update with the wrong role
update_url = location + "?api_key=" + acc3.api_key
resp = requests.put(update_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 403
# try an update with the wrong format data
update_url = location + "?api_key=" + acc.api_key
resp = requests.put(update_url, data=json.dumps({"random" : "junk"}), headers={"Content-Type" : "application/json"})
assert resp.status_code == 400
def test_06_delete_request(self):
# Create and publish a record, then issue a delete request
pub_dao = models.PublicAPC()
# create an account with the correct rights
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save(blocking=True)
# first, create a record via the API
record = fixtures.RequestFixtureFactory.record()
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 201
j = resp.json()
# execute the back-end workflow, which should publish the update
time.sleep(2)
api.WorkflowApi.process_requests()
time.sleep(2)
# check that publication took place
pubs = pub_dao.find_by_doi(j.get("public_id"))
assert len(pubs) == 1
# now send a delete request
delete_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key
resp2 = requests.delete(delete_url)
assert resp2.status_code == 200
j = resp2.json()
assert j.get("status") == "deleted"
assert j.get("public_id") == "10.1234/me"
assert j.get("request_id") is not None
# execute the back-end workflow, which should delete the public record
time.sleep(2)
api.WorkflowApi.process_requests()
time.sleep(2)
# request the record, and discover that it is no longer there (even though, secretly, it's still on
# the request queue)
retrieve_url = self.api_base + "apc/" + j.get("public_id") + "?api_key=" + acc.api_key
resp3 = requests.get(retrieve_url)
assert resp3.status_code == 404
def test_07_delete_error(self):
# Create and publish a record, then issue a delete request with an error
# create an account with the correct rights
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save()
# and a third account without the correct role
acc2 = models.MonitorUKAccount()
acc2.generate_api_key()
acc.add_role("write_apc")
acc2.save(blocking=True)
# and a third account without the correct role
acc3 = models.MonitorUKAccount()
acc3.generate_api_key()
acc3.save(blocking=True)
# first, create a record via the API
record = fixtures.RequestFixtureFactory.record()
create_url = self.api_base + "apc?api_key=" + acc.api_key
resp = requests.post(create_url, data=json.dumps(record), headers={"Content-Type" : "application/json"})
assert resp.status_code == 201
j = resp.json()
# execute the back-end workflow, which should publish the update
time.sleep(2)
api.WorkflowApi.process_requests()
time.sleep(2)
location = self.api_base + "apc/" + j.get("public_id")
# try a delete without an API key
delete_url = location
resp = requests.delete(delete_url)
assert resp.status_code == 401
# try a delete with the wrong role
delete_url = location + "?api_key=" + acc3.api_key
resp = requests.delete(delete_url)
assert resp.status_code == 403
# try a delete with an account which has no stake in this record
delete_url = location + "?api_key=" + acc2.api_key
resp = requests.delete(delete_url)
assert resp.status_code == 403
class TestSearch(ESTestCase):
def setUp(self):
super(TestSearch, self).setUp()
self.config = {
"PORT" : get_first_free_port(),
"ELASTIC_SEARCH_INDEX" : app.config['ELASTIC_SEARCH_INDEX'],
"THREADED" : True,
"FUNCTIONAL_TEST_MODE" : True
}
self.cfg_file = paths.rel2abs(__file__, "..", "resources", "test-server.cfg")
make_config(self.config, self.cfg_file)
self.test_server = TestServer(port=None, index=None, python_app_module_path=os.path.abspath(web.__file__), cfg_file=self.cfg_file)
self.test_server.spawn_with_config()
self.appurl = "http://*****:*****@context" in r
assert r.get("dcterms:dateSubmitted") == "2001-01-01T00:00:00Z"
assert r.get("dc:title") == "Title A"
assert r.get("dcterms:dateAccepted") == "2002-01-01T00:00:00Z"
# 2. more advanced query, with sorting
query = "dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-02T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=desc")
assert resp.status_code == 200
j = resp.json()
assert j.get("total") == 2
assert j.get("page") == 1
assert j.get("pageSize") == 10
assert len(j.get("results")) == 2
r1 = j.get("results")[0]
assert r1.get("dc:title") == "Title B", r1.get("dc:title")
r2 = j.get("results")[1]
assert r2.get("dc:title") == "Title A"
# 3. advanced query, with sorting and paging
query = "dcterms\\:dateSubmitted:2001-01-03T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-04T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=asc&page=2&pageSize=1")
assert resp.status_code == 200
j = resp.json()
assert j.get("total") == 2
assert j.get("page") == 2
assert j.get("pageSize") == 1
assert len(j.get("results")) == 1
r1 = j.get("results")[0]
assert r1.get("dc:title") == "Title D", r1.get("dc:title")
def test_02_public_error(self):
# query against the public search api which results in an error
# make a user account which can only read generically from public
acc2 = models.MonitorUKAccount()
acc2.generate_api_key()
acc2.save()
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("read_apc")
acc.save(blocking=True)
# 1. query with incorrect page parameter
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&page=fourteen")
assert resp.status_code == 400, resp.status_code
# 2. query with incorrect page size
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=huge")
assert resp.status_code == 400
# 3. query with incorrect sort_dir
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=10&sortBy=dc:title&sortDir=up")
assert resp.status_code == 400
# 4. wildcard query attempt
query = "record.dc\\:title:Ti*le"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query)
assert resp.status_code == 400
# 5. Proximity query attempt
query = "record.dc\\:title:Title~5"
resp = requests.get(self.api_base + "public?api_key=" + acc.api_key + "&q=" + query)
assert resp.status_code == 400
# 6. No API key
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?q=" + query)
assert resp.status_code == 401
# 7. Incorrect user role
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "public?api_key=" + acc2.api_key + "&q=" + query)
assert resp.status_code == 403
def test_03_private_success(self):
# Successful query against the private search API
# make user accounts with the correct usernames to query stuff we're about to create then search on
acc = models.MonitorUKAccount()
acc.id = "test1"
acc.generate_api_key()
acc.add_role("write_apc")
acc.save()
acc2 = models.MonitorUKAccount()
acc2.id = "test2"
acc2.generate_api_key()
acc2.add_role("write_apc")
acc2.save()
# first make a small dataset for us to search over
source1 = fixtures.PublicAPCFixtureFactory.make_record("test1", "Title A", "2001-01-01T00:00:00Z", "2002-01-01T00:00:00Z")
pub1 = models.PublicAPC(source1)
pub1.save()
source2 = fixtures.PublicAPCFixtureFactory.make_record("test1", "Title B", "2001-01-02T00:00:00Z", "2002-01-02T00:00:00Z")
pub2 = models.PublicAPC(source2)
pub2.save()
source3 = fixtures.PublicAPCFixtureFactory.make_record("test2", "Title C", "2001-01-03T00:00:00Z", "2002-01-03T00:00:00Z")
pub3 = models.PublicAPC(source3)
pub3.save()
source4 = fixtures.PublicAPCFixtureFactory.make_record("test2", "Title D", "2001-01-04T00:00:00Z", "2002-01-04T00:00:00Z")
pub4 = models.PublicAPC(source4)
pub4.save(blocking=True)
# 1. simple query for a field, with no sorting and no paging, done by the correct user account
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query)
assert resp.status_code == 200
j = resp.json()
assert j.get("total") == 1
assert j.get("page") == 1
assert j.get("pageSize") == 10
assert len(j.get("results")) == 1
r = j.get("results")[0]
assert r.get("dc:title") == "Title A"
# 2. more advanced query, with sorting, which would match 2 if it weren't for the user account
query = "dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-03T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=desc")
assert resp.status_code == 200
j = resp.json()
assert j.get("total") == 1
assert j.get("page") == 1
assert j.get("pageSize") == 10
assert len(j.get("results")) == 1
r1 = j.get("results")[0]
assert r1.get("dc:title") == "Title A", r1.get("dc:title")
# 3. now do the same query as above with a different account, and get the opposite result
query = "dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z OR dcterms\\:dateAccepted:2002-01-03T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc2.api_key + "&q=" + query + "&sortBy=dc:title&sortDir=desc")
assert resp.status_code == 200
j = resp.json()
assert j.get("total") == 1
assert j.get("page") == 1
assert j.get("pageSize") == 10
assert len(j.get("results")) == 1
r1 = j.get("results")[0]
assert r1.get("dc:title") == "Title C", r1.get("dc:title")
def test_04_private_error(self):
# Query against the private search API that results in an error
# make a user account which can only read generically from public
acc2 = models.MonitorUKAccount()
acc2.generate_api_key()
acc2.add_role("read_apc")
acc2.save()
acc = models.MonitorUKAccount()
acc.generate_api_key()
acc.add_role("write_apc")
acc.save(blocking=True)
# 1. query with incorrect page parameter
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&page=fourteen")
assert resp.status_code == 400, resp.status_code
# 2. query with incorrect page size
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=huge")
assert resp.status_code == 400
# 3. query with incorrect sort_dir
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query + "&page=1&pageSize=10&sortBy=dc:title&sortDir=up")
assert resp.status_code == 400
# 4. wildcard query attempt
query = "record.dc\\:title:Ti*le"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query)
assert resp.status_code == 400
# 5. Proximity query attempt
query = "record.dc\\:title:Title~5"
resp = requests.get(self.api_base + "private?api_key=" + acc.api_key + "&q=" + query)
assert resp.status_code == 400
# 6. No API key
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?q=" + query)
assert resp.status_code == 401
# 7. Incorrect user role
query = "record.dcterms\\:dateSubmitted:2001-01-01T00\\:00\\:00Z"
resp = requests.get(self.api_base + "private?api_key=" + acc2.api_key + "&q=" + query)
assert resp.status_code == 403