def test_02_update(self):
# Do updates on the ApiRequest object
source = RequestFixtureFactory.record()
csource = deepcopy(source)
csource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
acc = MonitorUKAccount()
acc.save()
# do an update on blank one (unlikely use case)
req = ApiRequest()
req.update(csource)
assert req.raw == source
assert req.account is None
assert isinstance(req.json(), basestring)
# make one, and then update it
source2 = deepcopy(source)
source2["dc:title"] = "An update"
req = ApiRequest(source, account=acc)
req.update(source2)
assert req.raw == source2
assert req.account.id == acc.id
assert isinstance(req.json(), basestring)
# make a valid one, and then do something invalid to it
req = ApiRequest(source, account=acc)
with self.assertRaises(dataobj.DataStructureException):
req.update({"whatever" : "here"})
def test_08_append(self):
# Check that append works, which is the same as update
source = RequestFixtureFactory.record()
acc = MonitorUKAccount()
acc.save()
# do an update on blank one (unlikely use case)
req = ApiRequest()
req.append(source)
assert req.raw == source
assert req.account is None
assert isinstance(req.json(), basestring)
# make one, and then update it
source2 = deepcopy(source)
source2["dc:title"] = "An update"
req = ApiRequest(source, account=acc)
req.append(source2)
assert req.raw == source2
assert req.account.id == acc.id
assert isinstance(req.json(), basestring)
# make a valid one, and then do something invalid to it
req = ApiRequest(source, account=acc)
with self.assertRaises(dataobj.DataStructureException):
req.append({"whatever" : "here"})
def test_01_create(self):
# Create instances of the ApiRequest object
# construct a blank one, just to make sure we can
req = ApiRequest()
assert req.raw is None
assert req.account is None
assert req.json() is None
# make one using valid source data
source = RequestFixtureFactory.record()
csource = deepcopy(source)
csource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
acc = MonitorUKAccount()
acc.save()
req = ApiRequest(csource, account=acc)
assert req.raw == source
assert req.account.id == acc.id
assert isinstance(req.json(), basestring)
# make one with some invalid data
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest({"whatever" : "here"}, account=acc)
def test_10_validate(self):
# Check the various validation routines
acc = MonitorUKAccount()
acc.save()
source = RequestFixtureFactory.record()
# first prove that our source record validates
csource = deepcopy(source)
csource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
req = ApiRequest(csource, account=acc)
assert req.raw == csource
# 1 - a record with no identifiers
asource = deepcopy(source)
del asource["dc:identifier"]
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 2 - a record with no apc
asource = deepcopy(source)
del asource["jm:apc"]
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 3 - an apc with no inc vat amount
asource = deepcopy(source)
del asource["jm:apc"][0]["amount_inc_vat_gbp"]
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 4 - a record with a date in the wrong format
asource = deepcopy(source)
asource["dcterms:dateAccepted"] = "sometime last week"
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 5 - a record with an unparseable number
asource = deepcopy(source)
asource["jm:apc"][0]["amount_inc_vat_gbp"] = "three fifty"
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 6 - a record with an invalid currency code
asource = deepcopy(source)
asource["jm:apc"][0]["currency"] = "XXX"
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
def test_04_pull_public(self):
# Pull a PublicAPC through the ApiRequest object
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save()
acc2 = MonitorUKAccount()
acc2.id = "qwerty"
acc.save(blocking=True)
# make a competing public version, and check that a pull on the doi works
pub_source = PublicAPCFixtureFactory.example()
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull("10.1234/me", account=acc)
assert result.raw == pub.clean_record
assert result.account.id == acc.id
assert result.public_id == pub.id
craw = deepcopy(pub.clean_record)
craw["@context"] = app.config.get("API_JSON_LD_CONTEXT")
assert json.loads(result.json()) == craw
# also try the pull with the wrong owner, which should work
result = ApiRequest.pull("10.1234/me", account=acc2)
assert result is not None
def check_jobs(cls):
"""
Check any existing Lantern jobs for progress, and process any that have completed
:return:
"""
dao = LanternJob()
delay = app.config.get("JOB_LOOKUP_DELAY_LANTERN", 3600)
cutoff = dates.before_now(delay)
gen = dao.list_active(cutoff, keepalive="30m")
for job in gen:
acc = MonitorUKAccount.pull(job.account)
lc = client.Lantern(api_key=acc.lantern_api_key)
prog = lc.get_progress(job.job_id)
if prog.get("status") == "success":
pc = prog.get("data", {}).get("progress", 0)
if pc != 100:
# this will update the last_updated date, which means we won't look at it again for a while
job.save()
continue
# if we get here, the job is complete so we need to retrieve it
results = lc.get_results(job.job_id)
if results.get("status") == "success":
for res in results.get("data", []):
enhancement = LanternApi._xwalk(res)
enhancement.save()
# set the job as complete
job.status = "complete"
job.save()
def test_07_save_delete(self):
# Work through acycle of saves and deletes to observe the outputs
source = RequestFixtureFactory.record()
acc = MonitorUKAccount()
acc.save(blocking=True)
req = ApiRequest(source, account=acc)
req.save()
dao = Request()
req2 = dao.pull(req.request.id)
assert req2 is not None
assert req2.owner == acc.id
assert req2.record == source
assert req2.action == "update"
# now publish the request
PublicApi.publish(req2)
time.sleep(2)
# now pull the object as identified by its API identifier (which should be the DOI)
source2 = deepcopy(source)
source2["dc:title"] = "An update"
next = ApiRequest.pull(req.id, account=acc)
next.update(source2)
next.save()
# now, at this point we should have 2 request objects in the index. One for the
# original save, and one for the new save
req3 = dao.pull(next.request.id)
assert req3 is not None
assert req3.owner == acc.id
assert req3.record == source2
assert req3.action == "update"
# now issue a delete on the same record
next.delete()
# by now we should have 3 request objects in the index, 2 for the above updates
# and one for the delete request
req4 = dao.pull(next.request.id)
assert req4 is not None
assert req4.owner == acc.id
assert req4.record == source2
assert req4.action == "delete"
def test_05_pull_then_update(self):
# Pull a record through the ApiRequest object and then udpate it
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save(blocking=True)
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
pub_source2 = deepcopy(pub_source.get("record"))
pub_source2["dc:title"] = "An update"
result = ApiRequest.pull(pub.id, account=acc)
result.update(pub_source2)
assert result.raw == pub_source2
assert result.account.id == acc.id
def test_09_responses(self):
# Check that we get appropriate JSON responses
source = RequestFixtureFactory.record()
acc = MonitorUKAccount()
acc.save(blocking=True)
req = ApiRequest(source, account=acc)
req.save()
# have a look at what we'd expect the responses to be if this was a create
cr = req.created_response()
assert cr.get("status") == "created"
assert cr.get("request_id") is not None
assert cr.get("public_id") == "10.1234/me"
# now look at the responses if this was an update/append
ur = req.updated_response()
assert ur.get("status") == "updated"
assert ur.get("request_id") is not None
assert ur.get("public_id") == "10.1234/me"
# now look at a delete
dr = req.deleted_response()
assert dr.get("status") == "deleted"
assert dr.get("request_id") is not None
assert dr.get("public_id") == "10.1234/me"
# just make a quick check to be sure that if no DOI is present, we get the right kind of info in the public_id
source2 = RequestFixtureFactory.record()
source2["dc:identifier"] = [{"type" : "pmcid", "id" : "PMC1234"}]
req2 = ApiRequest(source2, account=acc)
req2.save()
cr = req.created_response()
assert cr.get("public_id") == req.id
ur = req.updated_response()
assert ur.get("public_id") == req.id
dr = req.deleted_response()
assert dr.get("public_id") == req.id
def test_03_pull_request(self):
# Pull a Request through the ApiRequest object
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save(blocking=True)
# first make a request which contains that doi
req_source = RequestFixtureFactory.example()
req = Request(req_source)
req.save(blocking=True)
# you can't pull a request object, so just show that that's true...
# pull by doi should fail
result = ApiRequest.pull("10.1234/me", account=acc)
assert result is None
# pull by request id should fail
result = ApiRequest.pull(req.id, account=acc)
assert result is None
def test_06_get_id(self):
# Get the operative ID of a record
acc = MonitorUKAccount()
acc.id = "abcdefghij"
source = RequestFixtureFactory.record()
# first, provide a record which has a doi, and ensure that's the id we get back
req = ApiRequest(source, account=acc)
assert req.id == "10.1234/me"
# now make a public record, and check we get the right ids for it
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
del pub_source["record"]["dc:identifier"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull(pub.id, account=acc)
assert result.id == pub.id
assert result.id == result.public_id
def make_new_jobs(cls):
"""
Send new requests to lantern, and record the jobs that are created.
This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those
accounts which could benefit from lookup in Lantern,
:return:
"""
dao = PublicAPC()
gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h")
for acc in gen:
gen2 = dao.list_by_owner(acc.id)
identifiers = []
for apc in gen2:
if LanternApi._needs_lantern_data(apc):
idents = LanternApi._get_identifiers(apc)
if idents is not None:
identifiers.append(idents)
apc.lantern_lookup = dates.now()
apc.save()
# if there are no identifiers, no need to do any more
if len(identifiers) == 0:
continue
# now check the user's quota
lc = client.Lantern(api_key=acc.lantern_api_key)
quota = lc.get_quota(acc.lantern_email)
available = quota.get("data", {}).get("available", 0)
if available == 0:
continue
if len(identifiers) > available:
identifiers = identifiers[:available]
batches = LanternApi._batch(identifiers)
for batch in batches:
resp = lc.create_job(acc.lantern_email, "monitor-uk", batch)
if resp.get("status") == "success":
job_id = resp.get("data", {}).get("job")
lj = LanternJob()
lj.job_id = job_id
lj.account = acc.id
lj.status = "active"
lj.save()
def test_05_low_quota(self):
# Check what happens when the use has a low quota on Lantern
global QUOTA
QUOTA = 1
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 1
assert len(CREATED_JOBS) == 1
job = CREATED_JOBS[0]
assert job["email"] == "*****@*****.**"
assert len(job["list"]) == 1
def test_16_lantern_accounts(self):
# Check that we can detect lantern accounts
acc1 = MonitorUKAccount()
acc1.email = "*****@*****.**"
acc1.save()
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
acc3 = MonitorUKAccount()
acc3.email = "*****@*****.**"
acc3.lantern_email = "*****@*****.**"
acc3.lantern_api_key = "987654321"
acc3.save(blocking=True)
count = 0
gen = MonitorUKAccount.list_lantern_enabled()
for acc in gen:
if acc.email == "*****@*****.**":
assert acc.lantern_api_key == "123456789"
assert acc.lantern_email == "*****@*****.**"
count += 1
elif acc.email == "*****@*****.**":
assert acc.lantern_api_key == "987654321"
assert acc.lantern_email == "*****@*****.**"
count += 10
else:
count += 100
assert count == 11
def test_07_check_jobs(self):
# Ensure that we can check existing jobs correctly
acc = MonitorUKAccount()
acc.email = "*****@*****.**"
acc.lantern_email = "*****@*****.**"
acc.lantern_api_key = "123456789"
acc.save()
lj1 = LanternJob()
lj1.job_id = "111111111"
lj1.account = acc.id
lj1.status = "complete"
lj1.save()
lj2 = LanternJob()
lj2.job_id = "222222222"
lj2.account = acc.id
lj2.status = "active"
lj2.last_updated = dates.format(dates.before_now(5000))
lj2.save(updated=False)
lj3 = LanternJob()
lj3.job_id = "333333333"
lj3.account = acc.id
lj3.status = "active"
lj3.last_updated = dates.format(dates.before_now(5000))
lj3.save(updated=False)
lj4 = LanternJob()
lj4.job_id = "444444444"
lj4.account = acc.id
lj4.status = "active"
lj4.last_updated = dates.format(dates.before_now(5000))
lj4.save(updated=False)
lj5 = LanternJob()
lj5.job_id = "555555555"
lj5.account = acc.id
lj5.status = "active"
lj5.save(blocking=True)
LanternApi.check_jobs()
# check that the progress requests we expected were made
assert len(PROGRESS_REQUESTS) == 3
assert "222222222" in PROGRESS_REQUESTS
assert "333333333" in PROGRESS_REQUESTS
assert "444444444" in PROGRESS_REQUESTS
# check that the job which received an error was just ignored
dao = LanternJob()
ignored = dao.pull(lj4.id)
assert ignored.last_updated == lj4.last_updated
assert ignored.status == "active"
# check that the record which was not complete was touched
touched = dao.pull(lj2.id)
assert touched.last_updated != lj2.last_updated
assert touched.status == "active"
# check that results were requested only for one item
assert len(RESULTS_REQUESTS) == 1
assert "333333333" in RESULTS_REQUESTS
# wait for a bit, so that enhancements have time to go in
time.sleep(2)
# check that an enhancement was registered
edao = Enhancement()
gen = edao.iterall()
enhancements = [e for e in gen]
assert len(enhancements) == 1
result = LanternFixtureFactory.xwalk_result()
assert enhancements[0].data["record"] == result["record"]
def test_04_create_job(self):
# Check we can create jobs correctly
acc1 = MonitorUKAccount()
acc1.email = "*****@*****.**"
acc1.save()
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
acc3 = MonitorUKAccount()
acc3.email = "*****@*****.**"
acc3.lantern_email = "*****@*****.**"
acc3.lantern_api_key = "987654321"
acc3.save(blocking=True)
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save()
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
assert len(CREATED_JOBS) == 2
count = 0
for job in CREATED_JOBS:
if job["email"] == "*****@*****.**":
count += 1
assert len(job["list"]) == 2
elif job["email"] == "*****@*****.**":
count += 10
assert len(job["list"]) == 2
assert count == 11
# now do the same thing again. The jobs should not change, as we've already created jobs
# for all the public records
LanternApi.make_new_jobs()
time.sleep(2)
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
