def test_02_update(self):
# Do updates on the ApiRequest object
source = RequestFixtureFactory.record()
csource = deepcopy(source)
csource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
acc = MonitorUKAccount()
acc.save()
# do an update on blank one (unlikely use case)
req = ApiRequest()
req.update(csource)
assert req.raw == source
assert req.account is None
assert isinstance(req.json(), basestring)
# make one, and then update it
source2 = deepcopy(source)
source2["dc:title"] = "An update"
req = ApiRequest(source, account=acc)
req.update(source2)
assert req.raw == source2
assert req.account.id == acc.id
assert isinstance(req.json(), basestring)
# make a valid one, and then do something invalid to it
req = ApiRequest(source, account=acc)
with self.assertRaises(dataobj.DataStructureException):
req.update({"whatever" : "here"})
def test_02_request(self):
# Check we can instantiate and work with a Request
# first make a blank one
req = Request()
# now make one around the fixture
source = RequestFixtureFactory.example()
req = Request(source)
# make one with a broken source
broken = {"whatever" : "broken"}
with self.assertRaises(dataobj.DataStructureException):
req = Request(broken)
# now make one bit by bit
req = Request()
req.record = source.get("record")
req.owner = "test1"
req.action = "update"
req.public_id = "abcdefg"
assert req.owner == "test1"
assert req.action == "update"
assert req.public_id == "abcdefg"
# now make it broken
req = Request()
with self.assertRaises(dataobj.DataStructureException):
req.record = {"random" : "stuff"}
def test_09_remove_permanent(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving no owners, thus deleting the record
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
pub = PublicAPC(source2)
pub.remove_apcs_by_owner("abcdefg") # clear the existing apc record
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub.add_apc_for_owner("test", apc_record) # add a new, known one
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert pub2 is None
def test_10_request_refs(self):
# Check that APC refs are handled correctly by Reuqests
# first check that refs are stripped automatically on construction
source = RequestFixtureFactory.example()
source["record"]["jm:apc"][0]["ref"] = "1234567890"
req = Request(source)
assert "ref" not in req.apc_records[0]
# now do it again, setting the record explicitly
source = RequestFixtureFactory.example()
record = source.get("record")
record["jm:apc"][0]["ref"] = "123456789"
req = Request()
req.record = record
assert "ref" not in req.apc_records[0]
def test_01_create(self):
# Create instances of the ApiRequest object
# construct a blank one, just to make sure we can
req = ApiRequest()
assert req.raw is None
assert req.account is None
assert req.json() is None
# make one using valid source data
source = RequestFixtureFactory.record()
csource = deepcopy(source)
csource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
acc = MonitorUKAccount()
acc.save()
req = ApiRequest(csource, account=acc)
assert req.raw == source
assert req.account.id == acc.id
assert isinstance(req.json(), basestring)
# make one with some invalid data
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest({"whatever" : "here"}, account=acc)
def test_08_append(self):
# Check that append works, which is the same as update
source = RequestFixtureFactory.record()
acc = MonitorUKAccount()
acc.save()
# do an update on blank one (unlikely use case)
req = ApiRequest()
req.append(source)
assert req.raw == source
assert req.account is None
assert isinstance(req.json(), basestring)
# make one, and then update it
source2 = deepcopy(source)
source2["dc:title"] = "An update"
req = ApiRequest(source, account=acc)
req.append(source2)
assert req.raw == source2
assert req.account.id == acc.id
assert isinstance(req.json(), basestring)
# make a valid one, and then do something invalid to it
req = ApiRequest(source, account=acc)
with self.assertRaises(dataobj.DataStructureException):
req.append({"whatever" : "here"})
def test_02_find_public_record(self):
# Find a public record with a variety of identifiers
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.save(blocking=True)
# document to form the basis of the queries
source2 = RequestFixtureFactory.example()
# create sources with one of each kind of identifier, then look them up using the
# find_public_record and find_public_record_by_identifier methods
pid = deepcopy(source2)
del pid["record"]["dc:identifier"]
req = Request(pid)
req.public_id = pub.id
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
doi = deepcopy(source2)
doi["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/me"}]
req = Request(doi)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/me")
assert pub11 is not None
pmid = deepcopy(source2)
pmid["record"]["dc:identifier"] = [{"type": "pmid", "id": "87654321"}]
req = Request(pmid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmid", "87654321")
assert pub11 is not None
pmcid = deepcopy(source2)
pmcid["record"]["dc:identifier"] = [{"type": "pmcid", "id": "PMC1234"}]
req = Request(pmcid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmcid", "PMC1234")
assert pub11 is not None
url = deepcopy(source2)
url["record"]["dc:identifier"] = [{"type": "url", "id": "http://example.com/whatever"}]
req = Request(url)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("url", "http://example.com/whatever")
assert pub11 is not None
# finally, ensure that you don't get a match when you shouldn't
null = deepcopy(source2)
null["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/another"}]
req = Request(null)
pub1 = PublicApi.find_public_record(req)
assert pub1 is None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/another")
assert pub11 is None
def test_03_publish_new(self):
# Publish a new request
source = RequestFixtureFactory.example()
req = Request(source)
pub = PublicApi.publish(req)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert pub2 is not None
def test_10_validate(self):
# Check the various validation routines
acc = MonitorUKAccount()
acc.save()
source = RequestFixtureFactory.record()
# first prove that our source record validates
csource = deepcopy(source)
csource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
req = ApiRequest(csource, account=acc)
assert req.raw == csource
# 1 - a record with no identifiers
asource = deepcopy(source)
del asource["dc:identifier"]
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 2 - a record with no apc
asource = deepcopy(source)
del asource["jm:apc"]
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 3 - an apc with no inc vat amount
asource = deepcopy(source)
del asource["jm:apc"][0]["amount_inc_vat_gbp"]
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 4 - a record with a date in the wrong format
asource = deepcopy(source)
asource["dcterms:dateAccepted"] = "sometime last week"
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 5 - a record with an unparseable number
asource = deepcopy(source)
asource["jm:apc"][0]["amount_inc_vat_gbp"] = "three fifty"
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
# 6 - a record with an invalid currency code
asource = deepcopy(source)
asource["jm:apc"][0]["currency"] = "XXX"
asource["@context"] = app.config.get("API_JSON_LD_CONTEXT")
with self.assertRaises(dataobj.DataStructureException):
req = ApiRequest(asource, account=acc)
def test_09_responses(self):
# Check that we get appropriate JSON responses
source = RequestFixtureFactory.record()
acc = MonitorUKAccount()
acc.save(blocking=True)
req = ApiRequest(source, account=acc)
req.save()
# have a look at what we'd expect the responses to be if this was a create
cr = req.created_response()
assert cr.get("status") == "created"
assert cr.get("request_id") is not None
assert cr.get("public_id") == "10.1234/me"
# now look at the responses if this was an update/append
ur = req.updated_response()
assert ur.get("status") == "updated"
assert ur.get("request_id") is not None
assert ur.get("public_id") == "10.1234/me"
# now look at a delete
dr = req.deleted_response()
assert dr.get("status") == "deleted"
assert dr.get("request_id") is not None
assert dr.get("public_id") == "10.1234/me"
# just make a quick check to be sure that if no DOI is present, we get the right kind of info in the public_id
source2 = RequestFixtureFactory.record()
source2["dc:identifier"] = [{"type" : "pmcid", "id" : "PMC1234"}]
req2 = ApiRequest(source2, account=acc)
req2.save()
cr = req.created_response()
assert cr.get("public_id") == req.id
ur = req.updated_response()
assert ur.get("public_id") == req.id
dr = req.deleted_response()
assert dr.get("public_id") == req.id
def test_05_request_dao(self):
# Check the DAO methods on the Request object
dao = Request()
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test1"
req.action = "update"
req.public_id = "abcdefg"
req.save(blocking=True)
req2 = dao.pull(req.id)
assert req2 is not None
# check successful queries for identifiers
res = dao.find_by_identifier("doi", "10.1234/me", "test1")
assert len(res) == 1
res = dao.find_by_identifier("pmcid", "PMC1234", "test1")
assert len(res) == 1
res = dao.find_by_identifier("pmid", "87654321", "test1")
assert len(res) == 1
res = dao.find_by_identifier("url", "http://example.com/whatever", "test1")
assert len(res) == 1
# check unsuccessful ones
res = dao.find_by_identifier("doi", "10.1234/you", "test1")
assert len(res) == 0
res = dao.find_by_identifier("pmcid", "PMC5678", "test1")
assert len(res) == 0
res = dao.find_by_identifier("pmid", "123456789", "test1")
assert len(res) == 0
res = dao.find_by_identifier("url", "http://example.com/this", "test1")
assert len(res) == 0
# and check using the wrong owner
res = dao.find_by_identifier("doi", "10.1234/me", "test2")
assert len(res) == 0
res = dao.find_by_identifier("pmcid", "PMC1234", "test2")
assert len(res) == 0
res = dao.find_by_identifier("pmid", "87654321", "test2")
assert len(res) == 0
res = dao.find_by_identifier("url", "http://example.com/whatever", "test2")
assert len(res) == 0
def test_10_find_request(self):
# Find a Request through a number of routes
source = RequestFixtureFactory.example()
req = Request(source)
req.save(blocking=True)
time.sleep(2)
source = RequestFixtureFactory.example()
req1 = Request(source)
req1.save(blocking=True)
# document to form the basis of the queries
source2 = RequestFixtureFactory.example()
# create sources with one of each kind of identifier, then look them up using the
# find_request_by_identifier method
result = RequestApi.find_request_by_identifier("doi", "10.1234/me", "abcdefghij")
assert result is not None
assert result.created_date == req1.created_date
result = RequestApi.find_request_by_identifier("pmid", "87654321", "abcdefghij")
assert result is not None
assert result.created_date == req1.created_date
result = RequestApi.find_request_by_identifier("pmcid", "PMC1234", "abcdefghij")
assert result is not None
assert result.created_date == req1.created_date
result = RequestApi.find_request_by_identifier("url", "http://example.com/whatever", "abcdefghij")
assert result is not None
assert result.created_date == req1.created_date
# finally, ensure that you don't get a match when you shouldn't
result = RequestApi.find_request_by_identifier("doi", "10.1234/another", "abcdefghij")
assert result is None
result = RequestApi.find_request_by_identifier("doi", "10.1234/me", "test")
assert result is None
def test_11_process_requests_exception(self):
# What happens when the process_reuests method fails for a variety of reasons
sources = RequestFixtureFactory.request_per_day("2001-01", 9)
dois = ["10.1234/first", "10.1234/second", "10.1234/third"]
# we're going to construct a series of requests for each doi
# starting with a create, then an update, followed by a delete
# (not that it matters, as we're going to pump them through a mock)
for i in range(len(sources)):
s = sources[i]
doi_idx = i % 3 # iterate over the dois 3 times
doi = dois[doi_idx]
s["record"]["dc:identifier"] = [{"type": "doi", "id": doi}]
if i < 3:
s["record"]["dc:title"] = "Create"
req = Request(s)
req.action = "update"
req.save()
elif i < 6:
s["record"]["dc:title"] = "Update"
req = Request(s)
req.action = "update"
req.save()
else:
s["record"]["dc:title"] = "Delete"
req = Request(s)
req.action = "delete"
req.save()
time.sleep(2)
# set up the mocks
PublicApi.publish = publish_mock
PublicApi.remove = delete_mock
# now run the process job back to the first day
with self.assertRaises(TestException):
WorkflowApi.process_requests()
# we know this died during the 6th update request being processed,
# so just check that the workflow state reflects that
wfs_dao = WorkflowState()
wfs = wfs_dao.pull("requests")
assert wfs.last_request == "2001-01-05T00:00:00Z"
assert len(wfs.already_processed) == 1
def test_07_save_delete(self):
# Work through acycle of saves and deletes to observe the outputs
source = RequestFixtureFactory.record()
acc = MonitorUKAccount()
acc.save(blocking=True)
req = ApiRequest(source, account=acc)
req.save()
dao = Request()
req2 = dao.pull(req.request.id)
assert req2 is not None
assert req2.owner == acc.id
assert req2.record == source
assert req2.action == "update"
# now publish the request
PublicApi.publish(req2)
time.sleep(2)
# now pull the object as identified by its API identifier (which should be the DOI)
source2 = deepcopy(source)
source2["dc:title"] = "An update"
next = ApiRequest.pull(req.id, account=acc)
next.update(source2)
next.save()
# now, at this point we should have 2 request objects in the index. One for the
# original save, and one for the new save
req3 = dao.pull(next.request.id)
assert req3 is not None
assert req3.owner == acc.id
assert req3.record == source2
assert req3.action == "update"
# now issue a delete on the same record
next.delete()
# by now we should have 3 request objects in the index, 2 for the above updates
# and one for the delete request
req4 = dao.pull(next.request.id)
assert req4 is not None
assert req4.owner == acc.id
assert req4.record == source2
assert req4.action == "delete"
def test_01_request_delete(self):
# Create a delete request
record = RequestFixtureFactory.record()
acc = BasicAccount()
acc.id = "test1"
r = RequestApi.delete(record, acc, "01010101")
# now check it was saved correctly
assert r is not None
r1 = r.pull(r.id)
assert r1 is not None
assert r1.owner == acc.id
assert r1.action == "delete"
assert r1.public_id == "01010101"
def test_07_request2public(self):
# Check the conversion of a Request to a PublicAPC
source = RequestFixtureFactory.example()
req = Request(source)
pub = req.make_public_apc()
assert pub is not None
assert pub.record is not None
assert len(pub.apc_records) == 1
setrefs = []
for apc in pub.apc_records:
assert apc.get("ref") is not None
setrefs.append(apc.get("ref"))
assert len(setrefs) == 1
refs = pub.get_apc_refs(req.owner)
assert len(refs) == 1
assert refs[0] == setrefs[0]
def test_03_pull_request(self):
# Pull a Request through the ApiRequest object
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save(blocking=True)
# first make a request which contains that doi
req_source = RequestFixtureFactory.example()
req = Request(req_source)
req.save(blocking=True)
# you can't pull a request object, so just show that that's true...
# pull by doi should fail
result = ApiRequest.pull("10.1234/me", account=acc)
assert result is None
# pull by request id should fail
result = ApiRequest.pull(req.id, account=acc)
assert result is None
def test_06_get_id(self):
# Get the operative ID of a record
acc = MonitorUKAccount()
acc.id = "abcdefghij"
source = RequestFixtureFactory.record()
# first, provide a record which has a doi, and ensure that's the id we get back
req = ApiRequest(source, account=acc)
assert req.id == "10.1234/me"
# now make a public record, and check we get the right ids for it
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
del pub_source["record"]["dc:identifier"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull(pub.id, account=acc)
assert result.id == pub.id
assert result.id == result.public_id
def test_04_record_methods(self):
# Check all the Record methods work
classes = [Request, PublicAPC]
source = RequestFixtureFactory.example()
record = source.get("record")
additional_apcs = deepcopy(record.get("jm:apc")) + deepcopy(record.get("jm:apc"))
for c in classes:
inst = c()
assert isinstance(inst, RecordMethods)
inst.record = deepcopy(record)
assert inst.doi == "10.1234/me"
assert inst.pmid == "87654321"
assert inst.pmcid == "PMC1234"
assert inst.url == "http://example.com/whatever"
assert len(inst.apc_records) == 1
inst.apc_records = additional_apcs
assert len(inst.apc_records) == 2
# now try adding and removing apc records
withref = deepcopy(record.get("jm:apc")[0])
withref["ref"] = "myref"
inst.add_apc_record(withref)
assert len(inst.apc_records) == 3
inst.remove_apc_by_ref("myref")
assert len(inst.apc_records) == 2
for apc in inst.apc_records:
assert "ref" not in apc
assert inst.has_apcs() is True
inst.apc_records = []
assert inst.has_apcs() is False
def test_01_request_update(self):
# Create an update Request
record = RequestFixtureFactory.record()
acc = BasicAccount()
acc.id = "test1"
# without a public id
r1 = RequestApi.update(record, acc)
# with a public id
r2 = RequestApi.update(record, acc, "01010101")
# now check they were saved correctly
assert r1 is not None
assert r2 is not None
r11 = r1.pull(r1.id)
assert r11 is not None
r21 = r2.pull(r2.id)
assert r21 is not None
def test_13_request_iterator(self):
# Check we can iterate successfully over lists of Requests
sources = RequestFixtureFactory.request_per_day("2001-01", 10)
for s in sources:
req = Request(s)
req.save()
time.sleep(2)
dao = Request()
gen = dao.list_all_since("2001-01-01T00:00:00Z", page_size=5) # set the page size small, to ensure the iterator has to work
results = [x for x in gen]
assert len(results) == 10
dates = [r.created_date for r in results]
comp = deepcopy(dates)
comp.sort() # this puts the dates in ascending order (i.e. oldest first)
# the point of this comparison is to show that the results came out in the right order.
# that is, oldest first
assert dates == comp
def test_08_remove_separate(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving only one owner behind
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub = PublicAPC(source2)
pub.add_apc_for_owner("test", apc_record)
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert len(pub2.get_apcs_by_owner("test")) == 0
assert len(pub2.get_apcs_by_owner("abcdefg")) == 1
def test_11_process_requests_cycle(self):
# Run through the process of processing a Request into a PublicAPC
source = RequestFixtureFactory.example()
if "id" in source:
del source["id"]
pub_dao = PublicAPC()
wfs_dao = WorkflowState()
# first make a record for the first time
first = deepcopy(source)
del first["record"]["dc:title"]
req = Request(first)
req.owner = "test"
req.action = "update"
req.save(blocking=True)
# run the job
WorkflowApi.process_requests()
time.sleep(2)
# first check that a public record was made
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") is None
# check that the workflow state was created
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req.created_date
assert wfs.already_processed == [req.id]
# now run an update with a different date
second = deepcopy(source)
second["record"]["dc:title"] = "Update"
second["created_date"] = "2002-01-01T00:00:00Z"
req2 = Request(second)
req2.owner = "test"
req2.action = "update"
req2.save(blocking=True)
# run the job again
WorkflowApi.process_requests()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") == "Update"
# check that the workflow state was updated
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req2.created_date
assert wfs.already_processed == [req2.id]
# now run an update with the same date, to observe the difference in the workflow state
third = deepcopy(source)
third["record"]["dc:title"] = "Update 2"
third["created_date"] = "2002-01-01T00:00:00Z"
req3 = Request(third)
req3.owner = "test"
req3.action = "update"
req3.save(blocking=True)
# run the job again
WorkflowApi.process_requests()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert (
pubs[0].record.get("dc:title") == "Update 2"
) # should have been updated, as there are only apc contributions from one source
# check that the workflow state was updated
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req3.created_date
assert wfs.already_processed == [req2.id, req3.id] # processed records should have been appended
# finally issue a delete request
fourth = deepcopy(source)
fourth["created_date"] = "2003-01-01T00:00:00Z"
req4 = Request(fourth)
req4.owner = "test"
req4.action = "delete"
req4.save(blocking=True)
# run the job again
WorkflowApi.process_requests()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 0
# check that the workflow state was updated
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req4.created_date
assert wfs.already_processed == [req4.id] # processed records should have been appended
