def test_09_remove_permanent(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving no owners, thus deleting the record
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
pub = PublicAPC(source2)
pub.remove_apcs_by_owner("abcdefg") # clear the existing apc record
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub.add_apc_for_owner("test", apc_record) # add a new, known one
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert pub2 is None
def test_15_public_indexing(self):
# Check that public records are indexed correctly
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
apc_record1 = PublicAPCFixtureFactory.apc_record()
apc_record2 = PublicAPCFixtureFactory.apc_record()
del apc_record1["amount_inc_vat_gbp"]
apc_record1["amount_ex_vat_gbp"] = 1000
apc_record1["vat_gbp"] = 200
apc_record1["additional_costs"] = 100
apc_record2["amount_inc_vat_gbp"] = 2400
apc_record2["amount_ex_vat_gbp"] = 2000
apc_record2["vat_gbp"] = 400
apc_record2["additional_costs"] = 200
pub.apc_records = [apc_record1, apc_record2]
pub.prep()
# first check that the amount_inc_vat_gbp was calculated or kept
assert pub.apc_records[0]["amount_inc_vat_gbp"] == 1200
# now check all the indexed amounts add up
assert pub.data.get("index", {}).get("additional_costs") == 300
assert pub.data.get("index", {}).get("vat") == 600
assert pub.data.get("index", {}).get("amount_ex_vat") == 3000
assert pub.data.get("index", {}).get("amount_inc_vat") == 3600
assert pub.data.get("index", {}).get("grand_total") == 3900
def test_06_publish_update(self):
# Publish an update to an existing PublicAPC
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
apc_record = PublicAPCFixtureFactory.apc_record()
result = PublicAPCFixtureFactory.record_merge_result()
del merge_source["jm:apc"]
del merge_target["jm:apc"]
del result["jm:apc"]
first = deepcopy(apc_record)
second = deepcopy(apc_record)
third = deepcopy(apc_record)
first["organisation_name"] = "First"
del first["ref"]
second["organisation_name"] = "Second"
del second["ref"]
third["organisation_name"] = "Third"
del third["ref"]
req = Request()
req.record = merge_source
req.add_apc_record(first)
req.owner = "11111"
pub = PublicAPC()
pub.record = merge_target
pub.add_apc_for_owner("22222", second)
pub.add_apc_for_owner("11111", third)
pub.save(blocking=True)
PublicApi.publish(req)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
# first check that the apcs are as we would expect
one = pub2.get_apcs_by_owner("11111")
two = pub2.get_apcs_by_owner("22222")
assert len(one) == 1
assert len(two) == 1
assert one[0]["organisation_name"] == "First"
assert two[0]["organisation_name"] == "Second"
# now check that the metadata merge proceeded correctly
record = pub2.record
del record["jm:apc"]
assert record == result
def test_04_pull_public(self):
# Pull a PublicAPC through the ApiRequest object
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save()
acc2 = MonitorUKAccount()
acc2.id = "qwerty"
acc.save(blocking=True)
# make a competing public version, and check that a pull on the doi works
pub_source = PublicAPCFixtureFactory.example()
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull("10.1234/me", account=acc)
assert result.raw == pub.clean_record
assert result.account.id == acc.id
assert result.public_id == pub.id
craw = deepcopy(pub.clean_record)
craw["@context"] = app.config.get("API_JSON_LD_CONTEXT")
assert json.loads(result.json()) == craw
# also try the pull with the wrong owner, which should work
result = ApiRequest.pull("10.1234/me", account=acc2)
assert result is not None
def test_03_public(self):
# Check we can instantiate and work with a PublicAPC
# first make a blank one
pub = PublicAPC()
# now make one around the fixture
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
# make one with a broken source
broken = {"whatever" : "broken"}
with self.assertRaises(dataobj.DataStructureException):
pub = PublicAPC(broken)
# now make one bit by bit
pub = PublicAPC()
pub.record = source.get("record")
pub.set_apc_ref("test1", "1111111111")
apc_refs = pub.get_apc_refs("test1")
assert len(apc_refs) == 1
assert apc_refs[0] == "1111111111"
# now make it broken
pub = PublicAPC()
with self.assertRaises(dataobj.DataStructureException):
pub.record = {"random" : "stuff"}
def test_02_find_public_record(self):
# Find a public record with a variety of identifiers
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.save(blocking=True)
# document to form the basis of the queries
source2 = RequestFixtureFactory.example()
# create sources with one of each kind of identifier, then look them up using the
# find_public_record and find_public_record_by_identifier methods
pid = deepcopy(source2)
del pid["record"]["dc:identifier"]
req = Request(pid)
req.public_id = pub.id
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
doi = deepcopy(source2)
doi["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/me"}]
req = Request(doi)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/me")
assert pub11 is not None
pmid = deepcopy(source2)
pmid["record"]["dc:identifier"] = [{"type": "pmid", "id": "87654321"}]
req = Request(pmid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmid", "87654321")
assert pub11 is not None
pmcid = deepcopy(source2)
pmcid["record"]["dc:identifier"] = [{"type": "pmcid", "id": "PMC1234"}]
req = Request(pmcid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmcid", "PMC1234")
assert pub11 is not None
url = deepcopy(source2)
url["record"]["dc:identifier"] = [{"type": "url", "id": "http://example.com/whatever"}]
req = Request(url)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("url", "http://example.com/whatever")
assert pub11 is not None
# finally, ensure that you don't get a match when you shouldn't
null = deepcopy(source2)
null["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/another"}]
req = Request(null)
pub1 = PublicApi.find_public_record(req)
assert pub1 is None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/another")
assert pub11 is None
def test_11_public_refs(self):
# Check that APC refs are handled correctly by PublicAPCs
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
assert pub.record.get("jm:apc")[0]["ref"] == "1111111111"
assert "ref" not in pub.clean_record.get("jm:apc")[0]
def test_08_public_apc_methods(self):
# Check all the PublcAPC object methods
pub = PublicAPC()
assert len(pub.get_apc_refs("11111")) == 0
pub.set_apc_ref("11111", "aaaaa")
assert pub.get_apc_refs("11111")[0] == "aaaaa"
pub.set_apc_ref("22222", "bbbbb")
assert len(pub.get_apc_refs("22222")) == 1
pub.remove_apc_refs("11111")
assert len(pub.get_apc_refs("22222")) == 1
assert len(pub.get_apc_refs("11111")) == 0
assert "11111" not in pub.list_owners()
assert "22222" in pub.list_owners()
pub.remove_apc_refs("22222")
assert len(pub.get_apcs_by_owner("11111")) == 0
apc_record = PublicAPCFixtureFactory.apc_record()
first = deepcopy(apc_record)
first["ref"] = "aaaaa"
pub.add_apc_for_owner("11111", first)
assert len(pub.get_apc_refs("11111")) == 1
assert pub.get_apc_refs("11111")[0] == "aaaaa"
assert len(pub.get_apcs_by_owner("11111")) == 1
assert pub.get_apcs_by_owner("11111")[0]["ref"] == "aaaaa"
second = deepcopy(apc_record)
second["ref"] = "bbbbb"
pub.add_apc_for_owner("22222", second)
assert len(pub.get_apc_refs("22222")) == 1
assert pub.get_apc_refs("22222")[0] == "bbbbb"
assert len(pub.get_apcs_by_owner("22222")) == 1
assert pub.get_apcs_by_owner("22222")[0]["ref"] == "bbbbb"
assert len(pub.apc_records) == 2
pub.remove_apcs_by_owner("11111")
assert len(pub.apc_records) == 1
assert pub.apc_records[0]["ref"] == "bbbbb"
assert len(pub.get_apc_refs("22222")) == 1
assert pub.get_apc_refs("22222")[0] == "bbbbb"
assert len(pub.get_apcs_by_owner("22222")) == 1
assert pub.get_apcs_by_owner("22222")[0]["ref"] == "bbbbb"
pub.remove_apcs_by_owner("22222")
assert len(pub.apc_records) == 0
assert len(pub.get_apc_refs("11111")) == 0
assert len(pub.get_apc_refs("22222")) == 0
def test_05_enhance_metadata(self):
# Enhance the bibliographic metadata on a PublicAPC record
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
result = PublicAPCFixtureFactory.record_merge_result()
source = PublicAPC()
source.record = merge_source
source.set_apc_ref("22222", "bbbbb")
target = PublicAPC()
target.record = merge_target
target.set_apc_ref("11111", "aaaaa")
PublicApi.enhance_metadata(source, target)
assert target.record == result
assert target.get_apc_refs("11111") == ["aaaaa"]
assert target.get_apc_refs("22222") == []
def test_14_copy_overwrite(self):
# Check the copy and overwrite cloning methods work
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub2 = pub.copy()
assert pub2.record == pub.record
assert pub2.admin == pub.admin
assert pub2.id == pub.id
source3 = PublicAPCFixtureFactory.example()
source3["record"]["dc:title"] = "Overwrite"
pub3 = PublicAPC(source3)
pub2.overwrite(pub3)
assert pub2.record == pub3.record
assert pub2.admin == pub3.admin
assert pub2.id == pub3.id
def test_05_low_quota(self):
# Check what happens when the use has a low quota on Lantern
global QUOTA
QUOTA = 1
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 1
assert len(CREATED_JOBS) == 1
job = CREATED_JOBS[0]
assert job["email"] == "*****@*****.**"
assert len(job["list"]) == 1
def test_09_merge_records(self):
# Check the merge of PublicAPC records
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
result = PublicAPCFixtureFactory.record_merge_result()
source = PublicAPC()
source.record = merge_source
source.set_apc_ref("22222", "bbbbb")
target = PublicAPC()
target.record = merge_target
target.set_apc_ref("11111", "aaaaa")
target.merge_records(source)
assert target.record == result
assert target.get_apc_refs("11111") == ["aaaaa"]
assert target.get_apc_refs("22222") == []
# now just try some basic error cases
with self.assertRaises(ModelException):
target.merge_records({"random" : "data"})
def test_08_remove_separate(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving only one owner behind
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub = PublicAPC(source2)
pub.add_apc_for_owner("test", apc_record)
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert len(pub2.get_apcs_by_owner("test")) == 0
assert len(pub2.get_apcs_by_owner("abcdefg")) == 1
def test_07_separate_records(self):
# Separate an incoming Request from its corresponding PublicAPC
apc_record = PublicAPCFixtureFactory.apc_record()
req = Request()
req.owner = "test"
pub = PublicAPC()
pub.add_apc_for_owner("test", apc_record)
pub.add_apc_for_owner("test", apc_record)
assert len(pub.apc_records) == 2
PublicApi.separate_records(req, pub)
assert not pub.has_apcs()
def test_06_public_dao(self):
# Check all the DAO methods on the PublicAPC object
dao = PublicAPC()
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.set_apc_ref("test1", "1111111111")
pub.save(blocking=True)
# first try the straight-forward pull
pub2 = dao.pull(pub.id)
assert pub2 is not None
# now do the successful queries
res = dao.find_by_doi("10.1234/me")
assert len(res) == 1
res = dao.find_by_pmid("87654321")
assert len(res) == 1
res = dao.find_by_pmcid("PMC1234")
assert len(res) == 1
res = dao.find_by_url("http://example.com/whatever")
assert len(res) == 1
# now to check those queries don't always return, make sure we can get 0 results
res = dao.find_by_doi("10.1234/whatever")
assert len(res) == 0
res = dao.find_by_pmid("88888888")
assert len(res) == 0
res = dao.find_by_pmcid("PMC1111")
assert len(res) == 0
res = dao.find_by_url("http://example.com/another")
assert len(res) == 0
gen = dao.list_by_owner("abcdefg")
count = 0
for apc in gen:
count += 1
assert count == 1
def test_05_pull_then_update(self):
# Pull a record through the ApiRequest object and then udpate it
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save(blocking=True)
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
pub_source2 = deepcopy(pub_source.get("record"))
pub_source2["dc:title"] = "An update"
result = ApiRequest.pull(pub.id, account=acc)
result.update(pub_source2)
assert result.raw == pub_source2
assert result.account.id == acc.id
def test_06_get_id(self):
# Get the operative ID of a record
acc = MonitorUKAccount()
acc.id = "abcdefghij"
source = RequestFixtureFactory.record()
# first, provide a record which has a doi, and ensure that's the id we get back
req = ApiRequest(source, account=acc)
assert req.id == "10.1234/me"
# now make a public record, and check we get the right ids for it
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
del pub_source["record"]["dc:identifier"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull(pub.id, account=acc)
assert result.id == pub.id
assert result.id == result.public_id
def test_04_merge_public_apcs(self):
# Merge two PublicAPC records together
source_source = PublicAPCFixtureFactory.example()
target_source = PublicAPCFixtureFactory.example()
# first try a merge with no apc records (this shouldn't ever happen in normal operation)
ss1 = deepcopy(source_source)
del ss1["record"]["jm:apc"]
source1 = PublicAPC(ss1)
ts1 = deepcopy(target_source)
del ts1["record"]["jm:apc"]
target1 = PublicAPC(ts1)
result = PublicApi.merge_public_apcs(source1, target1)
assert len(result.apc_records) == 0
# next try a merge with only apc records in the source (again, shouldn't really happen in real life)
ss2 = deepcopy(source_source)
source2 = PublicAPC(ss2)
ts2 = deepcopy(target_source)
del ts2["record"]["jm:apc"]
target2 = PublicAPC(ts2)
result = PublicApi.merge_public_apcs(source2, target2)
assert len(result.apc_records) == 1
# next try a merge with only apc records in the target (also shouldn't happen in real life)
ss3 = deepcopy(source_source)
source3 = PublicAPC(ss3)
ts3 = deepcopy(target_source)
del ts3["record"]["jm:apc"]
target3 = PublicAPC(ts3)
result = PublicApi.merge_public_apcs(source3, target3)
assert len(result.apc_records) == 1
# finally try a merge with the following criteria:
# replacements apcs and new apcs in the source record
# existing apcs and other apcs in the target record
apc_record = PublicAPCFixtureFactory.apc_record()
first = deepcopy(apc_record)
first["ref"] = "aaaaa"
second = deepcopy(apc_record)
second["ref"] = "bbbbb"
ss4 = deepcopy(source_source)
del ss4["record"]["jm:apc"]
source4 = PublicAPC(ss4)
source4.add_apc_for_owner("11111", first)
source4.add_apc_for_owner("11111", second)
third = deepcopy(apc_record)
third["ref"] = "ccccc"
fourth = deepcopy(apc_record)
fourth["ref"] = "ddddd"
ts4 = deepcopy(target_source)
del ts4["record"]["jm:apc"]
target4 = PublicAPC(ts4)
target4.add_apc_for_owner("11111", third)
target4.add_apc_for_owner("22222", fourth)
result = PublicApi.merge_public_apcs(source4, target4)
assert len(result.apc_records) == 3
ones = result.get_apcs_by_owner("11111")
assert len(ones) == 2
refs = [o.get("ref") for o in ones]
assert "aaaaa" in refs
assert "bbbbb" in refs
assert "ccccc" not in refs
assert "ddddd" not in refs
twos = result.get_apcs_by_owner("22222")
assert len(twos) == 1
refs = [o.get("ref") for o in twos]
assert "aaaaa" not in refs
assert "bbbbb" not in refs
assert "ccccc" not in refs
assert "ddddd" in refs
def test_04_create_job(self):
# Check we can create jobs correctly
acc1 = MonitorUKAccount()
acc1.email = "*****@*****.**"
acc1.save()
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
acc3 = MonitorUKAccount()
acc3.email = "*****@*****.**"
acc3.lantern_email = "*****@*****.**"
acc3.lantern_api_key = "987654321"
acc3.save(blocking=True)
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save()
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
assert len(CREATED_JOBS) == 2
count = 0
for job in CREATED_JOBS:
if job["email"] == "*****@*****.**":
count += 1
assert len(job["list"]) == 2
elif job["email"] == "*****@*****.**":
count += 10
assert len(job["list"]) == 2
assert count == 11
# now do the same thing again. The jobs should not change, as we've already created jobs
# for all the public records
LanternApi.make_new_jobs()
time.sleep(2)
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
def test_13_process_ehnancements_cycle(self):
# Run through the process of processing an enhancement
source = EnhancementFixtureFactory.example()
if "id" in source:
del source["id"]
pub_dao = PublicAPC()
wfs_dao = WorkflowState()
# first make a public record for us to enhance
first = PublicAPCFixtureFactory.example()
del first["record"]["dc:title"]
pub = PublicAPC(first)
pub.save(blocking=True)
# now create an enhancements on the record
second = deepcopy(source)
second["record"]["dc:title"] = "Update"
second["created_date"] = "2002-01-01T00:00:00Z"
en = Enhancement(second)
en.public_id = pub.id
en.save(blocking=True)
# run the job
WorkflowApi.process_enhancements()
time.sleep(2)
# check that the workflow state was created
wfs = wfs_dao.pull("enhancements")
assert wfs is not None
assert wfs.last_request == en.created_date
assert wfs.already_processed == [en.id]
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") == "Update"
# now run an update with the same date, to observe the difference in the workflow state
third = deepcopy(source)
third["record"]["dc:title"] = "Update 2"
third["created_date"] = "2002-01-01T00:00:00Z"
en2 = Enhancement(third)
en2.public_id = pub.id
en2.save(blocking=True)
# run the job again
WorkflowApi.process_enhancements()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert (
pubs[0].record.get("dc:title") == "Update"
) # should not have been updated, since data was already present
# check that the workflow state was updated
wfs = wfs_dao.pull("enhancements")
assert wfs is not None
assert wfs.last_request == en2.created_date
assert wfs.already_processed == [en.id, en2.id] # processed records should have been appended
