def test_04_pull_public(self):
# Pull a PublicAPC through the ApiRequest object
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save()
acc2 = MonitorUKAccount()
acc2.id = "qwerty"
acc.save(blocking=True)
# make a competing public version, and check that a pull on the doi works
pub_source = PublicAPCFixtureFactory.example()
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull("10.1234/me", account=acc)
assert result.raw == pub.clean_record
assert result.account.id == acc.id
assert result.public_id == pub.id
craw = deepcopy(pub.clean_record)
craw["@context"] = app.config.get("API_JSON_LD_CONTEXT")
assert json.loads(result.json()) == craw
# also try the pull with the wrong owner, which should work
result = ApiRequest.pull("10.1234/me", account=acc2)
assert result is not None
def test_09_remove_permanent(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving no owners, thus deleting the record
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
pub = PublicAPC(source2)
pub.remove_apcs_by_owner("abcdefg") # clear the existing apc record
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub.add_apc_for_owner("test", apc_record) # add a new, known one
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert pub2 is None
def test_02_find_public_record(self):
# Find a public record with a variety of identifiers
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.save(blocking=True)
# document to form the basis of the queries
source2 = RequestFixtureFactory.example()
# create sources with one of each kind of identifier, then look them up using the
# find_public_record and find_public_record_by_identifier methods
pid = deepcopy(source2)
del pid["record"]["dc:identifier"]
req = Request(pid)
req.public_id = pub.id
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
doi = deepcopy(source2)
doi["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/me"}]
req = Request(doi)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/me")
assert pub11 is not None
pmid = deepcopy(source2)
pmid["record"]["dc:identifier"] = [{"type": "pmid", "id": "87654321"}]
req = Request(pmid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmid", "87654321")
assert pub11 is not None
pmcid = deepcopy(source2)
pmcid["record"]["dc:identifier"] = [{"type": "pmcid", "id": "PMC1234"}]
req = Request(pmcid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmcid", "PMC1234")
assert pub11 is not None
url = deepcopy(source2)
url["record"]["dc:identifier"] = [{"type": "url", "id": "http://example.com/whatever"}]
req = Request(url)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("url", "http://example.com/whatever")
assert pub11 is not None
# finally, ensure that you don't get a match when you shouldn't
null = deepcopy(source2)
null["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/another"}]
req = Request(null)
pub1 = PublicApi.find_public_record(req)
assert pub1 is None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/another")
assert pub11 is None
def test_06_publish_update(self):
# Publish an update to an existing PublicAPC
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
apc_record = PublicAPCFixtureFactory.apc_record()
result = PublicAPCFixtureFactory.record_merge_result()
del merge_source["jm:apc"]
del merge_target["jm:apc"]
del result["jm:apc"]
first = deepcopy(apc_record)
second = deepcopy(apc_record)
third = deepcopy(apc_record)
first["organisation_name"] = "First"
del first["ref"]
second["organisation_name"] = "Second"
del second["ref"]
third["organisation_name"] = "Third"
del third["ref"]
req = Request()
req.record = merge_source
req.add_apc_record(first)
req.owner = "11111"
pub = PublicAPC()
pub.record = merge_target
pub.add_apc_for_owner("22222", second)
pub.add_apc_for_owner("11111", third)
pub.save(blocking=True)
PublicApi.publish(req)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
# first check that the apcs are as we would expect
one = pub2.get_apcs_by_owner("11111")
two = pub2.get_apcs_by_owner("22222")
assert len(one) == 1
assert len(two) == 1
assert one[0]["organisation_name"] == "First"
assert two[0]["organisation_name"] == "Second"
# now check that the metadata merge proceeded correctly
record = pub2.record
del record["jm:apc"]
assert record == result
def test_06_public_dao(self):
# Check all the DAO methods on the PublicAPC object
dao = PublicAPC()
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.set_apc_ref("test1", "1111111111")
pub.save(blocking=True)
# first try the straight-forward pull
pub2 = dao.pull(pub.id)
assert pub2 is not None
# now do the successful queries
res = dao.find_by_doi("10.1234/me")
assert len(res) == 1
res = dao.find_by_pmid("87654321")
assert len(res) == 1
res = dao.find_by_pmcid("PMC1234")
assert len(res) == 1
res = dao.find_by_url("http://example.com/whatever")
assert len(res) == 1
# now to check those queries don't always return, make sure we can get 0 results
res = dao.find_by_doi("10.1234/whatever")
assert len(res) == 0
res = dao.find_by_pmid("88888888")
assert len(res) == 0
res = dao.find_by_pmcid("PMC1111")
assert len(res) == 0
res = dao.find_by_url("http://example.com/another")
assert len(res) == 0
gen = dao.list_by_owner("abcdefg")
count = 0
for apc in gen:
count += 1
assert count == 1
def test_05_pull_then_update(self):
# Pull a record through the ApiRequest object and then udpate it
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save(blocking=True)
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
pub_source2 = deepcopy(pub_source.get("record"))
pub_source2["dc:title"] = "An update"
result = ApiRequest.pull(pub.id, account=acc)
result.update(pub_source2)
assert result.raw == pub_source2
assert result.account.id == acc.id
def test_06_get_id(self):
# Get the operative ID of a record
acc = MonitorUKAccount()
acc.id = "abcdefghij"
source = RequestFixtureFactory.record()
# first, provide a record which has a doi, and ensure that's the id we get back
req = ApiRequest(source, account=acc)
assert req.id == "10.1234/me"
# now make a public record, and check we get the right ids for it
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
del pub_source["record"]["dc:identifier"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull(pub.id, account=acc)
assert result.id == pub.id
assert result.id == result.public_id
def test_05_low_quota(self):
# Check what happens when the use has a low quota on Lantern
global QUOTA
QUOTA = 1
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 1
assert len(CREATED_JOBS) == 1
job = CREATED_JOBS[0]
assert job["email"] == "*****@*****.**"
assert len(job["list"]) == 1
def test_08_remove_separate(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving only one owner behind
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub = PublicAPC(source2)
pub.add_apc_for_owner("test", apc_record)
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert len(pub2.get_apcs_by_owner("test")) == 0
assert len(pub2.get_apcs_by_owner("abcdefg")) == 1
def test_04_create_job(self):
# Check we can create jobs correctly
acc1 = MonitorUKAccount()
acc1.email = "*****@*****.**"
acc1.save()
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
acc3 = MonitorUKAccount()
acc3.email = "*****@*****.**"
acc3.lantern_email = "*****@*****.**"
acc3.lantern_api_key = "987654321"
acc3.save(blocking=True)
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save()
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
assert len(CREATED_JOBS) == 2
count = 0
for job in CREATED_JOBS:
if job["email"] == "*****@*****.**":
count += 1
assert len(job["list"]) == 2
elif job["email"] == "*****@*****.**":
count += 10
assert len(job["list"]) == 2
assert count == 11
# now do the same thing again. The jobs should not change, as we've already created jobs
# for all the public records
LanternApi.make_new_jobs()
time.sleep(2)
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
def test_13_process_ehnancements_cycle(self):
# Run through the process of processing an enhancement
source = EnhancementFixtureFactory.example()
if "id" in source:
del source["id"]
pub_dao = PublicAPC()
wfs_dao = WorkflowState()
# first make a public record for us to enhance
first = PublicAPCFixtureFactory.example()
del first["record"]["dc:title"]
pub = PublicAPC(first)
pub.save(blocking=True)
# now create an enhancements on the record
second = deepcopy(source)
second["record"]["dc:title"] = "Update"
second["created_date"] = "2002-01-01T00:00:00Z"
en = Enhancement(second)
en.public_id = pub.id
en.save(blocking=True)
# run the job
WorkflowApi.process_enhancements()
time.sleep(2)
# check that the workflow state was created
wfs = wfs_dao.pull("enhancements")
assert wfs is not None
assert wfs.last_request == en.created_date
assert wfs.already_processed == [en.id]
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") == "Update"
# now run an update with the same date, to observe the difference in the workflow state
third = deepcopy(source)
third["record"]["dc:title"] = "Update 2"
third["created_date"] = "2002-01-01T00:00:00Z"
en2 = Enhancement(third)
en2.public_id = pub.id
en2.save(blocking=True)
# run the job again
WorkflowApi.process_enhancements()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert (
pubs[0].record.get("dc:title") == "Update"
) # should not have been updated, since data was already present
# check that the workflow state was updated
wfs = wfs_dao.pull("enhancements")
assert wfs is not None
assert wfs.last_request == en2.created_date
assert wfs.already_processed == [en.id, en2.id] # processed records should have been appended
