def test_15_public_indexing(self):
# Check that public records are indexed correctly
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
apc_record1 = PublicAPCFixtureFactory.apc_record()
apc_record2 = PublicAPCFixtureFactory.apc_record()
del apc_record1["amount_inc_vat_gbp"]
apc_record1["amount_ex_vat_gbp"] = 1000
apc_record1["vat_gbp"] = 200
apc_record1["additional_costs"] = 100
apc_record2["amount_inc_vat_gbp"] = 2400
apc_record2["amount_ex_vat_gbp"] = 2000
apc_record2["vat_gbp"] = 400
apc_record2["additional_costs"] = 200
pub.apc_records = [apc_record1, apc_record2]
pub.prep()
# first check that the amount_inc_vat_gbp was calculated or kept
assert pub.apc_records[0]["amount_inc_vat_gbp"] == 1200
# now check all the indexed amounts add up
assert pub.data.get("index", {}).get("additional_costs") == 300
assert pub.data.get("index", {}).get("vat") == 600
assert pub.data.get("index", {}).get("amount_ex_vat") == 3000
assert pub.data.get("index", {}).get("amount_inc_vat") == 3600
assert pub.data.get("index", {}).get("grand_total") == 3900
def test_04_pull_public(self):
# Pull a PublicAPC through the ApiRequest object
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save()
acc2 = MonitorUKAccount()
acc2.id = "qwerty"
acc.save(blocking=True)
# make a competing public version, and check that a pull on the doi works
pub_source = PublicAPCFixtureFactory.example()
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull("10.1234/me", account=acc)
assert result.raw == pub.clean_record
assert result.account.id == acc.id
assert result.public_id == pub.id
craw = deepcopy(pub.clean_record)
craw["@context"] = app.config.get("API_JSON_LD_CONTEXT")
assert json.loads(result.json()) == craw
# also try the pull with the wrong owner, which should work
result = ApiRequest.pull("10.1234/me", account=acc2)
assert result is not None
def test_03_public(self):
# Check we can instantiate and work with a PublicAPC
# first make a blank one
pub = PublicAPC()
# now make one around the fixture
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
# make one with a broken source
broken = {"whatever" : "broken"}
with self.assertRaises(dataobj.DataStructureException):
pub = PublicAPC(broken)
# now make one bit by bit
pub = PublicAPC()
pub.record = source.get("record")
pub.set_apc_ref("test1", "1111111111")
apc_refs = pub.get_apc_refs("test1")
assert len(apc_refs) == 1
assert apc_refs[0] == "1111111111"
# now make it broken
pub = PublicAPC()
with self.assertRaises(dataobj.DataStructureException):
pub.record = {"random" : "stuff"}
def test_02_find_public_record(self):
# Find a public record with a variety of identifiers
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.save(blocking=True)
# document to form the basis of the queries
source2 = RequestFixtureFactory.example()
# create sources with one of each kind of identifier, then look them up using the
# find_public_record and find_public_record_by_identifier methods
pid = deepcopy(source2)
del pid["record"]["dc:identifier"]
req = Request(pid)
req.public_id = pub.id
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
doi = deepcopy(source2)
doi["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/me"}]
req = Request(doi)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/me")
assert pub11 is not None
pmid = deepcopy(source2)
pmid["record"]["dc:identifier"] = [{"type": "pmid", "id": "87654321"}]
req = Request(pmid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmid", "87654321")
assert pub11 is not None
pmcid = deepcopy(source2)
pmcid["record"]["dc:identifier"] = [{"type": "pmcid", "id": "PMC1234"}]
req = Request(pmcid)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("pmcid", "PMC1234")
assert pub11 is not None
url = deepcopy(source2)
url["record"]["dc:identifier"] = [{"type": "url", "id": "http://example.com/whatever"}]
req = Request(url)
pub1 = PublicApi.find_public_record(req)
assert pub1 is not None
pub11 = PublicApi.find_public_record_by_identifier("url", "http://example.com/whatever")
assert pub11 is not None
# finally, ensure that you don't get a match when you shouldn't
null = deepcopy(source2)
null["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/another"}]
req = Request(null)
pub1 = PublicApi.find_public_record(req)
assert pub1 is None
pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/another")
assert pub11 is None
def test_03_publish_new(self):
# Publish a new request
source = RequestFixtureFactory.example()
req = Request(source)
pub = PublicApi.publish(req)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert pub2 is not None
def test_09_remove_permanent(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving no owners, thus deleting the record
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
pub = PublicAPC(source2)
pub.remove_apcs_by_owner("abcdefg") # clear the existing apc record
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub.add_apc_for_owner("test", apc_record) # add a new, known one
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert pub2 is None
def test_12_publish_enhancement(self):
# Publish an Enhancement where no public record exists
source = EnhancementFixtureFactory.example()
en = Enhancement(source)
pub = PublicApi.publish(en)
# check that no public record was saved
assert pub.id is None
dao = PublicAPC()
pubs = [p for p in dao.iterall()]
assert len(pubs) == 0
def test_06_publish_update(self):
# Publish an update to an existing PublicAPC
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
apc_record = PublicAPCFixtureFactory.apc_record()
result = PublicAPCFixtureFactory.record_merge_result()
del merge_source["jm:apc"]
del merge_target["jm:apc"]
del result["jm:apc"]
first = deepcopy(apc_record)
second = deepcopy(apc_record)
third = deepcopy(apc_record)
first["organisation_name"] = "First"
del first["ref"]
second["organisation_name"] = "Second"
del second["ref"]
third["organisation_name"] = "Third"
del third["ref"]
req = Request()
req.record = merge_source
req.add_apc_record(first)
req.owner = "11111"
pub = PublicAPC()
pub.record = merge_target
pub.add_apc_for_owner("22222", second)
pub.add_apc_for_owner("11111", third)
pub.save(blocking=True)
PublicApi.publish(req)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
# first check that the apcs are as we would expect
one = pub2.get_apcs_by_owner("11111")
two = pub2.get_apcs_by_owner("22222")
assert len(one) == 1
assert len(two) == 1
assert one[0]["organisation_name"] == "First"
assert two[0]["organisation_name"] == "Second"
# now check that the metadata merge proceeded correctly
record = pub2.record
del record["jm:apc"]
assert record == result
def find_public_record(cls, req):
"""
Find a single public record which could be considered the same as the document in the
supplied request.
This will look at identifiers in the following order of precedence:
1. by public id
2. By DOI
3. By PMID
4. By PMCID
5. By URL
If more than one match is made at any point, a warning will be written to the logs, and the
first result (which is basically arbitrary) will be returned.
:param req: Request object containing data to be used to find the public record
:return: The public record, or None if no candidate found
"""
dao = PublicAPC()
if req.public_id is not None:
pub = dao.pull(req.public_id)
if pub is not None:
return pub
if req.doi is not None:
pub = PublicApi.find_public_record_by_identifier("doi", req.doi)
if pub is not None:
return pub
if req.pmid is not None:
pub = PublicApi.find_public_record_by_identifier("pmid", req.pmid)
if pub is not None:
return pub
if req.pmcid is not None:
pub = PublicApi.find_public_record_by_identifier("pmcid", req.pmcid)
if pub is not None:
return pub
if req.url is not None:
pub = PublicApi.find_public_record_by_identifier("url", req.url)
if pub is not None:
return pub
# if we get to here, there is no record for this id
return None
def make_new_jobs(cls):
"""
Send new requests to lantern, and record the jobs that are created.
This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those
accounts which could benefit from lookup in Lantern,
:return:
"""
dao = PublicAPC()
gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h")
for acc in gen:
gen2 = dao.list_by_owner(acc.id)
identifiers = []
for apc in gen2:
if LanternApi._needs_lantern_data(apc):
idents = LanternApi._get_identifiers(apc)
if idents is not None:
identifiers.append(idents)
apc.lantern_lookup = dates.now()
apc.save()
# if there are no identifiers, no need to do any more
if len(identifiers) == 0:
continue
# now check the user's quota
lc = client.Lantern(api_key=acc.lantern_api_key)
quota = lc.get_quota(acc.lantern_email)
available = quota.get("data", {}).get("available", 0)
if available == 0:
continue
if len(identifiers) > available:
identifiers = identifiers[:available]
batches = LanternApi._batch(identifiers)
for batch in batches:
resp = lc.create_job(acc.lantern_email, "monitor-uk", batch)
if resp.get("status") == "success":
job_id = resp.get("data", {}).get("job")
lj = LanternJob()
lj.job_id = job_id
lj.account = acc.id
lj.status = "active"
lj.save()
def test_07_separate_records(self):
# Separate an incoming Request from its corresponding PublicAPC
apc_record = PublicAPCFixtureFactory.apc_record()
req = Request()
req.owner = "test"
pub = PublicAPC()
pub.add_apc_for_owner("test", apc_record)
pub.add_apc_for_owner("test", apc_record)
assert len(pub.apc_records) == 2
PublicApi.separate_records(req, pub)
assert not pub.has_apcs()
def test_05_pull_then_update(self):
# Pull a record through the ApiRequest object and then udpate it
acc = MonitorUKAccount()
acc.id = "abcdefghij"
acc.save(blocking=True)
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
pub_source2 = deepcopy(pub_source.get("record"))
pub_source2["dc:title"] = "An update"
result = ApiRequest.pull(pub.id, account=acc)
result.update(pub_source2)
assert result.raw == pub_source2
assert result.account.id == acc.id
def find_public_record_by_identifier(cls, type, id):
"""
Find a single public record which corresponds to the given identifier. If more than one record are present,
this will return the first one encountered, which is essentially random.
:param type: the identifier type (e.g. doi, pmid, pmcid, url)
:param id: the identifier
:return: a PublicAPC record if one is found otherwise None
"""
dao = PublicAPC()
if type == "doi":
pubs = dao.find_by_doi(id)
if len(pubs) > 1:
app.logger.warn(u"Multiple public records found for DOI {x}".format(x=id))
return None
if len(pubs) > 0:
return pubs[0]
if type == "pmid":
pubs = dao.find_by_pmid(id)
if len(pubs) > 1:
app.logger.warn(u"Multiple public records found for PMID {x}".format(x=id))
return None
if len(pubs) > 0:
return pubs[0]
if type == "pmcid":
pubs = dao.find_by_pmcid(id)
if len(pubs) > 1:
app.logger.warn(u"Multiple public records found for PMCID {x}".format(x=id))
return None
if len(pubs) > 0:
return pubs[0]
if type == "url":
pubs = dao.find_by_url(id)
if len(pubs) > 1:
app.logger.warn(u"Multiple public records found for URL {x}".format(x=id))
return None
if len(pubs) > 0:
return pubs[0]
def test_14_copy_overwrite(self):
# Check the copy and overwrite cloning methods work
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub2 = pub.copy()
assert pub2.record == pub.record
assert pub2.admin == pub.admin
assert pub2.id == pub.id
source3 = PublicAPCFixtureFactory.example()
source3["record"]["dc:title"] = "Overwrite"
pub3 = PublicAPC(source3)
pub2.overwrite(pub3)
assert pub2.record == pub3.record
assert pub2.admin == pub3.admin
assert pub2.id == pub3.id
def test_06_get_id(self):
# Get the operative ID of a record
acc = MonitorUKAccount()
acc.id = "abcdefghij"
source = RequestFixtureFactory.record()
# first, provide a record which has a doi, and ensure that's the id we get back
req = ApiRequest(source, account=acc)
assert req.id == "10.1234/me"
# now make a public record, and check we get the right ids for it
pub_source = PublicAPCFixtureFactory.example()
del pub_source["id"]
del pub_source["record"]["dc:identifier"]
pub = PublicAPC(pub_source)
pub.save(blocking=True)
result = ApiRequest.pull(pub.id, account=acc)
assert result.id == pub.id
assert result.id == result.public_id
def test_05_enhance_metadata(self):
# Enhance the bibliographic metadata on a PublicAPC record
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
result = PublicAPCFixtureFactory.record_merge_result()
source = PublicAPC()
source.record = merge_source
source.set_apc_ref("22222", "bbbbb")
target = PublicAPC()
target.record = merge_target
target.set_apc_ref("11111", "aaaaa")
PublicApi.enhance_metadata(source, target)
assert target.record == result
assert target.get_apc_refs("11111") == ["aaaaa"]
assert target.get_apc_refs("22222") == []
def test_08_remove_separate(self):
# Separate an incoming Request from its corresponding PublicAPC, leaving only one owner behind
source = RequestFixtureFactory.example()
req = Request(source)
req.owner = "test"
# create a record with 2 distinct apcs from different owners
source2 = PublicAPCFixtureFactory.example()
apc_record = PublicAPCFixtureFactory.apc_record()
del apc_record["ref"] # do this so that the ref gets created correctly later
pub = PublicAPC(source2)
pub.add_apc_for_owner("test", apc_record)
pub.save(blocking=True)
# now request the removal
PublicApi.remove(req)
time.sleep(2)
dao = PublicAPC()
pub2 = dao.pull(pub.id)
assert len(pub2.get_apcs_by_owner("test")) == 0
assert len(pub2.get_apcs_by_owner("abcdefg")) == 1
def test_05_low_quota(self):
# Check what happens when the use has a low quota on Lantern
global QUOTA
QUOTA = 1
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 1
assert len(CREATED_JOBS) == 1
job = CREATED_JOBS[0]
assert job["email"] == "*****@*****.**"
assert len(job["list"]) == 1
def test_04_create_job(self):
# Check we can create jobs correctly
acc1 = MonitorUKAccount()
acc1.email = "*****@*****.**"
acc1.save()
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
acc3 = MonitorUKAccount()
acc3.email = "*****@*****.**"
acc3.lantern_email = "*****@*****.**"
acc3.lantern_api_key = "987654321"
acc3.save(blocking=True)
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save()
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
assert len(CREATED_JOBS) == 2
count = 0
for job in CREATED_JOBS:
if job["email"] == "*****@*****.**":
count += 1
assert len(job["list"]) == 2
elif job["email"] == "*****@*****.**":
count += 10
assert len(job["list"]) == 2
assert count == 11
# now do the same thing again. The jobs should not change, as we've already created jobs
# for all the public records
LanternApi.make_new_jobs()
time.sleep(2)
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
def test_04_merge_public_apcs(self):
# Merge two PublicAPC records together
source_source = PublicAPCFixtureFactory.example()
target_source = PublicAPCFixtureFactory.example()
# first try a merge with no apc records (this shouldn't ever happen in normal operation)
ss1 = deepcopy(source_source)
del ss1["record"]["jm:apc"]
source1 = PublicAPC(ss1)
ts1 = deepcopy(target_source)
del ts1["record"]["jm:apc"]
target1 = PublicAPC(ts1)
result = PublicApi.merge_public_apcs(source1, target1)
assert len(result.apc_records) == 0
# next try a merge with only apc records in the source (again, shouldn't really happen in real life)
ss2 = deepcopy(source_source)
source2 = PublicAPC(ss2)
ts2 = deepcopy(target_source)
del ts2["record"]["jm:apc"]
target2 = PublicAPC(ts2)
result = PublicApi.merge_public_apcs(source2, target2)
assert len(result.apc_records) == 1
# next try a merge with only apc records in the target (also shouldn't happen in real life)
ss3 = deepcopy(source_source)
source3 = PublicAPC(ss3)
ts3 = deepcopy(target_source)
del ts3["record"]["jm:apc"]
target3 = PublicAPC(ts3)
result = PublicApi.merge_public_apcs(source3, target3)
assert len(result.apc_records) == 1
# finally try a merge with the following criteria:
# replacements apcs and new apcs in the source record
# existing apcs and other apcs in the target record
apc_record = PublicAPCFixtureFactory.apc_record()
first = deepcopy(apc_record)
first["ref"] = "aaaaa"
second = deepcopy(apc_record)
second["ref"] = "bbbbb"
ss4 = deepcopy(source_source)
del ss4["record"]["jm:apc"]
source4 = PublicAPC(ss4)
source4.add_apc_for_owner("11111", first)
source4.add_apc_for_owner("11111", second)
third = deepcopy(apc_record)
third["ref"] = "ccccc"
fourth = deepcopy(apc_record)
fourth["ref"] = "ddddd"
ts4 = deepcopy(target_source)
del ts4["record"]["jm:apc"]
target4 = PublicAPC(ts4)
target4.add_apc_for_owner("11111", third)
target4.add_apc_for_owner("22222", fourth)
result = PublicApi.merge_public_apcs(source4, target4)
assert len(result.apc_records) == 3
ones = result.get_apcs_by_owner("11111")
assert len(ones) == 2
refs = [o.get("ref") for o in ones]
assert "aaaaa" in refs
assert "bbbbb" in refs
assert "ccccc" not in refs
assert "ddddd" not in refs
twos = result.get_apcs_by_owner("22222")
assert len(twos) == 1
refs = [o.get("ref") for o in twos]
assert "aaaaa" not in refs
assert "bbbbb" not in refs
assert "ccccc" not in refs
assert "ddddd" in refs
def test_09_merge_records(self):
# Check the merge of PublicAPC records
merge_source = PublicAPCFixtureFactory.record_merge_source()
merge_target = PublicAPCFixtureFactory.record_merge_target()
result = PublicAPCFixtureFactory.record_merge_result()
source = PublicAPC()
source.record = merge_source
source.set_apc_ref("22222", "bbbbb")
target = PublicAPC()
target.record = merge_target
target.set_apc_ref("11111", "aaaaa")
target.merge_records(source)
assert target.record == result
assert target.get_apc_refs("11111") == ["aaaaa"]
assert target.get_apc_refs("22222") == []
# now just try some basic error cases
with self.assertRaises(ModelException):
target.merge_records({"random" : "data"})
def test_08_public_apc_methods(self):
# Check all the PublcAPC object methods
pub = PublicAPC()
assert len(pub.get_apc_refs("11111")) == 0
pub.set_apc_ref("11111", "aaaaa")
assert pub.get_apc_refs("11111")[0] == "aaaaa"
pub.set_apc_ref("22222", "bbbbb")
assert len(pub.get_apc_refs("22222")) == 1
pub.remove_apc_refs("11111")
assert len(pub.get_apc_refs("22222")) == 1
assert len(pub.get_apc_refs("11111")) == 0
assert "11111" not in pub.list_owners()
assert "22222" in pub.list_owners()
pub.remove_apc_refs("22222")
assert len(pub.get_apcs_by_owner("11111")) == 0
apc_record = PublicAPCFixtureFactory.apc_record()
first = deepcopy(apc_record)
first["ref"] = "aaaaa"
pub.add_apc_for_owner("11111", first)
assert len(pub.get_apc_refs("11111")) == 1
assert pub.get_apc_refs("11111")[0] == "aaaaa"
assert len(pub.get_apcs_by_owner("11111")) == 1
assert pub.get_apcs_by_owner("11111")[0]["ref"] == "aaaaa"
second = deepcopy(apc_record)
second["ref"] = "bbbbb"
pub.add_apc_for_owner("22222", second)
assert len(pub.get_apc_refs("22222")) == 1
assert pub.get_apc_refs("22222")[0] == "bbbbb"
assert len(pub.get_apcs_by_owner("22222")) == 1
assert pub.get_apcs_by_owner("22222")[0]["ref"] == "bbbbb"
assert len(pub.apc_records) == 2
pub.remove_apcs_by_owner("11111")
assert len(pub.apc_records) == 1
assert pub.apc_records[0]["ref"] == "bbbbb"
assert len(pub.get_apc_refs("22222")) == 1
assert pub.get_apc_refs("22222")[0] == "bbbbb"
assert len(pub.get_apcs_by_owner("22222")) == 1
assert pub.get_apcs_by_owner("22222")[0]["ref"] == "bbbbb"
pub.remove_apcs_by_owner("22222")
assert len(pub.apc_records) == 0
assert len(pub.get_apc_refs("11111")) == 0
assert len(pub.get_apc_refs("22222")) == 0
def test_06_public_dao(self):
# Check all the DAO methods on the PublicAPC object
dao = PublicAPC()
source = PublicAPCFixtureFactory.example()
pub = PublicAPC(source)
pub.set_apc_ref("test1", "1111111111")
pub.save(blocking=True)
# first try the straight-forward pull
pub2 = dao.pull(pub.id)
assert pub2 is not None
# now do the successful queries
res = dao.find_by_doi("10.1234/me")
assert len(res) == 1
res = dao.find_by_pmid("87654321")
assert len(res) == 1
res = dao.find_by_pmcid("PMC1234")
assert len(res) == 1
res = dao.find_by_url("http://example.com/whatever")
assert len(res) == 1
# now to check those queries don't always return, make sure we can get 0 results
res = dao.find_by_doi("10.1234/whatever")
assert len(res) == 0
res = dao.find_by_pmid("88888888")
assert len(res) == 0
res = dao.find_by_pmcid("PMC1111")
assert len(res) == 0
res = dao.find_by_url("http://example.com/another")
assert len(res) == 0
gen = dao.list_by_owner("abcdefg")
count = 0
for apc in gen:
count += 1
assert count == 1
def test_13_process_ehnancements_cycle(self):
# Run through the process of processing an enhancement
source = EnhancementFixtureFactory.example()
if "id" in source:
del source["id"]
pub_dao = PublicAPC()
wfs_dao = WorkflowState()
# first make a public record for us to enhance
first = PublicAPCFixtureFactory.example()
del first["record"]["dc:title"]
pub = PublicAPC(first)
pub.save(blocking=True)
# now create an enhancements on the record
second = deepcopy(source)
second["record"]["dc:title"] = "Update"
second["created_date"] = "2002-01-01T00:00:00Z"
en = Enhancement(second)
en.public_id = pub.id
en.save(blocking=True)
# run the job
WorkflowApi.process_enhancements()
time.sleep(2)
# check that the workflow state was created
wfs = wfs_dao.pull("enhancements")
assert wfs is not None
assert wfs.last_request == en.created_date
assert wfs.already_processed == [en.id]
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") == "Update"
# now run an update with the same date, to observe the difference in the workflow state
third = deepcopy(source)
third["record"]["dc:title"] = "Update 2"
third["created_date"] = "2002-01-01T00:00:00Z"
en2 = Enhancement(third)
en2.public_id = pub.id
en2.save(blocking=True)
# run the job again
WorkflowApi.process_enhancements()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert (
pubs[0].record.get("dc:title") == "Update"
) # should not have been updated, since data was already present
# check that the workflow state was updated
wfs = wfs_dao.pull("enhancements")
assert wfs is not None
assert wfs.last_request == en2.created_date
assert wfs.already_processed == [en.id, en2.id] # processed records should have been appended
def test_11_process_requests_cycle(self):
# Run through the process of processing a Request into a PublicAPC
source = RequestFixtureFactory.example()
if "id" in source:
del source["id"]
pub_dao = PublicAPC()
wfs_dao = WorkflowState()
# first make a record for the first time
first = deepcopy(source)
del first["record"]["dc:title"]
req = Request(first)
req.owner = "test"
req.action = "update"
req.save(blocking=True)
# run the job
WorkflowApi.process_requests()
time.sleep(2)
# first check that a public record was made
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") is None
# check that the workflow state was created
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req.created_date
assert wfs.already_processed == [req.id]
# now run an update with a different date
second = deepcopy(source)
second["record"]["dc:title"] = "Update"
second["created_date"] = "2002-01-01T00:00:00Z"
req2 = Request(second)
req2.owner = "test"
req2.action = "update"
req2.save(blocking=True)
# run the job again
WorkflowApi.process_requests()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert pubs[0].record.get("dc:title") == "Update"
# check that the workflow state was updated
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req2.created_date
assert wfs.already_processed == [req2.id]
# now run an update with the same date, to observe the difference in the workflow state
third = deepcopy(source)
third["record"]["dc:title"] = "Update 2"
third["created_date"] = "2002-01-01T00:00:00Z"
req3 = Request(third)
req3.owner = "test"
req3.action = "update"
req3.save(blocking=True)
# run the job again
WorkflowApi.process_requests()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 1
assert (
pubs[0].record.get("dc:title") == "Update 2"
) # should have been updated, as there are only apc contributions from one source
# check that the workflow state was updated
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req3.created_date
assert wfs.already_processed == [req2.id, req3.id] # processed records should have been appended
# finally issue a delete request
fourth = deepcopy(source)
fourth["created_date"] = "2003-01-01T00:00:00Z"
req4 = Request(fourth)
req4.owner = "test"
req4.action = "delete"
req4.save(blocking=True)
# run the job again
WorkflowApi.process_requests()
time.sleep(2)
# check the public record was updated
pubs = pub_dao.find_by_doi("10.1234/me")
assert len(pubs) == 0
# check that the workflow state was updated
wfs = wfs_dao.pull("requests")
assert wfs is not None
assert wfs.last_request == req4.created_date
assert wfs.already_processed == [req4.id] # processed records should have been appended
