def lantern_check():
"""
Check any existing Lantern jobs
"""
if not app.config.get("ENABLE_LANTERN", False):
print "[{x}] Not checking Lantern jobs - interface disabled".format(x=dates.now())
return
print "[{x}] Checking Lantern jobs".format(x=dates.now())
LanternApi.check_jobs()
def lantern_jobs():
"""
Create any new jobs with Lantern that are requried
"""
if not app.config.get("ENABLE_LANTERN", False):
print "[{x}] Not sending Lantern jobs - interface disabled".format(x=dates.now())
return
print "[{x}] Sending Lantern jobs".format(x=dates.now())
LanternApi.make_new_jobs()
def process_updates():
"""
Process all new Requests and all new Enhancements
"""
print "[{x}] Processing Requests".format(x=dates.now())
WorkflowApi.process_requests()
WorkflowApi.process_enhancements()
def type_map(t):
type = DO_TYPE_TO_JSON_TYPE.get(t, "string")
if type == "timestamp":
return dates.now()
elif type == "datestamp":
return dates.format(datetime.utcnow(), "%Y-%m-%d")
return type
def make_new_jobs(cls):
"""
Send new requests to lantern, and record the jobs that are created.
This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those
accounts which could benefit from lookup in Lantern,
:return:
"""
dao = PublicAPC()
gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h")
for acc in gen:
gen2 = dao.list_by_owner(acc.id)
identifiers = []
for apc in gen2:
if LanternApi._needs_lantern_data(apc):
idents = LanternApi._get_identifiers(apc)
if idents is not None:
identifiers.append(idents)
apc.lantern_lookup = dates.now()
apc.save()
# if there are no identifiers, no need to do any more
if len(identifiers) == 0:
continue
# now check the user's quota
lc = client.Lantern(api_key=acc.lantern_api_key)
quota = lc.get_quota(acc.lantern_email)
available = quota.get("data", {}).get("available", 0)
if available == 0:
continue
if len(identifiers) > available:
identifiers = identifiers[:available]
batches = LanternApi._batch(identifiers)
for batch in batches:
resp = lc.create_job(acc.lantern_email, "monitor-uk", batch)
if resp.get("status") == "success":
job_id = resp.get("data", {}).get("job")
lj = LanternJob()
lj.job_id = job_id
lj.account = acc.id
lj.status = "active"
lj.save()
def test_04_create_job(self):
# Check we can create jobs correctly
acc1 = MonitorUKAccount()
acc1.email = "*****@*****.**"
acc1.save()
acc2 = MonitorUKAccount()
acc2.email = "*****@*****.**"
acc2.lantern_email = "*****@*****.**"
acc2.lantern_api_key = "123456789"
acc2.save()
acc3 = MonitorUKAccount()
acc3.email = "*****@*****.**"
acc3.lantern_email = "*****@*****.**"
acc3.lantern_api_key = "987654321"
acc3.save(blocking=True)
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save()
# a record which does not need lantern
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.now()
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because of a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
del source["admin"]["lantern_lookup"]
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern because it has timed out and has a missing field
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
pub = PublicAPC(source)
pub.save()
# a record that needs lantern but has no identifiers
source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
del source["record"]["rioxxterms:publication_date"]
del source["record"]["dc:identifier"]
pub = PublicAPC(source)
pub.save(blocking=True)
LanternApi.make_new_jobs()
time.sleep(2)
dao = LanternJob()
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
assert len(CREATED_JOBS) == 2
count = 0
for job in CREATED_JOBS:
if job["email"] == "*****@*****.**":
count += 1
assert len(job["list"]) == 2
elif job["email"] == "*****@*****.**":
count += 10
assert len(job["list"]) == 2
assert count == 11
# now do the same thing again. The jobs should not change, as we've already created jobs
# for all the public records
LanternApi.make_new_jobs()
time.sleep(2)
jobs = [job for job in dao.iterall()]
assert len(jobs) == 2
def test_01_needs_lantern(self):
# Check that we can tell when an object needs to be looked up in Lantern
apc = {
"record" : {
"rioxxterms:publication_date" : "2001-01-01",
"rioxxterms:version" : "AAM",
"dc:source" : {
"name" : "Journal Title",
"identifier" : [
{"type" : "issn", "id" : "XXXX-XXXX"}
],
"oa_type" : "hybrid",
"self_archiving" : {
"preprint" : {
"embargo" : 10,
"policy" : "can"
},
"postprint" : {
"embargo" : 20,
"policy" : "cannot"
},
"publisher" : {
"embargo" : 30,
"policy" : "maybe"
}
}
},
"rioxxterms:project" : [
{"funder_name" : "BBSRC"}
],
"ali:license_ref" : [
{ "type" : "CC BY" }
],
"jm:repository" : [
{"repo_name" : "arXiv"}
]
}
}
obj = PublicAPC(apc)
needs = LanternApi._needs_lantern_data(obj)
assert needs is False
clone = deepcopy(apc)
clone["record"]["rioxxterms:publication_date"] = ""
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["rioxxterms:version"] = ""
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["name"] = ""
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["identifier"][0]["type"] = "other"
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
del clone["record"]["dc:source"]["oa_type"]
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["self_archiving"]["preprint"]["embargo"] = None
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["self_archiving"]["preprint"]["policy"] = None
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["self_archiving"]["postprint"]["embargo"] = None
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["self_archiving"]["postprint"]["policy"] = None
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["self_archiving"]["publisher"]["embargo"] = None
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
clone["record"]["dc:source"]["self_archiving"]["publisher"]["policy"] = None
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
del clone["record"]["rioxxterms:project"][0]["funder_name"]
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
del clone["record"]["ali:license_ref"][0]["type"]
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
clone = deepcopy(apc)
del clone["record"]["jm:repository"][0]["repo_name"]
cobj = PublicAPC(clone)
needs = LanternApi._needs_lantern_data(cobj)
assert needs is True
# now check that having a date cut-off works
apc = {
"admin" : {
"lantern_lookup" : dates.now()
}
}
obj = PublicAPC(apc)
needs = LanternApi._needs_lantern_data(obj)
assert needs is False
apc["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago
obj = PublicAPC(apc)
needs = LanternApi._needs_lantern_data(obj)
assert needs is True