def make_new_jobs(cls):
"""
Send new requests to lantern, and record the jobs that are created.
This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those
accounts which could benefit from lookup in Lantern,
:return:
"""
dao = PublicAPC()
gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h")
for acc in gen:
gen2 = dao.list_by_owner(acc.id)
identifiers = []
for apc in gen2:
if LanternApi._needs_lantern_data(apc):
idents = LanternApi._get_identifiers(apc)
if idents is not None:
identifiers.append(idents)
apc.lantern_lookup = dates.now()
apc.save()
# if there are no identifiers, no need to do any more
if len(identifiers) == 0:
continue
# now check the user's quota
lc = client.Lantern(api_key=acc.lantern_api_key)
quota = lc.get_quota(acc.lantern_email)
available = quota.get("data", {}).get("available", 0)
if available == 0:
continue
if len(identifiers) > available:
identifiers = identifiers[:available]
batches = LanternApi._batch(identifiers)
for batch in batches:
resp = lc.create_job(acc.lantern_email, "monitor-uk", batch)
if resp.get("status") == "success":
job_id = resp.get("data", {}).get("job")
lj = LanternJob()
lj.job_id = job_id
lj.account = acc.id
lj.status = "active"
lj.save()
def test_17_lantern_jobs(self):
# Check we can create and work with Lantern model objects
lj = LanternJob()
lj.job_id = "123456789"
lj.account = "abcdefg"
lj.status = "active"
lj.save(blocking=True)
dao = LanternJob()
lj2 = dao.pull(lj.id)
assert lj2.job_id == "123456789"
assert lj2.account == "abcdefg"
assert lj2.status == "active"
lj2.status = "complete"
with self.assertRaises(dataobj.DataSchemaException):
lj2.status = "other"
def test_19_lantern_active_jobs(self):
# Check we can list active Lantern jobs
lj1 = LanternJob()
lj1.job_id = "123456789"
lj1.account = "abcdefg"
lj1.status = "complete"
lj1.save()
lj2 = LanternJob()
lj2.job_id = "123456789"
lj2.account = "abcdefg"
lj2.status = "active"
lj2.save()
time.sleep(2)
lj3 = LanternJob()
lj3.job_id = "987654321"
lj3.account = "abcdefg"
lj3.status = "active"
lj3.save(blocking=True)
dao = LanternJob()
# list all the active jobs, without worrying about cutting them off by date
gen = dao.list_active()
jobs = [job for job in gen]
assert len(jobs) == 2
# now check that we can do a date cut-off, by setting the before date to just after
# the oldest one
lu = lj2.last_updated
ds = dates.parse(lu)
ds = ds + timedelta(seconds=1)
gen = dao.list_active(checked_before=ds)
jobs = [job for job in gen]
assert len(jobs) == 1
def test_07_check_jobs(self):
# Ensure that we can check existing jobs correctly
acc = MonitorUKAccount()
acc.email = "*****@*****.**"
acc.lantern_email = "*****@*****.**"
acc.lantern_api_key = "123456789"
acc.save()
lj1 = LanternJob()
lj1.job_id = "111111111"
lj1.account = acc.id
lj1.status = "complete"
lj1.save()
lj2 = LanternJob()
lj2.job_id = "222222222"
lj2.account = acc.id
lj2.status = "active"
lj2.last_updated = dates.format(dates.before_now(5000))
lj2.save(updated=False)
lj3 = LanternJob()
lj3.job_id = "333333333"
lj3.account = acc.id
lj3.status = "active"
lj3.last_updated = dates.format(dates.before_now(5000))
lj3.save(updated=False)
lj4 = LanternJob()
lj4.job_id = "444444444"
lj4.account = acc.id
lj4.status = "active"
lj4.last_updated = dates.format(dates.before_now(5000))
lj4.save(updated=False)
lj5 = LanternJob()
lj5.job_id = "555555555"
lj5.account = acc.id
lj5.status = "active"
lj5.save(blocking=True)
LanternApi.check_jobs()
# check that the progress requests we expected were made
assert len(PROGRESS_REQUESTS) == 3
assert "222222222" in PROGRESS_REQUESTS
assert "333333333" in PROGRESS_REQUESTS
assert "444444444" in PROGRESS_REQUESTS
# check that the job which received an error was just ignored
dao = LanternJob()
ignored = dao.pull(lj4.id)
assert ignored.last_updated == lj4.last_updated
assert ignored.status == "active"
# check that the record which was not complete was touched
touched = dao.pull(lj2.id)
assert touched.last_updated != lj2.last_updated
assert touched.status == "active"
# check that results were requested only for one item
assert len(RESULTS_REQUESTS) == 1
assert "333333333" in RESULTS_REQUESTS
# wait for a bit, so that enhancements have time to go in
time.sleep(2)
# check that an enhancement was registered
edao = Enhancement()
gen = edao.iterall()
enhancements = [e for e in gen]
assert len(enhancements) == 1
result = LanternFixtureFactory.xwalk_result()
assert enhancements[0].data["record"] == result["record"]
