def check_jobs(cls):
"""
Check any existing Lantern jobs for progress, and process any that have completed
:return:
"""
dao = LanternJob()
delay = app.config.get("JOB_LOOKUP_DELAY_LANTERN", 3600)
cutoff = dates.before_now(delay)
gen = dao.list_active(cutoff, keepalive="30m")
for job in gen:
acc = MonitorUKAccount.pull(job.account)
lc = client.Lantern(api_key=acc.lantern_api_key)
prog = lc.get_progress(job.job_id)
if prog.get("status") == "success":
pc = prog.get("data", {}).get("progress", 0)
if pc != 100:
# this will update the last_updated date, which means we won't look at it again for a while
job.save()
continue
# if we get here, the job is complete so we need to retrieve it
results = lc.get_results(job.job_id)
if results.get("status") == "success":
for res in results.get("data", []):
enhancement = LanternApi._xwalk(res)
enhancement.save()
# set the job as complete
job.status = "complete"
job.save()
def test_19_lantern_active_jobs(self):
# Check we can list active Lantern jobs
lj1 = LanternJob()
lj1.job_id = "123456789"
lj1.account = "abcdefg"
lj1.status = "complete"
lj1.save()
lj2 = LanternJob()
lj2.job_id = "123456789"
lj2.account = "abcdefg"
lj2.status = "active"
lj2.save()
time.sleep(2)
lj3 = LanternJob()
lj3.job_id = "987654321"
lj3.account = "abcdefg"
lj3.status = "active"
lj3.save(blocking=True)
dao = LanternJob()
# list all the active jobs, without worrying about cutting them off by date
gen = dao.list_active()
jobs = [job for job in gen]
assert len(jobs) == 2
# now check that we can do a date cut-off, by setting the before date to just after
# the oldest one
lu = lj2.last_updated
ds = dates.parse(lu)
ds = ds + timedelta(seconds=1)
gen = dao.list_active(checked_before=ds)
jobs = [job for job in gen]
assert len(jobs) == 1