def test_19_lantern_active_jobs(self): # Check we can list active Lantern jobs lj1 = LanternJob() lj1.job_id = "123456789" lj1.account = "abcdefg" lj1.status = "complete" lj1.save() lj2 = LanternJob() lj2.job_id = "123456789" lj2.account = "abcdefg" lj2.status = "active" lj2.save() time.sleep(2) lj3 = LanternJob() lj3.job_id = "987654321" lj3.account = "abcdefg" lj3.status = "active" lj3.save(blocking=True) dao = LanternJob() # list all the active jobs, without worrying about cutting them off by date gen = dao.list_active() jobs = [job for job in gen] assert len(jobs) == 2 # now check that we can do a date cut-off, by setting the before date to just after # the oldest one lu = lj2.last_updated ds = dates.parse(lu) ds = ds + timedelta(seconds=1) gen = dao.list_active(checked_before=ds) jobs = [job for job in gen] assert len(jobs) == 1
def make_new_jobs(cls): """ Send new requests to lantern, and record the jobs that are created. This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those accounts which could benefit from lookup in Lantern, :return: """ dao = PublicAPC() gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h") for acc in gen: gen2 = dao.list_by_owner(acc.id) identifiers = [] for apc in gen2: if LanternApi._needs_lantern_data(apc): idents = LanternApi._get_identifiers(apc) if idents is not None: identifiers.append(idents) apc.lantern_lookup = dates.now() apc.save() # if there are no identifiers, no need to do any more if len(identifiers) == 0: continue # now check the user's quota lc = client.Lantern(api_key=acc.lantern_api_key) quota = lc.get_quota(acc.lantern_email) available = quota.get("data", {}).get("available", 0) if available == 0: continue if len(identifiers) > available: identifiers = identifiers[:available] batches = LanternApi._batch(identifiers) for batch in batches: resp = lc.create_job(acc.lantern_email, "monitor-uk", batch) if resp.get("status") == "success": job_id = resp.get("data", {}).get("job") lj = LanternJob() lj.job_id = job_id lj.account = acc.id lj.status = "active" lj.save()
def test_17_lantern_jobs(self): # Check we can create and work with Lantern model objects lj = LanternJob() lj.job_id = "123456789" lj.account = "abcdefg" lj.status = "active" lj.save(blocking=True) dao = LanternJob() lj2 = dao.pull(lj.id) assert lj2.job_id == "123456789" assert lj2.account == "abcdefg" assert lj2.status == "active" lj2.status = "complete" with self.assertRaises(dataobj.DataSchemaException): lj2.status = "other"
def test_07_check_jobs(self): # Ensure that we can check existing jobs correctly acc = MonitorUKAccount() acc.email = "*****@*****.**" acc.lantern_email = "*****@*****.**" acc.lantern_api_key = "123456789" acc.save() lj1 = LanternJob() lj1.job_id = "111111111" lj1.account = acc.id lj1.status = "complete" lj1.save() lj2 = LanternJob() lj2.job_id = "222222222" lj2.account = acc.id lj2.status = "active" lj2.last_updated = dates.format(dates.before_now(5000)) lj2.save(updated=False) lj3 = LanternJob() lj3.job_id = "333333333" lj3.account = acc.id lj3.status = "active" lj3.last_updated = dates.format(dates.before_now(5000)) lj3.save(updated=False) lj4 = LanternJob() lj4.job_id = "444444444" lj4.account = acc.id lj4.status = "active" lj4.last_updated = dates.format(dates.before_now(5000)) lj4.save(updated=False) lj5 = LanternJob() lj5.job_id = "555555555" lj5.account = acc.id lj5.status = "active" lj5.save(blocking=True) LanternApi.check_jobs() # check that the progress requests we expected were made assert len(PROGRESS_REQUESTS) == 3 assert "222222222" in PROGRESS_REQUESTS assert "333333333" in PROGRESS_REQUESTS assert "444444444" in PROGRESS_REQUESTS # check that the job which received an error was just ignored dao = LanternJob() ignored = dao.pull(lj4.id) assert ignored.last_updated == lj4.last_updated assert ignored.status == "active" # check that the record which was not complete was touched touched = dao.pull(lj2.id) assert touched.last_updated != lj2.last_updated assert touched.status == "active" # check that results were requested only for one item assert len(RESULTS_REQUESTS) == 1 assert "333333333" in RESULTS_REQUESTS # wait for a bit, so that enhancements have time to go in time.sleep(2) # check that an enhancement was registered edao = Enhancement() gen = edao.iterall() enhancements = [e for e in gen] assert len(enhancements) == 1 result = LanternFixtureFactory.xwalk_result() assert enhancements[0].data["record"] == result["record"]