예제 #1
0
    def check_jobs(cls):
        """
        Check any existing Lantern jobs for progress, and process any that have completed

        :return:
        """
        dao = LanternJob()

        delay = app.config.get("JOB_LOOKUP_DELAY_LANTERN", 3600)
        cutoff = dates.before_now(delay)
        gen = dao.list_active(cutoff, keepalive="30m")

        for job in gen:
            acc = MonitorUKAccount.pull(job.account)
            lc = client.Lantern(api_key=acc.lantern_api_key)
            prog = lc.get_progress(job.job_id)
            if prog.get("status") == "success":
                pc = prog.get("data",  {}).get("progress", 0)
                if pc != 100:
                    # this will update the last_updated date, which means we won't look at it again for a while
                    job.save()
                    continue

                # if we get here, the job is complete so we need to retrieve it
                results = lc.get_results(job.job_id)
                if results.get("status") == "success":
                    for res in results.get("data", []):
                        enhancement = LanternApi._xwalk(res)
                        enhancement.save()

                # set the job as complete
                job.status = "complete"
                job.save()
예제 #2
0
    def test_19_lantern_active_jobs(self):
        # Check we can list active Lantern jobs

        lj1 = LanternJob()
        lj1.job_id = "123456789"
        lj1.account = "abcdefg"
        lj1.status = "complete"
        lj1.save()

        lj2 = LanternJob()
        lj2.job_id = "123456789"
        lj2.account = "abcdefg"
        lj2.status = "active"
        lj2.save()

        time.sleep(2)

        lj3 = LanternJob()
        lj3.job_id = "987654321"
        lj3.account = "abcdefg"
        lj3.status = "active"
        lj3.save(blocking=True)

        dao = LanternJob()

        # list all the active jobs, without worrying about cutting them off by date
        gen = dao.list_active()
        jobs = [job for job in gen]

        assert len(jobs) == 2

        # now check that we can do a date cut-off, by setting the before date to just after
        # the oldest one
        lu = lj2.last_updated
        ds = dates.parse(lu)
        ds = ds + timedelta(seconds=1)

        gen = dao.list_active(checked_before=ds)
        jobs = [job for job in gen]

        assert len(jobs) == 1