Esempio n. 1
0
    def test_19_lantern_active_jobs(self):
        # Check we can list active Lantern jobs

        lj1 = LanternJob()
        lj1.job_id = "123456789"
        lj1.account = "abcdefg"
        lj1.status = "complete"
        lj1.save()

        lj2 = LanternJob()
        lj2.job_id = "123456789"
        lj2.account = "abcdefg"
        lj2.status = "active"
        lj2.save()

        time.sleep(2)

        lj3 = LanternJob()
        lj3.job_id = "987654321"
        lj3.account = "abcdefg"
        lj3.status = "active"
        lj3.save(blocking=True)

        dao = LanternJob()

        # list all the active jobs, without worrying about cutting them off by date
        gen = dao.list_active()
        jobs = [job for job in gen]

        assert len(jobs) == 2

        # now check that we can do a date cut-off, by setting the before date to just after
        # the oldest one
        lu = lj2.last_updated
        ds = dates.parse(lu)
        ds = ds + timedelta(seconds=1)

        gen = dao.list_active(checked_before=ds)
        jobs = [job for job in gen]

        assert len(jobs) == 1
Esempio n. 2
0
    def make_new_jobs(cls):
        """
        Send new requests to lantern, and record the jobs that are created.

        This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those
        accounts which could benefit from lookup in Lantern,

        :return:
        """
        dao = PublicAPC()

        gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h")
        for acc in gen:
            gen2 = dao.list_by_owner(acc.id)
            identifiers = []
            for apc in gen2:
                if LanternApi._needs_lantern_data(apc):
                    idents = LanternApi._get_identifiers(apc)
                    if idents is not None:
                        identifiers.append(idents)
                    apc.lantern_lookup = dates.now()
                    apc.save()

            # if there are no identifiers, no need to do any more
            if len(identifiers) == 0:
                continue

            # now check the user's quota
            lc = client.Lantern(api_key=acc.lantern_api_key)
            quota = lc.get_quota(acc.lantern_email)
            available = quota.get("data", {}).get("available", 0)
            if available == 0:
                continue

            if len(identifiers) > available:
                identifiers = identifiers[:available]

            batches = LanternApi._batch(identifiers)
            for batch in batches:
                resp = lc.create_job(acc.lantern_email, "monitor-uk", batch)
                if resp.get("status") == "success":
                    job_id = resp.get("data", {}).get("job")
                    lj = LanternJob()
                    lj.job_id = job_id
                    lj.account = acc.id
                    lj.status = "active"
                    lj.save()
Esempio n. 3
0
    def test_17_lantern_jobs(self):
        # Check we can create and work with Lantern model objects
        lj = LanternJob()
        lj.job_id = "123456789"
        lj.account = "abcdefg"
        lj.status = "active"
        lj.save(blocking=True)

        dao = LanternJob()
        lj2 = dao.pull(lj.id)
        assert lj2.job_id == "123456789"
        assert lj2.account == "abcdefg"
        assert lj2.status == "active"

        lj2.status = "complete"

        with self.assertRaises(dataobj.DataSchemaException):
            lj2.status = "other"
Esempio n. 4
0
    def test_07_check_jobs(self):
        # Ensure that we can check existing jobs correctly

        acc = MonitorUKAccount()
        acc.email = "*****@*****.**"
        acc.lantern_email = "*****@*****.**"
        acc.lantern_api_key = "123456789"
        acc.save()

        lj1 = LanternJob()
        lj1.job_id = "111111111"
        lj1.account = acc.id
        lj1.status = "complete"
        lj1.save()

        lj2 = LanternJob()
        lj2.job_id = "222222222"
        lj2.account = acc.id
        lj2.status = "active"
        lj2.last_updated = dates.format(dates.before_now(5000))
        lj2.save(updated=False)

        lj3 = LanternJob()
        lj3.job_id = "333333333"
        lj3.account = acc.id
        lj3.status = "active"
        lj3.last_updated = dates.format(dates.before_now(5000))
        lj3.save(updated=False)

        lj4 = LanternJob()
        lj4.job_id = "444444444"
        lj4.account = acc.id
        lj4.status = "active"
        lj4.last_updated = dates.format(dates.before_now(5000))
        lj4.save(updated=False)

        lj5 = LanternJob()
        lj5.job_id = "555555555"
        lj5.account = acc.id
        lj5.status = "active"
        lj5.save(blocking=True)

        LanternApi.check_jobs()

        # check that the progress requests we expected were made
        assert len(PROGRESS_REQUESTS) == 3
        assert "222222222" in PROGRESS_REQUESTS
        assert "333333333" in PROGRESS_REQUESTS
        assert "444444444" in PROGRESS_REQUESTS

        # check that the job which received an error was just ignored
        dao = LanternJob()
        ignored = dao.pull(lj4.id)
        assert ignored.last_updated == lj4.last_updated
        assert ignored.status == "active"

        # check that the record which was not complete was touched
        touched = dao.pull(lj2.id)
        assert touched.last_updated != lj2.last_updated
        assert touched.status == "active"

        # check that results were requested only for one item
        assert len(RESULTS_REQUESTS) == 1
        assert "333333333" in RESULTS_REQUESTS

        # wait for a bit, so that enhancements have time to go in
        time.sleep(2)

        # check that an enhancement was registered
        edao = Enhancement()
        gen = edao.iterall()
        enhancements = [e for e in gen]

        assert len(enhancements) == 1

        result = LanternFixtureFactory.xwalk_result()
        assert enhancements[0].data["record"] == result["record"]