示例#1
0
def lantern_check():
    """
    Check any existing Lantern jobs
    """
    if not app.config.get("ENABLE_LANTERN", False):
        print "[{x}] Not checking Lantern jobs - interface disabled".format(x=dates.now())
        return
    print "[{x}] Checking Lantern jobs".format(x=dates.now())
    LanternApi.check_jobs()
示例#2
0
def lantern_jobs():
    """
    Create any new jobs with Lantern that are requried
    """
    if not app.config.get("ENABLE_LANTERN", False):
        print "[{x}] Not sending Lantern jobs - interface disabled".format(x=dates.now())
        return
    print "[{x}] Sending Lantern jobs".format(x=dates.now())
    LanternApi.make_new_jobs()
示例#3
0
def process_updates():
    """
    Process all new Requests and all new Enhancements
    """
    print "[{x}] Processing Requests".format(x=dates.now())
    WorkflowApi.process_requests()
    WorkflowApi.process_enhancements()
示例#4
0
def type_map(t):
    type = DO_TYPE_TO_JSON_TYPE.get(t, "string")
    if type == "timestamp":
        return dates.now()
    elif type == "datestamp":
        return dates.format(datetime.utcnow(), "%Y-%m-%d")
    return type
示例#5
0
    def make_new_jobs(cls):
        """
        Send new requests to lantern, and record the jobs that are created.

        This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those
        accounts which could benefit from lookup in Lantern,

        :return:
        """
        dao = PublicAPC()

        gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h")
        for acc in gen:
            gen2 = dao.list_by_owner(acc.id)
            identifiers = []
            for apc in gen2:
                if LanternApi._needs_lantern_data(apc):
                    idents = LanternApi._get_identifiers(apc)
                    if idents is not None:
                        identifiers.append(idents)
                    apc.lantern_lookup = dates.now()
                    apc.save()

            # if there are no identifiers, no need to do any more
            if len(identifiers) == 0:
                continue

            # now check the user's quota
            lc = client.Lantern(api_key=acc.lantern_api_key)
            quota = lc.get_quota(acc.lantern_email)
            available = quota.get("data", {}).get("available", 0)
            if available == 0:
                continue

            if len(identifiers) > available:
                identifiers = identifiers[:available]

            batches = LanternApi._batch(identifiers)
            for batch in batches:
                resp = lc.create_job(acc.lantern_email, "monitor-uk", batch)
                if resp.get("status") == "success":
                    job_id = resp.get("data", {}).get("job")
                    lj = LanternJob()
                    lj.job_id = job_id
                    lj.account = acc.id
                    lj.status = "active"
                    lj.save()
示例#6
0
    def test_04_create_job(self):
        # Check we can create jobs correctly

        acc1 = MonitorUKAccount()
        acc1.email = "*****@*****.**"
        acc1.save()

        acc2 = MonitorUKAccount()
        acc2.email = "*****@*****.**"
        acc2.lantern_email = "*****@*****.**"
        acc2.lantern_api_key = "123456789"
        acc2.save()

        acc3 = MonitorUKAccount()
        acc3.email = "*****@*****.**"
        acc3.lantern_email = "*****@*****.**"
        acc3.lantern_api_key = "987654321"
        acc3.save(blocking=True)

        # a record which does not need lantern
        source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
        source["admin"]["lantern_lookup"] = dates.now()
        pub = PublicAPC(source)
        pub.save()

        # a record that needs lantern because of a missing field
        source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
        del source["admin"]["lantern_lookup"]
        del source["record"]["rioxxterms:publication_date"]
        pub = PublicAPC(source)
        pub.save()

        # a record that needs lantern because it has timed out and has a missing field
        source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
        source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000))   # a year ago
        del source["record"]["rioxxterms:publication_date"]
        pub = PublicAPC(source)
        pub.save()

        # a record that needs lantern but has no identifiers
        source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None)
        source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000))   # a year ago
        del source["record"]["rioxxterms:publication_date"]
        del source["record"]["dc:identifier"]
        pub = PublicAPC(source)
        pub.save()

        # a record which does not need lantern
        source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
        source["admin"]["lantern_lookup"] = dates.now()
        pub = PublicAPC(source)
        pub.save()

        # a record that needs lantern because of a missing field
        source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
        del source["admin"]["lantern_lookup"]
        del source["record"]["rioxxterms:publication_date"]
        pub = PublicAPC(source)
        pub.save()

        # a record that needs lantern because it has timed out and has a missing field
        source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
        source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000))   # a year ago
        del source["record"]["rioxxterms:publication_date"]
        pub = PublicAPC(source)
        pub.save()

        # a record that needs lantern but has no identifiers
        source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None)
        source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000))   # a year ago
        del source["record"]["rioxxterms:publication_date"]
        del source["record"]["dc:identifier"]
        pub = PublicAPC(source)
        pub.save(blocking=True)

        LanternApi.make_new_jobs()

        time.sleep(2)

        dao = LanternJob()
        jobs = [job for job in dao.iterall()]
        assert len(jobs) == 2

        assert len(CREATED_JOBS) == 2
        count = 0
        for job in CREATED_JOBS:
            if job["email"] == "*****@*****.**":
                count += 1
                assert len(job["list"]) == 2
            elif job["email"] == "*****@*****.**":
                count += 10
                assert len(job["list"]) == 2
        assert count == 11

        # now do the same thing again.  The jobs should not change, as we've already created jobs
        # for all the public records
        LanternApi.make_new_jobs()
        time.sleep(2)
        jobs = [job for job in dao.iterall()]
        assert len(jobs) == 2
示例#7
0
    def test_01_needs_lantern(self):
        # Check that we can tell when an object needs to be looked up in Lantern

        apc = {
            "record" : {
                "rioxxterms:publication_date" : "2001-01-01",
                "rioxxterms:version" : "AAM",
                "dc:source" : {
                    "name" : "Journal Title",
                    "identifier" : [
                        {"type" : "issn", "id" : "XXXX-XXXX"}
                    ],
                    "oa_type" : "hybrid",
                    "self_archiving" : {
                        "preprint" : {
                            "embargo" : 10,
                            "policy" : "can"
                        },
                        "postprint" : {
                            "embargo" : 20,
                            "policy" : "cannot"
                        },
                        "publisher" : {
                            "embargo" : 30,
                            "policy" : "maybe"
                        }
                    }
                },
                "rioxxterms:project" : [
                    {"funder_name" : "BBSRC"}
                ],
                "ali:license_ref" : [
                    { "type" : "CC BY" }
                ],
                "jm:repository" : [
                    {"repo_name" : "arXiv"}
                ]
            }
        }
        obj = PublicAPC(apc)

        needs = LanternApi._needs_lantern_data(obj)
        assert needs is False

        clone = deepcopy(apc)
        clone["record"]["rioxxterms:publication_date"] = ""
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["rioxxterms:version"] = ""
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["name"] = ""
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["identifier"][0]["type"] = "other"
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        del clone["record"]["dc:source"]["oa_type"]
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["self_archiving"]["preprint"]["embargo"] = None
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["self_archiving"]["preprint"]["policy"] = None
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["self_archiving"]["postprint"]["embargo"] = None
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["self_archiving"]["postprint"]["policy"] = None
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["self_archiving"]["publisher"]["embargo"] = None
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        clone["record"]["dc:source"]["self_archiving"]["publisher"]["policy"] = None
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        del clone["record"]["rioxxterms:project"][0]["funder_name"]
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        del clone["record"]["ali:license_ref"][0]["type"]
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        clone = deepcopy(apc)
        del clone["record"]["jm:repository"][0]["repo_name"]
        cobj = PublicAPC(clone)
        needs = LanternApi._needs_lantern_data(cobj)
        assert needs is True

        # now check that having a date cut-off works
        apc = {
            "admin" : {
                "lantern_lookup" : dates.now()
            }
        }
        obj = PublicAPC(apc)
        needs = LanternApi._needs_lantern_data(obj)
        assert needs is False

        apc["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000))   # a year ago
        obj = PublicAPC(apc)
        needs = LanternApi._needs_lantern_data(obj)
        assert needs is True