def test_02_update(self): # Do updates on the ApiRequest object source = RequestFixtureFactory.record() csource = deepcopy(source) csource["@context"] = app.config.get("API_JSON_LD_CONTEXT") acc = MonitorUKAccount() acc.save() # do an update on blank one (unlikely use case) req = ApiRequest() req.update(csource) assert req.raw == source assert req.account is None assert isinstance(req.json(), basestring) # make one, and then update it source2 = deepcopy(source) source2["dc:title"] = "An update" req = ApiRequest(source, account=acc) req.update(source2) assert req.raw == source2 assert req.account.id == acc.id assert isinstance(req.json(), basestring) # make a valid one, and then do something invalid to it req = ApiRequest(source, account=acc) with self.assertRaises(dataobj.DataStructureException): req.update({"whatever" : "here"})
def test_08_append(self): # Check that append works, which is the same as update source = RequestFixtureFactory.record() acc = MonitorUKAccount() acc.save() # do an update on blank one (unlikely use case) req = ApiRequest() req.append(source) assert req.raw == source assert req.account is None assert isinstance(req.json(), basestring) # make one, and then update it source2 = deepcopy(source) source2["dc:title"] = "An update" req = ApiRequest(source, account=acc) req.append(source2) assert req.raw == source2 assert req.account.id == acc.id assert isinstance(req.json(), basestring) # make a valid one, and then do something invalid to it req = ApiRequest(source, account=acc) with self.assertRaises(dataobj.DataStructureException): req.append({"whatever" : "here"})
def test_01_create(self): # Create instances of the ApiRequest object # construct a blank one, just to make sure we can req = ApiRequest() assert req.raw is None assert req.account is None assert req.json() is None # make one using valid source data source = RequestFixtureFactory.record() csource = deepcopy(source) csource["@context"] = app.config.get("API_JSON_LD_CONTEXT") acc = MonitorUKAccount() acc.save() req = ApiRequest(csource, account=acc) assert req.raw == source assert req.account.id == acc.id assert isinstance(req.json(), basestring) # make one with some invalid data with self.assertRaises(dataobj.DataStructureException): req = ApiRequest({"whatever" : "here"}, account=acc)
def test_10_validate(self): # Check the various validation routines acc = MonitorUKAccount() acc.save() source = RequestFixtureFactory.record() # first prove that our source record validates csource = deepcopy(source) csource["@context"] = app.config.get("API_JSON_LD_CONTEXT") req = ApiRequest(csource, account=acc) assert req.raw == csource # 1 - a record with no identifiers asource = deepcopy(source) del asource["dc:identifier"] asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 2 - a record with no apc asource = deepcopy(source) del asource["jm:apc"] asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 3 - an apc with no inc vat amount asource = deepcopy(source) del asource["jm:apc"][0]["amount_inc_vat_gbp"] asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 4 - a record with a date in the wrong format asource = deepcopy(source) asource["dcterms:dateAccepted"] = "sometime last week" asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 5 - a record with an unparseable number asource = deepcopy(source) asource["jm:apc"][0]["amount_inc_vat_gbp"] = "three fifty" asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 6 - a record with an invalid currency code asource = deepcopy(source) asource["jm:apc"][0]["currency"] = "XXX" asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc)
def test_04_pull_public(self): # Pull a PublicAPC through the ApiRequest object acc = MonitorUKAccount() acc.id = "abcdefghij" acc.save() acc2 = MonitorUKAccount() acc2.id = "qwerty" acc.save(blocking=True) # make a competing public version, and check that a pull on the doi works pub_source = PublicAPCFixtureFactory.example() pub = PublicAPC(pub_source) pub.save(blocking=True) result = ApiRequest.pull("10.1234/me", account=acc) assert result.raw == pub.clean_record assert result.account.id == acc.id assert result.public_id == pub.id craw = deepcopy(pub.clean_record) craw["@context"] = app.config.get("API_JSON_LD_CONTEXT") assert json.loads(result.json()) == craw # also try the pull with the wrong owner, which should work result = ApiRequest.pull("10.1234/me", account=acc2) assert result is not None
def check_jobs(cls): """ Check any existing Lantern jobs for progress, and process any that have completed :return: """ dao = LanternJob() delay = app.config.get("JOB_LOOKUP_DELAY_LANTERN", 3600) cutoff = dates.before_now(delay) gen = dao.list_active(cutoff, keepalive="30m") for job in gen: acc = MonitorUKAccount.pull(job.account) lc = client.Lantern(api_key=acc.lantern_api_key) prog = lc.get_progress(job.job_id) if prog.get("status") == "success": pc = prog.get("data", {}).get("progress", 0) if pc != 100: # this will update the last_updated date, which means we won't look at it again for a while job.save() continue # if we get here, the job is complete so we need to retrieve it results = lc.get_results(job.job_id) if results.get("status") == "success": for res in results.get("data", []): enhancement = LanternApi._xwalk(res) enhancement.save() # set the job as complete job.status = "complete" job.save()
def test_07_save_delete(self): # Work through acycle of saves and deletes to observe the outputs source = RequestFixtureFactory.record() acc = MonitorUKAccount() acc.save(blocking=True) req = ApiRequest(source, account=acc) req.save() dao = Request() req2 = dao.pull(req.request.id) assert req2 is not None assert req2.owner == acc.id assert req2.record == source assert req2.action == "update" # now publish the request PublicApi.publish(req2) time.sleep(2) # now pull the object as identified by its API identifier (which should be the DOI) source2 = deepcopy(source) source2["dc:title"] = "An update" next = ApiRequest.pull(req.id, account=acc) next.update(source2) next.save() # now, at this point we should have 2 request objects in the index. One for the # original save, and one for the new save req3 = dao.pull(next.request.id) assert req3 is not None assert req3.owner == acc.id assert req3.record == source2 assert req3.action == "update" # now issue a delete on the same record next.delete() # by now we should have 3 request objects in the index, 2 for the above updates # and one for the delete request req4 = dao.pull(next.request.id) assert req4 is not None assert req4.owner == acc.id assert req4.record == source2 assert req4.action == "delete"
def test_05_pull_then_update(self): # Pull a record through the ApiRequest object and then udpate it acc = MonitorUKAccount() acc.id = "abcdefghij" acc.save(blocking=True) pub_source = PublicAPCFixtureFactory.example() del pub_source["id"] pub = PublicAPC(pub_source) pub.save(blocking=True) pub_source2 = deepcopy(pub_source.get("record")) pub_source2["dc:title"] = "An update" result = ApiRequest.pull(pub.id, account=acc) result.update(pub_source2) assert result.raw == pub_source2 assert result.account.id == acc.id
def test_09_responses(self): # Check that we get appropriate JSON responses source = RequestFixtureFactory.record() acc = MonitorUKAccount() acc.save(blocking=True) req = ApiRequest(source, account=acc) req.save() # have a look at what we'd expect the responses to be if this was a create cr = req.created_response() assert cr.get("status") == "created" assert cr.get("request_id") is not None assert cr.get("public_id") == "10.1234/me" # now look at the responses if this was an update/append ur = req.updated_response() assert ur.get("status") == "updated" assert ur.get("request_id") is not None assert ur.get("public_id") == "10.1234/me" # now look at a delete dr = req.deleted_response() assert dr.get("status") == "deleted" assert dr.get("request_id") is not None assert dr.get("public_id") == "10.1234/me" # just make a quick check to be sure that if no DOI is present, we get the right kind of info in the public_id source2 = RequestFixtureFactory.record() source2["dc:identifier"] = [{"type" : "pmcid", "id" : "PMC1234"}] req2 = ApiRequest(source2, account=acc) req2.save() cr = req.created_response() assert cr.get("public_id") == req.id ur = req.updated_response() assert ur.get("public_id") == req.id dr = req.deleted_response() assert dr.get("public_id") == req.id
def test_03_pull_request(self): # Pull a Request through the ApiRequest object acc = MonitorUKAccount() acc.id = "abcdefghij" acc.save(blocking=True) # first make a request which contains that doi req_source = RequestFixtureFactory.example() req = Request(req_source) req.save(blocking=True) # you can't pull a request object, so just show that that's true... # pull by doi should fail result = ApiRequest.pull("10.1234/me", account=acc) assert result is None # pull by request id should fail result = ApiRequest.pull(req.id, account=acc) assert result is None
def test_06_get_id(self): # Get the operative ID of a record acc = MonitorUKAccount() acc.id = "abcdefghij" source = RequestFixtureFactory.record() # first, provide a record which has a doi, and ensure that's the id we get back req = ApiRequest(source, account=acc) assert req.id == "10.1234/me" # now make a public record, and check we get the right ids for it pub_source = PublicAPCFixtureFactory.example() del pub_source["id"] del pub_source["record"]["dc:identifier"] pub = PublicAPC(pub_source) pub.save(blocking=True) result = ApiRequest.pull(pub.id, account=acc) assert result.id == pub.id assert result.id == result.public_id
def make_new_jobs(cls): """ Send new requests to lantern, and record the jobs that are created. This method looks for accounts who have Lantern credentials, looks for PublicAPC records belonging to those accounts which could benefit from lookup in Lantern, :return: """ dao = PublicAPC() gen = MonitorUKAccount.list_lantern_enabled(keepalive="1h") for acc in gen: gen2 = dao.list_by_owner(acc.id) identifiers = [] for apc in gen2: if LanternApi._needs_lantern_data(apc): idents = LanternApi._get_identifiers(apc) if idents is not None: identifiers.append(idents) apc.lantern_lookup = dates.now() apc.save() # if there are no identifiers, no need to do any more if len(identifiers) == 0: continue # now check the user's quota lc = client.Lantern(api_key=acc.lantern_api_key) quota = lc.get_quota(acc.lantern_email) available = quota.get("data", {}).get("available", 0) if available == 0: continue if len(identifiers) > available: identifiers = identifiers[:available] batches = LanternApi._batch(identifiers) for batch in batches: resp = lc.create_job(acc.lantern_email, "monitor-uk", batch) if resp.get("status") == "success": job_id = resp.get("data", {}).get("job") lj = LanternJob() lj.job_id = job_id lj.account = acc.id lj.status = "active" lj.save()
def test_05_low_quota(self): # Check what happens when the use has a low quota on Lantern global QUOTA QUOTA = 1 acc2 = MonitorUKAccount() acc2.email = "*****@*****.**" acc2.lantern_email = "*****@*****.**" acc2.lantern_api_key = "123456789" acc2.save() # a record that needs lantern because of a missing field source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None) del source["admin"]["lantern_lookup"] del source["record"]["rioxxterms:publication_date"] pub = PublicAPC(source) pub.save() # a record that needs lantern because it has timed out and has a missing field source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None) source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago del source["record"]["rioxxterms:publication_date"] pub = PublicAPC(source) pub.save(blocking=True) LanternApi.make_new_jobs() time.sleep(2) dao = LanternJob() jobs = [job for job in dao.iterall()] assert len(jobs) == 1 assert len(CREATED_JOBS) == 1 job = CREATED_JOBS[0] assert job["email"] == "*****@*****.**" assert len(job["list"]) == 1
def test_16_lantern_accounts(self): # Check that we can detect lantern accounts acc1 = MonitorUKAccount() acc1.email = "*****@*****.**" acc1.save() acc2 = MonitorUKAccount() acc2.email = "*****@*****.**" acc2.lantern_email = "*****@*****.**" acc2.lantern_api_key = "123456789" acc2.save() acc3 = MonitorUKAccount() acc3.email = "*****@*****.**" acc3.lantern_email = "*****@*****.**" acc3.lantern_api_key = "987654321" acc3.save(blocking=True) count = 0 gen = MonitorUKAccount.list_lantern_enabled() for acc in gen: if acc.email == "*****@*****.**": assert acc.lantern_api_key == "123456789" assert acc.lantern_email == "*****@*****.**" count += 1 elif acc.email == "*****@*****.**": assert acc.lantern_api_key == "987654321" assert acc.lantern_email == "*****@*****.**" count += 10 else: count += 100 assert count == 11
def test_07_check_jobs(self): # Ensure that we can check existing jobs correctly acc = MonitorUKAccount() acc.email = "*****@*****.**" acc.lantern_email = "*****@*****.**" acc.lantern_api_key = "123456789" acc.save() lj1 = LanternJob() lj1.job_id = "111111111" lj1.account = acc.id lj1.status = "complete" lj1.save() lj2 = LanternJob() lj2.job_id = "222222222" lj2.account = acc.id lj2.status = "active" lj2.last_updated = dates.format(dates.before_now(5000)) lj2.save(updated=False) lj3 = LanternJob() lj3.job_id = "333333333" lj3.account = acc.id lj3.status = "active" lj3.last_updated = dates.format(dates.before_now(5000)) lj3.save(updated=False) lj4 = LanternJob() lj4.job_id = "444444444" lj4.account = acc.id lj4.status = "active" lj4.last_updated = dates.format(dates.before_now(5000)) lj4.save(updated=False) lj5 = LanternJob() lj5.job_id = "555555555" lj5.account = acc.id lj5.status = "active" lj5.save(blocking=True) LanternApi.check_jobs() # check that the progress requests we expected were made assert len(PROGRESS_REQUESTS) == 3 assert "222222222" in PROGRESS_REQUESTS assert "333333333" in PROGRESS_REQUESTS assert "444444444" in PROGRESS_REQUESTS # check that the job which received an error was just ignored dao = LanternJob() ignored = dao.pull(lj4.id) assert ignored.last_updated == lj4.last_updated assert ignored.status == "active" # check that the record which was not complete was touched touched = dao.pull(lj2.id) assert touched.last_updated != lj2.last_updated assert touched.status == "active" # check that results were requested only for one item assert len(RESULTS_REQUESTS) == 1 assert "333333333" in RESULTS_REQUESTS # wait for a bit, so that enhancements have time to go in time.sleep(2) # check that an enhancement was registered edao = Enhancement() gen = edao.iterall() enhancements = [e for e in gen] assert len(enhancements) == 1 result = LanternFixtureFactory.xwalk_result() assert enhancements[0].data["record"] == result["record"]
def test_04_create_job(self): # Check we can create jobs correctly acc1 = MonitorUKAccount() acc1.email = "*****@*****.**" acc1.save() acc2 = MonitorUKAccount() acc2.email = "*****@*****.**" acc2.lantern_email = "*****@*****.**" acc2.lantern_api_key = "123456789" acc2.save() acc3 = MonitorUKAccount() acc3.email = "*****@*****.**" acc3.lantern_email = "*****@*****.**" acc3.lantern_api_key = "987654321" acc3.save(blocking=True) # a record which does not need lantern source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None) source["admin"]["lantern_lookup"] = dates.now() pub = PublicAPC(source) pub.save() # a record that needs lantern because of a missing field source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None) del source["admin"]["lantern_lookup"] del source["record"]["rioxxterms:publication_date"] pub = PublicAPC(source) pub.save() # a record that needs lantern because it has timed out and has a missing field source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None) source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago del source["record"]["rioxxterms:publication_date"] pub = PublicAPC(source) pub.save() # a record that needs lantern but has no identifiers source = PublicAPCFixtureFactory.make_record(acc2.id, None, None, None) source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago del source["record"]["rioxxterms:publication_date"] del source["record"]["dc:identifier"] pub = PublicAPC(source) pub.save() # a record which does not need lantern source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None) source["admin"]["lantern_lookup"] = dates.now() pub = PublicAPC(source) pub.save() # a record that needs lantern because of a missing field source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None) del source["admin"]["lantern_lookup"] del source["record"]["rioxxterms:publication_date"] pub = PublicAPC(source) pub.save() # a record that needs lantern because it has timed out and has a missing field source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None) source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago del source["record"]["rioxxterms:publication_date"] pub = PublicAPC(source) pub.save() # a record that needs lantern but has no identifiers source = PublicAPCFixtureFactory.make_record(acc3.id, None, None, None) source["admin"]["lantern_lookup"] = dates.format(dates.before_now(31104000)) # a year ago del source["record"]["rioxxterms:publication_date"] del source["record"]["dc:identifier"] pub = PublicAPC(source) pub.save(blocking=True) LanternApi.make_new_jobs() time.sleep(2) dao = LanternJob() jobs = [job for job in dao.iterall()] assert len(jobs) == 2 assert len(CREATED_JOBS) == 2 count = 0 for job in CREATED_JOBS: if job["email"] == "*****@*****.**": count += 1 assert len(job["list"]) == 2 elif job["email"] == "*****@*****.**": count += 10 assert len(job["list"]) == 2 assert count == 11 # now do the same thing again. The jobs should not change, as we've already created jobs # for all the public records LanternApi.make_new_jobs() time.sleep(2) jobs = [job for job in dao.iterall()] assert len(jobs) == 2