def test_02_update(self): # Do updates on the ApiRequest object source = RequestFixtureFactory.record() csource = deepcopy(source) csource["@context"] = app.config.get("API_JSON_LD_CONTEXT") acc = MonitorUKAccount() acc.save() # do an update on blank one (unlikely use case) req = ApiRequest() req.update(csource) assert req.raw == source assert req.account is None assert isinstance(req.json(), basestring) # make one, and then update it source2 = deepcopy(source) source2["dc:title"] = "An update" req = ApiRequest(source, account=acc) req.update(source2) assert req.raw == source2 assert req.account.id == acc.id assert isinstance(req.json(), basestring) # make a valid one, and then do something invalid to it req = ApiRequest(source, account=acc) with self.assertRaises(dataobj.DataStructureException): req.update({"whatever" : "here"})
def test_02_request(self): # Check we can instantiate and work with a Request # first make a blank one req = Request() # now make one around the fixture source = RequestFixtureFactory.example() req = Request(source) # make one with a broken source broken = {"whatever" : "broken"} with self.assertRaises(dataobj.DataStructureException): req = Request(broken) # now make one bit by bit req = Request() req.record = source.get("record") req.owner = "test1" req.action = "update" req.public_id = "abcdefg" assert req.owner == "test1" assert req.action == "update" assert req.public_id == "abcdefg" # now make it broken req = Request() with self.assertRaises(dataobj.DataStructureException): req.record = {"random" : "stuff"}
def test_09_remove_permanent(self): # Separate an incoming Request from its corresponding PublicAPC, leaving no owners, thus deleting the record source = RequestFixtureFactory.example() req = Request(source) req.owner = "test" # create a record with 2 distinct apcs from different owners source2 = PublicAPCFixtureFactory.example() pub = PublicAPC(source2) pub.remove_apcs_by_owner("abcdefg") # clear the existing apc record apc_record = PublicAPCFixtureFactory.apc_record() del apc_record["ref"] # do this so that the ref gets created correctly later pub.add_apc_for_owner("test", apc_record) # add a new, known one pub.save(blocking=True) # now request the removal PublicApi.remove(req) time.sleep(2) dao = PublicAPC() pub2 = dao.pull(pub.id) assert pub2 is None
def test_10_request_refs(self): # Check that APC refs are handled correctly by Reuqests # first check that refs are stripped automatically on construction source = RequestFixtureFactory.example() source["record"]["jm:apc"][0]["ref"] = "1234567890" req = Request(source) assert "ref" not in req.apc_records[0] # now do it again, setting the record explicitly source = RequestFixtureFactory.example() record = source.get("record") record["jm:apc"][0]["ref"] = "123456789" req = Request() req.record = record assert "ref" not in req.apc_records[0]
def test_01_create(self): # Create instances of the ApiRequest object # construct a blank one, just to make sure we can req = ApiRequest() assert req.raw is None assert req.account is None assert req.json() is None # make one using valid source data source = RequestFixtureFactory.record() csource = deepcopy(source) csource["@context"] = app.config.get("API_JSON_LD_CONTEXT") acc = MonitorUKAccount() acc.save() req = ApiRequest(csource, account=acc) assert req.raw == source assert req.account.id == acc.id assert isinstance(req.json(), basestring) # make one with some invalid data with self.assertRaises(dataobj.DataStructureException): req = ApiRequest({"whatever" : "here"}, account=acc)
def test_08_append(self): # Check that append works, which is the same as update source = RequestFixtureFactory.record() acc = MonitorUKAccount() acc.save() # do an update on blank one (unlikely use case) req = ApiRequest() req.append(source) assert req.raw == source assert req.account is None assert isinstance(req.json(), basestring) # make one, and then update it source2 = deepcopy(source) source2["dc:title"] = "An update" req = ApiRequest(source, account=acc) req.append(source2) assert req.raw == source2 assert req.account.id == acc.id assert isinstance(req.json(), basestring) # make a valid one, and then do something invalid to it req = ApiRequest(source, account=acc) with self.assertRaises(dataobj.DataStructureException): req.append({"whatever" : "here"})
def test_02_find_public_record(self): # Find a public record with a variety of identifiers source = PublicAPCFixtureFactory.example() pub = PublicAPC(source) pub.save(blocking=True) # document to form the basis of the queries source2 = RequestFixtureFactory.example() # create sources with one of each kind of identifier, then look them up using the # find_public_record and find_public_record_by_identifier methods pid = deepcopy(source2) del pid["record"]["dc:identifier"] req = Request(pid) req.public_id = pub.id pub1 = PublicApi.find_public_record(req) assert pub1 is not None doi = deepcopy(source2) doi["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/me"}] req = Request(doi) pub1 = PublicApi.find_public_record(req) assert pub1 is not None pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/me") assert pub11 is not None pmid = deepcopy(source2) pmid["record"]["dc:identifier"] = [{"type": "pmid", "id": "87654321"}] req = Request(pmid) pub1 = PublicApi.find_public_record(req) assert pub1 is not None pub11 = PublicApi.find_public_record_by_identifier("pmid", "87654321") assert pub11 is not None pmcid = deepcopy(source2) pmcid["record"]["dc:identifier"] = [{"type": "pmcid", "id": "PMC1234"}] req = Request(pmcid) pub1 = PublicApi.find_public_record(req) assert pub1 is not None pub11 = PublicApi.find_public_record_by_identifier("pmcid", "PMC1234") assert pub11 is not None url = deepcopy(source2) url["record"]["dc:identifier"] = [{"type": "url", "id": "http://example.com/whatever"}] req = Request(url) pub1 = PublicApi.find_public_record(req) assert pub1 is not None pub11 = PublicApi.find_public_record_by_identifier("url", "http://example.com/whatever") assert pub11 is not None # finally, ensure that you don't get a match when you shouldn't null = deepcopy(source2) null["record"]["dc:identifier"] = [{"type": "doi", "id": "10.1234/another"}] req = Request(null) pub1 = PublicApi.find_public_record(req) assert pub1 is None pub11 = PublicApi.find_public_record_by_identifier("doi", "10.1234/another") assert pub11 is None
def test_03_publish_new(self): # Publish a new request source = RequestFixtureFactory.example() req = Request(source) pub = PublicApi.publish(req) dao = PublicAPC() pub2 = dao.pull(pub.id) assert pub2 is not None
def test_10_validate(self): # Check the various validation routines acc = MonitorUKAccount() acc.save() source = RequestFixtureFactory.record() # first prove that our source record validates csource = deepcopy(source) csource["@context"] = app.config.get("API_JSON_LD_CONTEXT") req = ApiRequest(csource, account=acc) assert req.raw == csource # 1 - a record with no identifiers asource = deepcopy(source) del asource["dc:identifier"] asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 2 - a record with no apc asource = deepcopy(source) del asource["jm:apc"] asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 3 - an apc with no inc vat amount asource = deepcopy(source) del asource["jm:apc"][0]["amount_inc_vat_gbp"] asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 4 - a record with a date in the wrong format asource = deepcopy(source) asource["dcterms:dateAccepted"] = "sometime last week" asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 5 - a record with an unparseable number asource = deepcopy(source) asource["jm:apc"][0]["amount_inc_vat_gbp"] = "three fifty" asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc) # 6 - a record with an invalid currency code asource = deepcopy(source) asource["jm:apc"][0]["currency"] = "XXX" asource["@context"] = app.config.get("API_JSON_LD_CONTEXT") with self.assertRaises(dataobj.DataStructureException): req = ApiRequest(asource, account=acc)
def test_09_responses(self): # Check that we get appropriate JSON responses source = RequestFixtureFactory.record() acc = MonitorUKAccount() acc.save(blocking=True) req = ApiRequest(source, account=acc) req.save() # have a look at what we'd expect the responses to be if this was a create cr = req.created_response() assert cr.get("status") == "created" assert cr.get("request_id") is not None assert cr.get("public_id") == "10.1234/me" # now look at the responses if this was an update/append ur = req.updated_response() assert ur.get("status") == "updated" assert ur.get("request_id") is not None assert ur.get("public_id") == "10.1234/me" # now look at a delete dr = req.deleted_response() assert dr.get("status") == "deleted" assert dr.get("request_id") is not None assert dr.get("public_id") == "10.1234/me" # just make a quick check to be sure that if no DOI is present, we get the right kind of info in the public_id source2 = RequestFixtureFactory.record() source2["dc:identifier"] = [{"type" : "pmcid", "id" : "PMC1234"}] req2 = ApiRequest(source2, account=acc) req2.save() cr = req.created_response() assert cr.get("public_id") == req.id ur = req.updated_response() assert ur.get("public_id") == req.id dr = req.deleted_response() assert dr.get("public_id") == req.id
def test_05_request_dao(self): # Check the DAO methods on the Request object dao = Request() source = RequestFixtureFactory.example() req = Request(source) req.owner = "test1" req.action = "update" req.public_id = "abcdefg" req.save(blocking=True) req2 = dao.pull(req.id) assert req2 is not None # check successful queries for identifiers res = dao.find_by_identifier("doi", "10.1234/me", "test1") assert len(res) == 1 res = dao.find_by_identifier("pmcid", "PMC1234", "test1") assert len(res) == 1 res = dao.find_by_identifier("pmid", "87654321", "test1") assert len(res) == 1 res = dao.find_by_identifier("url", "http://example.com/whatever", "test1") assert len(res) == 1 # check unsuccessful ones res = dao.find_by_identifier("doi", "10.1234/you", "test1") assert len(res) == 0 res = dao.find_by_identifier("pmcid", "PMC5678", "test1") assert len(res) == 0 res = dao.find_by_identifier("pmid", "123456789", "test1") assert len(res) == 0 res = dao.find_by_identifier("url", "http://example.com/this", "test1") assert len(res) == 0 # and check using the wrong owner res = dao.find_by_identifier("doi", "10.1234/me", "test2") assert len(res) == 0 res = dao.find_by_identifier("pmcid", "PMC1234", "test2") assert len(res) == 0 res = dao.find_by_identifier("pmid", "87654321", "test2") assert len(res) == 0 res = dao.find_by_identifier("url", "http://example.com/whatever", "test2") assert len(res) == 0
def test_10_find_request(self): # Find a Request through a number of routes source = RequestFixtureFactory.example() req = Request(source) req.save(blocking=True) time.sleep(2) source = RequestFixtureFactory.example() req1 = Request(source) req1.save(blocking=True) # document to form the basis of the queries source2 = RequestFixtureFactory.example() # create sources with one of each kind of identifier, then look them up using the # find_request_by_identifier method result = RequestApi.find_request_by_identifier("doi", "10.1234/me", "abcdefghij") assert result is not None assert result.created_date == req1.created_date result = RequestApi.find_request_by_identifier("pmid", "87654321", "abcdefghij") assert result is not None assert result.created_date == req1.created_date result = RequestApi.find_request_by_identifier("pmcid", "PMC1234", "abcdefghij") assert result is not None assert result.created_date == req1.created_date result = RequestApi.find_request_by_identifier("url", "http://example.com/whatever", "abcdefghij") assert result is not None assert result.created_date == req1.created_date # finally, ensure that you don't get a match when you shouldn't result = RequestApi.find_request_by_identifier("doi", "10.1234/another", "abcdefghij") assert result is None result = RequestApi.find_request_by_identifier("doi", "10.1234/me", "test") assert result is None
def test_11_process_requests_exception(self): # What happens when the process_reuests method fails for a variety of reasons sources = RequestFixtureFactory.request_per_day("2001-01", 9) dois = ["10.1234/first", "10.1234/second", "10.1234/third"] # we're going to construct a series of requests for each doi # starting with a create, then an update, followed by a delete # (not that it matters, as we're going to pump them through a mock) for i in range(len(sources)): s = sources[i] doi_idx = i % 3 # iterate over the dois 3 times doi = dois[doi_idx] s["record"]["dc:identifier"] = [{"type": "doi", "id": doi}] if i < 3: s["record"]["dc:title"] = "Create" req = Request(s) req.action = "update" req.save() elif i < 6: s["record"]["dc:title"] = "Update" req = Request(s) req.action = "update" req.save() else: s["record"]["dc:title"] = "Delete" req = Request(s) req.action = "delete" req.save() time.sleep(2) # set up the mocks PublicApi.publish = publish_mock PublicApi.remove = delete_mock # now run the process job back to the first day with self.assertRaises(TestException): WorkflowApi.process_requests() # we know this died during the 6th update request being processed, # so just check that the workflow state reflects that wfs_dao = WorkflowState() wfs = wfs_dao.pull("requests") assert wfs.last_request == "2001-01-05T00:00:00Z" assert len(wfs.already_processed) == 1
def test_07_save_delete(self): # Work through acycle of saves and deletes to observe the outputs source = RequestFixtureFactory.record() acc = MonitorUKAccount() acc.save(blocking=True) req = ApiRequest(source, account=acc) req.save() dao = Request() req2 = dao.pull(req.request.id) assert req2 is not None assert req2.owner == acc.id assert req2.record == source assert req2.action == "update" # now publish the request PublicApi.publish(req2) time.sleep(2) # now pull the object as identified by its API identifier (which should be the DOI) source2 = deepcopy(source) source2["dc:title"] = "An update" next = ApiRequest.pull(req.id, account=acc) next.update(source2) next.save() # now, at this point we should have 2 request objects in the index. One for the # original save, and one for the new save req3 = dao.pull(next.request.id) assert req3 is not None assert req3.owner == acc.id assert req3.record == source2 assert req3.action == "update" # now issue a delete on the same record next.delete() # by now we should have 3 request objects in the index, 2 for the above updates # and one for the delete request req4 = dao.pull(next.request.id) assert req4 is not None assert req4.owner == acc.id assert req4.record == source2 assert req4.action == "delete"
def test_01_request_delete(self): # Create a delete request record = RequestFixtureFactory.record() acc = BasicAccount() acc.id = "test1" r = RequestApi.delete(record, acc, "01010101") # now check it was saved correctly assert r is not None r1 = r.pull(r.id) assert r1 is not None assert r1.owner == acc.id assert r1.action == "delete" assert r1.public_id == "01010101"
def test_07_request2public(self): # Check the conversion of a Request to a PublicAPC source = RequestFixtureFactory.example() req = Request(source) pub = req.make_public_apc() assert pub is not None assert pub.record is not None assert len(pub.apc_records) == 1 setrefs = [] for apc in pub.apc_records: assert apc.get("ref") is not None setrefs.append(apc.get("ref")) assert len(setrefs) == 1 refs = pub.get_apc_refs(req.owner) assert len(refs) == 1 assert refs[0] == setrefs[0]
def test_03_pull_request(self): # Pull a Request through the ApiRequest object acc = MonitorUKAccount() acc.id = "abcdefghij" acc.save(blocking=True) # first make a request which contains that doi req_source = RequestFixtureFactory.example() req = Request(req_source) req.save(blocking=True) # you can't pull a request object, so just show that that's true... # pull by doi should fail result = ApiRequest.pull("10.1234/me", account=acc) assert result is None # pull by request id should fail result = ApiRequest.pull(req.id, account=acc) assert result is None
def test_06_get_id(self): # Get the operative ID of a record acc = MonitorUKAccount() acc.id = "abcdefghij" source = RequestFixtureFactory.record() # first, provide a record which has a doi, and ensure that's the id we get back req = ApiRequest(source, account=acc) assert req.id == "10.1234/me" # now make a public record, and check we get the right ids for it pub_source = PublicAPCFixtureFactory.example() del pub_source["id"] del pub_source["record"]["dc:identifier"] pub = PublicAPC(pub_source) pub.save(blocking=True) result = ApiRequest.pull(pub.id, account=acc) assert result.id == pub.id assert result.id == result.public_id
def test_04_record_methods(self): # Check all the Record methods work classes = [Request, PublicAPC] source = RequestFixtureFactory.example() record = source.get("record") additional_apcs = deepcopy(record.get("jm:apc")) + deepcopy(record.get("jm:apc")) for c in classes: inst = c() assert isinstance(inst, RecordMethods) inst.record = deepcopy(record) assert inst.doi == "10.1234/me" assert inst.pmid == "87654321" assert inst.pmcid == "PMC1234" assert inst.url == "http://example.com/whatever" assert len(inst.apc_records) == 1 inst.apc_records = additional_apcs assert len(inst.apc_records) == 2 # now try adding and removing apc records withref = deepcopy(record.get("jm:apc")[0]) withref["ref"] = "myref" inst.add_apc_record(withref) assert len(inst.apc_records) == 3 inst.remove_apc_by_ref("myref") assert len(inst.apc_records) == 2 for apc in inst.apc_records: assert "ref" not in apc assert inst.has_apcs() is True inst.apc_records = [] assert inst.has_apcs() is False
def test_01_request_update(self): # Create an update Request record = RequestFixtureFactory.record() acc = BasicAccount() acc.id = "test1" # without a public id r1 = RequestApi.update(record, acc) # with a public id r2 = RequestApi.update(record, acc, "01010101") # now check they were saved correctly assert r1 is not None assert r2 is not None r11 = r1.pull(r1.id) assert r11 is not None r21 = r2.pull(r2.id) assert r21 is not None
def test_13_request_iterator(self): # Check we can iterate successfully over lists of Requests sources = RequestFixtureFactory.request_per_day("2001-01", 10) for s in sources: req = Request(s) req.save() time.sleep(2) dao = Request() gen = dao.list_all_since("2001-01-01T00:00:00Z", page_size=5) # set the page size small, to ensure the iterator has to work results = [x for x in gen] assert len(results) == 10 dates = [r.created_date for r in results] comp = deepcopy(dates) comp.sort() # this puts the dates in ascending order (i.e. oldest first) # the point of this comparison is to show that the results came out in the right order. # that is, oldest first assert dates == comp
def test_08_remove_separate(self): # Separate an incoming Request from its corresponding PublicAPC, leaving only one owner behind source = RequestFixtureFactory.example() req = Request(source) req.owner = "test" # create a record with 2 distinct apcs from different owners source2 = PublicAPCFixtureFactory.example() apc_record = PublicAPCFixtureFactory.apc_record() del apc_record["ref"] # do this so that the ref gets created correctly later pub = PublicAPC(source2) pub.add_apc_for_owner("test", apc_record) pub.save(blocking=True) # now request the removal PublicApi.remove(req) time.sleep(2) dao = PublicAPC() pub2 = dao.pull(pub.id) assert len(pub2.get_apcs_by_owner("test")) == 0 assert len(pub2.get_apcs_by_owner("abcdefg")) == 1
def test_11_process_requests_cycle(self): # Run through the process of processing a Request into a PublicAPC source = RequestFixtureFactory.example() if "id" in source: del source["id"] pub_dao = PublicAPC() wfs_dao = WorkflowState() # first make a record for the first time first = deepcopy(source) del first["record"]["dc:title"] req = Request(first) req.owner = "test" req.action = "update" req.save(blocking=True) # run the job WorkflowApi.process_requests() time.sleep(2) # first check that a public record was made pubs = pub_dao.find_by_doi("10.1234/me") assert len(pubs) == 1 assert pubs[0].record.get("dc:title") is None # check that the workflow state was created wfs = wfs_dao.pull("requests") assert wfs is not None assert wfs.last_request == req.created_date assert wfs.already_processed == [req.id] # now run an update with a different date second = deepcopy(source) second["record"]["dc:title"] = "Update" second["created_date"] = "2002-01-01T00:00:00Z" req2 = Request(second) req2.owner = "test" req2.action = "update" req2.save(blocking=True) # run the job again WorkflowApi.process_requests() time.sleep(2) # check the public record was updated pubs = pub_dao.find_by_doi("10.1234/me") assert len(pubs) == 1 assert pubs[0].record.get("dc:title") == "Update" # check that the workflow state was updated wfs = wfs_dao.pull("requests") assert wfs is not None assert wfs.last_request == req2.created_date assert wfs.already_processed == [req2.id] # now run an update with the same date, to observe the difference in the workflow state third = deepcopy(source) third["record"]["dc:title"] = "Update 2" third["created_date"] = "2002-01-01T00:00:00Z" req3 = Request(third) req3.owner = "test" req3.action = "update" req3.save(blocking=True) # run the job again WorkflowApi.process_requests() time.sleep(2) # check the public record was updated pubs = pub_dao.find_by_doi("10.1234/me") assert len(pubs) == 1 assert ( pubs[0].record.get("dc:title") == "Update 2" ) # should have been updated, as there are only apc contributions from one source # check that the workflow state was updated wfs = wfs_dao.pull("requests") assert wfs is not None assert wfs.last_request == req3.created_date assert wfs.already_processed == [req2.id, req3.id] # processed records should have been appended # finally issue a delete request fourth = deepcopy(source) fourth["created_date"] = "2003-01-01T00:00:00Z" req4 = Request(fourth) req4.owner = "test" req4.action = "delete" req4.save(blocking=True) # run the job again WorkflowApi.process_requests() time.sleep(2) # check the public record was updated pubs = pub_dao.find_by_doi("10.1234/me") assert len(pubs) == 0 # check that the workflow state was updated wfs = wfs_dao.pull("requests") assert wfs is not None assert wfs.last_request == req4.created_date assert wfs.already_processed == [req4.id] # processed records should have been appended