def test_14_process_enhancements_exception(self): # What happens when processing an enhancement fails sources = EnhancementFixtureFactory.request_per_day("2001-01", 9) dois = ["10.1234/first", "10.1234/second", "10.1234/third"] # we're going to construct a series of enhancements for each doi for i in range(len(sources)): s = sources[i] doi_idx = i % 3 # iterate over the dois 3 times doi = dois[doi_idx] s["record"]["dc:identifier"] = [{"type": "doi", "id": doi}] en = Enhancement(s) en.save() time.sleep(2) # set up the mock PublicApi.publish = publish_mock # now run the process job back to the first day with self.assertRaises(TestException): WorkflowApi.process_enhancements() time.sleep(2) # we know this died during the 6th update request being processed, # so just check that the workflow state reflects that wfs_dao = WorkflowState() wfs = wfs_dao.pull("enhancements") assert wfs.last_request == "2001-01-05T00:00:00Z" assert len(wfs.already_processed) == 1
def test_20_enhancement_iterator(self): # Check we can successfully iterate over enhancements sources = EnhancementFixtureFactory.request_per_day("2001-01", 10) for s in sources: req = Enhancement(s) req.save() time.sleep(2) dao = Enhancement() gen = dao.list_all_since("2001-01-01T00:00:00Z", page_size=5) # set the page size small, to ensure the iterator has to work results = [x for x in gen] assert len(results) == 10 dates = [r.created_date for r in results] comp = deepcopy(dates) comp.sort() # this puts the dates in ascending order (i.e. oldest first) # the point of this comparison is to show that the results came out in the right order. # that is, oldest first assert dates == comp
def test_13_process_ehnancements_cycle(self): # Run through the process of processing an enhancement source = EnhancementFixtureFactory.example() if "id" in source: del source["id"] pub_dao = PublicAPC() wfs_dao = WorkflowState() # first make a public record for us to enhance first = PublicAPCFixtureFactory.example() del first["record"]["dc:title"] pub = PublicAPC(first) pub.save(blocking=True) # now create an enhancements on the record second = deepcopy(source) second["record"]["dc:title"] = "Update" second["created_date"] = "2002-01-01T00:00:00Z" en = Enhancement(second) en.public_id = pub.id en.save(blocking=True) # run the job WorkflowApi.process_enhancements() time.sleep(2) # check that the workflow state was created wfs = wfs_dao.pull("enhancements") assert wfs is not None assert wfs.last_request == en.created_date assert wfs.already_processed == [en.id] # check the public record was updated pubs = pub_dao.find_by_doi("10.1234/me") assert len(pubs) == 1 assert pubs[0].record.get("dc:title") == "Update" # now run an update with the same date, to observe the difference in the workflow state third = deepcopy(source) third["record"]["dc:title"] = "Update 2" third["created_date"] = "2002-01-01T00:00:00Z" en2 = Enhancement(third) en2.public_id = pub.id en2.save(blocking=True) # run the job again WorkflowApi.process_enhancements() time.sleep(2) # check the public record was updated pubs = pub_dao.find_by_doi("10.1234/me") assert len(pubs) == 1 assert ( pubs[0].record.get("dc:title") == "Update" ) # should not have been updated, since data was already present # check that the workflow state was updated wfs = wfs_dao.pull("enhancements") assert wfs is not None assert wfs.last_request == en2.created_date assert wfs.already_processed == [en.id, en2.id] # processed records should have been appended