def process_enhancements(cls): """ Go through any new Enhancements (since this method last ran) and process them. :return: """ # first, pick up our current state from storage workflow_dao = WorkflowState() wfs = workflow_dao.pull("enhancements") # if we don't have a current state, make one if wfs is None: wfs = WorkflowState() wfs.id = "enhancements" # get the oldest page of enhancements and process them dao = Enhancement() enhancements = dao.list_all_since(wfs.last_request) # produces a generator for e in enhancements: try: # if the request was created at the time of the last request processed, it is possible it arrived # before or after the cut-off. As we don't have any more than second-level granularity in the timing, # we also need to check to see whether it was one of the ids processed during that second if e.created_date == wfs.last_request and wfs.is_processed(e.id): # if it was created at that time, and it was one of the ones processed, we can skip it continue # if the request is from a later time, or was not processed during the last run, then do the usual # processing, which in this case is just to publish the data, and let the merge handle it PublicApi.publish(e) # now, revisit the timing of this request. If the time is the same as the last request date, this is a # request which came in during that same second, but was not processed at the time because it was at the # wrong end of the second. In that case, we just need to add the id to the list of records from that second # which have now been processed if e.created_date == wfs.last_request: wfs.add_processed(e.id) else: # otherwise, this is a whole new second, and we can forget everything that went before and start afresh. wfs.last_request = e.created_date wfs.already_processed = [e.id] except: wfs.save(blocking=True) raise wfs.save(blocking=True)
def test_20_enhancement_iterator(self): # Check we can successfully iterate over enhancements sources = EnhancementFixtureFactory.request_per_day("2001-01", 10) for s in sources: req = Enhancement(s) req.save() time.sleep(2) dao = Enhancement() gen = dao.list_all_since("2001-01-01T00:00:00Z", page_size=5) # set the page size small, to ensure the iterator has to work results = [x for x in gen] assert len(results) == 10 dates = [r.created_date for r in results] comp = deepcopy(dates) comp.sort() # this puts the dates in ascending order (i.e. oldest first) # the point of this comparison is to show that the results came out in the right order. # that is, oldest first assert dates == comp