def test_12_workflow_state(self): # Check we can construct and work with WorkflowState objects # make a blank one just in case we need to wfs = WorkflowState() # now make one from source source = WorkflowStateFixtureFactory.example() wfs = WorkflowState(source) assert wfs.last_request == "2003-01-01T00:00:00Z" assert wfs.already_processed == ["123456789", "987654321"] # now hit the setters, and check the round-trip wfs.last_request = "2004-01-01T00:00:00Z" wfs.already_processed = ["abcdefg"] assert wfs.last_request == "2004-01-01T00:00:00Z" assert wfs.already_processed == ["abcdefg"] wfs.add_processed("qwerty") assert wfs.already_processed == ["abcdefg", "qwerty"] assert wfs.is_processed("qwerty") assert wfs.is_processed("abcdefg") assert not wfs.is_processed("random") # now make one with broken content with self.assertRaises(dataobj.DataStructureException): wfs = WorkflowState({"junk" : "data"})
def process_requests(cls): """ Go through any new requests (since this method last ran) and process them. This will apply the creates, updates and deletes to the public space :return: """ # first, pick up our current state from storage workflow_dao = WorkflowState() wfs = workflow_dao.pull("requests") # if we don't have a current state, make one if wfs is None: wfs = WorkflowState() wfs.id = "requests" # get the oldest page of requests and process them dao = Request() requests = dao.list_all_since(wfs.last_request) # produces a generator for r in requests: try: # if the request was created at the time of the last request processed, it is possible it arrived # before or after the cut-off. As we don't have any more than second-level granularity in the timing, # we also need to check to see whether it was one of the ids processed during that second if r.created_date == wfs.last_request and wfs.is_processed(r.id): # if it was created at that time, and it was one of the ones processed, we can skip it continue # if the request is from a later time, or was not processed during the last run, then do the usual # processing if r.action == "update": PublicApi.publish(r) elif r.action == "delete": PublicApi.remove(r) # now, revisit the timing of this request. If the time is the same as the last request date, this is a # request which came in during that same second, but was not processed at the time because it was at the # wrong end of the second. In that case, we just need to add the id to the list of records from that second # which have now been processed if r.created_date == wfs.last_request: wfs.add_processed(r.id) else: # otherwise, this is a whole new second, and we can forget everything that went before and start afresh. wfs.last_request = r.created_date wfs.already_processed = [r.id] except: wfs.save(blocking=True) raise wfs.save(blocking=True)