def process_enhancements(cls):
"""
Go through any new Enhancements (since this method last ran) and process them.
:return:
"""
# first, pick up our current state from storage
workflow_dao = WorkflowState()
wfs = workflow_dao.pull("enhancements")
# if we don't have a current state, make one
if wfs is None:
wfs = WorkflowState()
wfs.id = "enhancements"
# get the oldest page of enhancements and process them
dao = Enhancement()
enhancements = dao.list_all_since(wfs.last_request) # produces a generator
for e in enhancements:
try:
# if the request was created at the time of the last request processed, it is possible it arrived
# before or after the cut-off. As we don't have any more than second-level granularity in the timing,
# we also need to check to see whether it was one of the ids processed during that second
if e.created_date == wfs.last_request and wfs.is_processed(e.id):
# if it was created at that time, and it was one of the ones processed, we can skip it
continue
# if the request is from a later time, or was not processed during the last run, then do the usual
# processing, which in this case is just to publish the data, and let the merge handle it
PublicApi.publish(e)
# now, revisit the timing of this request. If the time is the same as the last request date, this is a
# request which came in during that same second, but was not processed at the time because it was at the
# wrong end of the second. In that case, we just need to add the id to the list of records from that second
# which have now been processed
if e.created_date == wfs.last_request:
wfs.add_processed(e.id)
else:
# otherwise, this is a whole new second, and we can forget everything that went before and start afresh.
wfs.last_request = e.created_date
wfs.already_processed = [e.id]
except:
wfs.save(blocking=True)
raise
wfs.save(blocking=True)
def test_20_enhancement_iterator(self):
# Check we can successfully iterate over enhancements
sources = EnhancementFixtureFactory.request_per_day("2001-01", 10)
for s in sources:
req = Enhancement(s)
req.save()
time.sleep(2)
dao = Enhancement()
gen = dao.list_all_since("2001-01-01T00:00:00Z", page_size=5) # set the page size small, to ensure the iterator has to work
results = [x for x in gen]
assert len(results) == 10
dates = [r.created_date for r in results]
comp = deepcopy(dates)
comp.sort() # this puts the dates in ascending order (i.e. oldest first)
# the point of this comparison is to show that the results came out in the right order.
# that is, oldest first
assert dates == comp