def periodic_tasks( hour=False, day=False ): """Tasks done once in a great while. .. todo:: Must track these changes in our LRU to avoid echo issues. :param hour: If the monitor has run for an hour. :param day: If the monitor has run for a day. """ global lru_fifo # Remove old status reports; not every time. Once per day. docs= status.remove_old(settings.db) seq= settings.db.info()['update_seq'] logger.info( "Status Removal {0!r}".format( docs ) ) for d in docs: lru_fifo.append( LRU(d['id'], seq) ) # Remove old feeds; not every time. Once per day. docs= feed.remove_old(settings.db) seq= settings.db.info()['update_seq'] logger.info( "Feed Removal {0!r}".format( docs ) ) for d in docs: lru_fifo.append( LRU(d['id'], seq) ) if day: # Compact database. settings.db.compact()
def build_status(): """Get the mappings and refresh the mappings cache. Then process all new feeds using :func:`caravel.feed.new_feed_iter` instead of using the change notification. """ Mapping.set_db(settings.db) Feed.set_db(settings.db) Route.set_db(settings.db) RouteStop.set_db(settings.db) Vehicle.set_db(settings.db) Stop.set_db(settings.db) # Mappings cache in the application server. mappings = {} # Remove all damaged feed documents; these cannot be processed. docs = feed.remove_damaged( settings.db, settings.db.view( "feed/new" ) ) print( "Cleanup", docs ) # If the change notification is a mapping... # Or. Do all new mappings. counts= mapping.refresh_mapping_cache(mappings, Mapping.view('mapping/new', descending=True)) print( "Mapping", dict(counts) ) # If the change notification is a feed... docs= status.remove_old(settings.db) print( "Status Removal", docs ) start= datetime.datetime.now() counts= feed.transform_new( mappings, feed.new_feed_iter(), status.track_arrival, status.track_location ) end= datetime.datetime.now() print( "Transform {0} reports in {1}".format( dict(counts), end-start ) ) # Not every time we receive a feed; only once per day. docs= feed.remove_old( settings.db ) print( "Feed Removal", docs )