def test_should_remove_old( self ):
global db
feeds = list( Feed.view("feed/old") )
self.assertEqual( 1, len(feeds) )
remove_old( db, today = datetime.date(2012,3,16) )
feeds = list( Feed.view("feed/old") )
self.assertEqual( 0, len(feeds) )
def periodic_tasks( hour=False, day=False ):
"""Tasks done once in a great while.
.. todo:: Must track these changes in our LRU to avoid echo issues.
:param hour: If the monitor has run for an hour.
:param day: If the monitor has run for a day.
"""
global lru_fifo
# Remove old status reports; not every time. Once per day.
docs= status.remove_old(settings.db)
seq= settings.db.info()['update_seq']
logger.info( "Status Removal {0!r}".format( docs ) )
for d in docs:
lru_fifo.append( LRU(d['id'], seq) )
# Remove old feeds; not every time. Once per day.
docs= feed.remove_old(settings.db)
seq= settings.db.info()['update_seq']
logger.info( "Feed Removal {0!r}".format( docs ) )
for d in docs:
lru_fifo.append( LRU(d['id'], seq) )
if day:
# Compact database.
settings.db.compact()
def build_status():
"""Get the mappings and refresh the mappings cache.
Then process all new feeds using :func:`caravel.feed.new_feed_iter`
instead of using the change notification.
"""
Mapping.set_db(settings.db)
Feed.set_db(settings.db)
Route.set_db(settings.db)
RouteStop.set_db(settings.db)
Vehicle.set_db(settings.db)
Stop.set_db(settings.db)
# Mappings cache in the application server.
mappings = {}
# Remove all damaged feed documents; these cannot be processed.
docs = feed.remove_damaged( settings.db, settings.db.view( "feed/new" ) )
print( "Cleanup", docs )
# If the change notification is a mapping...
# Or. Do all new mappings.
counts= mapping.refresh_mapping_cache(mappings, Mapping.view('mapping/new', descending=True))
print( "Mapping", dict(counts) )
# If the change notification is a feed...
docs= status.remove_old(settings.db)
print( "Status Removal", docs )
start= datetime.datetime.now()
counts= feed.transform_new( mappings, feed.new_feed_iter(), status.track_arrival, status.track_location )
end= datetime.datetime.now()
print( "Transform {0} reports in {1}".format( dict(counts), end-start ) )
# Not every time we receive a feed; only once per day.
docs= feed.remove_old( settings.db )
print( "Feed Removal", docs )
