def test_should_set_mappings( self ):
cache= {}
mappings = refresh_mapping_cache( cache, Mapping.view('mapping/new', descending=True), today=datetime.date(2012,4,2) )
self.assertNotIn( 'vehicle', mappings )
self.assertNotIn( 'stop', mappings )
self.assertIn( 'route', mappings )
self.assertEqual( cache, mappings )
expected= {'2': '2', '5': '17'} # route 2 on the
self.assertEqual( expected, mappings['route'].map )
def build_status():
"""Get the mappings and refresh the mappings cache.
Then process all new feeds using :func:`caravel.feed.new_feed_iter`
instead of using the change notification.
"""
Mapping.set_db(settings.db)
Feed.set_db(settings.db)
Route.set_db(settings.db)
RouteStop.set_db(settings.db)
Vehicle.set_db(settings.db)
Stop.set_db(settings.db)
# Mappings cache in the application server.
mappings = {}
# Remove all damaged feed documents; these cannot be processed.
docs = feed.remove_damaged( settings.db, settings.db.view( "feed/new" ) )
print( "Cleanup", docs )
# If the change notification is a mapping...
# Or. Do all new mappings.
counts= mapping.refresh_mapping_cache(mappings, Mapping.view('mapping/new', descending=True))
print( "Mapping", dict(counts) )
# If the change notification is a feed...
docs= status.remove_old(settings.db)
print( "Status Removal", docs )
start= datetime.datetime.now()
counts= feed.transform_new( mappings, feed.new_feed_iter(), status.track_arrival, status.track_location )
end= datetime.datetime.now()
print( "Transform {0} reports in {1}".format( dict(counts), end-start ) )
# Not every time we receive a feed; only once per day.
docs= feed.remove_old( settings.db )
print( "Feed Removal", docs )
def main():
"""Main CoucnDB consumer processing loop.
Use a long poll query for database changes.
Invoke the :func:`long_poll_callback` function for each change.
Once each hour of elapsed run-time do :func:`periodic_tasks` with hour true.
This does not happen at the the top of the hour, but instead happens
after an hour has passed.
Once each day do :func:`periodic_tasks` with day true. This happens
right around midnight.
"""
global last_seq
Mapping.set_db(settings.db)
Feed.set_db(settings.db)
Route.set_db(settings.db)
RouteStop.set_db(settings.db)
Vehicle.set_db(settings.db)
Stop.set_db(settings.db)
daily = hourly = datetime.datetime.now()
h, m, s = settings.change_notification_daily_task_time
daily_sched = h*60+m
# Seed processing status with last known sequence number.
try:
with open("last_seq.json") as status_file:
proc_state= json.load( status_file )
except (IOError, ValueError):
proc_state= {'last_seq': 0}
last_seq= proc_state['last_seq']
# Was the database reset? If so, reset the sequence number.
db_seq= settings.db.info()['update_seq']
if db_seq < last_seq:
last_seq= 0
# Seed mappings with last known good mappings.
try:
mapping.refresh_mapping_cache(mappings)
except couchdbkit.exceptions.ResourceNotFound:
log.error( "Database does not have proper view definitions" )
sys.exit(2)
consumer= SyncConsumer(settings.db)
while True:
try:
consumer.wait_once(cb=long_poll_callback, since=last_seq)
now= datetime.datetime.now()
logger.debug( "State time={0} last_seq={1}".format( now, last_seq ) )
proc_state['last_seq']= last_seq
proc_state['time']= now.strftime("%Y-%m-%dT%H:%M:%S%Z")
with open("last_seq.json",'w') as status_file:
json.dump( proc_state, status_file )
if now.date() > daily.date():
# Midnight passed, in a new day.
now_minute = now.time().hour*60+now.time().minute
if now_minute >= daily_sched:
daily= now
periodic_tasks( day=True )
elif (now-hourly).seconds > 3600:
# An hour has gone by.
hourly= now
periodic_tasks( hour=True )
except (KeyboardInterrupt, SystemExit):
logger.info( "Interrupted" )
break
def mapping_notification( new_mapping ):
"""The change notification is a complete mapping with an attachment."""
global mappings
mapping_cache= mapping.refresh_mapping_cache(mappings, [new_mapping] )
logger.info( "Mapping {0!r}".format( dict(mapping_cache) ) )
