def test_should_set_mappings( self ): cache= {} mappings = refresh_mapping_cache( cache, Mapping.view('mapping/new', descending=True), today=datetime.date(2012,4,2) ) self.assertNotIn( 'vehicle', mappings ) self.assertNotIn( 'stop', mappings ) self.assertIn( 'route', mappings ) self.assertEqual( cache, mappings ) expected= {'2': '2', '5': '17'} # route 2 on the self.assertEqual( expected, mappings['route'].map )
def build_status(): """Get the mappings and refresh the mappings cache. Then process all new feeds using :func:`caravel.feed.new_feed_iter` instead of using the change notification. """ Mapping.set_db(settings.db) Feed.set_db(settings.db) Route.set_db(settings.db) RouteStop.set_db(settings.db) Vehicle.set_db(settings.db) Stop.set_db(settings.db) # Mappings cache in the application server. mappings = {} # Remove all damaged feed documents; these cannot be processed. docs = feed.remove_damaged( settings.db, settings.db.view( "feed/new" ) ) print( "Cleanup", docs ) # If the change notification is a mapping... # Or. Do all new mappings. counts= mapping.refresh_mapping_cache(mappings, Mapping.view('mapping/new', descending=True)) print( "Mapping", dict(counts) ) # If the change notification is a feed... docs= status.remove_old(settings.db) print( "Status Removal", docs ) start= datetime.datetime.now() counts= feed.transform_new( mappings, feed.new_feed_iter(), status.track_arrival, status.track_location ) end= datetime.datetime.now() print( "Transform {0} reports in {1}".format( dict(counts), end-start ) ) # Not every time we receive a feed; only once per day. docs= feed.remove_old( settings.db ) print( "Feed Removal", docs )
def main(): """Main CoucnDB consumer processing loop. Use a long poll query for database changes. Invoke the :func:`long_poll_callback` function for each change. Once each hour of elapsed run-time do :func:`periodic_tasks` with hour true. This does not happen at the the top of the hour, but instead happens after an hour has passed. Once each day do :func:`periodic_tasks` with day true. This happens right around midnight. """ global last_seq Mapping.set_db(settings.db) Feed.set_db(settings.db) Route.set_db(settings.db) RouteStop.set_db(settings.db) Vehicle.set_db(settings.db) Stop.set_db(settings.db) daily = hourly = datetime.datetime.now() h, m, s = settings.change_notification_daily_task_time daily_sched = h*60+m # Seed processing status with last known sequence number. try: with open("last_seq.json") as status_file: proc_state= json.load( status_file ) except (IOError, ValueError): proc_state= {'last_seq': 0} last_seq= proc_state['last_seq'] # Was the database reset? If so, reset the sequence number. db_seq= settings.db.info()['update_seq'] if db_seq < last_seq: last_seq= 0 # Seed mappings with last known good mappings. try: mapping.refresh_mapping_cache(mappings) except couchdbkit.exceptions.ResourceNotFound: log.error( "Database does not have proper view definitions" ) sys.exit(2) consumer= SyncConsumer(settings.db) while True: try: consumer.wait_once(cb=long_poll_callback, since=last_seq) now= datetime.datetime.now() logger.debug( "State time={0} last_seq={1}".format( now, last_seq ) ) proc_state['last_seq']= last_seq proc_state['time']= now.strftime("%Y-%m-%dT%H:%M:%S%Z") with open("last_seq.json",'w') as status_file: json.dump( proc_state, status_file ) if now.date() > daily.date(): # Midnight passed, in a new day. now_minute = now.time().hour*60+now.time().minute if now_minute >= daily_sched: daily= now periodic_tasks( day=True ) elif (now-hourly).seconds > 3600: # An hour has gone by. hourly= now periodic_tasks( hour=True ) except (KeyboardInterrupt, SystemExit): logger.info( "Interrupted" ) break
def mapping_notification( new_mapping ): """The change notification is a complete mapping with an attachment.""" global mappings mapping_cache= mapping.refresh_mapping_cache(mappings, [new_mapping] ) logger.info( "Mapping {0!r}".format( dict(mapping_cache) ) )