def feed_notification( new_feed ): """The change notification is a feed with an attachment.""" global mappings start= datetime.datetime.now() counts= feed.transform_new( mappings, [new_feed], status.track_arrival, status.track_location ) end= datetime.datetime.now() logger.info( "Transform {0!r} reports in {1}".format( dict(counts), end-start ) )
def test_should_process_feed( self ): arrivals = [] def track_arrival( mappings, report ): arrivals.append( report ) locations = [] def track_location( mappings, report ): locations.append( report ) mappings = {} counts= transform_new( mappings, new_feed_iter(), track_arrival, track_location, today=datetime.date(2012,3,15) ) self.assertEqual( 1, counts['all'] ) self.assertEqual( 1, counts['arrival'] ) self.assertEqual( 0, counts['location'] ) self.assertEqual( 1, len(arrivals) ) self.assertEqual( 0, len(locations) ) feeds2 = list( new_feed_iter() ) self.assertEqual( 0, len(feeds2) ) old = list( Feed.view('feed/old') ) self.assertEqual( 1, len(old) ) self.assertEqual( "processed", old[0].status )
def build_status(): """Get the mappings and refresh the mappings cache. Then process all new feeds using :func:`caravel.feed.new_feed_iter` instead of using the change notification. """ Mapping.set_db(settings.db) Feed.set_db(settings.db) Route.set_db(settings.db) RouteStop.set_db(settings.db) Vehicle.set_db(settings.db) Stop.set_db(settings.db) # Mappings cache in the application server. mappings = {} # Remove all damaged feed documents; these cannot be processed. docs = feed.remove_damaged( settings.db, settings.db.view( "feed/new" ) ) print( "Cleanup", docs ) # If the change notification is a mapping... # Or. Do all new mappings. counts= mapping.refresh_mapping_cache(mappings, Mapping.view('mapping/new', descending=True)) print( "Mapping", dict(counts) ) # If the change notification is a feed... docs= status.remove_old(settings.db) print( "Status Removal", docs ) start= datetime.datetime.now() counts= feed.transform_new( mappings, feed.new_feed_iter(), status.track_arrival, status.track_location ) end= datetime.datetime.now() print( "Transform {0} reports in {1}".format( dict(counts), end-start ) ) # Not every time we receive a feed; only once per day. docs= feed.remove_old( settings.db ) print( "Feed Removal", docs )