def test_should_see_bad_attachment( self ):
global db
feeds1 = list( Feed.view("feed/new") )
self.assertEqual( 1, len(feeds1) )
feeds2 = list( new_feed_iter() )
self.assertEqual( 1, len(feeds2) )
f, rdr= validate_and_open( feeds2[0] )
self.assertIsNone( rdr )
self.assertEquals( "wrong format attachment", f.status )
def test_should_process_feed( self ):
arrivals = []
def track_arrival( mappings, report ):
arrivals.append( report )
locations = []
def track_location( mappings, report ):
locations.append( report )
mappings = {}
counts= transform_new(
mappings, new_feed_iter(),
track_arrival, track_location,
today=datetime.date(2012,3,15) )
self.assertEqual( 1, counts['all'] )
self.assertEqual( 1, counts['arrival'] )
self.assertEqual( 0, counts['location'] )
self.assertEqual( 1, len(arrivals) )
self.assertEqual( 0, len(locations) )
feeds2 = list( new_feed_iter() )
self.assertEqual( 0, len(feeds2) )
old = list( Feed.view('feed/old') )
self.assertEqual( 1, len(old) )
self.assertEqual( "processed", old[0].status )
def test_should_open_and_read_feed( self ):
global db
feeds2 = list( new_feed_iter() )
self.assertEqual( 1, len(feeds2) )
f, rdr= validate_and_open( feeds2[0] )
self.assertEqual( self.feed_headings, rdr.source.fieldnames )
data = list( rdr )
self.assertEqual( 1, len(data) )
#print( data[0] )
self.assertEqual( datetime.datetime(2012,3,15,2,3,4), data[0].timestamp )
self.assertEqual( '123', data[0].id )
self.assertAlmostEqual( 36.12345, data[0].lat )
self.assertAlmostEqual( -76.23456, data[0].lon )
self.assertEqual( "V", data[0].ll_valid )
self.assertEqual( -1, data[0].adher )
self.assertEqual( "V", data[0].adher_valid )
self.assertEqual( "2", data[0].rte )
self.assertEqual( "3", data[0].dir )
self.assertEqual( "5", data[0].stop )
def build_status():
"""Get the mappings and refresh the mappings cache.
Then process all new feeds using :func:`caravel.feed.new_feed_iter`
instead of using the change notification.
"""
Mapping.set_db(settings.db)
Feed.set_db(settings.db)
Route.set_db(settings.db)
RouteStop.set_db(settings.db)
Vehicle.set_db(settings.db)
Stop.set_db(settings.db)
# Mappings cache in the application server.
mappings = {}
# Remove all damaged feed documents; these cannot be processed.
docs = feed.remove_damaged( settings.db, settings.db.view( "feed/new" ) )
print( "Cleanup", docs )
# If the change notification is a mapping...
# Or. Do all new mappings.
counts= mapping.refresh_mapping_cache(mappings, Mapping.view('mapping/new', descending=True))
print( "Mapping", dict(counts) )
# If the change notification is a feed...
docs= status.remove_old(settings.db)
print( "Status Removal", docs )
start= datetime.datetime.now()
counts= feed.transform_new( mappings, feed.new_feed_iter(), status.track_arrival, status.track_location )
end= datetime.datetime.now()
print( "Transform {0} reports in {1}".format( dict(counts), end-start ) )
# Not every time we receive a feed; only once per day.
docs= feed.remove_old( settings.db )
print( "Feed Removal", docs )
