def update(url, filename): """ Updates a file by appending current information to already known information """ flights = scraper.load(filename) # load currently known flights from file if flights is None: flights = [] newflights = scraper.readFlights(url) # scrape newly available flight data from site for flight in newflights: if flight not in flights: flights.append(flight) # append each new flight to the list of known flights scraper.save(flights, filename) # save the updated list of flights to file
def update(url, filename):
"""
Updates a file by appending current information to already known information
"""
flights = scraper.load(filename) # load currently known flights from file
if flights is None:
flights = []
newflights = scraper.readFlights(
url) # scrape newly available flight data from site
for flight in newflights:
if flight not in flights:
flights.append(
flight) # append each new flight to the list of known flights
scraper.save(flights, filename) # save the updated list of flights to file
def test_save(self):
with open(r'data/geocode.json') as f:
sample = f.read()
responses.add(responses.GET,
'https://maps.googleapis.com/maps/api/geocode/json',
body=sample,
status=200,
content_type='application/json')
with open(r'data/directions.json') as f:
sample = f.read()
responses.add(
responses.GET,
'https://maps.googleapis.com/maps/api/directions/json',
body=sample,
status=200,
content_type='application/json')
enriched = scraper.enrich_records(RECORDS)
result = scraper.save(enriched)
self.assertTrue(result)
results = self.session.query(RealEstate).all()
self.assertEqual(6, len(results))
passengers = 0
for flight in flights[:]:
# if the flight's arrival time is at most config.interval seconds ago and less than now
if int(time.time()) - config.interval < flight.arrival and flight.arrival < int(time.time()):
numflights += 1
flightcap = flightmod.findSeats(flight)
if flightcap != None:
capacity += flightcap
passengers += config.ratio(flight) * flightcap
else:
unknown += 1
# if the flight is old enough to be archived
elif flight.arrival < int(time.time()) - config.archiveage:
archives.append(flight)
flights.remove(flight)
scraper.save(flights, config.flightdata)
scraper.save(archives, config.archivedata)
if numflights != 0 and numflights != unknown:
capacity = int(capacity * numflights / (numflights - unknown))
passengers = int(passengers * numflights / (numflights - unknown))
print numflights, capacity, passengers
#base = Firebase("https://vpc.firebaseio.com")
#base.child("dashboard/flightsRef").set(numflights)
#base.child("dashboard/capacityRef").set(capacity)
#base.child("dashboard/passengerRef").set(passengers)
flights = scraper.load(config.flightdata)
archives = scraper.load(config.archivedata)
numflights = 0
unknown = 0
passengers = 0
for flight in flights[:]:
# if the flight's arrival time is at most config.interval seconds ago and less than now
if int(time.time()
) - config.interval < flight.arrival and flight.arrival < int(
time.time()):
numflights += 1
flightpass = flightmod.findSeats(flight)
if flightpass != None:
passengers += flightpass
else:
unknown += 1
# if the flight is old enough to be archived
elif flight.arrival < int(time.time()) - config.archiveage:
archives.append(flight)
flights.remove(flight)
scraper.save(flights, config.flightdata)
scraper.save(archives, config.archivedata)
print "\nSince " + time.ctime(time.time() - config.interval) + ", "\
+ str(numflights) + " airplanes have entered Marco Polo Airport. Of those flights, "\
+ str(numflights - unknown) + " carried a total of " + str(passengers)\
+ " seats' worth of passengers.\n"