def scrape_full(country, city, file): print("starting...") local_url = url(country, city) restaurant_url = restaurants_url(country, city) hotel_url = hotels_url(country, city) activities = scrape_one_event_type(country, city, local_url, activity) restaurants = scrape_one_event_type(country, city, restaurant_url, restaurant) hotels = filter(lambda l: l.longitude != "None" , scrape_one_event_type(country, city, hotel_url, hotel)) all_locations = activities + restaurants + hotels if(os.path.isfile(file)): os.remove(file) write_all_locations_to_disk(all_locations, file) print("done.") #scrape_full('usa', 'chicago') #scrape_full('china', 'beijing') #scrape_full('usa', 'new-york-city', file_name) #scrape_full('china', 'shanghai', file_name) #scrape_full('france', 'paris', file_name) #scrape_full('usa', 'chicago', file_name)
def update_data(read_from, write_to): all_locations = read_all_from_disk(read_from) updated_locations = [] for location in all_locations: updated_locations.append(update_one_location(location)) write_all_locations_to_disk(updated_locations, write_to)