def get_connections(origin, dest, date): trip_data_request = TripDataRequest(origin, dest, [date], [date]) two_one_ways_trip_flight = TwoOneWaysFlightType(trip_data_request) flights_to_search = two_one_ways_trip_flight.get_trip_data_requests() flight1 = flights_to_search[0] response1 = flight_provider.search_flight(flight1) if response1: connections= get_connections_list(response1) area = get_area(origin, dest) flight_resp_dal.add_connections_to_area(area, connections)
def scrap_connections(): areas=set() airports = [] with open('csv_files/top_100_airports.csv', 'rb') as csvfile: reader = csv.reader(csvfile) for row in reader: for airport in row: airports.append(airport) origins = dests = airports dates = [] for i in range(100): new_date = datetime.date(2014, 11, 02) + timedelta(days=i) dates.append(new_date) pool = ThreadPool(20) pool.start() for dest in dests: for origin in origins: if dest != origin: date = dates[randint(0, len(dates)-1)] pool.add_task(get_connections, origin, dest, date) areas.add(get_area(origin, dest)) pool.wait_completion() #arrange all connections in single set per area instead of list areas_conn = dict() for area in areas: conn_list = flight_resp_dal.get_connections_in_area(area) connections = set() for conn in conn_list: connections.update(set(conn)) areas_conn[area] = connections flight_resp_dal.clean_areas_to_connections_table() for area in areas: flight_resp_dal.add_connections_to_area(area, areas_conn[area])