def handle_query_flight(self):
"""return a QueryFlight model object"""
QueryFlight = models.get_model("skyscanner_scraper", "QueryFlight")
query_info = self.route_date_dict.get("Query", {})
query_id = query_info['RequestId']
#get the origin stations, can be multiple if the origin is city with many airport or a country
origin_stations_id = query_info["OriginPlaceInfo"].get("AirportIds", list())
origin_stations_qs = models.get_model("skyscanner_scraper", "Station").objects.all().filter(code__in=origin_stations_id)
#get the destination stations,
destination_stations_id = query_info["DestinationPlaceInfo"].get("AirportIds", list())
destination_stations_qs = models.get_model("skyscanner_scraper", "Station").objects.all().filter(code__in=destination_stations_id)
outbound_date = self._get_datetime(query_info.get("OutboundDate")).date()
inbound_date = None
inbound_datetime = self._get_datetime(query_info.get("InboundDate"))
#outbound date can me None if the flight is a one way
if inbound_datetime:
inbound_date = inbound_datetime.date()
defaults = {
"inbound_date":inbound_date,
"outbound_date":outbound_date,
}
query_flight, created, merged = merge_or_create(
QueryFlight,
request_id = query_id,
defaults = defaults,
)
query_flight.origin_station_set = origin_stations_qs
query_flight.destination_station_set = destination_stations_qs
self.query_flight = query_flight
return query_flight
def handle_quotes(self):
"""return quotes object seen in feed"""
object_set = set()
Quote = models.get_model("skyscanner_scraper", "Quote")
quote_info_list = self.route_date_dict.get("Quotes", {})
for quote_info in quote_info_list:
quote_id = quote_info["Id"]
defaults = {
"price":
quote_info.get("Price"),
"request_time":
self._get_datetime(quote_info.get("RequestDateTime")),
"agent":
self._get_agent(quote_info.get("QuoteRequestId")),
"is_return":
quote_info.get("IsReturn"),
}
instance, created, merged = merge_or_create(
Quote,
id=quote_id,
defaults=defaults,
)
object_set.add(instance)
return object_set
def handle_flights(self):
"""return flights (inbound and outbound) object seen in feed"""
object_set = set()
Flight = models.get_model("skyscanner_scraper", "Flight")
inbound_info_list = self.route_date_dict.get("InboundItineraryLegs", {})
#add a marker to recognize the inbound and outbound flight after the list merge
for inbound_itinerary_leg in inbound_info_list:
inbound_itinerary_leg["is_inbound"] = True
outbound_info_list = self.route_date_dict.get("OutboundItineraryLegs", {})
#merge the lists
inbound_info_list.extend(outbound_info_list)
flights_info_list = inbound_info_list
for flight_info in flights_info_list:
flight_id = flight_info["Id"]
origin_station = models.get_model("skyscanner_scraper", "Station").objects.get_or_create(id=flight_info.get("OriginStation"))[0]
destination_station = models.get_model("skyscanner_scraper", "Station").objects.get_or_create(id=flight_info.get("DestinationStation"))[0]
defaults = {
"origin_station":origin_station,
"destination_station":destination_station,
"departure_time":self._get_datetime(flight_info.get("DepartureDateTime")),
"arrival_time":self._get_datetime(flight_info.get("ArrivalDateTime")),
"duration":flight_info.get("Duration"),
"stop_count":flight_info.get("StopsCount"),
"inbound_itinerary_leg":flight_info.get("is_inbound", False),
"query_flight":self.query_flight,
}
instance, created, merged = merge_or_create(
Flight,
id = flight_id,
defaults = defaults,
)
#handle the stop_station_set
station_pk_set = flight_info.get("StopIds", list())
instance.stop_station_set = models.get_model("skyscanner_scraper", "Station").objects.all().filter(id__in=station_pk_set)
#handle the pricing_options
self.handle_pricing_options(flight_info.get("PricingOptions", list()), instance)
#handle the carriers
carrier_pk_set = flight_info.get("MarketingCarrierIds")
instance.carrier_set = models.get_model('skyscanner_scraper', 'carrier').objects.all().filter(pk__in=carrier_pk_set)
object_set.add(instance)
return object_set
def handle_carriers(self):
"""return the carriers seen in feed"""
object_set = set()
Carrier = models.get_model("skyscanner_scraper", "Carrier")
carrier_info_list = self.route_date_dict.get("Carriers", {})
for carrier_info in carrier_info_list:
carrier_id = carrier_info["Id"]
defaults = {
"name":carrier_info.get("Name"),
}
instance, created, merged = merge_or_create(
Carrier,
id = carrier_id,
defaults = defaults,
)
object_set.add(instance)
return object_set
def handle_carriers(self):
"""return the carriers seen in feed"""
object_set = set()
Carrier = models.get_model("skyscanner_scraper", "Carrier")
carrier_info_list = self.route_date_dict.get("Carriers", {})
for carrier_info in carrier_info_list:
carrier_id = carrier_info["Id"]
defaults = {
"name": carrier_info.get("Name"),
}
instance, created, merged = merge_or_create(
Carrier,
id=carrier_id,
defaults=defaults,
)
object_set.add(instance)
return object_set
def handle_stations(self):
"""return stations object seen in feed"""
stations_set = set()
Station = models.get_model("skyscanner_scraper", "Station")
station_info_list = self.route_date_dict.get("Stations", {})
for station_info in station_info_list:
station_code = station_info["Code"]
defaults = {
"name":station_info.get("Name"),
"id":station_info.get("Id"),
}
station, created, merged = merge_or_create(
Station,
code = station_code,
defaults = defaults,
)
stations_set.add(station)
return stations_set
def handle_stations(self):
"""return stations object seen in feed"""
stations_set = set()
Station = models.get_model("skyscanner_scraper", "Station")
station_info_list = self.route_date_dict.get("Stations", {})
for station_info in station_info_list:
station_code = station_info["Code"]
defaults = {
"name": station_info.get("Name"),
"id": station_info.get("Id"),
}
station, created, merged = merge_or_create(
Station,
code=station_code,
defaults=defaults,
)
stations_set.add(station)
return stations_set
def handle_agents(self):
"""return a list of the seen agents"""
agents_set = set()
Agent = models.get_model("skyscanner_scraper", "Agent")
agent_info_list = self.route_date_dict.get("Agents", {})
for agent_info in agent_info_list:
agent_id = agent_info["Id"]
defaults = {
"name":agent_info.get("Name"),
"default_url":agent_info.get("DefaultUrl"),
"booking_number":agent_info.get("BookingNumber"),
"is_carrier":agent_info.get("IsCarrier"),
}
agent, created, merged = merge_or_create(
Agent,
id = agent_id,
defaults = defaults,
)
agents_set.add(agent)
return agents_set
def handle_agents(self):
"""return a list of the seen agents"""
agents_set = set()
Agent = models.get_model("skyscanner_scraper", "Agent")
agent_info_list = self.route_date_dict.get("Agents", {})
for agent_info in agent_info_list:
agent_id = agent_info["Id"]
defaults = {
"name": agent_info.get("Name"),
"default_url": agent_info.get("DefaultUrl"),
"booking_number": agent_info.get("BookingNumber"),
"is_carrier": agent_info.get("IsCarrier"),
}
agent, created, merged = merge_or_create(
Agent,
id=agent_id,
defaults=defaults,
)
agents_set.add(agent)
return agents_set
def handle_quotes(self):
"""return quotes object seen in feed"""
object_set = set()
Quote = models.get_model("skyscanner_scraper", "Quote")
quote_info_list = self.route_date_dict.get("Quotes", {})
for quote_info in quote_info_list:
quote_id = quote_info["Id"]
defaults = {
"price":quote_info.get("Price"),
"request_time":self._get_datetime(quote_info.get("RequestDateTime")),
"agent":self._get_agent(quote_info.get("QuoteRequestId")),
"is_return":quote_info.get("IsReturn"),
}
instance, created, merged = merge_or_create(
Quote,
id = quote_id,
defaults = defaults,
)
object_set.add(instance)
return object_set
def handle_query_flight(self):
"""return a QueryFlight model object"""
QueryFlight = models.get_model("skyscanner_scraper", "QueryFlight")
query_info = self.route_date_dict.get("Query", {})
query_id = query_info['RequestId']
#get the origin stations, can be multiple if the origin is city with many airport or a country
origin_stations_id = query_info["OriginPlaceInfo"].get(
"AirportIds", list())
origin_stations_qs = models.get_model(
"skyscanner_scraper",
"Station").objects.all().filter(code__in=origin_stations_id)
#get the destination stations,
destination_stations_id = query_info["DestinationPlaceInfo"].get(
"AirportIds", list())
destination_stations_qs = models.get_model(
"skyscanner_scraper",
"Station").objects.all().filter(code__in=destination_stations_id)
outbound_date = self._get_datetime(
query_info.get("OutboundDate")).date()
inbound_date = None
inbound_datetime = self._get_datetime(query_info.get("InboundDate"))
#outbound date can me None if the flight is a one way
if inbound_datetime:
inbound_date = inbound_datetime.date()
defaults = {
"inbound_date": inbound_date,
"outbound_date": outbound_date,
}
query_flight, created, merged = merge_or_create(
QueryFlight,
request_id=query_id,
defaults=defaults,
)
query_flight.origin_station_set = origin_stations_qs
query_flight.destination_station_set = destination_stations_qs
self.query_flight = query_flight
return query_flight
def handle_pricing_options(self, pricing_options_info_list, flight):
""" merge or create the each pricing options """
PricingOption = models.get_model("skyscanner_scraper", "PricingOption")
pricing_option_set = set()
for pricing_option_info in pricing_options_info_list:
#get the fisrt quote, it's a simplification
quote_id = pricing_option_info.get("QuoteIds", list())[0]
quote, created = models.get_model(
'skyscanner_scraper',
'Quote').objects.get_or_create(id=quote_id)
#get the opposing flight
opposing_flight_id = pricing_option_info.get("OpposingLegId")
opposing_flight = None
if opposing_flight_id:
opposing_flight, created = models.get_model(
"skyscanner_scraper",
"Flight").objects.get_or_create(id=opposing_flight_id,
defaults={
"query_flight":
self.query_flight,
})
inbound_flight = None
outbound_flight = None
if flight.inbound_itinerary_leg:
inbound_flight = flight
outbound_flight = opposing_flight
else:
inbound_flight = opposing_flight
outbound_flight = flight
instance, created, merged = merge_or_create(
PricingOption,
quote=quote,
inbound_flight=inbound_flight,
outbound_flight=outbound_flight,
defaults={},
)
pricing_option_set.add(instance)
return pricing_option_set
def handle_pricing_options(self, pricing_options_info_list, flight):
""" merge or create the each pricing options """
PricingOption = models.get_model("skyscanner_scraper", "PricingOption")
pricing_option_set = set()
for pricing_option_info in pricing_options_info_list:
#get the fisrt quote, it's a simplification
quote_id = pricing_option_info.get("QuoteIds", list())[0]
quote, created = models.get_model('skyscanner_scraper', 'Quote').objects.get_or_create(id=quote_id)
#get the opposing flight
opposing_flight_id = pricing_option_info.get("OpposingLegId")
opposing_flight = None
if opposing_flight_id:
opposing_flight, created = models.get_model("skyscanner_scraper", "Flight").objects.get_or_create(
id=opposing_flight_id,
defaults={
"query_flight":self.query_flight,
}
)
inbound_flight = None
outbound_flight = None
if flight.inbound_itinerary_leg:
inbound_flight = flight
outbound_flight = opposing_flight
else:
inbound_flight = opposing_flight
outbound_flight = flight
instance, created, merged = merge_or_create(
PricingOption,
quote = quote,
inbound_flight=inbound_flight,
outbound_flight=outbound_flight,
defaults = {},
)
pricing_option_set.add(instance)
return pricing_option_set
def handle_flights(self):
"""return flights (inbound and outbound) object seen in feed"""
object_set = set()
Flight = models.get_model("skyscanner_scraper", "Flight")
inbound_info_list = self.route_date_dict.get("InboundItineraryLegs",
{})
#add a marker to recognize the inbound and outbound flight after the list merge
for inbound_itinerary_leg in inbound_info_list:
inbound_itinerary_leg["is_inbound"] = True
outbound_info_list = self.route_date_dict.get("OutboundItineraryLegs",
{})
#merge the lists
inbound_info_list.extend(outbound_info_list)
flights_info_list = inbound_info_list
for flight_info in flights_info_list:
flight_id = flight_info["Id"]
origin_station = models.get_model(
"skyscanner_scraper", "Station").objects.get_or_create(
id=flight_info.get("OriginStation"))[0]
destination_station = models.get_model(
"skyscanner_scraper", "Station").objects.get_or_create(
id=flight_info.get("DestinationStation"))[0]
defaults = {
"origin_station":
origin_station,
"destination_station":
destination_station,
"departure_time":
self._get_datetime(flight_info.get("DepartureDateTime")),
"arrival_time":
self._get_datetime(flight_info.get("ArrivalDateTime")),
"duration":
flight_info.get("Duration"),
"stop_count":
flight_info.get("StopsCount"),
"inbound_itinerary_leg":
flight_info.get("is_inbound", False),
"query_flight":
self.query_flight,
}
instance, created, merged = merge_or_create(
Flight,
id=flight_id,
defaults=defaults,
)
#handle the stop_station_set
station_pk_set = flight_info.get("StopIds", list())
instance.stop_station_set = models.get_model(
"skyscanner_scraper",
"Station").objects.all().filter(id__in=station_pk_set)
#handle the pricing_options
self.handle_pricing_options(
flight_info.get("PricingOptions", list()), instance)
#handle the carriers
carrier_pk_set = flight_info.get("MarketingCarrierIds")
instance.carrier_set = models.get_model(
'skyscanner_scraper',
'carrier').objects.all().filter(pk__in=carrier_pk_set)
object_set.add(instance)
return object_set