Ejemplo n.º 1
0
 def handle_query_flight(self):
     """return a QueryFlight model object"""
     QueryFlight = models.get_model("skyscanner_scraper", "QueryFlight")
     query_info = self.route_date_dict.get("Query", {})
     
     query_id = query_info['RequestId']
     #get the origin stations, can be multiple if the origin is city with many airport or a country
     origin_stations_id = query_info["OriginPlaceInfo"].get("AirportIds", list())
     origin_stations_qs = models.get_model("skyscanner_scraper", "Station").objects.all().filter(code__in=origin_stations_id)
     #get the destination stations,
     destination_stations_id = query_info["DestinationPlaceInfo"].get("AirportIds", list())
     destination_stations_qs = models.get_model("skyscanner_scraper", "Station").objects.all().filter(code__in=destination_stations_id)
     
     outbound_date = self._get_datetime(query_info.get("OutboundDate")).date()
     inbound_date = None
     inbound_datetime = self._get_datetime(query_info.get("InboundDate"))
     #outbound date can me None if the flight is a one way
     if inbound_datetime:
         inbound_date = inbound_datetime.date()
         
         
         
     defaults = {
         "inbound_date":inbound_date,
         "outbound_date":outbound_date,
     }
     query_flight, created, merged = merge_or_create(
        QueryFlight,
        request_id = query_id,
        defaults = defaults,
     )
     query_flight.origin_station_set = origin_stations_qs
     query_flight.destination_station_set = destination_stations_qs        
     self.query_flight = query_flight
     return query_flight
Ejemplo n.º 2
0
    def handle_quotes(self):
        """return quotes object seen in feed"""
        object_set = set()
        Quote = models.get_model("skyscanner_scraper", "Quote")

        quote_info_list = self.route_date_dict.get("Quotes", {})
        for quote_info in quote_info_list:
            quote_id = quote_info["Id"]
            defaults = {
                "price":
                quote_info.get("Price"),
                "request_time":
                self._get_datetime(quote_info.get("RequestDateTime")),
                "agent":
                self._get_agent(quote_info.get("QuoteRequestId")),
                "is_return":
                quote_info.get("IsReturn"),
            }
            instance, created, merged = merge_or_create(
                Quote,
                id=quote_id,
                defaults=defaults,
            )
            object_set.add(instance)
        return object_set
Ejemplo n.º 3
0
 def handle_flights(self):
     """return flights (inbound and outbound) object seen in feed"""
     object_set = set()
     Flight = models.get_model("skyscanner_scraper", "Flight")
             
     inbound_info_list = self.route_date_dict.get("InboundItineraryLegs", {})
     #add a marker to recognize the inbound and outbound flight after the list merge
     for inbound_itinerary_leg in inbound_info_list:
         inbound_itinerary_leg["is_inbound"] = True
     
     outbound_info_list = self.route_date_dict.get("OutboundItineraryLegs", {})
     #merge the lists
     inbound_info_list.extend(outbound_info_list)
     flights_info_list = inbound_info_list
     for flight_info in flights_info_list:
         flight_id = flight_info["Id"]
         
         origin_station = models.get_model("skyscanner_scraper", "Station").objects.get_or_create(id=flight_info.get("OriginStation"))[0]
         destination_station = models.get_model("skyscanner_scraper", "Station").objects.get_or_create(id=flight_info.get("DestinationStation"))[0]
         defaults = {
             "origin_station":origin_station,
             "destination_station":destination_station,
             "departure_time":self._get_datetime(flight_info.get("DepartureDateTime")),
             "arrival_time":self._get_datetime(flight_info.get("ArrivalDateTime")),
             "duration":flight_info.get("Duration"),
             "stop_count":flight_info.get("StopsCount"),
             "inbound_itinerary_leg":flight_info.get("is_inbound", False),
             "query_flight":self.query_flight,
         }
         instance, created, merged = merge_or_create(
            Flight,
            id = flight_id,
            defaults = defaults,
         )
         
         #handle the stop_station_set
         station_pk_set = flight_info.get("StopIds", list())
         instance.stop_station_set = models.get_model("skyscanner_scraper", "Station").objects.all().filter(id__in=station_pk_set)
         #handle the pricing_options
         self.handle_pricing_options(flight_info.get("PricingOptions", list()), instance)
         #handle the carriers
         carrier_pk_set = flight_info.get("MarketingCarrierIds")
         instance.carrier_set = models.get_model('skyscanner_scraper', 'carrier').objects.all().filter(pk__in=carrier_pk_set)
         
         object_set.add(instance)
     return object_set
Ejemplo n.º 4
0
 def handle_carriers(self):
     """return the carriers seen in feed"""
     object_set = set()
     Carrier = models.get_model("skyscanner_scraper", "Carrier")
     
     carrier_info_list = self.route_date_dict.get("Carriers", {})
     for carrier_info in carrier_info_list:
         carrier_id = carrier_info["Id"]
         defaults = {
             "name":carrier_info.get("Name"),
         }
         instance, created, merged = merge_or_create(
            Carrier,
            id = carrier_id,
            defaults = defaults,
         )
         object_set.add(instance)
     return object_set
Ejemplo n.º 5
0
    def handle_carriers(self):
        """return the carriers seen in feed"""
        object_set = set()
        Carrier = models.get_model("skyscanner_scraper", "Carrier")

        carrier_info_list = self.route_date_dict.get("Carriers", {})
        for carrier_info in carrier_info_list:
            carrier_id = carrier_info["Id"]
            defaults = {
                "name": carrier_info.get("Name"),
            }
            instance, created, merged = merge_or_create(
                Carrier,
                id=carrier_id,
                defaults=defaults,
            )
            object_set.add(instance)
        return object_set
Ejemplo n.º 6
0
 def handle_stations(self):
     """return stations object seen in feed"""
     stations_set = set()
     Station = models.get_model("skyscanner_scraper", "Station")
     
     station_info_list = self.route_date_dict.get("Stations", {})
     for station_info in station_info_list:
         station_code = station_info["Code"]
         defaults = {
             "name":station_info.get("Name"),
             "id":station_info.get("Id"),
         }
         station, created, merged = merge_or_create(
            Station,
            code = station_code,
            defaults = defaults,
         )
         stations_set.add(station)
     return stations_set
Ejemplo n.º 7
0
    def handle_stations(self):
        """return stations object seen in feed"""
        stations_set = set()
        Station = models.get_model("skyscanner_scraper", "Station")

        station_info_list = self.route_date_dict.get("Stations", {})
        for station_info in station_info_list:
            station_code = station_info["Code"]
            defaults = {
                "name": station_info.get("Name"),
                "id": station_info.get("Id"),
            }
            station, created, merged = merge_or_create(
                Station,
                code=station_code,
                defaults=defaults,
            )
            stations_set.add(station)
        return stations_set
Ejemplo n.º 8
0
 def handle_agents(self):
     """return a list of the seen agents"""
     agents_set = set()
     Agent = models.get_model("skyscanner_scraper", "Agent")
     agent_info_list = self.route_date_dict.get("Agents", {})
     for agent_info in agent_info_list:
         agent_id = agent_info["Id"]
         defaults = {
             "name":agent_info.get("Name"),
             "default_url":agent_info.get("DefaultUrl"),
             "booking_number":agent_info.get("BookingNumber"),
             "is_carrier":agent_info.get("IsCarrier"),
         }
         agent, created, merged = merge_or_create(
            Agent,
            id = agent_id,
            defaults = defaults,
         )
         agents_set.add(agent)
     return agents_set
Ejemplo n.º 9
0
 def handle_agents(self):
     """return a list of the seen agents"""
     agents_set = set()
     Agent = models.get_model("skyscanner_scraper", "Agent")
     agent_info_list = self.route_date_dict.get("Agents", {})
     for agent_info in agent_info_list:
         agent_id = agent_info["Id"]
         defaults = {
             "name": agent_info.get("Name"),
             "default_url": agent_info.get("DefaultUrl"),
             "booking_number": agent_info.get("BookingNumber"),
             "is_carrier": agent_info.get("IsCarrier"),
         }
         agent, created, merged = merge_or_create(
             Agent,
             id=agent_id,
             defaults=defaults,
         )
         agents_set.add(agent)
     return agents_set
Ejemplo n.º 10
0
 def handle_quotes(self):
     """return quotes object seen in feed"""
     object_set = set()
     Quote = models.get_model("skyscanner_scraper", "Quote")
     
     quote_info_list = self.route_date_dict.get("Quotes", {})
     for quote_info in quote_info_list:
         quote_id = quote_info["Id"]
         defaults = {
             "price":quote_info.get("Price"),
             "request_time":self._get_datetime(quote_info.get("RequestDateTime")),
             "agent":self._get_agent(quote_info.get("QuoteRequestId")),
             "is_return":quote_info.get("IsReturn"),
         }
         instance, created, merged = merge_or_create(
            Quote,
            id = quote_id,
            defaults = defaults,
         )
         object_set.add(instance)
     return object_set
Ejemplo n.º 11
0
    def handle_query_flight(self):
        """return a QueryFlight model object"""
        QueryFlight = models.get_model("skyscanner_scraper", "QueryFlight")
        query_info = self.route_date_dict.get("Query", {})

        query_id = query_info['RequestId']
        #get the origin stations, can be multiple if the origin is city with many airport or a country
        origin_stations_id = query_info["OriginPlaceInfo"].get(
            "AirportIds", list())
        origin_stations_qs = models.get_model(
            "skyscanner_scraper",
            "Station").objects.all().filter(code__in=origin_stations_id)
        #get the destination stations,
        destination_stations_id = query_info["DestinationPlaceInfo"].get(
            "AirportIds", list())
        destination_stations_qs = models.get_model(
            "skyscanner_scraper",
            "Station").objects.all().filter(code__in=destination_stations_id)

        outbound_date = self._get_datetime(
            query_info.get("OutboundDate")).date()
        inbound_date = None
        inbound_datetime = self._get_datetime(query_info.get("InboundDate"))
        #outbound date can me None if the flight is a one way
        if inbound_datetime:
            inbound_date = inbound_datetime.date()

        defaults = {
            "inbound_date": inbound_date,
            "outbound_date": outbound_date,
        }
        query_flight, created, merged = merge_or_create(
            QueryFlight,
            request_id=query_id,
            defaults=defaults,
        )
        query_flight.origin_station_set = origin_stations_qs
        query_flight.destination_station_set = destination_stations_qs
        self.query_flight = query_flight
        return query_flight
Ejemplo n.º 12
0
    def handle_pricing_options(self, pricing_options_info_list, flight):
        """ merge or create the each pricing options """
        PricingOption = models.get_model("skyscanner_scraper", "PricingOption")
        pricing_option_set = set()
        for pricing_option_info in pricing_options_info_list:
            #get the fisrt quote, it's a simplification
            quote_id = pricing_option_info.get("QuoteIds", list())[0]
            quote, created = models.get_model(
                'skyscanner_scraper',
                'Quote').objects.get_or_create(id=quote_id)
            #get the opposing flight
            opposing_flight_id = pricing_option_info.get("OpposingLegId")
            opposing_flight = None
            if opposing_flight_id:
                opposing_flight, created = models.get_model(
                    "skyscanner_scraper",
                    "Flight").objects.get_or_create(id=opposing_flight_id,
                                                    defaults={
                                                        "query_flight":
                                                        self.query_flight,
                                                    })

            inbound_flight = None
            outbound_flight = None
            if flight.inbound_itinerary_leg:
                inbound_flight = flight
                outbound_flight = opposing_flight
            else:
                inbound_flight = opposing_flight
                outbound_flight = flight

            instance, created, merged = merge_or_create(
                PricingOption,
                quote=quote,
                inbound_flight=inbound_flight,
                outbound_flight=outbound_flight,
                defaults={},
            )
            pricing_option_set.add(instance)
        return pricing_option_set
Ejemplo n.º 13
0
 def handle_pricing_options(self, pricing_options_info_list, flight):
     """ merge or create the each pricing options """
     PricingOption = models.get_model("skyscanner_scraper", "PricingOption")
     pricing_option_set = set()
     for pricing_option_info in pricing_options_info_list:
         #get the fisrt quote, it's a simplification
         quote_id = pricing_option_info.get("QuoteIds", list())[0]
         quote, created = models.get_model('skyscanner_scraper', 'Quote').objects.get_or_create(id=quote_id)
         #get the opposing flight
         opposing_flight_id = pricing_option_info.get("OpposingLegId")
         opposing_flight = None
         if opposing_flight_id:
             opposing_flight, created = models.get_model("skyscanner_scraper", "Flight").objects.get_or_create(
                 id=opposing_flight_id,
                 defaults={
                       "query_flight":self.query_flight,
                 }
             )
         
         inbound_flight = None
         outbound_flight = None            
         if flight.inbound_itinerary_leg:
             inbound_flight = flight
             outbound_flight = opposing_flight
         else:
             inbound_flight = opposing_flight
             outbound_flight = flight
             
         instance, created, merged = merge_or_create(
            PricingOption,
            quote = quote,
            inbound_flight=inbound_flight,
            outbound_flight=outbound_flight,
            defaults = {},
         )
         pricing_option_set.add(instance)
     return pricing_option_set
Ejemplo n.º 14
0
    def handle_flights(self):
        """return flights (inbound and outbound) object seen in feed"""
        object_set = set()
        Flight = models.get_model("skyscanner_scraper", "Flight")

        inbound_info_list = self.route_date_dict.get("InboundItineraryLegs",
                                                     {})
        #add a marker to recognize the inbound and outbound flight after the list merge
        for inbound_itinerary_leg in inbound_info_list:
            inbound_itinerary_leg["is_inbound"] = True

        outbound_info_list = self.route_date_dict.get("OutboundItineraryLegs",
                                                      {})
        #merge the lists
        inbound_info_list.extend(outbound_info_list)
        flights_info_list = inbound_info_list
        for flight_info in flights_info_list:
            flight_id = flight_info["Id"]

            origin_station = models.get_model(
                "skyscanner_scraper", "Station").objects.get_or_create(
                    id=flight_info.get("OriginStation"))[0]
            destination_station = models.get_model(
                "skyscanner_scraper", "Station").objects.get_or_create(
                    id=flight_info.get("DestinationStation"))[0]
            defaults = {
                "origin_station":
                origin_station,
                "destination_station":
                destination_station,
                "departure_time":
                self._get_datetime(flight_info.get("DepartureDateTime")),
                "arrival_time":
                self._get_datetime(flight_info.get("ArrivalDateTime")),
                "duration":
                flight_info.get("Duration"),
                "stop_count":
                flight_info.get("StopsCount"),
                "inbound_itinerary_leg":
                flight_info.get("is_inbound", False),
                "query_flight":
                self.query_flight,
            }
            instance, created, merged = merge_or_create(
                Flight,
                id=flight_id,
                defaults=defaults,
            )

            #handle the stop_station_set
            station_pk_set = flight_info.get("StopIds", list())
            instance.stop_station_set = models.get_model(
                "skyscanner_scraper",
                "Station").objects.all().filter(id__in=station_pk_set)
            #handle the pricing_options
            self.handle_pricing_options(
                flight_info.get("PricingOptions", list()), instance)
            #handle the carriers
            carrier_pk_set = flight_info.get("MarketingCarrierIds")
            instance.carrier_set = models.get_model(
                'skyscanner_scraper',
                'carrier').objects.all().filter(pk__in=carrier_pk_set)

            object_set.add(instance)
        return object_set