def get_or_fetch(self, city_name):
"""
get or fetch a station from a city name
if multiple stations are found, this function return the first
"""
#try to get a place on db
stations = None
stations = models.get_model("skyscanner_scraper", "Station").objects.filter(name__iexact=city_name)
if len(stations) == 0:
#else fetch the results
client = SkyscannerClient()
stations = client.get_stations(city_name)
if len(stations) > 0:
#return the first
return stations[0]
def get_or_fetch(self, city_name):
"""
get or fetch a station from a city name
if multiple stations are found, this function return the first
"""
#try to get a place on db
stations = None
stations = models.get_model(
"skyscanner_scraper",
"Station").objects.filter(name__iexact=city_name)
if len(stations) == 0:
#else fetch the results
client = SkyscannerClient()
stations = client.get_stations(city_name)
if len(stations) > 0:
#return the first
return stations[0]
def get_any_flights_page(self, client=None, **kwargs):
"""return a flights page"""
client = client or SkyscannerClient()
now = timezone.now()
params = {
"short_from": "NTE",
"short_to": "EDI",
"depart_date": now,
"return_date": now + datetime.timedelta(days=5)
}
params.update(kwargs)
return client._get_flights_page(**params)
def get_flights(self, client=None, **kwargs):
"""return some quotes"""
client = client or SkyscannerClient()
today = datetime.date.today()
params = {
"short_from": "NTE",
"short_to": "EDI",
"depart_date": today,
"return_date": today + datetime.timedelta(days=5)
}
params.update(kwargs)
query_flight, flights = client.get_flights(**params)
return flights
def get_stations(self, client=None, city_name=""):
"""retur a place instance from a city name using the autosuggest client methode"""
client = client or SkyscannerClient()
return client.get_stations(city_name)
def any_client(self):
"""return a skyscanner client"""
return SkyscannerClient()
def handle(self, *args, **options):
# make sure origin and destination options are present
if options['from_city'] == None:
raise CommandError("Option `--from_city=...` must be specified.")
if options['to_city'] == None:
raise CommandError("Option `--to_city=...` must be specified.")
origin_city_name = options['from_city']
destination_city_name = options['to_city']
print_limit = options['print_limit']
depart_date = datetime.datetime.strptime(options['depart_date'],
ARGUMENT_DATE_FORMAT).date()
#if no return date given, the flight is a on way
return_date = ""
if options['return_date']:
return_date = datetime.datetime.strptime(
options['return_date'], ARGUMENT_DATE_FORMAT).date()
#instantiate a skyscanner client
client = SkyscannerClient()
now = timezone.now()
origin_station = models.get_model(
"skyscanner_scraper",
"Station").objects.get_or_fetch(origin_city_name)
destination_station = models.get_model(
"skyscanner_scraper",
"Station").objects.get_or_fetch(destination_city_name)
search_params = {
"short_from": origin_station.code,
"short_to": destination_station.code,
"depart_date": depart_date,
"return_date": return_date,
}
#get the result, if no result found at first call, wait and retry (retry max = 6)`
pricing_option_qs = None
i = 0
max_val = 5
pbar = ProgressBar(maxval=max_val).start()
while i < max_val:
i += 1
query_flight, flights = client.get_flights(**search_params)
seen_flights_pk = [f.pk for f in flights]
pricing_option_qs = models.get_model(
'skyscanner_scraper', 'PricingOption').objects.all()
#One way flight
if not query_flight.inbound_date:
pricing_option_qs = pricing_option_qs.filter(
outbound_flight_id__in=seen_flights_pk,
inbound_flight__isnull=True,
quote__is_return=False,
)
#if the query for a return flight, remove the one way flights
if query_flight.outbound_date and query_flight.inbound_date:
pricing_option_qs = pricing_option_qs.filter(
Q(inbound_flight_id__in=seen_flights_pk)
| Q(outbound_flight_id__in=seen_flights_pk),
quote__is_return=True)
pricing_option_qs = pricing_option_qs.exclude(
Q(inbound_flight__isnull=True)
| Q(outbound_flight__isnull=True), )
if len(pricing_option_qs) > 0:
break
pbar.update(i)
time.sleep(1)
pbar.finish()
pricing_option_qs = pricing_option_qs.order_by('quote__price')
#then print the result
print "---------------------------------------------------------"
print "search on SkyScanner from %s(%s) to %s(%s)" % (
origin_station.name, origin_station.code, destination_station.name,
destination_station.code)
if return_date:
print "%s - %s" % (depart_date.strftime(PRINT_DATE_FORMAT),
return_date.strftime(PRINT_DATE_FORMAT))
else:
print "one way from %s" % (depart_date.strftime(PRINT_DATE_FORMAT))
print "---------------------------------------------------------"
limit_number = print_limit
print_count = 0
#to remove the duplicate flight we will create a key with outbound_flight.pk+inboud_flight.pk
#if this id is already seen, don't add the price_option
printed_combination = set()
for price_option in pricing_option_qs:
#outbound
outbound_flight_id = price_option.outbound_flight_id
#inbound
inbound_flight_id = price_option.inbound_flight_id
#printed_combination pk
combination = "%s_%s" % (outbound_flight_id, inbound_flight_id)
if not combination in printed_combination:
self.print_flight_details(price_option)
#print the informations
printed_combination.add(combination)
print_count += 1
if print_count >= limit_number:
break
print "%s of %s results" % (limit_number, len(pricing_option_qs))
#TODO : fix this bug(maybe by adding the query_flight in quote and price_option)
#delete all quote and pricing_option to avoid complexe mistakes
models.get_model('skyscanner_scraper', 'Quote').objects.all().delete()
models.get_model('skyscanner_scraper',
'PricingOption').objects.all().delete()
def handle(self, *args, **options):
# make sure origin and destination options are present
if options['from_city'] == None :
raise CommandError("Option `--from_city=...` must be specified.")
if options['to_city'] == None :
raise CommandError("Option `--to_city=...` must be specified.")
origin_city_name = options['from_city']
destination_city_name = options['to_city']
print_limit = options['print_limit']
depart_date = datetime.datetime.strptime(options['depart_date'], ARGUMENT_DATE_FORMAT).date()
#if no return date given, the flight is a on way
return_date = ""
if options['return_date']:
return_date = datetime.datetime.strptime(options['return_date'], ARGUMENT_DATE_FORMAT).date()
#instantiate a skyscanner client
client = SkyscannerClient()
now = timezone.now()
origin_station = models.get_model("skyscanner_scraper", "Station").objects.get_or_fetch(origin_city_name)
destination_station = models.get_model("skyscanner_scraper", "Station").objects.get_or_fetch(destination_city_name)
search_params = {
"short_from":origin_station.code,
"short_to":destination_station.code,
"depart_date":depart_date,
"return_date":return_date,
}
#get the result, if no result found at first call, wait and retry (retry max = 6)`
pricing_option_qs = None
i = 0
max_val = 5
pbar = ProgressBar(maxval=max_val).start()
while i < max_val:
i += 1
query_flight, flights = client.get_flights(**search_params)
seen_flights_pk = [f.pk for f in flights]
pricing_option_qs = models.get_model('skyscanner_scraper', 'PricingOption').objects.all()
#One way flight
if not query_flight.inbound_date:
pricing_option_qs = pricing_option_qs.filter(
outbound_flight_id__in=seen_flights_pk,
inbound_flight__isnull=True,
quote__is_return=False,
)
#if the query for a return flight, remove the one way flights
if query_flight.outbound_date and query_flight.inbound_date:
pricing_option_qs = pricing_option_qs.filter(
Q(inbound_flight_id__in=seen_flights_pk) | Q(outbound_flight_id__in=seen_flights_pk),
quote__is_return=True
)
pricing_option_qs = pricing_option_qs.exclude(
Q(inbound_flight__isnull=True) | Q(outbound_flight__isnull=True),
)
if len(pricing_option_qs) > 0:
break
pbar.update(i)
time.sleep(1)
pbar.finish()
pricing_option_qs = pricing_option_qs.order_by('quote__price')
#then print the result
print "---------------------------------------------------------"
print "search on SkyScanner from %s(%s) to %s(%s)" % (origin_station.name, origin_station.code, destination_station.name, destination_station.code)
if return_date:
print "%s - %s" % (depart_date.strftime(PRINT_DATE_FORMAT), return_date.strftime(PRINT_DATE_FORMAT) )
else:
print "one way from %s" % (depart_date.strftime(PRINT_DATE_FORMAT))
print "---------------------------------------------------------"
limit_number = print_limit
print_count = 0
#to remove the duplicate flight we will create a key with outbound_flight.pk+inboud_flight.pk
#if this id is already seen, don't add the price_option
printed_combination = set()
for price_option in pricing_option_qs:
#outbound
outbound_flight_id = price_option.outbound_flight_id
#inbound
inbound_flight_id = price_option.inbound_flight_id
#printed_combination pk
combination = "%s_%s" % (outbound_flight_id, inbound_flight_id)
if not combination in printed_combination:
self.print_flight_details(price_option)
#print the informations
printed_combination.add(combination)
print_count += 1
if print_count >= limit_number:
break
print "%s of %s results" % (limit_number, len(pricing_option_qs))
#TODO : fix this bug(maybe by adding the query_flight in quote and price_option)
#delete all quote and pricing_option to avoid complexe mistakes
models.get_model('skyscanner_scraper', 'Quote').objects.all().delete()
models.get_model('skyscanner_scraper', 'PricingOption').objects.all().delete()