def export_whole_month_all_dest():
"""
Fetch data from SkyScanner.com for all the detentions from TLV,
and save the data as json in Data\\Flights folder.
"""
date_selected = datetime.today()
dates = []
flight_data = []
depart_list = [Airport(code=o) for o in DataManager.depart_list]
destination_list = [
Airport(code=o) for o in DataManager.destination_list_skyscanner
]
for i in range(11):
dates.append(date_selected)
date_selected = date_selected + relativedelta(months=1)
t_progress_bar_destination = tqdm(dates, leave=True)
for date in t_progress_bar_destination:
for depart in depart_list:
for destination in destination_list:
t_progress_bar_destination.set_description(
"SkyScanner " + date.strftime('%Y-%m') + " " +
destination.name)
t_progress_bar_destination.refresh()
flight_to_dest = export_whole_month(depart=depart,
destination=destination,
date=date)
if len(flight_to_dest) != 0:
flight_data.extend(flight_to_dest)
time.sleep(0.6)
general.update_most_updated_flights(flight_data)
def __init__(self,
flying_out: Airport = None,
flying_back: Airport = None,
flying_out_date: str = '',
flying_back_date: str = '',
price_per_adult: int = -1,
source_site: str = 'unknown',
rating_of_flight: int = -1,
data_set_of_flight: str = 'unknown',
**json_text: dict):
if flying_out is not None:
self.__departure = flying_out
self.__destination = flying_back
self.__depart_date = flying_out_date
self.__return_date = flying_back_date
self.__price = price_per_adult
self.__label = rating_of_flight
self.__source = source_site
self.__data_set = data_set_of_flight
self.__calculated_value = -1
else:
try:
json_text = json_text["json_text"]
except Exception:
pass
self.__depart_date = str(json_text["depart date"])
self.__return_date = str(json_text["return date"])
self.__price = json_text["price"]
self.__label = json_text["label"]
self.__source = str(json_text["source"])
self.__data_set = str(json_text["data set"])
self.__destination = Airport(**json_text.pop("destination"))
self.__departure = Airport(**json_text.pop("departure"))
try:
self.__destination_value = self.__rate_dest[
self.__destination.code]
except Exception:
self.__destination_value = -1
if self.__depart_date != '' and self.__return_date != '':
self.__days = (
datetime.strptime(self.__return_date, '%Y-%m-%d') -
datetime.strptime(self.__depart_date, '%Y-%m-%d')).days
else:
self.__days = -1
def export_whole_months_all_dest():
"""
Fetch data from Easyjet.com for all the detentions from TLV,
and save the data as json in Data\\Flights folder.
"""
depart_list = [Airport(code=o) for o in DataManager.depart_list]
destination_list = [
Airport(code=o) for o in DataManager.destination_list_easyjet
]
flights_data = []
for depart in depart_list:
t_progress_bar_destination = tqdm(destination_list, leave=True)
for destination in t_progress_bar_destination:
t_progress_bar_destination.set_description("EasyJet " +
destination.name)
t_progress_bar_destination.refresh()
flights_to_dest = export_whole_months(depart=depart,
destination=destination)
if len(flights_to_dest) != 0:
flights_data.extend(flights_to_dest)
time.sleep(0.6)
general.update_most_updated_flights(flights_data)
def get_list_of_all_destinations():
"""
get list of all destinations in data folder
:return:list of all airports in data
:rtype:list[Airport]
"""
airports_code = set()
airports_code.update(DataManager.destination_list_wizzair)
airports_code.update(DataManager.destination_list_skyscanner)
airports_code.update(DataManager.destination_list_easyjet)
airports = []
for airport_code in airports_code:
airports.append(
Airport(
DataManager.transfer_airport_cod_names_to_all(airport_code)))
return airports
def update_json_files(flights, year_month_date_depart, destination):
"""
Update the data folder with the fetched flights
:param flights: flights that need to update in json files
:type flights: list[Flight]
:param year_month_date_depart:month and year in format:YYYY-MM for folder name
:type year_month_date_depart:Datetime
:param destination:The destination of the flight
:type destination: Airport
"""
if len(flights) != 0:
file_name = datetime.today().strftime('%Y-%m-%d')
dest_all_for_name = Airport(
DataManager.transfer_airport_cod_names_to_all(destination.code))
Path(
os.path.dirname(__file__) + '/../Data/Flights/Whole Month/' +
year_month_date_depart).mkdir(parents=True, exist_ok=True)
Path(os.path.dirname(
__file__) + '/../Data/Flights/Whole Month/' + year_month_date_depart + '/' + dest_all_for_name.name) \
.mkdir(parents=True, exist_ok=True)
my_file = Path(
os.path.dirname(__file__) + '/../Data/Flights/Whole Month/' +
year_month_date_depart + '/' + dest_all_for_name.name + '/' +
file_name + ' ' + dest_all_for_name.name + ' ' +
year_month_date_depart + ".json")
append_data_flights = []
if my_file.is_file():
with open(os.path.dirname(__file__) +
'/../Data/Flights/Whole Month/' +
year_month_date_depart + '/' + dest_all_for_name.name +
'/' + file_name + ' ' + dest_all_for_name.name + ' ' +
year_month_date_depart + ".json",
'r',
encoding='utf-8') as f:
append_data_flights = json.load(f)
flights = flights + append_data_flights
with open(os.path.dirname(__file__) + '/../Data/Flights/Whole Month/' +
year_month_date_depart + '/' + dest_all_for_name.name + '/' +
file_name + ' ' + dest_all_for_name.name + ' ' +
year_month_date_depart + ".json",
'w',
encoding='utf-8') as f:
json.dump(flights,
f,
ensure_ascii=False,
default=obj_dict,
indent=4)
with open(os.path.dirname(__file__) +
'/../Data/Flights/json_files.json',
'r',
encoding='utf-8') as f:
append_data = json.load(f)
json_str = '/Data/Flights/Whole Month/' + year_month_date_depart + '/' + \
dest_all_for_name.name + '/' + file_name + ' ' + dest_all_for_name.name + ' ' \
+ year_month_date_depart + '.json'
append_data.append(json_str)
append_data = list(set(append_data))
with open(os.path.dirname(__file__) +
'/../Data/Flights/json_files.json',
'w',
encoding='utf-8') as f:
json.dump(append_data,
f,
ensure_ascii=False,
default=obj_dict,
indent=4)
add_to_json_dict(json_str)
class Flight:
"""
Attributes:
departure The departure airport
destination The destination airport
depart_date The date of the flying out
return_date The date of the flying back
days The duration between the flights
price The price of round-trip for one adult person
label How cheap the flight is 1-expensive 4-cheap
source From where the data Skyscanner,Wizzair,etc
data_set Which Data-set this flight belongs-"train-set","test-set"
calculated_value mathematical value for estimate the value of flight - temporarily
destination_value The rating of destination-how attractive this destination
"""
__departure: Airport
__destination: Airport
__depart_date: str
__return_date: str
__days: int
__price: float
__label: int
__source: str
__data_set: str
__calculated_value: int
__destination_value: int
with open(os.path.dirname(__file__) +
"/../Data/Flights/dict_rate_dest.json",
'r',
encoding="utf-8") as f:
__rate_dest = json.load(f)
def __init__(self,
flying_out: Airport = None,
flying_back: Airport = None,
flying_out_date: str = '',
flying_back_date: str = '',
price_per_adult: int = -1,
source_site: str = 'unknown',
rating_of_flight: int = -1,
data_set_of_flight: str = 'unknown',
**json_text: dict):
if flying_out is not None:
self.__departure = flying_out
self.__destination = flying_back
self.__depart_date = flying_out_date
self.__return_date = flying_back_date
self.__price = price_per_adult
self.__label = rating_of_flight
self.__source = source_site
self.__data_set = data_set_of_flight
self.__calculated_value = -1
else:
try:
json_text = json_text["json_text"]
except Exception:
pass
self.__depart_date = str(json_text["depart date"])
self.__return_date = str(json_text["return date"])
self.__price = json_text["price"]
self.__label = json_text["label"]
self.__source = str(json_text["source"])
self.__data_set = str(json_text["data set"])
self.__destination = Airport(**json_text.pop("destination"))
self.__departure = Airport(**json_text.pop("departure"))
try:
self.__destination_value = self.__rate_dest[
self.__destination.code]
except Exception:
self.__destination_value = -1
if self.__depart_date != '' and self.__return_date != '':
self.__days = (
datetime.strptime(self.__return_date, '%Y-%m-%d') -
datetime.strptime(self.__depart_date, '%Y-%m-%d')).days
else:
self.__days = -1
def to_json(self):
"""
:return dictionary with all attributes
:rtype: dict
"""
return {
"destination": self.__destination.to_json(),
"departure": self.__departure.to_json(),
"depart date": self.__depart_date,
"return date": self.__return_date,
"price": self.__price,
"label": self.__label,
"source": self.__source,
"data set": self.__data_set
}
@property
def departure(self) -> Airport:
return self.__departure
@property
def source(self):
return self.__source
@property
def days(self) -> int:
if self.__days != -1:
return self.__days
elif self.__depart_date != '' and self.__return_date != '':
self.__days = (
datetime.strptime(self.return_date, '%Y-%m-%d') -
datetime.strptime(self.depart_date, '%Y-%m-%d')).days
return self.__days
@property
def destination(self) -> Airport:
return self.__destination
@property
def destination_value(self) -> int:
if self.__destination_value == -1 and self.__destination is not None:
self.__destination_value = self.__rate_dest[
DataManager.transfer_airport_cod_names_to_all(
self.__destination.code)]
return self.__destination_value
@property
def depart_date(self) -> str:
return self.__depart_date
@property
def return_date(self) -> str:
return self.__return_date
@property
def price(self) -> float:
return self.__price
@property
def label(self) -> int:
return self.__label
@label.setter
def label(self, value):
if isinstance(value, int):
if -1 < value < 5:
self.__label = value
@property
def data_set(self) -> str:
return self.__data_set
@data_set.setter
def data_set(self, value):
if isinstance(value, str):
if value == "Train" or value == "Test":
self.__data_set = value
def __str__(self):
if self.__destination.city != '':
return "\ndestination: " + str(self.__destination.city) + "(" + str(self.__destination.code) + ") " \
+ str(self.__destination.country.name) + "\ndepart date: " + str(self.__depart_date) \
+ " return date: " + str(self.__return_date) + "\nprice: " + str(self.__price) + " site source: " \
+ str(self.__source)
else:
return "\ndestination: " + str(self.__destination.name) + "(" + str(self.__destination.code) + ") " \
+ str(self.__destination.country.name) + "\ndepart date: " + str(self.__depart_date) \
+ " return date: " + str(self.__return_date) + "\nprice: " + str(self.__price) + " site source: " \
+ str(self.__source)
def __eq__(self, other):
if isinstance(other, Flight):
if self.__depart_date == other.__depart_date and self.__departure == other.__departure \
and self.__destination == other.__destination and self.__return_date == other.__return_date \
and self.__price == other.__price:
return True
return False
def __lt__(self, other):
if isinstance(other, Flight):
return self.__price < other.__price
def __gt__(self, other):
if isinstance(other, Flight):
return self.__price > other.__price
def __ne__(self, other):
return not self.__eq__(other)
def __repr__(self):
return "Flight:: departure: " + str(self.__departure) + "\ndestination: " + str(self.__destination) + \
"\ndepart date: " + str(self.__depart_date) + " return date: " + str(self.__return_date) + \
"\nprice: " + str(self.__price) + " label: " + str(self.__label) + " data set: " + str(
self.__data_set)
def export_whole_month_all_dest():
"""
Fetch data from Wizzair.com for all the detentions from TLV,
parse him to Flight format and save the data as json in Data\\Flights folder.
"""
flights_data = fetch_data()
destinations = DataManager.destination_list_wizzair
departs = DataManager.depart_list
flights_data_most_updated = []
for depart in departs:
depart_flight = Airport(depart)
t_progress_bar_destination = tqdm(destinations, leave=True)
for destination in t_progress_bar_destination:
destination_flight = Airport(destination)
depart_destination_list = []
destination_depart_list = []
t_progress_bar_destination.set_description("Wizzair processing " +
destination_flight.name)
t_progress_bar_destination.refresh()
for flight_data in flights_data:
if flight_data["departureStation"] == depart_flight.code and \
flight_data["arrivalStation"] == destination_flight.code:
depart_destination_list.append(flight_data)
elif flight_data["departureStation"] == destination_flight.code \
and flight_data["arrivalStation"] == depart_flight.code:
destination_depart_list.append(flight_data)
for flight in depart_destination_list:
flight["departureDate"] = flight["departureDate"][0:10]
for flight in destination_depart_list:
flight["departureDate"] = flight["departureDate"][0:10]
combination_flights = []
for item_depart in depart_destination_list:
for item_return in destination_depart_list:
combination_flights.append(
Flight(flying_out=depart_flight,
flying_back=destination_flight,
flying_out_date=item_depart["departureDate"],
flying_back_date=item_return["departureDate"],
price_per_adult=item_depart["price"]["amount"] +
item_return["price"]["amount"],
source_site="Wizzair"))
flights_filter = [
flight for flight in combination_flights
if 3 <= flight.days < 7
]
flights_per_year_month_dict = {}
if len(flights_filter) != 0:
flights_data_most_updated.extend(flights_filter)
for flight_item in flights_filter:
depart_date_dt = datetime.strptime(flight_item.depart_date,
'%Y-%m-%d')
if datetime.strftime(depart_date_dt,
"%Y-%m") in flights_per_year_month_dict:
flights_per_year_month_dict[datetime.strftime(
depart_date_dt, "%Y-%m")].append(flight_item)
else:
flights_per_year_month_dict[datetime.strftime(
depart_date_dt, "%Y-%m")] = []
flights_per_year_month_dict[datetime.strftime(
depart_date_dt, "%Y-%m")].append(flight_item)
# update json files
for key in flights_per_year_month_dict:
general.update_json_files(
flights=flights_per_year_month_dict[key],
year_month_date_depart=key,
destination=destination_flight)
general.update_most_updated_flights(flights_data_most_updated)