def raw_data_to_training_days(raw_data_path, training_days_path, cutoff_times):
#df_flight_history = flighthistory.get_df_flight_history_from_raw_format(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"))
print(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"))
days_flight_ids = flighthistory.process_flight_history_to_train_day_files(
input_path = os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"),
output_path = training_days_path,
output_folder_name = "FlightHistory",
output_file_name = "flighthistory.csv",
cutoff_times = cutoff_times,
start_hours_offset = -9)
print("Flight History Events")
utilities.split_file_based_on_times_filter_on_ids_streaming(
os.path.join(raw_data_path, "FlightHistory", "flighthistoryevents.csv"),
training_days_path,
"FlightHistory",
"flighthistoryevents.csv",
"flight_history_id",
days_flight_ids)
print("ASDI Flight Plan")
days_flight_plan_ids = utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiflightplan.csv"), training_days_path,
"ASDI", "asdiflightplan.csv", "flighthistoryid", days_flight_ids, ids_to_track_column_name="asdiflightplanid")
print("ASDI Position")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiposition.csv"), training_days_path,
"ASDI", "asdiposition.csv", "flighthistoryid", days_flight_ids)
print("ASDI Airway")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiairway.csv"), training_days_path,
"ASDI", "asdiairway.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPFix")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpfix.csv"), training_days_path,
"ASDI", "asdifpfix.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPCenter")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpcenter.csv"), training_days_path,
"ASDI", "asdifpcenter.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPSector")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpsector.csv"), training_days_path,
"ASDI", "asdifpsector.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPWaypoint")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpwaypoint.csv"), training_days_path,
"ASDI", "asdifpwaypoint.csv", "asdiflightplanid", days_flight_plan_ids)
for ct in cutoff_times:
print ct
day_output_path = os.path.join(training_days_path, utilities.get_day_str(ct, -9))
day_beginning, day_end = utilities.get_day_boundaries(ct, -9)
if not os.path.exists(day_output_path):
os.makedirs(day_output_path)
weather.process_one_day(raw_data_path, day_output_path, day_beginning, day_end, "train")
def raw_data_to_training_days(raw_data_path, training_days_path, cutoff_times):
#df_flight_history = flighthistory.get_df_flight_history_from_raw_format(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"))
print(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"))
days_flight_ids = flighthistory.process_flight_history_to_train_day_files(
input_path = os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"),
output_path = training_days_path,
output_folder_name = "FlightHistory",
output_file_name = "flighthistory.csv",
cutoff_times = cutoff_times,
start_hours_offset = -9)
print("Flight History Events")
utilities.split_file_based_on_times_filter_on_ids_streaming(
os.path.join(raw_data_path, "FlightHistory", "flighthistoryevents.csv"),
training_days_path,
"FlightHistory",
"flighthistoryevents.csv",
"flight_history_id",
days_flight_ids)
print("ASDI Flight Plan")
days_flight_plan_ids = utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiflightplan.csv"), training_days_path,
"ASDI", "asdiflightplan.csv", "flighthistoryid", days_flight_ids, ids_to_track_column_name="asdiflightplanid")
print("ASDI Position")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiposition.csv"), training_days_path,
"ASDI", "asdiposition.csv", "flighthistoryid", days_flight_ids)
print("ASDI Airway")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiairway.csv"), training_days_path,
"ASDI", "asdiairway.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPFix")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpfix.csv"), training_days_path,
"ASDI", "asdifpfix.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPCenter")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpcenter.csv"), training_days_path,
"ASDI", "asdifpcenter.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPSector")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpsector.csv"), training_days_path,
"ASDI", "asdifpsector.csv", "asdiflightplanid", days_flight_plan_ids)
print("ASDI FPWaypoint")
utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpwaypoint.csv"), training_days_path,
"ASDI", "asdifpwaypoint.csv", "asdiflightplanid", days_flight_plan_ids)
for ct in cutoff_times:
print ct
day_output_path = os.path.join(output_path, utilities.get_day_str(ct, -9))
day_beginning, day_end = utilities.get_day_boundaries(ct, -9)
if not os.path.exists(day_output_path):
os.makedirs(day_output_path)
weather.process_one_day(raw_data_path, day_output_path, day_beginning, day_end, "train")
def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time):
flighthistory.write_flight_history_test_day_file(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"),
cutoff_time)
flighthistory.write_flight_history_test_day_and_solution_test_flights_only(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(test_day_path, "test_flights.csv"),
os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"),
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"),
"date_time_recorded",
utilities.parse_datetime_format3,
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"),
"received",
utilities.parse_datetime_format1,
cutoff_time)
flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"),
"updatetimeutc",
utilities.parse_datetime_format2,
cutoff_time,
ids_to_track_column_name = "asdiflightplanid")
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"),
"asdiflightplanid",
flight_plan_ids)
day_beginning, day_end = utilities.get_day_boundaries(cutoff_time)
weather.process_one_day(
training_day_path,
test_day_path,
day_beginning,
cutoff_time,
"test",
cutoff_time = cutoff_time)
def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time):
flighthistory.write_flight_history_test_day_file(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"),
cutoff_time)
flighthistory.write_flight_history_test_day_and_solution_test_flights_only(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(test_day_path, "test_flights.csv"),
os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"),
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"),
"date_time_recorded",
utilities.parse_datetime_format3,
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"),
"received",
utilities.parse_datetime_format1,
cutoff_time)
flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"),
"updatetimeutc",
utilities.parse_datetime_format2,
cutoff_time,
ids_to_track_column_name = "asdiflightplanid")
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"),
"asdiflightplanid",
flight_plan_ids)
day_beginning, day_end = utilities.get_day_boundaries(cutoff_time)
weather.process_one_day(
training_day_path,
test_day_path,
day_beginning,
cutoff_time,
"test",
cutoff_time = cutoff_time)
