def raw_data_to_training_days(raw_data_path, training_days_path, cutoff_times): #df_flight_history = flighthistory.get_df_flight_history_from_raw_format(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv")) print(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv")) days_flight_ids = flighthistory.process_flight_history_to_train_day_files( input_path = os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"), output_path = training_days_path, output_folder_name = "FlightHistory", output_file_name = "flighthistory.csv", cutoff_times = cutoff_times, start_hours_offset = -9) print("Flight History Events") utilities.split_file_based_on_times_filter_on_ids_streaming( os.path.join(raw_data_path, "FlightHistory", "flighthistoryevents.csv"), training_days_path, "FlightHistory", "flighthistoryevents.csv", "flight_history_id", days_flight_ids) print("ASDI Flight Plan") days_flight_plan_ids = utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiflightplan.csv"), training_days_path, "ASDI", "asdiflightplan.csv", "flighthistoryid", days_flight_ids, ids_to_track_column_name="asdiflightplanid") print("ASDI Position") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiposition.csv"), training_days_path, "ASDI", "asdiposition.csv", "flighthistoryid", days_flight_ids) print("ASDI Airway") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiairway.csv"), training_days_path, "ASDI", "asdiairway.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPFix") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpfix.csv"), training_days_path, "ASDI", "asdifpfix.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPCenter") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpcenter.csv"), training_days_path, "ASDI", "asdifpcenter.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPSector") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpsector.csv"), training_days_path, "ASDI", "asdifpsector.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPWaypoint") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpwaypoint.csv"), training_days_path, "ASDI", "asdifpwaypoint.csv", "asdiflightplanid", days_flight_plan_ids) for ct in cutoff_times: print ct day_output_path = os.path.join(training_days_path, utilities.get_day_str(ct, -9)) day_beginning, day_end = utilities.get_day_boundaries(ct, -9) if not os.path.exists(day_output_path): os.makedirs(day_output_path) weather.process_one_day(raw_data_path, day_output_path, day_beginning, day_end, "train")
def raw_data_to_training_days(raw_data_path, training_days_path, cutoff_times): #df_flight_history = flighthistory.get_df_flight_history_from_raw_format(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv")) print(os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv")) days_flight_ids = flighthistory.process_flight_history_to_train_day_files( input_path = os.path.join(raw_data_path, "FlightHistory", "flighthistory.csv"), output_path = training_days_path, output_folder_name = "FlightHistory", output_file_name = "flighthistory.csv", cutoff_times = cutoff_times, start_hours_offset = -9) print("Flight History Events") utilities.split_file_based_on_times_filter_on_ids_streaming( os.path.join(raw_data_path, "FlightHistory", "flighthistoryevents.csv"), training_days_path, "FlightHistory", "flighthistoryevents.csv", "flight_history_id", days_flight_ids) print("ASDI Flight Plan") days_flight_plan_ids = utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiflightplan.csv"), training_days_path, "ASDI", "asdiflightplan.csv", "flighthistoryid", days_flight_ids, ids_to_track_column_name="asdiflightplanid") print("ASDI Position") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiposition.csv"), training_days_path, "ASDI", "asdiposition.csv", "flighthistoryid", days_flight_ids) print("ASDI Airway") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdiairway.csv"), training_days_path, "ASDI", "asdiairway.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPFix") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpfix.csv"), training_days_path, "ASDI", "asdifpfix.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPCenter") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpcenter.csv"), training_days_path, "ASDI", "asdifpcenter.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPSector") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpsector.csv"), training_days_path, "ASDI", "asdifpsector.csv", "asdiflightplanid", days_flight_plan_ids) print("ASDI FPWaypoint") utilities.split_file_based_on_times_filter_on_ids_streaming(os.path.join(raw_data_path, "ASDI", "asdifpwaypoint.csv"), training_days_path, "ASDI", "asdifpwaypoint.csv", "asdiflightplanid", days_flight_plan_ids) for ct in cutoff_times: print ct day_output_path = os.path.join(output_path, utilities.get_day_str(ct, -9)) day_beginning, day_end = utilities.get_day_boundaries(ct, -9) if not os.path.exists(day_output_path): os.makedirs(day_output_path) weather.process_one_day(raw_data_path, day_output_path, day_beginning, day_end, "train")
def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time): flighthistory.write_flight_history_test_day_file( os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"), cutoff_time) flighthistory.write_flight_history_test_day_and_solution_test_flights_only( os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"), os.path.join(test_day_path, "test_flights.csv"), os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"), cutoff_time) utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"), "date_time_recorded", utilities.parse_datetime_format3, cutoff_time) utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"), "received", utilities.parse_datetime_format1, cutoff_time) flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"), "updatetimeutc", utilities.parse_datetime_format2, cutoff_time, ids_to_track_column_name = "asdiflightplanid") utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"), "asdiflightplanid", flight_plan_ids) day_beginning, day_end = utilities.get_day_boundaries(cutoff_time) weather.process_one_day( training_day_path, test_day_path, day_beginning, cutoff_time, "test", cutoff_time = cutoff_time)