def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time): flighthistory.write_flight_history_test_day_file( os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"), cutoff_time) flighthistory.write_flight_history_test_day_and_solution_test_flights_only( os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"), os.path.join(test_day_path, "test_flights.csv"), os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"), cutoff_time) utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"), "date_time_recorded", utilities.parse_datetime_format3, cutoff_time) utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"), "received", utilities.parse_datetime_format1, cutoff_time) flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"), "updatetimeutc", utilities.parse_datetime_format2, cutoff_time, ids_to_track_column_name = "asdiflightplanid") utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"), "asdiflightplanid", flight_plan_ids)
def test_filter_file_based_on_cutoff_time_streaming(self): start_path = os.tempnam() os.mkdir(start_path) input_path = os.path.join(start_path, "input.csv") f = open(input_path, "w") f.write("".join([ "id,name,time\n", "1,Ben,2012-11-12 01:00:03+00\n", "2,David,2012-11-12 05:00:03+00\n", "3,Andrew,2012-11-12 10:00:03+00" ])) f.close() output_path = os.path.join(start_path, "output.csv") ids = utilities.filter_file_based_on_cutoff_time_streaming( input_path=input_path, output_path=output_path, date_column_name="time", date_parser=utilities.parse_datetime_format1, cutoff_time=datetime(2012, 11, 12, 8, 00, tzinfo=tzutc()), ids_to_track_column_name="id") expected_data = "".join([ "id,name,time\n", "1,Ben,2012-11-12 01:00:03+00:00\n", "2,David,2012-11-12 05:00:03+00:00\n" ]) f = open(output_path) actual_data = f.read() f.close() self.assertEqual(actual_data, expected_data) self.assertEqual(ids, set(["1", "2"])) ids = utilities.filter_file_based_on_cutoff_time_streaming( input_path=input_path, output_path=output_path, date_column_name="time", date_parser=utilities.parse_datetime_format1, cutoff_time=datetime(2012, 11, 12, 5, 0, 3, tzinfo=tzutc())) f = open(output_path) actual_data = f.read() f.close() self.assertEqual(actual_data, expected_data) self.assertIsNone(ids) os.remove(input_path) os.remove(output_path) os.rmdir(start_path)
def test_filter_file_based_on_cutoff_time_streaming(self): start_path = os.tempnam() os.mkdir(start_path) input_path = os.path.join(start_path, "input.csv") f = open(input_path, "w") f.write("".join(["id,name,time\n", "1,Ben,2012-11-12 01:00:03+00\n", "2,David,2012-11-12 05:00:03+00\n", "3,Andrew,2012-11-12 10:00:03+00"])) f.close() output_path = os.path.join(start_path, "output.csv") ids = utilities.filter_file_based_on_cutoff_time_streaming( input_path = input_path, output_path = output_path, date_column_name = "time", date_parser = utilities.parse_datetime_format1, cutoff_time = datetime(2012, 11, 12, 8, 00, tzinfo=tzutc()), ids_to_track_column_name = "id") expected_data = "".join(["id,name,time\n", "1,Ben,2012-11-12 01:00:03+00:00\n", "2,David,2012-11-12 05:00:03+00:00\n"]) f = open(output_path) actual_data = f.read() f.close() self.assertEqual(actual_data, expected_data) self.assertEqual(ids, set(["1","2"])) ids = utilities.filter_file_based_on_cutoff_time_streaming( input_path = input_path, output_path = output_path, date_column_name = "time", date_parser = utilities.parse_datetime_format1, cutoff_time = datetime(2012, 11, 12, 5, 0, 3, tzinfo=tzutc())) f = open(output_path) actual_data = f.read() f.close() self.assertEqual(actual_data, expected_data) self.assertIsNone(ids) os.remove(input_path) os.remove(output_path) os.rmdir(start_path)
def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time): flighthistory.write_flight_history_test_day_file( os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"), cutoff_time) flighthistory.write_flight_history_test_day_and_solution_test_flights_only( os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"), os.path.join(test_day_path, "test_flights.csv"), os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"), cutoff_time) utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"), "date_time_recorded", utilities.parse_datetime_format3, cutoff_time) utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"), "received", utilities.parse_datetime_format1, cutoff_time) flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"), "updatetimeutc", utilities.parse_datetime_format2, cutoff_time, ids_to_track_column_name = "asdiflightplanid") utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"), "asdiflightplanid", flight_plan_ids) utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"), os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"), "asdiflightplanid", flight_plan_ids) day_beginning, day_end = utilities.get_day_boundaries(cutoff_time) weather.process_one_day( training_day_path, test_day_path, day_beginning, cutoff_time, "test", cutoff_time = cutoff_time)