def test_filter_file_based_on_ids_streaming(self):
start_path = os.tempnam()
os.mkdir(start_path)
input_path = os.path.join(start_path, "input.csv")
f = open(input_path, "w")
f.write("".join([
"id,name,time\n", "1,Ben,2012-11-12 01:00:03+00\n",
"2,David,2012-11-12 05:00:03+00\n",
"3,Andrew,2012-11-12 10:00:03+00"
]))
f.close()
output_path = os.path.join(start_path, "output.csv")
utilities.filter_file_based_on_ids_streaming(input_path=input_path,
output_path=output_path,
id_column_name="id",
valid_ids=set(["1", "2"]))
expected_data = "".join([
"id,name,time\n", "1,Ben,2012-11-12 01:00:03+00\n",
"2,David,2012-11-12 05:00:03+00\n"
])
f = open(output_path)
actual_data = f.read()
f.close()
self.assertEqual(actual_data, expected_data)
os.remove(input_path)
os.remove(output_path)
os.rmdir(start_path)
def test_filter_file_based_on_ids_streaming(self):
start_path = os.tempnam()
os.mkdir(start_path)
input_path = os.path.join(start_path, "input.csv")
f = open(input_path, "w")
f.write("".join(["id,name,time\n",
"1,Ben,2012-11-12 01:00:03+00\n",
"2,David,2012-11-12 05:00:03+00\n",
"3,Andrew,2012-11-12 10:00:03+00"]))
f.close()
output_path = os.path.join(start_path, "output.csv")
utilities.filter_file_based_on_ids_streaming(
input_path = input_path,
output_path = output_path,
id_column_name = "id",
valid_ids = set(["1","2"]))
expected_data = "".join(["id,name,time\n",
"1,Ben,2012-11-12 01:00:03+00\n",
"2,David,2012-11-12 05:00:03+00\n"])
f = open(output_path)
actual_data = f.read()
f.close()
self.assertEqual(actual_data, expected_data)
os.remove(input_path)
os.remove(output_path)
os.rmdir(start_path)
def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time):
flighthistory.write_flight_history_test_day_file(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"),
cutoff_time)
flighthistory.write_flight_history_test_day_and_solution_test_flights_only(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(test_day_path, "test_flights.csv"),
os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"),
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"),
"date_time_recorded",
utilities.parse_datetime_format3,
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"),
"received",
utilities.parse_datetime_format1,
cutoff_time)
flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"),
"updatetimeutc",
utilities.parse_datetime_format2,
cutoff_time,
ids_to_track_column_name = "asdiflightplanid")
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"),
"asdiflightplanid",
flight_plan_ids)
def training_day_to_test_day(training_day_path, test_day_path, solution_path, cutoff_time):
flighthistory.write_flight_history_test_day_file(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistory.csv"),
cutoff_time)
flighthistory.write_flight_history_test_day_and_solution_test_flights_only(
os.path.join(training_day_path, "FlightHistory", "flighthistory.csv"),
os.path.join(test_day_path, "test_flights.csv"),
os.path.join(solution_path, utilities.get_day_str(cutoff_time) + "_solution.csv"),
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "FlightHistory", "flighthistoryevents.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "FlightHistory"), "flighthistoryevents.csv"),
"date_time_recorded",
utilities.parse_datetime_format3,
cutoff_time)
utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiposition.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiposition.csv"),
"received",
utilities.parse_datetime_format1,
cutoff_time)
flight_plan_ids = utilities.filter_file_based_on_cutoff_time_streaming(os.path.join(training_day_path, "ASDI", "asdiflightplan.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiflightplan.csv"),
"updatetimeutc",
utilities.parse_datetime_format2,
cutoff_time,
ids_to_track_column_name = "asdiflightplanid")
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdiairway.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdiairway.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpfix.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpfix.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpcenter.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpcenter.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpsector.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpsector.csv"),
"asdiflightplanid",
flight_plan_ids)
utilities.filter_file_based_on_ids_streaming(os.path.join(training_day_path, "ASDI", "asdifpwaypoint.csv"),
os.path.join(utilities.get_output_subdirectory(test_day_path, "ASDI"), "asdifpwaypoint.csv"),
"asdiflightplanid",
flight_plan_ids)
day_beginning, day_end = utilities.get_day_boundaries(cutoff_time)
weather.process_one_day(
training_day_path,
test_day_path,
day_beginning,
cutoff_time,
"test",
cutoff_time = cutoff_time)
