def load_csv_files(self):
print "FlightDay Initializing: {}, {} in {} mode".format(self.folder_name, self.data_set_name, self.mode)
print "\tLoading flight_history.csv...",
self.flight_history = \
pd.read_csv("../Data/" + self.data_set_name + \
"/" + self.folder_name + "/" + "FlightHistory/flighthistory.csv",
na_values=["MISSING", "HIDDEN", ""], keep_default_na=True,
converters=dut.get_flight_history_date_converter())
print "done"
def load_csv_files(self):
print "FlightDay Initializing: {}, {} in {} mode".format(
self.folder_name, self.data_set_name, self.mode)
print "\tLoading flight_history.csv...",
self.flight_history = \
pd.read_csv("../Data/" + self.data_set_name + \
"/" + self.folder_name + "/" + "FlightHistory/flighthistory.csv",
na_values=["MISSING", "HIDDEN", ""], keep_default_na=True,
converters=dut.get_flight_history_date_converter())
print "done"
def __init__(self, data):
data_set_name = "InitialTrainingSet_rev1"
self.flight_history = pd.DataFrame(None)
self.parsed_fhe = pd.DataFrame(None)
if data == "flight_history":
print "AllTrainingData Initializing: using data {}".format(data)
for f in fn.folder_names_init_set():
print "\tLoading flight_history.csv folder {}...".format(f),
temp = \
pd.read_csv("../Data/" + data_set_name + \
"/" + f + "/" + "FlightHistory/flighthistory.csv",
converters = dut.get_flight_history_date_converter())
self.flight_history = pd.concat([self.flight_history, temp])
print "done"
if data == "parsed_fhe":
print "AllTrainingData Initializing: using data {}".format(data)
for f in fn.folder_names_init_set():
print "\tLoading parsed_fhe.csv file {}...".format(f),
temp = \
pd.read_csv('output_csv/parsed_fhe_' + f + '_' + "all" + '_filtered.csv',
# might have to fix to work with test data?
na_values=["MISSING"], keep_default_na=False,
parse_dates=[9,10,11,12,13,14,15,16,17,18,27,28,29,30,31,32,33,34,35,36,37,38,43,47])
self.parsed_fhe = pd.concat([self.parsed_fhe, temp])
print "done"
if data == "parsed_fhe_test":
print "AllTrainingData Initializing: using data {}".format(data)
for f in fn.folder_names_init_set():
print "\tLoading parsed_fhe.csv file {}...".format(f),
temp = \
pd.read_csv('output_csv/parsed_fhe_' + f + '_' + "test" + '_filtered.csv',
na_values=["MISSING"], keep_default_na=False,
parse_dates=[9,10,11,12,13,14,15,16,17,18,27,28,29,30,31,32,33,34,35,36,37,38,43,47])
self.parsed_fhe = pd.concat([self.parsed_fhe, temp])
print "done"
if data == "parsed_fhe_no_dates":
print "AllTrainingData Initializing: using data {}".format(data)
for f in fn.folder_names_init_set():
print "\tLoading parsed_fhe.csv file {}...".format(f),
temp = \
pd.read_csv('output_csv/parsed_fhe_' + f + '_' + "all" + '_filtered.csv',
# might have to fix to work with test data?
na_values=["MISSING"], keep_default_na=False)
self.parsed_fhe = pd.concat([self.parsed_fhe, temp])
print "done"
if data == "parsed_fhe_test_no_dates":
print "AllTrainingData Initializing: using data {}".format(data)
for f in fn.folder_names_test_set():
print "\tLoading parsed_fhe_test.csv file {}...".format(f),
temp = \
pd.read_csv('output_csv/parsed_fhe_' + f + '_' + "test" + '_filtered.csv',
# might have to fix to work with test data?
na_values=["MISSING"], keep_default_na=False)
self.parsed_fhe = pd.concat([self.parsed_fhe, temp])
print "done"
if data == "parsed_fhe_test_no_dates_with_best":
print "AllTrainingData Initializing: using data {}".format(data)
for f in fn.folder_names_test_set():
print "\tLoading parsed_fhe_test.csv file {}...".format(f),
temp = \
pd.read_csv('output_csv/parsed_fhe_' + f + '_' + "test" + \
'_filtered_with_dates_with_best_prediction.csv',
# might have to fix to work with test data?
na_values=["MISSING"], keep_default_na=False)
self.parsed_fhe = pd.concat([self.parsed_fhe, temp])
print "done"
