def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) file_amount = len(all_file_param) plot_files = ['Training set Microclimate (2 hour intervals)'] yield_file = 'Target Variable Water Yield' # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) # Read yield file for micro training file. yield_df = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == yield_file: yield_path = dataset_path + file.file_path yield_df = pd.read_csv(yield_path) break # Traverse all the dataset. for k in xrange(file_amount): file = all_file_param[k] if file.data_name not in plot_files: continue print '==========' + file.data_name + '==========' path = dataset_path + file.file_path df = pd.read_csv(path) # Split the micro training file into training dataset and test dataset. X_train, X_test, y_train, y_test, train_data, test_data = split_and_build_class( df.values, yield_df.values) # [train_data, test_data] = split_and_build_class(df.values, yield_df.values) # Run Ridge Regression. clf = run_regression(X_train[:, 1:], y_train) y_hat_test = clf.predict(X_test[:, 1:]) cmap = plt.get_cmap('jet_r') plt.figure(figsize=(10, 10)) interval = file.interval intervel_minute = read_dataset.get_interval_minute(interval) test_size = y_hat_test.shape[0] plt.plot([i for i in xrange(test_size)], y_hat_test) plt.plot([i for i in xrange(test_size)], y_test) plt.legend(['Prediction', 'Real']) plt.suptitle('Cross validation + Bagging Regressor') plt.savefig('Cross validation + Bagging Regressor.png', bbox_inches='tight') loss = np.sqrt(mean_squared_error(y_test, y_hat_test)) print 'Cross validation + Bagging Regressor loss =', loss
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) file_amount = len(all_file_param) plot_files = ['Training set Microclimate (2 hour intervals)'] yield_file = 'Target Variable Water Yield' # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) # Read yield file for micro training file. yield_df = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == yield_file: yield_path = dataset_path + file.file_path yield_df = pd.read_csv(yield_path) break # Traverse all the dataset. for k in xrange(file_amount): file = all_file_param[k] if file.data_name not in plot_files: continue print '==========' + file.data_name + '==========' path = dataset_path + file.file_path df = pd.read_csv(path) # Split the micro training file into training dataset and test dataset. X_train, X_test, y_train, y_test, train_data, test_data = split_and_build_class(df.values, yield_df.values) # [train_data, test_data] = split_and_build_class(df.values, yield_df.values) # Run Ridge Regression. clf = run_regression(X_train[:, 1:], y_train) y_hat_test = clf.predict(X_test[:, 1:]) cmap = plt.get_cmap('jet_r') plt.figure(figsize=(10, 10)) interval = file.interval intervel_minute = read_dataset.get_interval_minute(interval) test_size = y_hat_test.shape[0] plt.plot([i for i in xrange(test_size)], y_hat_test) plt.plot([i for i in xrange(test_size)], y_test) plt.legend(['Prediction', 'Real']) plt.suptitle('Cross validation + Bagging Regressor') plt.savefig('Cross validation + Bagging Regressor.png', bbox_inches='tight') loss = np.sqrt(mean_squared_error(y_test, y_hat_test)) print 'Cross validation + Bagging Regressor loss =', loss
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) plot_files = ['Training set Microclimate (2 hour intervals)'] # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) for feature in micro_features: plot_one_feature(plot_files, all_file_param, dataset_path, feature)
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) plot_files = ['Training set Microclimate (2 hour intervals)'] # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) for feature in micro_features: plot_one_feature(plot_files, all_file_param, dataset_path, feature)
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) file_amount = len(all_file_param) plot_files = ['Training set Microclimate (2 hour intervals)'] yield_file = 'Target Variable Water Yield' # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) # Read yield file for micro training file. yield_df = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == yield_file: yield_path = dataset_path + file.file_path yield_df = pd.read_csv(yield_path) break # Traverse all the dataset. for k in xrange(file_amount): file = all_file_param[k] if file.data_name not in plot_files: continue print '==========' + file.data_name + '==========' path = dataset_path + file.file_path df = pd.read_csv(path) # Split the micro training file into training dataset and test dataset. X_train, X_test, y_train, y_test, train_data, test_data = split_and_build_class(df.values, yield_df.values) # [train_data, test_data] = split_and_build_class(df.values, yield_df.values) # Run Ridge Regression. clf = run_regression(X_train[:, 1:], y_train) y_hat_test = clf.predict(X_test[:, 1:]) cmap = plt.get_cmap('jet_r') plt.figure(figsize=(10, 10)) interval = file.interval intervel_minute = read_dataset.get_interval_minute(interval) test_size = y_hat_test.shape[0] plt.plot([i for i in xrange(test_size)], y_hat_test) plt.plot([i for i in xrange(test_size)], y_test) plt.legend(['Prediction', 'Real']) plt.suptitle('Cross validation + Random Forest Regressor') plt.savefig('Cross validation + Random Forest Regressor.png', bbox_inches='tight') loss = np.sqrt(mean_squared_error(y_test, y_hat_test)) print 'Cross validation + Random Forest Regressor loss =', loss ''' ======================================================================= ''' # Predict test and write submission submission_file_name = 'Submission format' submission_file = None test_file_name = 'Test set Microclimate (2 hour intervals)' test_file = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == submission_file_name: submission_file = file break submission_path = dataset_path + submission_file.file_path df_submission = pd.read_csv(submission_path, index_col=0, parse_dates=[0]) for k in xrange(file_amount): file = all_file_param[k] if file.data_name == test_file_name: test_file = file break test_path = dataset_path + test_file.file_path df_test = pd.read_csv(test_path, index_col=0, parse_dates=[0]) X_combined = write_submission.combine_table(df_submission, df_test) imp = preprocessing.Imputer(missing_values='NaN', strategy='mean', axis=0) fixed_X = X_combined.values[:, 0:] imp.fit(fixed_X) X_combined.values[:, 0:] = imp.transform(fixed_X) preprocessing.normalize(X_combined.values, copy=False) y_submission = write_submission.write_submission( X_combined, clf, df_submission, 'Cross Valication + Random Forest Regressor Submission')
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) file_amount = len(all_file_param) # Initiate the plot. cmap = plt.get_cmap('jet_r') plt.figure(figsize=(10, 10)) plot_for_legend = plt.subplot() # Traverse all the dataset. for k in xrange(file_amount): file = all_file_param[k] # Use this block to set which dataset you want to find missing intervals. # if file.data_name != 'Macroclimate Guelmim Airport': # continue print '==========' + file.data_name + '==========' path = dataset_path + file.file_path df = pd.read_csv(path) data_time = df.values[:, 0] interval = file.interval start_time = datetime.datetime.strptime(file.start_time,'%H:%M') end_time = datetime.datetime.strptime(file.end_time,'%H:%M') missing_data = [] prev_time = read_dataset.assign_time(data_time[0]) current_time = None color = cmap(float(k) / file_amount) for i in range(1, data_time.shape[0]): tmp = data_time[i] current_time = read_dataset.assign_time(tmp) diff = current_time - prev_time # Find out missing intervals with gap larger that the default interval. if diff.days or diff.seconds / 60 > read_dataset.get_interval_minute(interval): missing_interval = [prev_time, current_time] missing_data += fix_interval(start_time, end_time, missing_interval) prev_time = current_time # Output all the missing intervals. for item in missing_data: print item[0].strftime('%Y-%m-%d %H:%M:%S'), \ item[1].strftime('%Y-%m-%d %H:%M:%S') print len(missing_data), ' missing intervals are found.' # Plot the missing intervals. for item in missing_data: plt.plot(item, [(k + 1) for j in xrange(2)], c=color) plt.ylim([0, file_amount + 1]) plot_for_legend.plot([], [], c=color, label=file.data_name) # Set the position, legend, and subtitle of the plot. box = plot_for_legend.get_position() plot_for_legend.set_position([box.x0, box.y0, box.width, box.height * 0.6]) plot_for_legend.set_position([box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8]) legend = plot_for_legend.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=2) plt.suptitle('Missing Intervals for the Dataset') plt.savefig('missing_intervals_for_the_dataset.png')
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) file_amount = len(all_file_param) plot_files = ['Training set Microclimate (2 hour intervals)'] yield_file = 'Target Variable Water Yield' # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) # Read yield file for micro training file. yield_df = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == yield_file: yield_path = dataset_path + file.file_path yield_df = pd.read_csv(yield_path) break # Traverse all the dataset. for k in xrange(file_amount): file = all_file_param[k] if file.data_name not in plot_files: continue print '==========' + file.data_name + '==========' path = dataset_path + file.file_path df = pd.read_csv(path) # Split the micro training file into training dataset and test dataset. X_train, X_test, y_train, y_test, train_data, test_data = split_and_build_class(df.values, yield_df.values) # [train_data, test_data] = split_and_build_class(df.values, yield_df.values) y_train_binary = Binary_Classification.transform_to_binary(y_train) # Run SVM. clf = run_regression(X_train[:, 1:], y_train_binary) y_hat_test_binary = clf.predict(X_test[:, 1:]) print 'Number of Class 1 in Training Data:', np.count_nonzero(y_train_binary) print 'Number of Class 1 in Test Data:', np.count_nonzero(y_test) print 'Number of Class 1 in Predicted Data:', np.count_nonzero(y_hat_test_binary) # Run Ridge Regression. X_train_regression = [] y_train_regression = [] X_test_regression = [] y_hat_test_regression = [] for i in xrange(len(y_train_binary)): if y_train_binary[i] != 0: X_train_regression.append(X_train[i]) y_train_regression.append(y_train[i]) X_train_regression = np.array(X_train_regression) ''' =====Regression===== ''' clf_regression = linear_model.Ridge(normalize=True) # clf_regression.fit(X_train_regression[:, 1:], y_train_regression) clf_regression.fit(X_train[:, 1:], y_train) ''' ==================== ''' for i in xrange(len(y_hat_test_binary)): if y_hat_test_binary[i] != 0: X_test_regression.append(X_test[i]) X_test_regression = np.array(X_test_regression) if len(X_test_regression): y_hat_test_regression = clf_regression.predict(X_test_regression[:, 1:]) j = 0 y_hat_test = [] if len(X_test_regression): for i in xrange(len(y_hat_test_binary)): if y_hat_test_binary[i] == 0: y_hat_test.append(y_hat_test_binary[i]) else: y_hat_test.append(y_hat_test_regression[j]) j += 1 y_hat_test = np.array(y_hat_test) else: y_hat_test = y_hat_test_binary cmap = plt.get_cmap('jet_r') plt.figure(figsize=(10, 10)) interval = file.interval intervel_minute = read_dataset.get_interval_minute(interval) test_size = y_hat_test.shape[0] plt.plot([i for i in xrange(test_size)], y_hat_test) plt.plot([i for i in xrange(test_size)], y_test) plt.legend(['Prediction', 'Real']) plt.suptitle('Cross Validation + Bagging Classifier + Ridge Regression') plt.savefig('Cross Validation + Bagging Classifier + Ridge Regression.png', bbox_inches='tight') loss = np.sqrt(mean_squared_error(y_test, y_hat_test)) print 'Cross Validation + Bagging Classifier + Ridge Regression loss =', loss ''' ======================================================================= ''' # Predict test and write submission submission_file_name = 'Submission format' submission_file = None test_file_name = 'Test set Microclimate (2 hour intervals)' test_file = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == submission_file_name: submission_file = file break submission_path = dataset_path + submission_file.file_path df_submission = pd.read_csv(submission_path, index_col=0, parse_dates=[0]) for k in xrange(file_amount): file = all_file_param[k] if file.data_name == test_file_name: test_file = file break test_path = dataset_path + test_file.file_path df_test = pd.read_csv(test_path, index_col=0, parse_dates=[0]) X_combined = write_submission.combine_table(df_submission, df_test) imp = preprocessing.Imputer(missing_values='NaN', strategy='mean', axis=0) fixed_X = X_combined.values[:, 0:] imp.fit(fixed_X) X_combined.values[:, 0:] = imp.transform(fixed_X) preprocessing.normalize(X_combined.values, copy=False) y_submission = write_submission.write_submission_binary_classifier_and_regression( X_combined, clf, clf_regression, df_submission, 'Cross Validation + Bagging Classifier + Ridge Regression Submission')
def main(): # Read the dataset. dataset_path = './dataset/' dataset_file_path = './dataset_file_path.csv' df_path = pd.read_csv(dataset_file_path) all_file_param = read_dataset.read_all_dataset(df_path) file_amount = len(all_file_param) plot_files = ['Training set Microclimate (2 hour intervals)'] yield_file = 'Target Variable Water Yield' # plot_files = ['Training set Microclimate (2 hour intervals)', # 'Training set Microclimate (5 minute intervals)'] # Set up features for micro files. micro_features = read_dataset.set_features(dataset_path, plot_files) # Read yield file for micro training file. yield_df = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == yield_file: yield_path = dataset_path + file.file_path yield_df = pd.read_csv(yield_path) break # Traverse all the dataset. for k in xrange(file_amount): file = all_file_param[k] if file.data_name not in plot_files: continue print '==========' + file.data_name + '==========' path = dataset_path + file.file_path df = pd.read_csv(path) # Split the micro training file into training dataset and test dataset. X_train, X_test, y_train, y_test, train_data, test_data = split_and_build_class( df.values, yield_df.values) # [train_data, test_data] = split_and_build_class(df.values, yield_df.values) y_train_binary = Binary_Classification.transform_to_binary(y_train) # Run SVM. clf = run_regression(X_train[:, 1:], y_train_binary) y_hat_test_binary = clf.predict(X_test[:, 1:]) print 'Number of Class 1 in Training Data:', np.count_nonzero( y_train_binary) print 'Number of Class 1 in Test Data:', np.count_nonzero(y_test) print 'Number of Class 1 in Predicted Data:', np.count_nonzero( y_hat_test_binary) # Run Ridge Regression. X_train_regression = [] y_train_regression = [] X_test_regression = [] y_hat_test_regression = [] for i in xrange(len(y_train_binary)): if y_train_binary[i] != 0: X_train_regression.append(X_train[i]) y_train_regression.append(y_train[i]) X_train_regression = np.array(X_train_regression) ''' =====Regression===== ''' clf_regression = linear_model.Ridge(normalize=True) # clf_regression.fit(X_train_regression[:, 1:], y_train_regression) clf_regression.fit(X_train[:, 1:], y_train) ''' ==================== ''' for i in xrange(len(y_hat_test_binary)): if y_hat_test_binary[i] != 0: X_test_regression.append(X_test[i]) X_test_regression = np.array(X_test_regression) if len(X_test_regression): y_hat_test_regression = clf_regression.predict( X_test_regression[:, 1:]) j = 0 y_hat_test = [] if len(X_test_regression): for i in xrange(len(y_hat_test_binary)): if y_hat_test_binary[i] == 0: y_hat_test.append(y_hat_test_binary[i]) else: y_hat_test.append(y_hat_test_regression[j]) j += 1 y_hat_test = np.array(y_hat_test) else: y_hat_test = y_hat_test_binary cmap = plt.get_cmap('jet_r') plt.figure(figsize=(10, 10)) interval = file.interval intervel_minute = read_dataset.get_interval_minute(interval) test_size = y_hat_test.shape[0] plt.plot([i for i in xrange(test_size)], y_hat_test) plt.plot([i for i in xrange(test_size)], y_test) plt.legend(['Prediction', 'Real']) plt.suptitle( 'Cross Validation + Bagging Classifier + Ridge Regression') plt.savefig( 'Cross Validation + Bagging Classifier + Ridge Regression.png', bbox_inches='tight') loss = np.sqrt(mean_squared_error(y_test, y_hat_test)) print 'Cross Validation + Bagging Classifier + Ridge Regression loss =', loss ''' ======================================================================= ''' # Predict test and write submission submission_file_name = 'Submission format' submission_file = None test_file_name = 'Test set Microclimate (2 hour intervals)' test_file = None for k in xrange(file_amount): file = all_file_param[k] if file.data_name == submission_file_name: submission_file = file break submission_path = dataset_path + submission_file.file_path df_submission = pd.read_csv(submission_path, index_col=0, parse_dates=[0]) for k in xrange(file_amount): file = all_file_param[k] if file.data_name == test_file_name: test_file = file break test_path = dataset_path + test_file.file_path df_test = pd.read_csv(test_path, index_col=0, parse_dates=[0]) X_combined = write_submission.combine_table(df_submission, df_test) imp = preprocessing.Imputer(missing_values='NaN', strategy='mean', axis=0) fixed_X = X_combined.values[:, 0:] imp.fit(fixed_X) X_combined.values[:, 0:] = imp.transform(fixed_X) preprocessing.normalize(X_combined.values, copy=False) y_submission = write_submission.write_submission_binary_classifier_and_regression( X_combined, clf, clf_regression, df_submission, 'Cross Validation + Bagging Classifier + Ridge Regression Submission' )