class TargetEncoderNSplits(BaseTransformer): def __init__(self, n_splits, **kwargs): self.k_folds = KFold(n_splits=n_splits) self.target_means_map = {} def _target_means_names(self, columns): confidence_rate_names = ['target_mean_{}'.format(column) for column in columns] return confidence_rate_names def _is_null_names(self, columns): is_null_names = ['target_mean_is_nan_{}'.format(column) for column in columns] return is_null_names def fit(self, categorical_features, target, **kwargs): feature_columns, target_column = categorical_features.columns, target.columns[0] X_target_means = [] self.k_folds.get_n_splits(target) for train_index, test_index in self.k_folds.split(target): X_train, y_train = categorical_features.iloc[train_index], target.iloc[train_index] X_test, y_test = categorical_features.iloc[test_index], target.iloc[test_index] train = pd.concat([X_train, y_train], axis=1) for column, target_mean_name in zip(feature_columns, self._target_means_names(feature_columns)): group_object = train.groupby(column) train_target_means = group_object[target_column].mean(). \ reset_index().rename(index=str, columns={target_column: target_mean_name}) X_test = X_test.merge(train_target_means, on=column, how='left') X_target_means.append(X_test) X_target_means = pd.concat(X_target_means, axis=0).astype(np.float32) for column, target_mean_name in zip(feature_columns, self._target_means_names(feature_columns)): group_object = X_target_means.groupby(column) self.target_means_map[column] = group_object[target_mean_name].mean().reset_index() return self def transform(self, categorical_features, **kwargs): columns = categorical_features.columns for column, target_mean_name, is_null_name in zip(columns, self._target_means_names(columns), self._is_null_names(columns)): categorical_features = categorical_features.merge(self.target_means_map[column], on=column, how='left').astype(np.float32) categorical_features[is_null_name] = pd.isnull(categorical_features[target_mean_name]).astype(int) categorical_features[target_mean_name].fillna(0, inplace=True) return {'numerical_features': categorical_features[self._target_means_names(columns)], 'categorical_features': categorical_features[self._is_null_names(columns)]} def load(self, filepath): self.target_means_map = joblib.load(filepath) return self def save(self, filepath): joblib.dump(self.target_means_map, filepath)
def kFolds(dataSet, k = 10): """ This is the k-fold method :param dataSet: of type DataFrame :param k: number of subsets to choose """ df_mx = dataSet.as_matrix() X = df_mx[:, 1:16] Y = df_mx[:, 0:1] lm = svm.SVC(gamma=0.001, C=100.) # Support Vector Machine kf = KFold(n_splits=10) # Define the split - into 10 folds i = 0 accuracies = numpy.zeros(kf.get_n_splits(X)) for train_index, test_index in kf.split(X): print("{}. TRAIN: {} TEST: {}".format(i+1, train_index, test_index)) X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] # train using X_Train model = lm.fit(X_train, Y_train) # evaluate against X_Test predictions = lm.predict(X_test) # save accuracy accuracies[i] = model.score(X_test, Y_test) i = i + 1 # find mean accuracy over all rounds print("Average accuracy of K-Folds (k={}): {}%".format(numpy.mean(accuracies) * 100, k))
def VTiter(self, *parsedArgs, **envars): largs, dictargs = self.full_parse(parsedArgs) # get arguments if 'splits' not in dictargs: raise functions.OperatorError(__name__.rsplit('.')[-1], "No splits argument.") else: self.n_splits = int(dictargs['splits']) # print largs # print dictargs self.data = [] if 'query' not in dictargs: raise functions.OperatorError(__name__.rsplit('.')[-1], "No query argument ") query = dictargs['query'] cur = envars['db'].cursor() c = cur.execute(query) for r in c: if r[0].isdigit(): self.data.append(r[0]) else: self.data.append(str(r[0])) yield [('rid',), ('idofset',)] # print "data", self.data X = np.array(self.data) # print X kf = KFold(self.n_splits) kf.get_n_splits(X) # print"KF", kf try: for train_index, test_index in kf.split(X): # print("TRAIN:", train_index ,"TEST:", test_index) j = 0 for train_index, test_index in kf.split(X): for k in test_index: yield (self.data[k], j) j += 1 except: yield (-1, "Cannot have number of splits greater than the number of samples")
from sklearn.model_selection import train_test_split from sklearn.model_selection import KFold import numpy as np from sklearn import metrics df = pd.read_csv("ols_dataset.csv") # print(df) target = df.iloc[:, 2].values #doesnot work we need to change it into arrays by adding .valuesS data = df.iloc[:, 3:10].values kfold_object = KFold( n_splits=4 ) #KFold only uses each value once, so we separate the dataset into 4 kfold_object.get_n_splits(data) for training_index, test_index in kfold_object.split(data): print(training_index) print(test_index) data_training, data_test = data[training_index], data[test_index] target_training, target_test = target[training_index], target[test_index] machine = linear_model.LinearRegression() machine.fit(data_training, target_training) prediction = machine.predict(data_test) print( metrics.r2_score(target_test, prediction) ) #r-square is the distance between test and prediction, the higher, the model is predicting more correctly # -----------------------------------------------------
weights_train)) training_scores.append(training_score) return training_scores, test_scores weights = X['age'].pipe(lambda s: s > train_cutoff).map({ True: .9, False: .1 }).values logistic_model = LogisticRegression() weighted_training_scores, weighted_test_scores = cross_val_scores_weighted( logistic_model, X, y, weights, cv=10) unweighted_test_scores = [] unweighted_train_scores = [] kf = KFold(n_splits=10) kf.get_n_splits(X) for train_index, test_index in kf.split(X.index): X_train, X_audit_holdout = X.loc[train_index], X.loc[test_index] y_train, y_test = y.loc[train_index], y.loc[test_index] clf.fit(X_train, y_train) y_pred = clf.predict_proba(X_audit_holdout)[:, 1] score = roc_auc_score(y_test, y_pred) unweighted_test_scores.append(score) y_pred = clf.predict_proba(X_train)[:, 1] score = roc_auc_score(y_train, y_pred) unweighted_train_scores.append(score) hard_weighted_test_scores = [] hard_weighted_train_scores = [] kf = KFold(n_splits=10) kf.get_n_splits(X)
X = X.T M = len(y) np.random.seed(42) p = np.random.permutation(M) perm_X = X[p] perm_y = y[p] num_test_ex = int(np.floor(0.15 * M)) test_data = X[0:num_test_ex] test_labels = y[0:num_test_ex] train_validate_data = X[num_test_ex:] train_validate_labels = y[num_test_ex:] #Try 6-fold cross validation instead? kf = KFold(n_splits=6, shuffle=True) kf.get_n_splits(train_validate_data) for train_index, validate_index in kf.split(train_validate_data): for i, params in enumerate(params_to_try): train_data, validate_data = train_validate_data[ train_index], train_validate_data[validate_index] train_labels, validate_labels = train_validate_labels[ train_index], train_validate_labels[validate_index] clf = LogisticRegression(penalty=params['penalty_type'], C=params['C'], solver=params['solver'], max_iter=100, random_state=42, multi_class='multinomial') clf.fit(train_data, train_labels) train_predict = clf.predict(train_data) train_accuracy = accuracy_score(train_labels, train_predict)
# ### Build and learn PCA model # + from sklearn.decomposition import PCA pca = PCA(n_components=3) # - x = dataset['train']['x'][0] # + from sklearn.model_selection import KFold # import KFold import sklearn as sk kf = KFold(n_splits=5) # Define the split - into 2 folds kf.get_n_splits( x) # returns the number of splitting iterations in the cross-validator for train_index, test_index in kf.split(x): #print('TRAIN:', train_index, 'TEST:', test_index) X_train, X_test = x[train_index], x[test_index] principalComponents = pca.fit_transform(X_train) X_test_pca = pca.transform(X_test) x_hat = pca.inverse_transform(X_test_pca) print("mae loss:") print(sk.metrics.mean_absolute_error(X_test, x_hat)) print("mse loss:") print(sk.metrics.mean_squared_error(X_test, x_hat)) # y_train, y_test = y[train_index], y[test_index] # - pca.explained_variance_ratio_
df2 = pd.read_csv('data/ip2.nmap.online.hosts.masscan.csv.optimised.som_win_map6.csv') df2_data=df2.drop(columns=['cluster']) df2_labels=df2['cluster'].values results=dict([ (8, {'accuracy':[],'recall':[],'f1':[],'precision':[]}), (12, {'accuracy':[],'recall':[],'f1':[],'precision':[]}), (16, {'accuracy':[],'recall':[],'f1':[],'precision':[]}), (32, {'accuracy':[],'recall':[],'f1':[],'precision':[]}), (64, {'accuracy':[],'recall':[],'f1':[],'precision':[]}), ]) kf = KFold(n_splits=4,shuffle=True) kf.get_n_splits(df2) next(kf.split(df2), None) for units in results.keys(): for train_index, test_index in kf.split(df2_data): # print('Train') # print(train_index) # print('Test') # print(test_index) X_train = df2_data.iloc[train_index] X_test = df2_data.iloc[test_index] y_train = to_categorical([df2_labels[i] for i in train_index],num_classes=64) y_test = to_categorical([df2_labels[i] for i in test_index],num_classes=64) # classifier = Sequential()
img_data_original = np.empty([50000, 1024]) for i in range(0, 50000): filename = '{0:05d}'.format(i) + '.png' img = rgb2gray(io.imread(train_set_dir + filename)) #Reading file, converting to Grayscale img_data_original[i, :] = img.flatten() for i in range(15, 1024, 16): pca = PCA(n_components=i, whiten=True) pca.fit(img_data_original) img_data = pca.transform(img_data_original) # Split the data using K-Folds, using 5 different sets kf = KFold(n_splits=5) kf.get_n_splits(img_data) count = 0 train_score = np.zeros(5) val_score = np.zeros(5) for train_index, val_index in kf.split(img_data): img_data_train, img_data_val = img_data[train_index], img_data[ val_index] img_labels_train, img_labels_val = img_labels[train_index], img_labels[ val_index] regr = LogisticRegression(multi_class='ovr') regr.fit(img_data_train, img_labels_train) count += 1 train_score[count - 1] = regr.score(img_data_train, img_labels_train)
y = [] with open(path) as f_train: for line in f_train: data_row = line.split(',') X.append(data_row[:-1]) y.append(data_row[-1]) X = np.asarray(X, dtype=np.float) y = np.asarray(y, dtype=np.float) return X, y if __name__ == '__main__': pen_based_recognition = DatasetBase(train_path='raw_data/pendigits.tra', test_path='raw_data/pendigits.tes') clf = neighbors.KNeighborsClassifier(n_neighbors=1, p=2) kf = KFold(n_splits=10, shuffle=True) kf.get_n_splits(pen_based_recognition.X_data, pen_based_recognition.y_data) for train_index, test_index in kf.split(pen_based_recognition.X_data): print("TRAIN:", train_index, "TEST:", test_index) # print(len(train_index)) # print(len(test_index)) X_train, X_test = pen_based_recognition.X_data[train_index], pen_based_recognition.X_data[test_index] y_train, y_test = pen_based_recognition.y_data[train_index], pen_based_recognition.y_data[test_index] clf.fit(X_train, y_train) print(clf.score(X_test, y_test))
print('nonlinearity: ' + str(nonlinearity)) print('model: ' + str(model)) # tr_fmri_net, tr_adj, tr_labels, val_fmri_net, val_adj, val_labels, test_fmri_net, test_adj, test_labels = \ # Data_processing.load_data('Data_BNF/'+dataset+'/','Data_BNF/'+dataset+'/labels.csv', augmentation) # tr_fmri_net, tr_adj, tr_labels, val_fmri_net, val_adj, val_labels, test_fmri_net, test_adj, test_labels = \ # Data_processing.load_sorted_data('Data_BNF/'+dataset+'/sort_data.pkl') fmri_signals, labels, graphs = \ Data_processing.load_ADHD('Data_BNF/'+dataset+'/','Data_BNF/'+dataset+'/labels.csv', augmentation) # while(True): kf = KFold(n_splits=5) kf.get_n_splits(fmri_signals) labels = np.array(labels) test_loss_avg = [] test_acc_avg = [] predict_labels = [] actual_labels = [] for train_ind, test_ind in kf.split(fmri_signals): train_fmri_rsignals, test_fmri_rsignals = fmri_signals[ train_ind], fmri_signals[test_ind] train_rlabels, test_rlabels = labels[train_ind], labels[test_ind] train_rgraphs, test_rgraphs = graphs[train_ind], graphs[test_ind] train_fmri_rsignals, val_fmri_rsignals, train_rlabels, val_rlabels, train_rgraphs, val_rgraphs = train_test_split( train_fmri_rsignals, train_rlabels, train_rgraphs, test_size=0.2)
print(model.score(test_X, test_y)) test_predict = model.predict(test_X) #avg_feature_importance.append(test_predict.feature_importances_) acc, precision, recall, f1, matrix = evaluation(test_y, test_predict) print("Fold: %d, Accuracy: %f, Precision: %f, Recall: %f, F1: %f" % (fold_count + 1, round(acc, 3), round(precision, 3), round( recall, 3), round(f1, 3))) avg_acc += acc avg_precision += precision avg_recall += recall avg_f1 += f1 avg_confusion_matrix.append(matrix) fold_count += 1 print( "=================================================================================" ) print("Avg Accuracy: %f, Avg Precision: %f, Avg Recall: %f, Avg F1: %f" % (round(avg_acc / kf.get_n_splits(), 3), \ round(avg_precision / kf.get_n_splits(), 3), \ round(avg_recall / kf.get_n_splits(), 3), \ round(avg_f1 / kf.get_n_splits(), 3))) ''' importance_dict = {} for col, importance in zip(train_X.columns, np.mean(np.array(avg_feature_importance), axis=0)): importance_dict[col] = importance print(sorted(importance_dict.items(), key=lambda x: -x[1])[:10]) '''
import pandas as pd from sklearn.model_selection import KFold, cross_val_predict from sklearn import svm, tree, neighbors, neural_network, naive_bayes import sklearn.metrics as met import matplotlib.pyplot as plt data = pd.read_csv('DataSets/tae.csv', sep=',', header=None) #print(data) train = data.ix[:, 0:4] target = data.ix[:, 5] #print(x) #print(target) kf = KFold(n_splits=10) print(kf) print(kf.get_n_splits(train)) for train_index, test_index in kf.split(train): print("TRAIN:", train_index, "TEST:", test_index) X_train, X_test = train.ix[train_index, :], train.ix[test_index, :] print(X_train) y_train, y_test = target[train_index], target[test_index] print(y_train) for i in range(5, 12): clf = tree.DecisionTreeClassifier(max_depth=i) print('--------------------{}---------------------------------'.format(i)) print('cross_val_predict') predicted = cross_val_predict( clf, train, target, cv=10,
df['flag'] = flag[0] df['target'] = tar[0] features = list(df.columns[0:39]) x = df[features] y = df["target"] xnmp=x.to_numpy() xnmp = np.array(xnmp, dtype=np.float64) ynmp=y.to_numpy() ynmp = np.array(ynmp, dtype=np.float64) kf = KFold(n_splits=3) kf.get_n_splits(xnmp) time = 0 for train_index, test_index in kf.split(xnmp): time = time + 1 x_train, x_test = xnmp[train_index], xnmp[test_index] y_train, y_test = ynmp[train_index], ynmp[test_index] print("-----Time: "+ str(time) + " Decision Tree-----") #decision tree classifier = tree.DecisionTreeClassifier(criterion = 'entropy' , max_depth=10, random_state=0) classifier.fit(x_train, y_train) tree.plot_tree(classifier) predict_y = classifier.predict(x_test) accuracy = metrics.accuracy_score(y_test, predict_y) cm = confusion_matrix(y_test, predict_y) print("Accuracy: "+ str(accuracy)) recall = cm[1,1]/(cm[0,1]+cm[1,1]) #true positives/ false negatives + true positives
def tachTestTrain(pathFolder): print("Create New DataSet") listFolderNameMSSV = timFolderName(pathFolder) dataX = [] # dataY = [] # dataName = [] for ia in listFolderNameMSSV: for ib in listdir(join(pathFolder, ia)): if isfile(join(pathFolder, ia, ib)): dataX.append(join(pathFolder, ia, ib)) # dataName.append(ia) shuffle(dataX) # for ia in dataX: # dataY.append(int(ia.split('/')[4].split('_')[0])) kf = KFold(n_splits=10, shuffle=True) kf.get_n_splits(dataX) tempDictXtrain = {} tempDictYtrain = {} tempDictXtest = {} tempDictYtest = {} # tempNtrain = [] # tempNtest = [] intDem = 0 for train_index, test_index in kf.split(dataX): y_train = [] y_test = [] X_train, X_test = np.array(dataX)[train_index], np.array( dataX)[test_index] shuffle(X_train) shuffle(X_test) for ia in X_train: y_train.append(int(ia.split('/')[4].split('_')[0])) for ia in X_test: y_test.append(int(ia.split('/')[4].split('_')[0])) # N_train, N_test = np.array(dataName)[train_index], np.array(dataName)[test_index] # Luu lai tempDictXtrain[intDem] = X_train tempDictYtrain[intDem] = y_train tempDictXtest[intDem] = X_test tempDictYtest[intDem] = y_test intDem += 1 # tempNtrain.append(N_train.tolist()) # tempNtest.append(N_test.tolist()) # print("TRAIN:", train_index, "TEST:", test_index) # print("TRAIN_len:", len(train_index), "TEST_len:", len(test_index)) dfXtrain = pd.DataFrame(tempDictXtrain) dfYtrain = pd.DataFrame(tempDictYtrain) dfXtest = pd.DataFrame(tempDictXtest) dfYtest = pd.DataFrame(tempDictYtest) if not os.path.exists("./tempLuu"): os.makedirs("./tempLuu") dfXtrain.to_csv('./tempLuu/tempDictXtrain.csv', index=None) dfYtrain.to_csv('./tempLuu/tempDictYtrain.csv', index=None) dfXtest.to_csv('./tempLuu/tempDictXtest.csv', index=None) dfYtest.to_csv('./tempLuu/tempDictYtest.csv', index=None)
def train_stage(df_path, lgb_path, xgb_path, cb_path): print('>> Loading training data...') df = pd.read_csv(df_path) y_df = np.array(df['Eclipse Duration (m)']) df_ids = np.array(df.index) df.drop(['Eclipse Duration (m)', 'Lunation Number'], axis=1, inplace=True) X = preprocess_data(df) print('>> Shape of train data:', X.shape) lgb_cv_result = np.zeros(df.shape[0]) xgb_cv_result = np.zeros(df.shape[0]) cb_cv_result = np.zeros(df.shape[0]) skf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42) print(">> K Fold with", skf.get_n_splits(df_ids, y_df), "splits") print('>> Model Fitting...') for counter, ids in enumerate(skf.split(df_ids, y_df)): print('>> Fold', counter + 1) X_fit, y_fit = X[ids[0]], y_df[ids[0]] X_val, y_val = X[ids[1]], y_df[ids[1]] if TRAIN_LGB: print('>>> LigthGBM...') lgb_cv_result[ids[1]] += fit_lgb(X_fit, y_fit, X_val, y_val, counter, lgb_path, name='lgb') if TRAIN_XGB: print('>>> XGBoost...') xgb_cv_result[ids[1]] += fit_xgb(X_fit, y_fit, X_val, y_val, counter, xgb_path, name='xgb') if TRAIN_CB: print('>>> CatBoost...') cb_cv_result[ids[1]] += fit_cb(X_fit, y_fit, X_val, y_val, counter, cb_path, name='cb') del X_fit, X_val, y_fit, y_val gc.collect() if TRAIN_LGB: rmse_lgb = round(sqrt(mean_squared_error(y_df, lgb_cv_result)), 4) print('>> LightGBM VAL RMSE:', rmse_lgb) if TRAIN_XGB: rmse_xgb = round(sqrt(mean_squared_error(y_df, xgb_cv_result)), 4) print('>> XGBoost VAL RMSE:', rmse_xgb) if TRAIN_CB: rmse_cb = round(sqrt(mean_squared_error(y_df, cb_cv_result)), 4) print('>> Catboost VAL RMSE:', rmse_cb) if TRAIN_LGB and TRAIN_XGB and TRAIN_CB: rmse_mean = round( sqrt( mean_squared_error( y_df, (lgb_cv_result + xgb_cv_result + cb_cv_result) / 3)), 4) print('>> Mean XGBoost+Catboost+LightGBM, VAL RMSE:', rmse_mean) if TRAIN_LGB and TRAIN_CB: rmse_mean_lgb_cb = round( sqrt(mean_squared_error(y_df, (lgb_cv_result + cb_cv_result) / 2)), 4) print('>> Mean Catboost+LightGBM VAL RMSE:', rmse_mean_lgb_cb) if TRAIN_LGB and TRAIN_XGB: rmse_mean_lgb_xgb = round( sqrt(mean_squared_error(y_df, (lgb_cv_result + xgb_cv_result) / 2)), 4) print('>> Mean XGBoost+LightGBM VAL RMSE:', rmse_mean_lgb_xgb) return 0
beta = np.zeros((P)) beta[causal_ind] = 1.0 X = np.random.randn(*(N,P)) noise = np.random.randn(N) y = X.dot(beta) print("Initialize the model") print("Option 1: use native glmnet `nfolds`") model = glmnet(l1_ratio=0.5, n_folds=10) print("Option 2: use `sklearn` `cv` syntax") from sklearn.model_selection import KFold n_folds =10 kf = KFold(n_folds) model = glmnet(l1_ratio=0.5, cv=kf.get_n_splits(y), keep=True) print("Fit in sklearn style") model.fit(X, y) print("Predict in sklearn style") y_hat = model.predict(X) print("penalty", model.alpha_) print("Use `.cross_val_score()` method in order to apply cross-validation metrics other than MSE") from sklearn import metrics print(model.cross_val_score(metrics.r2_score)) print("plot native R graphs") model.rplot()
def main(): parser = ArgumentParser() parser.add_argument("-m", "--model", default="vectorize", help="Choose between 'vectorize' and 'doc_to_vec'") parser.add_argument( "-d", "--distribution-graph", default=False, help="Draw a graph of the distribution of the content item pageviews") parser.add_argument( "-i", "--feature-importance", default=False, action='store_true', help="Show graph of the importance of various features") parser.add_argument("-c", "--confusion-matrix", default=False, action='store_true', help="Show confusion matrices during training") parser.add_argument("-k", "--k-fold", default=False, action='store_true', help="Perform K-Fold") parser.add_argument("-t", "--test", default=False, action='store_true', help="Split into training and test") args = vars(parser.parse_args()) model_file = "vectorize" if args["model"] != "vectorize": model_file = "doc_to_vec" with open("data/processed/" + model_file + "_X", 'rb') as fp: X = pickle.load(fp) with open("data/processed/" + model_file + "_y", 'rb') as fp: y = np.asarray(pickle.load(fp)) if args["distribution_graph"]: pageviews = utils.load_pageviews() discretizer, view_numbers = utils.generate_discretizer(pageviews) draw_content_item_distribution_graph(pageviews, discretizer, view_numbers) if args["feature_importance"]: show_feature_importance(X, y) if args["k_fold"]: kf = KFold(n_splits=5) kf.get_n_splits(X) f1_scores = [] accuracy_scores = [] confusion_matrix = np.zeros((utils.number_bins(), utils.number_bins())) for train_index, test_index in kf.split(X): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] f1_score, accuracy_score, fold_confusion_matrix = utils.train_and_test_logistic_regression( X_train, y_train, X_test, y_test, args["confusion_matrix"]) confusion_matrix = confusion_matrix + fold_confusion_matrix f1_scores.append(f1_score) accuracy_scores.append(accuracy_score) if args["confusion_matrix"]: utils.plot_confusion_matrix(confusion_matrix) print("Average f1 score :" + str(np.mean(f1_scores))) print("Average accuracy score :" + str(np.mean(accuracy_scores))) if args["test"]: count = len(X) split_index = math.floor(count * 0.8) X_train = X[0:split_index] X_test = X[split_index:] y_train = y[0:split_index] y_test = y[split_index:] f1_score, accuracy_score, confusion_matrix = utils.train_and_test_logistic_regression( X_train, y_train, X_test, y_test, args["confusion_matrix"]) confusion_matrix if args["confusion_matrix"]: utils.plot_confusion_matrix(confusion_matrix) print("Accuracy score for test data :" + str(accuracy_score)) print("F1 score for test data :" + str(f1_score)) model = utils.train_logistic_regression(X, y) model_filename = "logistic_regression_model.pkl" pickle.dump(model, open(model_filename, 'wb')) print("Model saved to " + model_filename)
def main(): # ============================================================================================================= # VGG-16 ARCHITECTURE # ============================================================================================================= model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=(20, 224, 224))) model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(Flatten()) model.add(Dense(4096, name='fc6', init='glorot_uniform')) # ============================================================================================================= # WEIGHT INITIALIZATION # ============================================================================================================= layerscaffe = [ 'conv1_1', 'conv1_2', 'conv2_1', 'conv2_2', 'conv3_1', 'conv3_2', 'conv3_3', 'conv4_1', 'conv4_2', 'conv4_3', 'conv5_1', 'conv5_2', 'conv5_3', 'fc6', 'fc7', 'fc8' ] h5 = h5py.File(vgg_16_weights) layer_dict = dict([(layer.name, layer) for layer in model.layers]) # Copy the weights stored in the 'vgg_16_weights' file to the feature extractor part of the VGG16 for layer in layerscaffe[:-3]: w2, b2 = h5['data'][layer]['0'], h5['data'][layer]['1'] w2 = np.transpose(np.asarray(w2), (0, 1, 2, 3)) w2 = w2[:, :, ::-1, ::-1] b2 = np.asarray(b2) layer_dict[layer].W.set_value(w2) layer_dict[layer].b.set_value(b2) # Copy the weights of the first fully-connected layer (fc6) layer = layerscaffe[-3] w2, b2 = h5['data'][layer]['0'], h5['data'][layer]['1'] w2 = np.transpose(np.asarray(w2), (1, 0)) b2 = np.asarray(b2) layer_dict[layer].W.set_value(w2) layer_dict[layer].b.set_value(b2) adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0005) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) # ============================================================================================================= # FEATURE EXTRACTION # ============================================================================================================= if save_features: saveFeatures(model, features_file, labels_file) # ============================================================================================================= # TRAINING # ============================================================================================================= do_training = True compute_metrics = True threshold = 0.5 if do_training: h5features = h5py.File(features_file, 'r') h5labels = h5py.File(labels_file, 'r') # X_full will contain all the feature vectors extracted from optical flow images X_full = h5features[features_key] _y_full = np.asarray(h5labels[labels_key]) zeroes = np.asarray(np.where(_y_full == 0)[0]) ones = np.asarray(np.where(_y_full == 1)[0]) zeroes.sort() ones.sort() kf_falls = KFold(n_splits=5) kf_falls.get_n_splits(X_full[zeroes, ...]) kf_nofalls = KFold(n_splits=5) kf_nofalls.get_n_splits(X_full[ones, ...]) sensitivities = [] specificities = [] accuracies = [] for (train_index_falls, test_index_falls), (train_index_nofalls, test_index_nofalls) in zip( kf_falls.split(X_full[zeroes, ...]), kf_nofalls.split(X_full[ones, ...])): train_index_falls = np.asarray(train_index_falls) test_index_falls = np.asarray(test_index_falls) train_index_nofalls = np.asarray(train_index_nofalls) test_index_nofalls = np.asarray(test_index_nofalls) train_index = np.concatenate( (train_index_falls, train_index_nofalls), axis=0) test_index = np.concatenate((test_index_falls, test_index_nofalls), axis=0) train_index.sort() test_index.sort() X = np.concatenate((X_full[train_index_falls, ...], X_full[train_index_nofalls, ...])) _y = np.concatenate((_y_full[train_index_falls, ...], _y_full[train_index_nofalls, ...])) X2 = np.concatenate((X_full[test_index_falls, ...], X_full[test_index_nofalls, ...])) _y2 = np.concatenate((_y_full[test_index_falls, ...], _y_full[test_index_nofalls, ...])) # Balance the number of positive and negative samples so that there is the same amount of each of them all0 = np.asarray(np.where(_y == 0)[0]) all1 = np.asarray(np.where(_y == 1)[0]) if len(all0) < len(all1): all1 = np.random.choice(all1, len(all0), replace=False) else: all0 = np.random.choice(all0, len(all1), replace=False) allin = np.concatenate((all0.flatten(), all1.flatten())) allin.sort() X = X[allin, ...] _y = _y[allin] # ==================== CLASSIFIER ======================== extracted_features = Input(shape=(4096, ), dtype='float32', name='input') if batch_norm: x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001)(extracted_features) x = ELU(alpha=1.0)(x) else: x = ELU(alpha=1.0)(extracted_features) x = Dropout(0.9)(x) x = Dense(4096, name='fc2', init='glorot_uniform')(x) if batch_norm: x = BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001)(x) x = Activation('relu')(x) else: x = ELU(alpha=1.0)(x) x = Dropout(0.8)(x) x = Dense(1, name='predictions', init='glorot_uniform')(x) x = Activation('sigmoid')(x) classifier = Model(input=extracted_features, output=x, name='classifier') classifier.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy']) # ==================== TRAINING ======================= # weighting of each class: only the fall class gets a different weight class_weight = {0: weight_0, 1: 1} # Batch training if mini_batch_size == 0: history = classifier.fit(X, _y, validation_data=(X2, _y2), batch_size=X.shape[0], nb_epoch=epochs, shuffle='batch', class_weight=class_weight) else: history = classifier.fit(X, _y, validation_data=(X2, _y2), batch_size=mini_batch_size, nb_epoch=epochs, shuffle='batch', class_weight=class_weight) plot_training_info(exp, ['accuracy', 'loss'], save_plots, history.history) # ==================== EVALUATION ======================== if compute_metrics: predicted = classifier.predict(np.asarray(X2)) for i in range(len(predicted)): if predicted[i] < threshold: predicted[i] = 0 else: predicted[i] = 1 # Array of predictions 0/1 predicted = np.asarray(predicted) # Compute metrics and print them cm = confusion_matrix(_y2, predicted, labels=[0, 1]) tp = cm[0][0] fn = cm[0][1] fp = cm[1][0] tn = cm[1][1] tpr = tp / float(tp + fn) fpr = fp / float(fp + tn) fnr = fn / float(fn + tp) tnr = tn / float(tn + fp) precision = tp / float(tp + fp) recall = tp / float(tp + fn) specificity = tn / float(tn + fp) f1 = 2 * float(precision * recall) / float(precision + recall) accuracy = accuracy_score(_y2, predicted) print('TP: {}, TN: {}, FP: {}, FN: {}'.format(tp, tn, fp, fn)) print('TPR: {}, TNR: {}, FPR: {}, FNR: {}'.format( tpr, tnr, fpr, fnr)) print('Sensitivity/Recall: {}'.format(recall)) print('Specificity: {}'.format(specificity)) print('Precision: {}'.format(precision)) print('F1-measure: {}'.format(f1)) print('Accuracy: {}'.format(accuracy)) # Store the metrics for this epoch sensitivities.append(tp / float(tp + fn)) specificities.append(tn / float(tn + fp)) accuracies.append(accuracy) print('5-FOLD CROSS-VALIDATION RESULTS ===================') print("Sensitivity: %.2f%% (+/- %.2f%%)" % (np.mean(sensitivities), np.std(sensitivities))) print("Specificity: %.2f%% (+/- %.2f%%)" % (np.mean(specificities), np.std(specificities))) print("Accuracy: %.2f%% (+/- %.2f%%)" % (np.mean(accuracies), np.std(accuracies)))
saver_dir_res = 'Results' file_name = os.path.join( saver_dir_res, 'Results_ger_age_epoch_{}_model_no_{}.xls'.format(epochs, model_no)) saver_dir_models = 'Trained_models/Start_ger_age' if not os.path.exists(saver_dir_models): os.mkdir(saver_dir_models) if not os.path.exists(saver_dir_res): os.mkdir(saver_dir_res) data, atribute, sensitive, output, pr_gr, un_gr = german_dataset() skf = KFold(n_splits=10) skf.get_n_splits(atribute, output) inp = atribute.shape[1] AUC_y = np.zeros(model_no) AUC_A = np.zeros(model_no) wb = Workbook() columns = [ "AUC_y", "AUC_A", 'bal_acc', 'avg_odds_diff', 'disp_imp', 'stat_par_diff', 'eq_opp_diff', 'theil_ind' ] alpha = np.linspace(0.1, 2.5, 5) sheets = [wb.add_sheet('{}'.format(i)) for i in alpha]
def main(): ######################################################### # DATA PREPARATION # The train set has 60k rows and 784 columns, so its shape is (60k,784). # Each row is a 28 by 28 pixel picture. # I will reshape the train set to have (60k,1) shape, i.e. each row will contain a 28 by 28 matrix of pixel color values. # Same for the test set. ######################################################### y_train_CNN = train.ix[:, 0].values.astype( 'int32') # only labels i.e targets digits X_train_CNN = np.array(train.iloc[:, 1:].values).reshape( train.shape[0], 1, 28, 28).astype(np.uint8) # reshape to be [samples][pixels][width][height] print('train shape after reshape: {}'.format(X_train_CNN.shape)) y_test_CNN = test.ix[:, 0].values.astype( 'int32') # only labels i.e targets digits X_test_CNN = np.array(test.iloc[:, 1:].values).reshape( (test.shape[0], 1, 28, 28)).astype(np.uint8) print('test shape after reshape: {}'.format(X_test_CNN.shape)) # normalize inputs from 0-255 to 0-1 X_train_CNN = X_train_CNN / 255 X_test_CNN = X_test_CNN / 255 #scaler = StandardScaler() #X_train_CNN = scaler.fit_transform(X_train_CNN) #X_test_CNN = scaler.fit_transform(X_test_CNN) # one hot encode outputs y_train_CNN = to_categorical(y_train_CNN) y_test_CNN = to_categorical(y_test_CNN) num_classes = y_train_CNN.shape[1] X_train = X_train_CNN X_val = X_test_CNN y_train = y_train_CNN y_val = y_test_CNN ######################################################### # BUILDE THE MODEL AND EVALUATE IT USING K-FOLD ######################################################### kf = KFold(n_splits=n_splits, random_state=seed, shuffle=True) kf.get_n_splits(X_train) acc_scores = list() for fold, (train_index, test_index) in enumerate(kf.split(X_train)): print('\n Fold %d' % (fold)) X_tr, X_v = X_train[train_index], X_train[test_index] y_tr, y_v = y_train[train_index], y_train[test_index] # build the model model = model_cnn(num_classes) # fit model model.fit(X_tr, y_tr, epochs=epochs, validation_data=(X_v, y_v), verbose=2, batch_size=batch_size, callbacks=callbacks, shuffle=True) acc = model.evaluate(X_v, y_v, verbose=0) acc_scores.append(acc[1]) print('Fold %d: Accuracy %.2f%%' % (fold, acc[1] * 100)) print('Accuracy scores: ', acc_scores) mean_acc = mean(acc_scores) standard_deviation_acc = mt.sqrt( sum_of_square_deviation(acc_scores, mean_acc)) print('=====================') print('Mean Accuracy %f' % mean_acc) print('=====================') print('=====================') print('Stdev Accuracy %f' % standard_deviation_acc) print('=====================') model = model_cnn(num_classes) # Fit the final model history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=epochs, batch_size=batch_size, callbacks=callbacks, verbose=2) # Final evaluation of the model scores = model.evaluate(X_val, y_val, verbose=0) print("Error: %.2f%%" % (100 - scores[1] * 100)) print("Accuracy: %.2f%%" % (scores[1] * 100)) # summarize history for accuracy fig_acc = plt.figure(figsize=(10, 10)) plt.plot(history.history['acc']) plt.plot(history.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() fig_acc.savefig("../../Output/model_accuracy_fm_cnn.png")
def ssnal_elastic_path(A, b, c_lam_vec=None, alpha=0.9, x0=None, y0=None, z0=None, Aty0=None, max_selected=None, cv=False, n_folds=10, sgm=5e-3, sgm_increase=5, sgm_change=1, step_reduce=0.5, mu=0.2, tol_ssn=1e-6, tol_ssnal=1e-6, maxiter_ssn=50, maxiter_ssnal=100, use_cg=True, r_exact=2e4, plot=False, print_lev=2): """ -------------------------------------------------------------------------- ssnal algorithm to solve the elastic net for a list of lambda1 and lambda2 -------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------- :param A: design matrix (m x n) :param b: response vector (m, ) :param c_lam_vec: np.array to determine all the values of lambda1 -- lambda1 = c_lam_vec * lambda1_max :param alpha: an array to determin lambda 2 -- lam2 = (1 - alpha) * lam1 :param max_selected: if given, the algorithm stops when selects a number of features > max_selected :param x0: initial value for the lagrangian multiplier (variable of the primal problem) (n, ) -- vector 0 if not given :param y0: initial value fot the first variable of the dual problem (m, ) -- vector of 0 if not given :param z0: initial value for the second variable of the dual problem (n, ) -- vector of 0 if not given :param Aty0: np.dot(A.transpose(), y0) (n,) :param max_selected: maximum number of features selected :param cv: True/False. I true, a cross validation is performed :param n_folds: number of folds to perform the cross validation :param sgm: starting value of the augmented lagrangian parameter sigma :param sgm_increase: increasing factor of sigma :param sgm_change: we increase sgm -- sgm *= sgm_increase -- every sgm_change iterations :param step_reduce: dividing factor of the step size during the linesearch :param mu: multiplicative factor fot the lhs of the linesearch condition :param tol_ssn: tolerance for the ssn algorithm :param tol_ssnal: global tolerance of the ssnal algorithm :param maxiter_ssn: maximum number of iterations for ssn :param maxiter_ssnal: maximum number of global iterations :param use_cg: True/False. If true, the conjugate gradient method is used to find the direction of the ssn :param r_exact: number of features such that we start using the exact method :param print_lev: different level of printing (0, 1, 2, 3, 4) :param plot: True/False. If true a plot of r_lm, gcv, extended bic and cv (if cv == True) is displayed ---------------------------------------------------------------------------------------------------------------------- --------------------------------------------------------------------------------------- :return[0] aic_vec: aic values for each lambda1 :return[1] ebic_vec: ebic values for each lambda1 :return[2] gcv_vec: gcv values for each lambda1 :return[3] cv_vec: cv values for each lambda1 :return[4] r_vec: number of selected features for each lambda1 :return[5] r_lm_vec: number of selected features by the debiased model for each lambda1 :return[6] iter_vec: number of ssnal iterations for each lambda1 :return[7] times_vec: ssnal time for each lambda1 :return[8] full_time: time for fitting the full lambda for all the sequence of lambda1 :return[9] cv_time: time for cross validation :return[10] total_time: total time --------------------------------------------------------------------------------------- REMEMBER: the output os ssnal_elastic_core is a list which contains: -------------------------------------------------------------------------------------------------- :return[0] x: optimal value of the primal variable :return[1] y: optimal value of the first dual variable :return[2] z: optimal value of the second dual variable :return[3] x_lm: linear regression estimates on the :return[4] r: number of selected features :return[5] r_lm: number of selected features by the debiased model :return[6] aic: aic computed on the debiased estimates :return[7] ebic: extended bic computed on the debiased estimates :return[8] gcv: gcv computed on the debiased estimates :return[9] sgm: final value of the augmented lagrangian parameter sigma :return[10] lam1: lasso penalization :return[11] lam2: ridge penalization :return[12] convergence_ssnal: True/False. If true, the ssnal has converged :return[13] ssnal_time: total time of ssnal :return[14] it_ssnal: total ssnal's iteration :return[15] Aty: np.dot(A.transpose(), y) computed at the optimal y. Useful to implement warmstart -------------------------------------------------------------------------------------------------- """ # -------------------------- # # initialize variables # # -------------------------- # m, n = A.shape lam1_max = LA.norm(np.dot(A.transpose(), b), np.inf) / alpha if x0 is None: x0 = np.zeros((n, )) if y0 is None: y0 = np.zeros((m, )) if z0 is None: z0 = np.zeros((n, )) if max_selected is None: max_selected = n if c_lam_vec is None: c_lam_vec = np.logspace(start=3, stop=1, num=100, base=10.0) / 1000 n_lam1 = c_lam_vec.shape[0] sgm0 = sgm # ---------------------- # # initialize flags # # ---------------------- # convergence = True reached_max = False n_lam1_stop = n_lam1 # ---------------------------- # # create output matrices # # ---------------------------- # aic_vec, ebic_vec, gcv_vec = -np.ones([n_lam1]), -np.ones( [n_lam1]), -np.ones([n_lam1]) times_vec, r_vec, r_lm_vec, iter_vec = -np.ones([n_lam1]), -np.ones( [n_lam1]), -np.ones([n_lam1]), -np.ones([n_lam1]) # ---------------------- # # solve full model # # ---------------------- # if print_lev > 0: print() print('--------------------------------------------------') print(' solving full model ') print('--------------------------------------------------') start_full = time.time() for i in range(n_lam1): lam1 = alpha * c_lam_vec[i] * lam1_max lam2 = (1 - alpha) * c_lam_vec[i] * lam1_max if print_lev > 2: print( '--------------------------------------------------------------------------------------------------' ) print( ' FULL MODEL: lambda1 (ratio) = %.2e (%.2e) | lambda2 = %.2e | sigma0 = %.2e' % (lam1, c_lam_vec[i], lam2, sgm)) print( '--------------------------------------------------------------------------------------------------' ) # ------------------- # # perform ssnal # # ------------------- # fit = ssnal_elastic_core(A=A, b=b, lam1=lam1, lam2=lam2, x0=x0, y0=y0, z0=z0, Aty0=Aty0, sgm=sgm, sgm_increase=sgm_increase, sgm_change=sgm_change, step_reduce=step_reduce, mu=mu, tol_ssn=tol_ssn, tol_ssnal=tol_ssnal, maxiter_ssn=maxiter_ssn, maxiter_ssnal=maxiter_ssnal, use_cg=use_cg, r_exact=r_exact, print_lev=print_lev - 3) # ----------------------- # # check convergence # # ----------------------- # if not fit[10]: convergence = False break # ---------------------------- # # update starting values # # ---------------------------- # x0, y0, z0, Aty0, sgm = fit[0], fit[1], fit[2], fit[15], fit[9] # ---------------------------- # # update output matrices # # ---------------------------- # times_vec[i], r_vec[i], r_lm_vec[i], iter_vec[i] = fit[13], fit[ 4], fit[5], fit[14] r_lm = fit[5] if r_lm > 0: aic_vec[i], ebic_vec[i], gcv_vec[i] = fit[6], fit[7], fit[8] # --------------------------------------- # # check number of selected features # # --------------------------------------- # r_lm = fit[5] if r_lm > max_selected: n_lam1_stop = i + 1 reached_max = True break # ------------------- # # end full model # # ------------------- # full_time = time.time() - start_full if not convergence: print('--------------------------------------------------') print(' snall has not converged for lam1 = %.4f, lam2 = %.4f ' % (lam1, lam2)) print('--------------------------------------------------') if reached_max: print('--------------------------------------------------') print(' max number of features has been selected') print('--------------------------------------------------') # -------------- # # start cv # # -------------- # cv_time = 0 cv_mat = -np.ones([n_lam1_stop, n_folds]) if cv and convergence: print('--------------------------------------------------') print(' performing cv ') print('--------------------------------------------------') x0_cv, z0_cv = np.zeros((n, )), np.zeros((n, )) Aty0_cv = None sgm_cv = sgm0 fold = 0 start_cv = time.time() # ------------- # # split A # # ------------- # kf = KFold(n_splits=n_folds) kf.get_n_splits(A) # -------------------- # # loop for folds # # -------------------- # for train_index, test_index in kf.split(A): A_train, A_test = A[train_index], A[test_index] b_train, b_test = b[train_index], b[test_index] y0_cv = np.zeros((np.shape(train_index)[0], )) # -------------------- # # loop for lam1 # # -------------------- # for i_cv in tqdm(range(n_lam1_stop)): lam1 = c_lam_vec[i_cv] * lam1_max lam2 = (1 - alpha) * lam1 # ------------------- # # perform ssnal # # ------------------- # fit_cv = ssnal_elastic_core(A=A_train, b=b_train, lam1=lam1, lam2=lam2, x0=x0_cv, y0=y0_cv, z0=z0_cv, Aty0=Aty0_cv, sgm=sgm_cv, sgm_increase=sgm_increase, sgm_change=sgm_change, step_reduce=step_reduce, mu=mu, tol_ssn=tol_ssn, tol_ssnal=tol_ssnal, maxiter_ssn=maxiter_ssn, maxiter_ssnal=maxiter_ssnal, use_cg=use_cg, r_exact=r_exact, print_lev=0) # ------------------- # # update cv mat # # ------------------- # cv_mat[i_cv, fold] = LA.norm(np.dot(A_test, fit_cv[3]) - b_test)**2 # ---------------------------- # # update starting values # # ---------------------------- # if i_cv == n_lam1_stop: x0_cv, y0_cv, z0_cv, Aty0_cv, sgm_cv = None, None, None, None, sgm0 else: x0_cv, y0_cv, z0_Cv, Aty0_cv, sgm_cv = fit_cv[0], fit_cv[ 1], fit_cv[2], fit_cv[15], fit_cv[9] # ------------------------ # # end loop for lam1 # # ------------------------ # fold += 1 # ------------ # # end cv # # ------------ # cv_time = time.time() - start_cv # ---------------------------- # # printing final results # # ---------------------------- # if cv: cv_vec = cv_mat.mean(1) / m else: cv_vec = -np.ones([n_lam1_stop]) total_time = full_time + cv_time time.sleep(0.1) print('') print('') print('------------------------------------------------------------') print(' total time: %.4f' % total_time) if cv: print('------------------------------------------------------------') print(' full time: %.4f' % full_time) print('------------------------------------------------------------') print(' cv time: %.4f' % cv_time) print('------------------------------------------------------------') if print_lev > 1: print('') # print('------------------------------------------------------------') print( ' c_lam lam1 lam2 r_lm gcv ebic cv ') print('------------------------------------------------------------') np.set_printoptions(formatter={'float': '{: 0.3f}'.format}) print( np.stack((c_lam_vec[:n_lam1_stop], alpha * c_lam_vec[:n_lam1_stop] * lam1_max, (1 - alpha) * c_lam_vec[:n_lam1_stop] * lam1_max, r_lm_vec[:n_lam1_stop], gcv_vec[:n_lam1_stop], ebic_vec[:n_lam1_stop], cv_vec), -1)) print('\n') if plot: plot_cv_ssnal_elstic(r_lm_vec, ebic_vec, gcv_vec, cv_vec, alpha, c_lam_vec) return aic_vec[:n_lam1_stop], \ ebic_vec[:n_lam1_stop], \ gcv_vec[:n_lam1_stop], \ cv_vec, \ r_vec[:n_lam1_stop], \ r_lm_vec[:n_lam1_stop], \ iter_vec[:n_lam1_stop], \ times_vec[:n_lam1_stop], \ full_time, cv_time, total_time
def main(batch_size): # read train data train_input = np.genfromtxt('ctg_data_cleaned.csv', delimiter=',') trainX, train_Y = train_input[1:, :21], train_input[1:, -1].astype(int) trainX = scale(trainX, np.min(trainX, axis=0), np.max(trainX, axis=0)) trainY = np.zeros((train_Y.shape[0], NUM_CLASSES)) trainY[np.arange(train_Y.shape[0]), train_Y - 1] = 1 # one hot matrix # experiment with small datasets # trainX = trainX[:1000] # trainY = trainY[:1000] # split the test and training data into 70:30 trainX, testX, trainY, testY = train_test_split(trainX, trainY, test_size=0.3, shuffle=True) # n = trainX.shape[0] # print(n) # Create the model x = tf.placeholder(tf.float32, [None, NUM_FEATURES]) y_ = tf.placeholder(tf.float32, [None, NUM_CLASSES]) logits, h_weights, weights = fnn(x, num_neurons) # Build the graph for the deep net cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=logits) L2_regularization = tf.nn.l2_loss(h_weights) + tf.nn.l2_loss(weights) loss = tf.reduce_mean(cross_entropy + beta * L2_regularization) # Create the gradient descent optimizer with the given learning rate. optimizer = tf.train.GradientDescentOptimizer(learning_rate) global_step = tf.Variable(0, name='global_step', trainable=False) train_op = optimizer.minimize(loss, global_step=global_step) correct_prediction = tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y_, 1)), tf.float32) accuracy = tf.reduce_mean(correct_prediction) kf = KFold(n_splits=5, shuffle=True) kf.get_n_splits(trainX) # print(kf) time_taken = [] for train_index, test_index in kf.split(trainX): # print("TRAIN:", train_index, "TEST:", test_index) trainX_, testX_ = trainX[train_index], trainX[test_index] trainY_, testY_ = trainY[train_index], trainY[test_index] N = len(trainX_) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) train_acc = [] test_acc = [] for i in range(epochs): start_time = time.time() for start, end in zip(range(0, N, batch_size), range(batch_size, N, batch_size)): train_op.run(feed_dict={x: trainX_[start:end], y_: trainY_[start:end]}) end_time = time.time() duration = end_time - start_time train_acc.append(accuracy.eval(feed_dict={x: trainX_, y_: trainY_})) test_acc.append(accuracy.eval(feed_dict={x: testX_, y_: testY_})) time_taken.append(duration) # print(time_taken) if i % 100 == 0: print('iter %d: training accuracy %g' % (i, train_acc[i])) print('iter %d: test accuracy %g' % (i, test_acc[i])) print(duration, '\n') avg_time_taken = np.mean(np.array(time_taken)) return avg_time_taken
def train(n_components, alpha, reg, n_samples): print("Loading data...") movie_titles, ratings, rating_indices, n_users, n_items = get_netflix_data(n_samples=n_samples) print("number of users with ratings: {}".format(len(np.unique(rating_indices[:,0])))) print("number of movies with ratings: {}".format(len(np.unique(rating_indices[:,1])))) n_splits = 5 kf = KFold(n_splits=n_splits, shuffle=True) kf.get_n_splits(rating_indices) ratings = scipy.sparse.dok_matrix(ratings) if not n_components or not alpha or not reg: alphas = [0.01, 0.1, 1, 10] regs = [0.01, 0.1] components = [5, 10, 15, 20, 30] print("Finding optimal parameters...") best_loss = float('inf') best_k, best_alpha, best_reg = 0, 0, 0 for n_components in components: for alpha in alphas: for reg in regs: mean_loss = 0.0 print("n_components: {}, alpha: {}, reg: {}".format(n_components, alpha, reg)) for k, (train_index, test_index) in enumerate(kf.split(rating_indices)): print("Fold {}".format(k)) test_indices = rating_indices[test_index] test_indices = test_indices[:,0], test_indices[:,1], test_indices[:,2] user_test_indices, item_test_indices = test_indices[0], test_indices[1] data_train = scipy.sparse.lil_matrix(ratings) data_train[user_test_indices, item_test_indices] = 0 data_train = scipy.sparse.csr_matrix(ratings) start = time.time() print("Fitting...") nmf = NMF(n_components=n_components, alpha=alpha, l1_ratio=reg, init='nndsvd', tol=0.001, verbose=1) P = nmf.fit_transform(data_train) Q = nmf.components_ acc, loss = evaluate(P, Q, test_indices) print("Elapsed time: {:.1f}s".format(time.time()-start)) print("loss: {:.4f} - acc: {:.4f}".format(loss, acc)) mean_loss = (mean_loss*k + loss) / (k+1) if mean_loss < best_loss: best_loss = mean_loss best_k = n_components best_alpha = alpha best_reg = reg print("best k: {}, best alpha: {}, best reg: {}, best loss: {:.4f}".format(best_k, best_alpha, best_reg, best_loss)) else: print("Performing cross validation...") mean_acc = 0.0 mean_loss = 0.0 for k, (train_index, test_index) in enumerate(kf.split(rating_indices)): print("Fold {}".format(k)) test_indices = rating_indices[test_index] test_indices = test_indices[:,0], test_indices[:,1], test_indices[:,2] user_test_indices, item_test_indices = test_indices[0], test_indices[1] data_train = scipy.sparse.lil_matrix(ratings) data_train[user_test_indices, item_test_indices] = 0 data_train = scipy.sparse.csr_matrix(data_train) start = time.time() print("Fitting...") nmf = NMF(n_components=n_components, alpha=alpha, l1_ratio=reg, init='nndsvd', tol=0.001, verbose=1) P = nmf.fit_transform(data_train) Q = nmf.components_ acc, loss = evaluate(P, Q, test_indices) print("Elapsed time: {:.4f}".format(time.time()-start)) print("loss: {:.4f} - acc: {:.4f}".format(loss, acc)) mean_acc = (mean_acc*k + acc) / (k+1) mean_loss = (mean_loss*k + loss) / (k+1) print("mean loss: {:.4f} - mean acc: {:.4f}".format(mean_loss, mean_acc))
print("Epoch: %d, Train: %.3f, Val: %.3f, Val IOU: %.3f" % (e, np.mean(train_loss), np.mean(val_loss), np.mean(val_iou))) if np.mean(val_iou) > best_iou: torch.save(model.state_dict(), '../torch_parameters/background_classifier.pt') #save i = 0 #reset best_iou = np.mean(val_iou) #reset #elif i > patience: # break scheduler.step() #load best model model = get_model(num_classes = 1, num_filters = 32, pretrained = True) model.load_state_dict(torch.load('../torch_parameters/background_classifier.pt')) #test predictions model.train(False) all_predictions = [] with torch.no_grad(): for image in tqdm(data.DataLoader(test_dataset, batch_size = 100)): image = image[0].type(torch.FloatTensor).cuda() y_pred = model(image).cpu().data.numpy() all_predictions.append(y_pred) all_predictions_stacked = np.vstack(all_predictions)[:, 0, :, :]/fold.get_n_splits()
for i in to_dummy: dummies = pd.get_dummies(X[i], prefix=i) #print dummies #dummies=dummies.iloc[:,1:] X = X.drop(i, 1) X = pd.concat([dummies, X], axis=1) num_cols = [ i for i in df.columns if i not in to_dummy + ['heating_load', 'cooling_load'] ] from sklearn.model_selection import KFold kf = KFold(n_splits=10) print("Using ", kf.get_n_splits(X), " folds") def Model(regressor, f): avg_r2_train = [] avg_r2_test = [] avg_meanS_train = [] avg_meanS_test = [] avg_r2_train_y2 = [] avg_r2_test_y2 = [] avg_meanS_train_y2 = [] avg_meanS_test_y2 = [] from sklearn.preprocessing import StandardScaler i = 1.0
temp.append(t) x_post = np.array(list(temp), dtype=np.int32) ori_id = np.array(list(ori_id)) # Randomly shuffle data shuffle_indices = np.random.permutation(np.arange(len(y))) x_pre_shuffled = x_pre[shuffle_indices] x_post_shuffled = x_post[shuffle_indices] y_shuffled = y[shuffle_indices] ori_id_shuffled = ori_id[shuffle_indices] # Split train/test set # TODO: This is very crude, should use cross-validation kf = KFold(n_splits=5) kf.get_n_splits(x_pre_shuffled) #dev_sample_index = int(0.2 * float(len(y))) avg_accuracy = 0.0 split = 0 for train_index, test_index in kf.split(x_pre_shuffled): out.write("*****************************************\n") out.write("*****************************************\n") out.write("*****************************************\n") out.write("This is the {} split\n".format(split + 1)) split += 1 out.write("*****************************************\n") x_pre_train, x_pre_test = x_pre_shuffled[train_index], x_pre_shuffled[ test_index]
def train(data, pre_embedding, config, output_dir): # x, y, x_test, y_test, vocab_processor # Training # ================================================== with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=config['allow_soft_placement'], log_device_placement=config['log_device_placement']) sess = tf.Session(config=session_conf) with sess.as_default(): model_config = _define_model_config(data, pre_embedding, config) cnn = build_model(model_config) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( "{}/grad/hist".format(v.name), g) sparsity_summary = tf.summary.scalar( "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.join(output_dir, timestamp) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.join(out_dir, "checkpoints") checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=config['num_checkpoints']) # Write vocabulary data['vocab_processor'].save(os.path.join(out_dir, "vocab")) # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: config['dropout_keep_prob'] } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) time_str = datetime.datetime.now().isoformat() print("Train: {}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("Test: {}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) def train_with_batch(saver, batches, x_dev, y_dev, evaluate_every, checkpoint_every, checkpoint_prefix): """ Train model with batches """ for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % evaluate_every == 0: dev_step(x_dev, y_dev, writer=dev_summary_writer) # Generate batches use_k_fold = config['use_k_fold'] if use_k_fold: kf = KFold(n_splits=10) fold_size = kf.get_n_splits() - 1 for train_idx, dev_idx in kf.split(x, y): x_train = data['x_train'][train_idx] x_dev = data['x_test'][dev_idx] y_train = data['y_train'][train_idx] y_dev = data['y_test'][dev_idx] # Generate batches batches = batch_iter(list(zip(x_train, y_train)), config['batch_size'], config['num_epochs']) # Training loop. For each batch... train_with_batch(saver, batches, x_dev, y_dev, fold_size, fold_size, checkpoint_prefix) else: batches = batch_iter( list(zip(data['x_train'], data['y_train'])), config['batch_size'], config['num_epochs']) train_with_batch(saver, batches, data['x_test'], data['y_test'], config['evaluate_every'], config['checkpoint_every'], checkpoint_prefix) # save predict results feed_dict = { cnn.input_x: data['x_test'], cnn.input_y: data['y_test'], cnn.dropout_keep_prob: 1.0 } predictions = sess.run([cnn.predictions], feed_dict) predictions, scores, input_y = sess.run( [cnn.predictions, cnn.scores, cnn.input_y], feed_dict) pickle.dump(scores, open(os.path.join(checkpoint_dir, 'scores.pkl'), 'wb')) pickle.dump( predictions, open(os.path.join(checkpoint_dir, 'predictions.pkl'), 'wb')) pickle.dump( input_y, open(os.path.join(checkpoint_dir, 'input_y.pkl'), 'wb')) # save model current_step = tf.train.global_step(sess, global_step) print('checkpoint_prefix: ', checkpoint_prefix) print('current_step:', current_step) path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
def RESNET_onehot(Dropout1=0,Epochs= 20,Batch_size=64): # 优化器选择 Adam 优化器。 # 损失函数使用 sparse_categorical_crossentropy, # 还有一个损失函数是 categorical_crossentropy,两者的区别在于输入的真实标签的形式, # sparse_categorical 输入的是整形的标签,例如 [1, 2, 3, 4],categorical 输入的是 one-hot 编码的标签。 Feature_test = np.load("../../data_all/TCRA_train_feature_array.npy") Label_array = np.load("../../data_all/TCRA_train_label_array.npy") X = Feature_test#[:,0:29,:] #提取one-hot特征 #print(X[0]) Y = Label_array#[:,1] X = X.reshape(len(X),-1) #loo = LeaveOneOut() kf = KFold(n_splits=5,shuffle=True,random_state=0) kf.get_n_splits(X) TN = FP = FN = TP = 0 aa = 1 for train_index, test_index in kf.split(X): np.random.shuffle(train_index) np.random.shuffle(test_index) X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] X_train= X_train.reshape([len(X_train),29,20,2]) X_test = X_test.reshape([len(X_test),29,20,2]) X_test=tf.cast(X_test, tf.float32) info = "using the one_hoot th model\n" modelfile = './tra_resnet_model.h5' Y_pred = resnet_attention_train_predict(29, 19 ,modelfile,info,2,X_train, Y_train,X_test, Y_test) Y_pred = np.argmax(Y_pred, axis=-1) Y_test = np.argmax(Y_test, axis=-1) confusion_matrix1 =confusion_matrix(Y_test,Y_pred) TP += confusion_matrix1[0,0] FN += confusion_matrix1[0,1] FP += confusion_matrix1[1,0] TN += confusion_matrix1[1,1] # accuracy = accuracy_score(Y_test,Y_pred) #准确率 # precision = precision_score(Y_test,Y_pred) #精确率 # recall = recall_score(Y_test,Y_pred) #召回率 # f1= f1_score(Y_test,Y_pred) #F1 # print('混淆矩阵\n',confusion_matrix1, # '\n准确率ACC:',accuracy, # '\n精确率precision:',precision, # '\n召回率recall:',recall, # '\nF1:',f1, # ) # y_predict = model.predict(X_test) # y_probs = model.predict_proba(X_test) #模型的预测得分 # #print(y_probs) # fpr, tpr, thresholds = metrics.roc_curve(Y_test,y_probs) # roc_auc = auc(fpr, tpr) #auc为Roc曲线下的面积 # #开始画ROC曲线 # plt.plot(fpr, tpr, 'b',label='AUC = %0.2f'% roc_auc) # plt.legend(loc='lower right') # plt.plot([0,1],[0,1],'r--') # plt.xlim([-0.1,1.1]) # plt.ylim([-0.1,1.1]) # plt.xlabel('False Positive Rate') #横坐标是fpr # plt.ylabel('True Positive Rate') #纵坐标是tpr # plt.title('Receiver operating characteristic example') # plt.show() #model.save('./data_625/model_'+str(aa)+'.h5') print(aa) if aa == 1: Y_test_all = Y_test Y_pred_all = Y_pred else: Y_test_all = np.append(Y_test_all, Y_test, axis=0) Y_pred_all = np.append(Y_pred_all, Y_pred, axis=0) aa += 1 print('\n\n总混淆矩阵') print(TP,FN) print(FP,TN) #print(Y_test_all[0]) accuracy = accuracy_score(Y_test_all,Y_pred_all) #准确率 precision = precision_score(Y_test_all,Y_pred_all) #精确率 recall = recall_score(Y_test_all,Y_pred_all) #召回率 f1= f1_score(Y_test_all,Y_pred_all) #F1 MCC = matthews_corrcoef(Y_test_all,Y_pred_all) #MCC print('\n准确率ACC:',accuracy, '\n精确率precision:',precision, '\n召回率recall:',recall, '\nF1:',f1, '\nMCC:',MCC )
def find_best_params_GridSearchCV(self, gridsearch, X_local, y_local): """ Find best set of parameters given a grid-search round across various folds. """ """ Generates all combination of hyperparameters """ hyperparam_space = list(ParameterGrid(gridsearch)) if len(hyperparam_space) == 1: return hyperparam_space[0] """ Splits local folds """ kf = KFold(n_splits=self.cross_validation) kf.get_n_splits(self.X_train) folds = [] for train_index, test_index in kf.split(X_local): folds.append([pd.Series(train_index), pd.Series(test_index)]) n_folds = len(folds) n_combinations = len(hyperparam_space) if self.verbose: msg = "Fitting %s models...\n" if self.verbose: print(msg % (n_combinations)) """ Performs gridsearch """ #Stores performance, Stores classification reports performance = [] for params_it, params in enumerate(hyperparam_space): time_start = time.time() #Evaluation rounds local_results = [] for fold_it, fold in enumerate(folds): X_train = X_local.iloc[fold[0]] X_test = X_local.iloc[fold[1]] y_train = y_local.iloc[fold[0]] y_test = y_local.iloc[fold[1]] params['eval_set'] = [(X_test, y_test)] alg = xgbw.XGBWrapper(**params) alg.fit(X_train, y_train) pred_test = alg.predict(X_test) local_report = class_report(y_true=y_test, y_pred=pred_test) local_results.append(local_report) #Stores performance evaluation given the performance-policy metric, statistic = self.performance_politic.split('__') local_performance = [] for local_report in local_results: local_report = local_report.drop('Global') metric_results = local_report[metric] if statistic == 'min': metric_stat = metric_results.min() elif statistic == 'max': metric_stat = metric_results.max() elif statistic == 'mean': metric_stat = metric_results.mean() local_performance.append(metric_stat) local_performance = pd.Series(local_performance) performance.append(local_performance) time_end = time.time() elapsed_time = (time_end - time_start) if self.gridsearch_verbose: msg = "%s of %s - %s: %s - %s s" % ( (params_it + 1), n_combinations, self.performance_politic, round(local_performance.mean(), 4), round(elapsed_time, 2)) if self.verbose: print(msg) for param_name in params.keys(): if param_name != 'eval_set': msg = "\t%s: %r" % (param_name, params[param_name]) if self.verbose: print(msg) if self.verbose: print('') performance = pd.DataFrame(performance) mean_performance = performance.mean(axis=1) idx_best = mean_performance.idxmax() best_parameters = hyperparam_space[idx_best] return best_parameters
from random import shuffle from torch.autograd import Variable from sklearn.model_selection import KFold # add manual_seed torch.manual_seed(1) shuffled = D.text_and_label shuffled = [[D.text_and_label[0][i], D.text_and_label[1][i]] for i in range(len(D.text_and_label[0]))] shuffle(shuffled) # definition for K-Folds kf = KFold(n_splits=10) kf.get_n_splits(shuffled) scores = [] k = 0 print('Training Start ...') for train_index, test_index in kf.split(shuffled): print('now : [%d] Fold' % k) CNN = Net.CnnNetwork() optimizer = optim.SGD(CNN.parameters(), lr=0.001, momentum=0.9) criterion = nn.BCELoss() CNN.cuda() k = k + 1 # @ Cross Validation : Train Part CNN.train() for epoch in range(10):
from sklearn.model_selection import KFold from Models import naive_bayes, logistic_regression, knn, cart from LoadDataset import load_dataset data, target = load_dataset("peersim.csv") kf = KFold(n_splits=10) kf.get_n_splits(data) print(kf) for train_index, test_index in kf.split(data): print("TRAIN:", train_index, "TEST:", test_index) X_train, X_test = data[train_index], data[test_index] y_train, y_test = target[train_index], target[test_index]
indices = np.column_stack(indices) indices = pd.DataFrame(indices) test = pd.read_csv(path+'test_start.csv') test = pd.concat([test, indices], axis=1) dtest = xgb.DMatrix(test) # y_pred = model.predict(dtest) # # test = pd.read_csv(path+'test_start.csv') # output = pd.DataFrame({'id': test['ID'].astype(np.int32), 'y': y_pred}) # output.to_csv(path+'xgboost-depth{}-pca-ica.csv'.format(xgb_params['max_depth']), index=False) n_splits = 5 kf = KFold(n_splits=n_splits) kf.get_n_splits(train) # dtest = xgb.DMatrix(test) predictions = np.zeros((test.shape[0], n_splits)) score = 0 oof_predictions = np.zeros(train.shape[0]) for fold, (train_index, test_index) in enumerate(kf.split(train)): X_train, X_valid = train.iloc[train_index, :], train.iloc[test_index, :] y_train, y_valid = y[train_index], y[test_index] d_train = xgb.DMatrix(X_train, label=y_train) d_valid = xgb.DMatrix(X_valid, label=y_valid) watchlist = [(d_train, 'train'), (d_valid, 'valid')] model = xgb.train(params, d_train, 1000, watchlist, early_stopping_rounds=50, feval=xgb_r2_score, maximize=True, verbose_eval=False)
# In[12]: scores = [] # In[13]: def get_score(model, X_train, X_test, y_train, y_test): model.fit(X_train, y_train) return abs(model.score(X_test, y_test)) # In[14]: kf = KFold(n_splits=5) kf.get_n_splits(X_train) # In[15]: KFold(n_splits=5, random_state=None, shuffle=False) for train_index, test_index in kf.split(df): x_train, x_test = df[train_index], df[test_index] y_train, y_test = labels[train_index], labels[test_index] scores.append(get_score(regressor, x_train, x_test, y_train, y_test)) # In[16]: scores # In[17]:
clf = model() # train the model clf.fit(X_train, y_train) # to compute training error, first make predictions on training set y_hat_train = clf.predict(X_train) # then compare our prediction with true labels using the metric training_error = r2_score(y_train, y_hat_train) # CROSS-VALIDATION ERROR from sklearn.model_selection import KFold from numpy import zeros, mean # 3-fold cross-validation n = 3 kf = KFold(n_splits=n) kf.get_n_splits(X_train) i=0 scores = zeros(n) for train_index, test_index in kf.split(X_train): Xtr, Xva = X_train[train_index], X_train[test_index] Ytr, Yva = y_train[train_index], y_train[test_index] M = model() M.fit(Xtr, Ytr) Yhat = M.predict(Xva) scores[i] = r2_score(Yva, Yhat) print ('Fold', i+1, 'example metric = ', scores[i]) i=i+1 cross_validation_error = mean(scores) # Print results print("\nThe scores are: ")
##reshaping it again to -1*size*size*1. 1 represents the color code as grayscale ######### images = labeledData(data, files, sz) shuffle(images) img_data = np.array([i[0] for i in images]).reshape(-1,sz,sz,1) lbl_data = np.array([i[1] for i in images]) ######### kernel = 3 strides = 2 seed = 5 #Split k-fold into desired number n_splits. kfold = KFold(n_splits=12, shuffle=True, random_state=seed) kfold.get_n_splits(img_data) #Store-variables scores = [] losses = [] probabilities = [] classes = [] label_data_1D = [] #Run validation #IMPORTANT NOTE: Validation will take a very long time. #It may appear frozen but is in fact processing. for train, test in kfold.split(img_data): #Use Keras Sequential and add convolution layers model = Sequential()