def plot(estimator, X, y, cv): title = "Learning curves" plotter.plot_learning_curve(estimator, title, X, y, ylim=(0.6, 1.1), n_jobs=-1, cv=cv) plt.show()
def plot_results(self, x, y): title = "Decision Tree - Reddit Data Set" return plot.plot_learning_curve(self.clf, title, x, y, cv=5, train_sizes=np.linspace(.1, 1.0))
def train(iteration): filename = agent.algo + "_" + env_id + "_" + str(n_games) + "games" + "_" + str(iteration) + '.png' figure_file = 'plots/' + filename best_score = env.reward_range[0] score_history = [] if load_checkpoint: agent.load_models() if os.environ.get('RENDER') == "t": env.render(mode='human') steps = 0 for i in range(n_games): observation = env.reset() done = False score = 0 # for every episode: while not done: if os.environ.get('RENDER') == "t": env.render() action = agent.choose_action(observation) observation_, reward, done, info = env.step(action) steps += 1 agent.remember(observation, action, reward, observation_, done) if not load_checkpoint: agent.learn() score += reward observation = observation_ score_history.append(score) avg_score = np.mean(score_history[-100:]) if avg_score > best_score: best_score = avg_score agent.save_models() print(env_id, "|", 'episode', i, "|", 'score %.1f' % score, "|", '100 games avg %.1f' % avg_score, "|", 'steps %d' % steps, "|" ) if not load_checkpoint: x = [i + 1 for i in range(n_games)] plot_learning_curve(x, score_history, figure_file, agent.algo, env_id)
def evaluate(X, Y, X_train, X_test, y_train, y_test): print("\n\nRunning neural network...") nn_clf = MLPClassifier() #define MLP and set learning rate to 0.01 MLPClassifier(solver='adam', activation='tanh', early_stopping=True, learning_rate=.01, alpha=1e-5, hidden_layer_sizes=(300, 200)) #traing MLP nn_clf.fit(X_train, y_train) #predict the test data y_pred = nn_clf.predict(X_test) print('accuracy on test data :', accuracy_score(y_test, y_pred)) # predict the train data y_pred = nn_clf.predict(X_train) print("accuracy on train data : ", accuracy_score(y_train, y_pred)) title = "Learning Curves (neural network)" #plot the learning curve plot_learning_curve(nn_clf, title, X, Y, ylim=(-0.1, 1.01), cv=5) plt.show() # ___________________ using naive bayes__________________# print("\n\nRunning Naive bayes...") NB_model = GaussianNB() NB_model = NB_model.fit(X=X_train, y=y_train) NB_model.fit(X_train, y_train) y_pred = NB_model.predict(X_test) print('accuracy on test data :', accuracy_score(y_test, y_pred)) y_pred = NB_model.predict(X_train) print("accuracy on train data : ", accuracy_score(y_train, y_pred)) title = "Learning Curves (Naive bayes)" plot_learning_curve(NB_model, title, X, Y, ylim=(-0.1, 1.01), cv=5) plt.show()
def plot(self, dirname): """ Generate plots """ # Plot learning curve print(f"{self.name}: Plotting learning curve") learning_curve = [ x for x in self.output if x["type"] == "learning_curve" ] if learning_curve: fig = plot.plot_learning_curve( self.title, learning_curve[0]["data"]["train_sizes"], learning_curve[0]["data"]["train_scores"], learning_curve[0]["data"]["test_scores"], learning_curve[0]["data"]["fit_times"], ) fig.savefig( os.path.join(dirname, f"{self.name}_learning_curve.png")) # Plot validation curve(s) valdiation_curve = [ x for x in self.output if x["type"] == "validation_curve" ] if valdiation_curve: for param in valdiation_curve: fig = plot.plot_validation_curve( self.title, param["data"]["parameter"], param["data"]["range"], param["data"]["train_scores"], param["data"]["test_scores"], ) fig.savefig( os.path.join( dirname, f'{self.name}_{param["data"]["parameter"]}_validation_curve.png', )) # If NN plot loss curve if self.title == "Neural Networks Classifier": fig = plot.plot_loss_curve(self.model) fig.savefig(os.path.join( dirname, f'{self.name}_loss_curve.png', )) # Plot confusion matrix confusion = [x for x in self.output if x["type"] == "confusion_matrix"] if confusion: fig = plot.plot_confusion_matrix(self.model, self.data.x_test, self.data.y_test) fig.savefig( os.path.join(dirname, f"{self.name}_confusion_matrix.png"))
# Save the architecture of the model json_string = model.to_json() open('model_architecture.json', 'w').write(json_string) # Save the best weights checkpointer = ModelCheckpoint(filepath='best_weights.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto') # Set early stopping early_stopping = EarlyStopping(monitor='val_loss', patience=20, verbose=1, mode='min') # 90% for traning and 10% for validation History = model.fit(patches_imgs_train, patches_masks_train, epochs=N_epochs, batch_size=batch_size, verbose=2, shuffle=True, validation_split=0.1, callbacks=[checkpointer, early_stopping]) # Plot the learning curve plot_learning_curve(History, save_path + 'learning_curve')
test_prop = 0.2 fmnist_trgX, fmnist_tstX, fmnist_trgY, fmnist_tstY = data_funcs.get_data( 'fmnist', data_prop, test_prop) chess_trgX, chess_tstX, chess_trgY, chess_tstY = data_funcs.get_data( 'chess', data_prop, test_prop) if learning_curve: if run_fmnist: # RELU # 50 layers # 0.002 tolerance plot.plot_learning_curve(MLPClassifier(hidden_layer_sizes=(50, ), activation='relu', tol=0.002), "fmnist_MLP_relu_learning", fmnist_trgX, fmnist_trgY, cv=5, n_jobs=-1) if run_chess: # RELU # (50, 10) plot.plot_learning_curve(MLPClassifier(max_iter=1000, hidden_layer_sizes=(50, 10), activation='relu'), "chess_MLP_relu_learning", chess_trgX, chess_trgY, cv=5, n_jobs=-1)
# Evaluate # out = clf.predict(fmnist_tstX) # Criterion = how to determine best attribute to split on # splitter = how to choose the split value (best vs. random) # max_depth = limits tree size (form of pruning by limiting growth) # min_samples_split = if learning_curve: if run_fmnist: # Best attrs: entropy, max depth = 20, min samples leaf = 5, min_impurity_decrease = 0.0005 plot.plot_learning_curve(tree.DecisionTreeClassifier( criterion='entropy', max_depth=20, min_samples_leaf=5, min_impurity_decrease=0.0005), "FMNIST_DT_entropy_learning", fmnist_trgX, fmnist_trgY, cv=5, n_jobs=-1) if run_chess: # Best attrs: entropy, max depth = 100, min samples leaf = 1, min_impurity_decrease = 0.0005 plot.plot_learning_curve(tree.DecisionTreeClassifier( criterion='entropy', max_depth=100, min_samples_leaf=1, min_impurity_decrease=0.0005), "chess_DT_entropy_learning", chess_trgX, chess_trgY, cv=5,
learn_iters = 0 avg_score = 0 n_steps = 0 for i in range(n_games): observation = env.reset() done = False score = 0 while not done: action, prob, val = agent.choose_action(observation) observation_, reward, done, info = env.step(action) n_steps += 1 score += reward agent.remember(observation, action, prob, val, reward, done) if n_steps % N == 0: agent.learn() learn_iters += 1 observation = observation_ score_history.append(score) avg_score = np.mean(score_history[-100:]) if avg_score > best_score: best_score = avg_score agent.save_models() print('episode', i, 'score %.1f' % score, 'avg score %.1f' % avg_score, 'time_steps', n_steps, 'learning_steps', learn_iters) x = [i + 1 for i in range(len(score_history))] plot_learning_curve(x, score_history, figure_file)
def gscv(model, params, x_train, x_test, y_train, y_test, cv=5, classes=2, crossval=True): global name print("Start_Classes{}".format(classes)) name = "{}{}".format(type(model).__name__, name) p = Path("./result") / name if not p.exists(): os.makedirs(p) global plot_lc global random if plot_lc: exploit_incremental_learning = False pl = plot.plot_learning_curve( model, name, np.concatenate((x_train, x_test)), np.concatenate((y_train, y_test)), (0, 1.01), custom_cv_2folds(x_train.shape[0], x_test.shape[0]), n_jobs=1, random_state=random, exploit_incremental_learning=exploit_incremental_learning) pl.savefig(p / "{}_learning_curve".format(name)) elif crossval: gscv = GridSearchCV(model, params, cv=cv, verbose=2, return_train_score=True, n_jobs=2) gscv.fit(x_train, y_train) print("Best params: {}".format(gscv.best_params_)) print("Best score: {}".format(gscv.best_score_)) res = pd.DataFrame.from_dict(gscv.cv_results_) print("Result: {}".format(res)) res.to_csv(p / "{}.csv".format(name)) res.to_html(p / "{}.html".format(name)) print("Accuracy on test:{}".format(gscv.score(x_test, y_test))) if classes == 2: if type(model).__name__ == 'RandomForestRegressor': probs = gscv.predict(x_test) print("F1 on test:{}".format( metrics.f1_score(y_test, np.round(probs)))) else: probs = gscv.predict_proba(x_test)[:, 1] print("F1 on test:{}".format( metrics.f1_score(y_test, gscv.predict(x_test)))) print("AUC on test:{}".format(metrics.roc_auc_score(y_test, probs))) fpr, tpr, thresholds = metrics.roc_curve(y_test, probs) save_roc_val(fpr, tpr, p / "{}_ROC.h5".format(name)) plt.plot(fpr, tpr) plt.title('ROC {}'.format(name)) plt.ylabel('TPR') plt.xlabel('FPR') plt.savefig(p / "{}_ROC".format(name)) else: probs = gscv.predict(x_test) #predict_proba(x_test)[:, 1] print("F1 on test (macro):{}".format( metrics.f1_score(y_test, probs, average='macro'))) else: model.fit(x_train, y_train) print("Accuracy on test:{}".format(model.score(x_test, y_test))) if classes == 2: probs = model.predict(x_test) # predict_proba(x_test)[:, 1] print("F1 on test:{}".format( metrics.f1_score(y_test, model.predict(x_test)))) print("AUC on test:{}".format(metrics.roc_auc_score(y_test, probs))) fpr, tpr, thresholds = metrics.roc_curve(y_test, probs) save_roc_val(fpr, tpr, p / "{}_ROC.h5".format(name)) plt.plot(fpr, tpr) plt.title('ROC {}'.format(name)) plt.ylabel('TPR') plt.xlabel('FPR') plt.savefig(p / "{}_ROC".format(name)) else: probs = model.predict(x_test) # .predict_proba(x_test)[:, 1]#gscv. print("F1 on test (macro):{}".format( metrics.f1_score(y_test, probs, average='macro'))) probs = model.predict_proba(x_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(classes): fpr[i], tpr[i], _ = metrics.roc_curve(y_test, probs[:, i], pos_label=i) roc_auc[i] = metrics.auc(fpr[i], tpr[i]) # Plot of a ROC curve for specific classes for i in range(classes): plt.figure() plt.plot(fpr[i], tpr[i], color='darkorange', label='ROC curve (area = %0.2f)' % roc_auc[i]) plt.plot([0, 1], [0, 1], 'k--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC {}'.format(name)) plt.legend(loc="lower right") plt.savefig(p / "{}_ROC_{}".format(name, i), dpi=1200) # Compute macro-average ROC curve and ROC area # First aggregate all false positive rates all_fpr = np.unique( np.concatenate([fpr[i] for i in range(classes)])) # Then interpolate all ROC curves at this points mean_tpr = np.zeros_like(all_fpr) for i in range(classes): mean_tpr += interp(all_fpr, fpr[i], tpr[i]) # Finally average it and compute AUC mean_tpr /= classes fpr["macro"] = all_fpr tpr["macro"] = mean_tpr roc_auc["macro"] = metrics.auc(fpr["macro"], tpr["macro"]) print("AUC on test:{}".format(roc_auc["macro"])) # Plot all ROC curves plt.figure() plt.plot(fpr["macro"], tpr["macro"], label='macro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["macro"]), color='navy', linestyle=':', linewidth=1) colors = cycle(['aqua', 'darkorange', 'cornflowerblue']) for i, color in zip(range(classes), colors): plt.plot(fpr[i], tpr[i], color=color, lw=1, label='ROC curve of class {0} (area = {1:0.2f})' ''.format(i, roc_auc[i])) plt.plot([0, 1], [0, 1], 'k--', lw=1) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title( 'Some extension of Receiver operating characteristic to multi-class' ) plt.legend(loc="lower right") plt.savefig(p / "{}_ROC_aggregate".format(name), dpi=1200)
Mask_train = read_data(path+'Mask/', Mask_train, img_h, img_w) Mask_val =read_data(path+'Mask/', Mask_val, img_h, img_w) Img_train = read_data(path+'Image/', Img_train, img_h, img_w) Img_val = read_data(path+'Image/', Img_val, img_h, img_w) model = get_UNet(img_shape=(img_h, img_w, 1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0, batchnorm=True, N=2, weight_use=False) model.compile(optimizer=Adam(lr=1e-5), loss=[dice_coef_loss], metrics=[dice_coef,precision,recall]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=1, validation_data=(Img_val, Mask_val)) plot_learning_curve(History, 'Task1_k={0}_loss_{1}_'.format(k,i+1)) plot_validation_metric(History, 'Task1_k={0}_metrics_{1}_'.format(k,i+1)) ''' Task 2 ''' k=3 # k=5 Mask = split_list(Mask,k) Img = split_list(Img,k) radius = 2 weight_strength=1 batch_size =8 for i in range(k): Mask_val = list(Mask[i])
Mask_train = read_data(path+'Mask/', Mask_train, img_h, img_w) Mask_val = read_data(path+'Mask/', Mask_val, img_h, img_w) Img_train = read_data(path+'Image/', Img_train, img_h, img_w) Img_val = read_data(path+'Image/', Img_val, img_h, img_w) # Train the model model = get_UNet(img_shape=(256,256,1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0.5, batchnorm=True) model.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_val, Mask_val)) # Plot the learning curve plot_learning_curve(History, 'Task1a') # Train the model model = get_UNet(img_shape=(256,256,1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0.5, batchnorm=True) model.compile(optimizer=Adam(lr=0.0001), loss=[dice_coef_loss], metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_val, Mask_val)) # Plot the learning curve plot_learning_curve(History, 'Task1b')
def main(): parser = argparse.ArgumentParser(description='Classification task') parser.add_argument('--normalize', action='store_true') parser.add_argument('--plot-pca', action='store_true') parser.add_argument('--plot-corr', action='store_true') parser.add_argument('--plot-feat-dist', action='store_true') parser.add_argument('--plot-curve', action='store_true') parser.add_argument('--classifier') parser.add_argument('datafile') args = parser.parse_args() data, X, y = preprocessing( args.datafile, #feature_discret=True, plot_feat_dist=args.plot_feat_dist, plot_corr=args.plot_corr, plot_PCA=args.plot_pca) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.4, random_state=0) X_dev, X_test, y_dev, y_test = train_test_split(X_test, y_test, stratify=y_test, test_size=0.5, random_state=0) if args.normalize: scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_dev = scaler.transform(X_dev) X_test = scaler.transform(X_test) """ #for neural network scaler = MinMaxScaler((-1, 1)) X_train = scaler.fit_transform(X_train) X_dev = scaler.transform(X_dev) X_test = scaler.transform(X_test) """ print("DataSet summary:") print("train X: ", X_train.shape, "y: ", y_train.shape) print("dev X: ", X_dev.shape, "y: ", y_dev.shape) print("test X:", X_test.shape, "y: ", y_test.shape) positive_weight = 1 - sum(y_train) / y_train.count() class_weight = {1: positive_weight, 0: 1 - positive_weight} print("Class weights: ", {1: positive_weight, 0: 1 - positive_weight}) if args.classifier == 'gbdt': clf = train_gbdt_classifier(X_train, y_train) elif args.classifier == 'rf': clf = train_random_forest_classifier(class_weight, X_train, y_train) elif args.classifier == 'nn': clf = train_MLP_classifier(X_train, y_train) else: clf = train_logit_classifier(class_weight, X_train, y_train) title = "Learning Curve" cv = StratifiedShuffleSplit(n_splits=10, test_size=0.33, random_state=0) X = pd.concat([X_train, X_dev]) y = pd.concat([y_train, y_dev]) plot_learning_curve(clf, title, X, y, cv=cv, n_jobs=2) plt.show() plot_feature_importance(clf, X_train, y_train, data.columns) y_train_pred = clf.predict(X_train) y_train_prop = clf.predict_proba(X_train) y_dev_pred = clf.predict(X_dev) y_dev_prop = clf.predict_proba(X_dev) y_test_pred = clf.predict(X_test) y_test_prop = clf.predict_proba(X_test) y_train_score = y_train_prop[:, 1] y_dev_score = y_dev_prop[:, 1] y_test_score = y_test_prop[:, 1] print('Training data report') print(classification_report(y_train, y_train_pred)) print('Dev data report') print(classification_report(y_dev, y_dev_pred)) #print('Test data report') #print(classification_report(y_test, y_test_pred)) if args.plot_curve: opthd = plot_curve(y_dev, y_dev_score, min_p=None, min_r=None) print("optimal threshold: ", opthd) y_test_pred = np.array(y_test_score > opthd) y_test_pred = y_test_pred.astype(int) print('Test data report (with optimal theshold)') print(classification_report(y_test, y_test_pred))
#Print the training evaluation print("The analysis of " + people) for t, l in zip(train, label_res): y_gold, y_pred, y_base = pred.train_and_dev(t, l, args.classification)#, args.use_string) pred.evaluation(args.classification, y_gold, y_pred, False) #Baseline pred.evaluation(args.classification, y_gold, y_base, True) elif args.mode == 'test': url_dic = pr.text_to_dic(args.people, table, label) train, label = pred.dic_to_feature(url_dic, args.length, args.part) train, test, label, label_t = train_test_split(train, label, test_size=0.33, random_state=42) pred.predict_test(train, label, test, label_t, args.classification)#, args.use_string) #PLot else: url_dic = pr.text_to_dic(args.people, table, label) train, label = pred.dic_to_feature(url_dic, args.length, args.part) #train, test, label, label_t = train_test_split(train, label, test_size=0.33, random_state=42) vec = CountVectorizer(analyzer='word', ngram_range=(1, 2)) train = vec.fit_transform(train) cv = ShuffleSplit(n_splits=50, test_size=0.2, random_state=42) if 'logistic' in args.classification: estimator = LogisticRegression(C=1000, solver='liblinear', max_iter=300, penalty='l1') else: estimator = LinearRegression() pltf.plot_learning_curve(estimator, "Learning Curves", train, label, axes=None, ylim=(0.1, 1.01), cv=cv, n_jobs=4)#, scoring = score) plt.show()
Mask_train, Mask_validation, Img_train, Img_validation = shuffle_split( Mask, Img, 80) # Image and mask distribution Mask_train = read_mask_onehot_encoding(path + 'Mask/', Mask_train, img_h, img_w) Mask_validation = read_mask_onehot_encoding(path + 'Mask/', Mask_validation, img_h, img_w) Img_train = read_data(path + 'Image/', Img_train, img_h, img_w) Img_validation = read_data(path + 'Image/', Img_validation, img_h, img_w) model = get_unet(input_img=(240, 240, 1), n_filters=16, kernel_size=3, dropout=0.5, batchnorm=True) model.compile(optimizer=Adam(lr=0.0001), loss=[dice_coef_loss], metrics=[dice_coef, precision, recall]) History = model.fit(Img_train, Mask_train, batch_size=4, epochs=100, verbose=1, validation_data=(Img_validation, Mask_validation)) plot_learning_curve(History, 'Task3_learning_curve_with_encoding') plot_validation_metric_task3(History, 'Task3_metrics_with_encoding') print('Task 7 done !')
batch_size=16, input_size=X_train.shape[1], input_dimension=1, Bi=False, drop=0.2) model.compile(optimizer=Adam(lr=0.001), loss='mean_squared_error', metrics=['mean_absolute_error']) History = model.fit(X_train, y_train, epochs=100, batch_size=16, verbose=1, validation_data=(X_val, y_val)) plot_learning_curve(History, 'Task1_units={0}_loss'.format(units)) plot_validation_metric_1(History, 'Task1_units={0}_metrics'.format(units)) ''' Task 2 ''' def load_streamlines(dataPath, subject_ids, bundles, n_tracts_per_bundle): X = [] y = [] for i in range(len(subject_ids)): for c in range((len(bundles))): filename = dataPath + subject_ids[i] + '/' + bundles[c] + '.trk' tfile = nib.streamlines.load(filename) streamlines = tfile.streamlines n_tracts_total = len(streamlines) ix_tracts = np.random.choice(range(n_tracts_total),
y_train = [int(name.split(os.path.sep)[-2]) for name in train] y_train = utils.to_categorical(y_train) class_weight = np.max(y_train.sum(axis=0)) / y_train.sum(axis=0) # Create and train the model model = resnet_18(n_class=2) epochs = 150 lr = 1e-5 model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=lr), metrics=['accuracy']) History = model.fit_generator( train_gen, steps_per_epoch=train_gen.n//Batch_size, epochs=epochs, verbose=2, validation_data=val_gen, validation_steps=val_gen.n//Batch_size, class_weight=class_weight) # Plot learning curves plot_learning_curve(History, 'loss') plot_validation_metric(History, 'metrics') # Save the trained model model.save('model.h5') print("Model saved")
Mask = gen_list(path, 'Mask') Img = gen_list(path,'Image') Mask_train, Mask_val, Img_train, Img_val = shuffle_split(Mask, Img, 0.8) Mask_train = read_data(path+'Mask/', Mask_train, img_h, img_w) Mask_val = read_data(path+'Mask/', Mask_val, img_h, img_w) Img_train = read_data(path+'Image/', Img_train, img_h, img_w) Img_val = read_data(path+'Image/', Img_val, img_h, img_w) model = get_UNet(img_shape=(256,256,1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0.5, batchnorm=True) model.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_val, Mask_val)) plot_learning_curve(History, 'Task5a_1') model = get_UNet(img_shape=(256,256,1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0.5, batchnorm=True) model.compile(optimizer=Adam(lr=0.0001), loss=[dice_coef_loss], metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_val, Mask_val)) plot_learning_curve(History, 'Task5a_2')
test_prop = 0.2 fmnist_trgX, fmnist_tstX, fmnist_trgY, fmnist_tstY = data_funcs.get_data( 'fmnist', data_prop, test_prop) chess_trgX, chess_tstX, chess_trgY, chess_tstY = data_funcs.get_data( 'chess', data_prop, test_prop) # Note: 100 seconds w/o early stopping, 18 w/ early stopping # Takes forever to do this with full set; 5 hours, 2 hours, 6 hours or something like that if learning_curve: if run_fmnist: # n_neighbors=5 weight = uniform plot.plot_learning_curve(KNeighborsClassifier(n_neighbors=5, weights='uniform'), "fmnist_kNN_learning", fmnist_trgX, fmnist_trgY, cv=5, n_jobs=-1) if run_chess: # n_neighbors=10 weight = uniform plot.plot_learning_curve(KNeighborsClassifier(n_neighbors=10, weights='uniform'), "chess_kNN_learning", chess_trgX, chess_trgY, cv=5, n_jobs=-1) if validation_curve: if run_fmnist:
from dataloader import gen_list, shuffle_split, read_data, read_data_onehot from Unet import get_UNet_multi from plot import plot_learning_curve, plot_validation_metric from metrics import dice_coef_loss, dice_coef, precision, recall # Read the data path = '/Lab1/Lab3/CT/' img_h, img_w = 256, 256 Mask = gen_list(path, 'Mask') Img = gen_list(path,'Image') Mask_train, Mask_val, Img_train, Img_val = shuffle_split(Mask, Img, 0.8) Mask_train = read_mask_onehot(path+'Mask/', Mask_train, img_h, img_w) Mask_val = read_mask_onehot(path+'Mask/', Mask_val, img_h, img_w) Img_train = read_data(path+'Image/', Img_train, img_h, img_w) Img_val = read_data(path+'Image/', Img_val, img_h, img_w) # Train the model model = get_UNet_multi(img_shape=(256,256,1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0.5, batchnorm=True, N=3) model.compile(optimizer=Adam(lr=0.0001), loss=[dice_coef_loss], metrics=[dice_coef, precision, recall]) History = model.fit(Img_train, Mask_train, batch_size=4, epochs=25, verbose=1, validation_data=(Img_val, Mask_val)) # Plot the learning curves plot_learning_curve(History, 'Task6_1') plot_validation_metric(History, 'Task6_2')
def main(batch_size: int = 100, log_path: Path = Path.home() / 'Desktop/thalnet/tensorboard' / timestamp()): mnist = input_data.read_data_sets('mnist', one_hot=True) num_classes = 10 num_rows, row_size = 28,28 summary_writer = tf.summary.FileWriter(str(log_path), graph=tf.get_default_graph()) def get_batch(train,batch_size): """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" if train: xs, ys = mnist.train.next_batch(batch_size) else: #rows = np.random.randint(1000,size=batch_size) #xs, ys = mnist.test.images[:rows,:], mnist.test.labels[:rows,:] xs, ys = mnist.test.images[:1000,:], mnist.test.labels[:1000,:] return {'x': xs, 'y': ys} def feed_dict(batch): return {data: batch['x'], target: batch['y'], dropout: 0} get_thalnet_cell = lambda: ThalNetCell(input_size=row_size, output_size=num_classes, context_input_size=32, center_size_per_module=32,num_modules=4) get_stacked_cell = lambda: GRUCell(num_hidden=50,num_layers=4) models = [lambda: MLPClassifier(data, target, dropout, num_hidden=64, num_layers=2), lambda: SequenceClassifier(data, target, dropout, get_rnn_cell=get_stacked_cell,num_rows=num_rows, row_size=row_size), lambda: SequenceClassifier(data, target, dropout, get_rnn_cell=get_thalnet_cell,num_rows=num_rows, row_size=row_size) ] labels = ['MLP-baseline','GRU-baseline','ThalNet-FF-GRU'] ys,zs = [],[] for model in models: with tf.Session() as session: data = tf.placeholder(tf.float32, [None, num_rows*row_size], name='data') target = tf.placeholder(tf.float32, [None, num_classes], name='target') dropout = tf.placeholder(tf.float32, name='dropout') model = model() # reproduce result under 60,000 total parameters for all three models print(f'{model.num_parameters} parameters') if model.num_parameters > 60000: session.close() return y,z = [],[] session.run(tf.global_variables_initializer()) for batch_index in itertools.count(): train_batch = get_batch(True,batch_size) _, train_cross_entropy, train_accuracy, train_summary = session.run( [model.optimize, model.cross_entropy, model.accuracy, model.train_summary], feed_dict=feed_dict(train_batch)) summary_writer.add_summary(train_summary, batch_index) if batch_index % 1 == 0: test_batch = get_batch(False,batch_size) test_cross_entropy, test_accuracy, test_summary = session.run( [model.cross_entropy, model.accuracy, model.test_summary], feed_dict=feed_dict(test_batch)) summary_writer.add_summary(test_summary, batch_index) summary_writer.add_summary(session.run(model.weights_summary), batch_index) y.append(test_accuracy) z.append(test_cross_entropy) print( f'Batch {batch_index}: train accuracy {train_accuracy:.3f} (cross entropy {train_cross_entropy:.3f}), test accuracy {test_accuracy:.3f} (cross entropy {test_cross_entropy:.3f})') if batch_index == 2000: ys.append(y) zs.append(z) break session.close() tf.reset_default_graph() plot_learning_curve('Sequential Mnist',ys,zs, labels,ylim=(0,1))
inc_rate=2, activation='relu', drop=0.5, batchnorm=False) model.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_val, Mask_val)) # Plot the learning curve plot_learning_curve(History, 'Task2a') # Train the model model = get_UNet(img_shape=(256, 256, 1), Base=16, depth=4, inc_rate=2, activation='relu', drop=0.5, batchnorm=False) model.compile(optimizer=Adam(lr=0.0001), loss=[dice_coef_loss], metrics=[dice_coef]) History = model.fit(Img_train,
chess_trgX, chess_tstX, chess_trgY, chess_tstY = data_funcs.get_data( 'chess', data_prop, test_prop) # Note: # With no max depth, get perfect fit on training but validation doesn't look any better than regular DT # With max depth =1 (default base_estimator) results are trash # 10 looks pretty good if learning_curve: # FMNIST max depth=20 (10 close but 87.5 vs. 85) and n_estimators=200 # Chess is 5 max depth 200 estimators with random split if run_fmnist: plot.plot_learning_curve(AdaBoostClassifier( base_estimator=DecisionTreeClassifier(max_depth=20, splitter='random'), n_estimators=200), "FMNIST_Boosting_ADA_SAMME.R_learning", fmnist_trgX, fmnist_trgY, cv=5, n_jobs=-1) if run_chess: plot.plot_learning_curve(AdaBoostClassifier( base_estimator=DecisionTreeClassifier(max_depth=5, splitter='random'), n_estimators=200), "chess_Boosting_ADA_SAMME.R_learning", chess_trgX, chess_trgY, cv=5, n_jobs=-1)
print("Accuracy Validation" + str(y.size) +": %0.2f (+/- %0.2f)" % (scoresVal.mean(), scoresVal.std() * 2)) return scoresTrain,scoresVal print("training fullset") #scoresT, scoresV = run_classifier(None,True) print("finished training fullset") #run_classifier(1000, False) train_sizes = [1000, 10000, 100000, 1000000] train_scores_mean = [] train_scores_std = [] test_scores_mean = [] test_scores_std = [] for s in train_sizes: scoresT, scoresV = run_classifier(s, False) train_scores_mean.append(scoresT.mean()) train_scores_std.append(scoresT.std()) test_scores_mean.append(scoresV.mean()) test_scores_std.append(scoresV.std()) import matplotlib matplotlib.use('Agg') import plot as pl title = "Learning_Curves_KNN_Classifier" if SVM: title = "Learning_Curves_SVM_Classifier" pl.plot_learning_curve(title, train_sizes = train_sizes, logX=True, test_scores_mean = test_scores_mean,test_scores_std = test_scores_std, train_scores_mean = train_scores_mean,train_scores_std = train_scores_std)
Mask_train = read_data(path+'Mask/', Mask_train, img_h, img_w) Mask_validation = read_data(path+'Mask/', Mask_validation, img_h, img_w) Img_train = read_data(path+'Image/', Img_train, img_h, img_w) Img_validation = read_data(path+'Image/', Img_validation, img_h, img_w) # calling the model model = get_unet(input_img=(256, 256, 1), n_filters=16, kernel_size=3, dropout=0.5, batchnorm=True) # Task 1a model.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_validation, Mask_validation)) plot_learning_curve(History, 'Lab3_task1a') print('Task1a done !') # Task 1b model.compile(optimizer=Adam(lr=0.0001), loss=[dice_coef_loss], metrics=[dice_coef]) History = model.fit(Img_train, Mask_train, batch_size=8, epochs=150, verbose=2, validation_data=(Img_validation, Mask_validation)) plot_learning_curve(History, 'Lab3_task1b') print('Task 1b done !') # Task 2a model = get_unet(input_img=(256, 256, 1), n_filters=16, kernel_size=3, dropout=0.5, batchnorm=False)