def NN_SA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) # SA HPs nodes = [128, 128, 128, 128] act = 'relu' seed = 1 sa_algo = 'simulated_annealing' sa_lr = 10 sa_iter = 10000 sa_temp = 10000 sa_decay = 0.92 sa_ma = 50 sa_clip = 10 temperature = [0.1, 1, 10, 100, 1000, 10000] plt.figure() for t in temperature: print('temperature', t) sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo, max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma, schedule=mlrose.GeomDecay(init_temp=t, decay=sa_decay)) sa_nn_model.fit(X_train, y_train) plt.plot(sa_nn_model.fitness_curve, label='temp =' + str(t)) plt.title("NN SA - Temperature") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.savefig("Images\\NN - SA - Temperature") plt.xscale('log') plt.savefig("Images\\NN - SA - Temperature - log") plt.show() decay_rates = [0.1, 0.2, 0.3, 0.4, 0.5, 0.8, 0.92] plt.figure() for dr in decay_rates: print('decay', dr) sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo, max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma, schedule=mlrose.GeomDecay(init_temp=sa_temp, decay=dr)) sa_nn_model.fit(X_train, y_train) plt.plot(sa_nn_model.fitness_curve, label='decay rate =' + str(dr)) plt.title("NN SA - Decay Rate") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.savefig("Images\\NN - SA - Decay Rate") plt.xscale('log') plt.savefig("Images\\NN - SA - Decay Rate - log") plt.show()
def main(): opts = util.parse_args() X, y = util.data_load(opts.dataset) fc_nn_model = fc.create_model() ada_model = AdaBoostClassifier(n_estimators=100, random_state=0) svm_model = SVC(C=1000, gamma=0.1) n = opts.upsamplen if opts.upsamplen is not None else 1 start = n if opts.upsamplestart is None else 1 if start > n: print("unsample range error") sys.exit() conf_fc, conf_ada, conf_svm = [], [], [] for t in np.arange(start, n + 1): needed = util.needed_n(X, y, t) temp_X, temp_y = util.upsample(X, y, needed) X_train, X_test, y_train, y_test = train_test_split(temp_X, temp_y, test_size=0.3, random_state=42) X_train, X_test = util.normalize(X_train, X_test) train_dset = tf.data.Dataset.from_tensor_slices( (X_train, y_train)).batch(64, drop_remainder=False).shuffle(buffer_size=10000) test_dset = tf.data.Dataset.from_tensor_slices( (X_test, y_test)).batch(64) ada_model.fit(X_train, y_train) svm_model.fit(X_train, y_train) fc_nn_model.fit(train_dset, epochs=10) pred_ada = ada_model.predict(X_test) pred_svm = svm_model.predict(X_test) conf_ada.append(confusion_matrix(y_test, pred_ada)) conf_svm.append(confusion_matrix(y_test, pred_svm)) temp = np.zeros((2, 2), dtype=int) for d, labels in test_dset: predictions = fc_nn_model(d) for i in range(len(d)): temp[labels[i]][np.argmax(predictions[i])] += 1 conf_fc.append(temp) recall_fc = list(map(lambda x: util.recall(x), conf_fc)) recall_ada = list(map(lambda x: util.recall(x), conf_ada)) recall_svm = list(map(lambda x: util.recall(x), conf_svm)) up_range = np.arange(start, n + 1) d = {"SVM": recall_svm, "Adaboost": recall_ada, "FC_NN": recall_fc} legends = ["SVM", "Adaboost", "FC_NN"] for key in d: plt.plot(up_range, d[key]) plt.title("Recall Vs Upsample Graph") plt.xlabel("Upsample rate") plt.ylabel("Recall") plt.legend(legends) plt.show()
def main(): opts = util.parse_args() X, y = util.data_load(opts.dataset) model = create_model() model_layers = create_model_more_layers() n = opts.upsamplen if opts.upsamplen is not None else 1 start = n if opts.upsamplestart is None else 1 all_conf = [] all_conf_layers = [] if start > n: print("Upsample start should be larger than end") sys.exit() for t in np.arange(start, n + 1): print("t", t) needed = util.needed_n(X, y, t) temp_X, temp_y = util.upsample(X, y, needed) X_train, X_test, y_train, y_test = train_test_split(temp_X, temp_y, test_size=0.3, random_state=42) X_train, X_test = util.normalize(X_train, X_test) train_dset = tf.data.Dataset.from_tensor_slices( (X_train, y_train)).batch(64, drop_remainder=False).shuffle(buffer_size=10000) test_dset = tf.data.Dataset.from_tensor_slices( (X_test, y_test)).batch(64) model.fit(train_dset, epochs=10) model_layers.fit(train_dset, epochs=10) conf_mat = np.zeros((2, 2), dtype=int) conf_mat_layers = np.zeros((2, 2), dtype=int) for d, labels in test_dset: predictions = model(d) predictions_layers = model_layers(d) for i in range(len(d)): conf_mat[labels[i]][np.argmax(predictions[i])] += 1 conf_mat_layers[labels[i]][np.argmax( predictions_layers[i])] += 1 all_conf.append(conf_mat) all_conf_layers.append(conf_mat_layers) re_fc, re_fc_layer = list(map(lambda x: util.recall(x), all_conf)), list( (map(lambda x: util.recall(x), all_conf_layers))) up_range = np.arange(start, n + 1) plt.plot(up_range, re_fc) plt.plot(up_range, re_fc_layer) plt.title("2-layer NN vs 3-layer NN") plt.legend(["2-layer", "3-layer"]) plt.xlabel("Upsample rate") plt.ylabel("Recall") plt.show()
def NN_GD(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) # GD HPs nodes = [128, 128, 128, 128] act = 'relu' seed = 1 gd_algo = 'gradient_descent' gd_lr = 0.00000009 gd_iter = 10000 gd_ma = 50 gd_clip = 5 learning_rates = [ 0.00000009, 0.00000001, 0.000000001, 0.0000001, 0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1 ] plt.figure() for lr in learning_rates: print('lr', lr) gd_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=gd_algo, max_iters=gd_iter, learning_rate=lr, clip_max=gd_clip, max_attempts=gd_ma) gd_nn_model.fit(X_train, y_train) gd_curve = gd_nn_model.fitness_curve inverted_gd_curve = np.array(gd_curve) * -1 plt.plot(inverted_gd_curve, label='lr =' + str(lr)) plt.title("NN GD - Learning Rates") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.savefig("Images\\NN - GD - Learning Rates") plt.xscale('log') plt.savefig("Images\\NN - GD - Learning Rates - log") plt.show()
def NN_GA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) activation = ['relu'] learning_rate = [5, 0.01, 0.1, 1, 2, 3, 4, 7, 10] algorithim = 'genetic_alg' iters = [1000, 10000, 50000, 100000] nodes = [128, 128, 128, 128] population = [2000, 2100,2200,2300] mutation = [ 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1] outcomes = [] max_attempts = [10, 50, 100, 200, 500, 1000] clips = [5, 10, 100, 1000, 10000, 100000] # GA HPs nodes = [128, 128, 128, 128] act = 'relu' seed = 1 ga_algo = 'genetic_alg' ga_lr = 5 ga_iter = 100 ga_pop = 1500 ga_mut = 0.1 ga_ma = 100 ga_clip = 5 Population = [100, 200, 300, 400, 500, 750, 1000, 1240, 1500] plt.figure() for p in Population: print('Population', p) ga_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=ga_iter, algorithm=ga_algo, pop_size=p, mutation_prob=ga_mut, bias=True, is_classifier=True, learning_rate=ga_lr, early_stopping=True, clip_max=ga_clip, max_attempts=ga_ma, random_state=seed, curve=True) ga_nn_model.fit(X_train, y_train) plt.plot(ga_nn_model.fitness_curve, label='pop =' + str(p)) plt.title("NN GA - Population") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.savefig("Images\\NN - GA - Population") plt.xscale('log') plt.savefig("Images\\NN - GA - Population - log") plt.show()
def main(): opts = util.parse_args() X, y = util.data_load(opts.dataset) #set upsample range n = opts.upsamplen if opts.upsamplen is not None else 1 start = n if opts.upsamplestart is None else 1 if start > n: print("unsample range error") sys.exit() for i in np.arange(start, n + 1): print("i", i) needed = util.needed_n(X, y, i) print("needed", needed) print("pre-upsampled len x", len(X)) upsampled_X, upsampled_y = util.upsample(X, y, needed) print("length of upsampled_X", len(upsampled_X)) #use a good param to improve speed X_train, X_test, y_train, y_test = train_test_split(upsampled_X, upsampled_y, test_size=0.3, random_state=42) X_train, X_test = util.normalize(X_train, X_test) model = SVC(C=1000, gamma=0.1) model.fit(X_train, y_train) predictions = model.predict(X_test) cm = confusion_matrix(y_test, predictions) report = classification_report(y_test, predictions) #precision-recall curve y_score = model.decision_function(X_test) average_precision = average_precision_score(y_test, y_score) disp = plot_precision_recall_curve(model, X_test, y_test) disp.ax_.set_title('Precision-Recall curve with 10x upsampling: ' 'AP={0:0.2f}'.format(average_precision)) plt.show() #roc curve # svc_roc=plot_roc_curve(model,X_test,y_test) # svc_roc.ax_.set_title('SVM ROC Curve with 10x upsampling') # plt.show() print(cm) print(report) '''
def main(): opts = util.parse_args() X, y = util.data_load(opts.dataset) skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42) for train_index, test_index in skf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] X_train, X_test = util.normalize(X_train, X_test) clf = AdaBoostClassifier(n_estimators=100, random_state=0) clf.fit(X_train, y_train) """ conf_mat = np.zeros((2, 2)) for i in range(len(X_test)): pred = clf.predict(X_test[i]) true = y_test[i] conf_mat[true][pred] += 1 print(conf_mat) """ predictions = clf.predict(X_test) conf_mat = confusion_matrix(y_test, predictions) print(conf_mat)
def main(): opts = util.parse_args() X, y = util.data_load(opts.dataset) skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42) model = create_model() for train_index, test_index in skf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] X_train, X_test = util.normalize(X_train, X_test) train_dset = tf.data.Dataset.from_tensor_slices( (X_train, y_train)).batch(64, drop_remainder=False).shuffle(buffer_size=10000) test_dset = tf.data.Dataset.from_tensor_slices( (X_test, y_test)).batch(64) model.fit(train_dset, epochs=10) conf_mat = np.zeros((2, 2), dtype=int) for d, labels in test_dset: predictions = model(d) for i in range(len(d)): conf_mat[labels[i]][np.argmax(predictions[i])] += 1 print(conf_mat)
def main(): opts = util.parse_args() X, y = util.data_load(opts.dataset) n = opts.upsamplen if opts.upsamplen is not None else 1 start = n if opts.upsamplestart is None else 1 if start > n: print("Upsample start should be larger than end") sys.exit() thresh = opts.threshold if opts.threshold is not None and opts.threshold >= 0.40 else None for t in np.arange(start, n + 1): needed = util.needed_n(X, y, t) temp_X, temp_y = util.upsample(X, y, needed) X_train, X_test, y_train, y_test = train_test_split(temp_X, temp_y, test_size=0.3, random_state=42) X_train, X_test = util.normalize(X_train, X_test) clf = AdaBoostClassifier(n_estimators=100, random_state=0) clf.fit(X_train, y_train) conf_upsample = [] if thresh is None: predictions = clf.predict(X_test) conf_mat = confusion_matrix(y_test, predictions) conf_upsample.append(conf_mat) print(conf_mat) else: conf_thresh = [] for i in np.arange(0.4, thresh + 0.01, 0.005): predictions = (clf.predict_proba(X_test)[:, 1] >= i).astype(int) conf_mat = confusion_matrix(y_test, predictions) conf_thresh.append(conf_mat) print(i) print(conf_mat) util.get_roc_curve(conf_thresh, "Adaboost", "threshold") plt.show()
def NN_GA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) activation = ['relu'] learning_rate = [0.00000002, 0.00000003, 0.00000004, 0.00000005, 0.00000006, 0.00000007, 0.00000008, 0.00000009] algorithim = 'gradient_descent' iters = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000] nodes = [128, 128, 128, 128] outcomes = [] max_attempts = [10, 50, 100, 200, 500, 1000] clips = [5, 10, 100, 1000, 10000, 100000] clips = [4, 5] act = 'relu' lr = 0.00000009 itera = 10000 ma = 100 clip = 5 seed = 1 while 1 == 1: iters_outs = {} for iter_test in iters: start = time.time() print(algorithim, act, lr, iter_test, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test, algorithm=algorithim, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) train_time = time.time() - start print('Train time', train_time) start = time.time() y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_train_query_time = time.time() - start print('y_train_query_time', y_train_query_time) start = time.time() y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) y_test_query_time = time.time() - start print('y_test_query_time', y_test_query_time) nn_loss = nn_model.loss print('loss', nn_loss) outcome = {} outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = iter_test outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = train_time + y_train_query_time + y_test_query_time outcome['Train time'] = train_time outcome['y_train_query_time'] = y_train_query_time outcome['y_test_query_time'] = y_test_query_time outcome['loss'] = nn_loss outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GD-itertests.csv') iters_outs[iter_test] = y_test_roc old_val = itera itera = max(iters_outs, key=iters_outs.get) print('best iter', itera, 'old', old_val) raise SystemExit(0) clips_outs = {} for clip_test in clips: start = time.time() print(algorithim, act, lr, itera, ma, clip_test) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip_test, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['max_attempts'] = ma outcome['clip'] = clip_test outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GD.csv') clips_outs[clip_test] = y_test_roc old_val = clip clip = max(clips_outs, key=clips_outs.get) print('best clip', clip, 'old', old_val) maxa_outs = {} for maxa_test in max_attempts: start = time.time() print(algorithim, act, lr, itera, maxa_test, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=maxa_test, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['max_attempts'] = maxa_test outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GD.csv') maxa_outs[maxa_test] = y_test_roc old_val = ma ma = max(maxa_outs, key=maxa_outs.get) print('best ma', ma, 'old', old_val) lr_outs = {} for lr_test in learning_rate: start = time.time() print(algorithim, act, lr_test, itera, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, bias=True, is_classifier=True, learning_rate=lr_test, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['activation'] = act outcome['learning_rate'] = lr_test outcome['max_iters'] = itera outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GD.csv') lr_outs[lr_test] = y_test_roc old_lr = lr lr = max(lr_outs, key=lr_outs.get) print('best lr', lr, 'old', old_lr)
def run_model(data_fraction=0.1, batch_size=1, num_epochs=10, multi_gpu=False, data=False, learning_rate=0.001, learning_rate_decay=False, dropout=False, tensorboard=False, save=False, describe=False, model_id=None): callbacks = None if tensorboard: print("\nUsing tensorboard\n") import datetime log_dir = "logs/fit/" + datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) callbacks = [tensorboard_callback] # load the data if (data): # data was provided print("\nUsing provided data\n") x_train, y_train, x_val, y_val, x_test, y_test = data else: x_train, y_train, x_val, y_val, x_test, y_test = u.data_load( f=data_fraction) if (model_id == 'baseline'): model_name = "v0_t" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) # get the model _model = get_baseline_conv_model_1() elif (model_id == 'baseline_vgg'): model_name = "vBVGG_t" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) # get the model _model = get_baseline_vgg_model_1(dropout=dropout) elif (model_id == 'baseline_vgg_block3'): model_name = "vBVGG3_t" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) # get the model _model = get_baseline_vgg_model_3(dropout=dropout) elif (model_id == 'baseline_vgg_block2'): model_name = "vBVGG2_t" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) # get the model _model = get_baseline_vgg_model_2(dropout=dropout) elif (model_id == 'vgg_co_baseline'): model_name = "vVCB" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) # get the model _model = vgg_co_baseline_model(dropout=dropout) elif (model_id == 'baseline_vgg_block4tr'): model_name = "vBVGG4tr_t" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) _model = get_baseline_vgg_model_4tr(dropout=dropout) elif (model_id == 'baseline_vgg_no_pool'): model_name = "vBVNP_t" + str(len(x_train)) + "_e" + str( num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate) _model = get_baseline_vgg_model_no_pool(dropout=dropout) else: raise Exception("Must specify model id!") if (learning_rate_decay): model_name += "_lrd" if (dropout): model_name = "{}_d{}".format(model_name, dropout) if (multi_gpu): print("Creating multi GPU model") model = keras.utils.multi_gpu_model(_model, gpus=2) else: model = _model print("\nRuning model:: {}\n".format(model_name)) # compile model if learning_rate_decay: print("\nUsing rate decay\n") model.compile(optimizer=tf.keras.optimizers.Adam( learning_rate=learning_rate, decay=learning_rate / num_epochs, beta_1=0.9, beta_2=0.999), loss='mean_squared_error', metrics=[IoU]) else: model.compile(optimizer=tf.keras.optimizers.Adam( learning_rate=learning_rate, beta_1=0.9, beta_2=0.999), loss='mean_squared_error', metrics=[IoU]) # describe the model if (describe): describe_model(model) # fit model print("\nFitting multi_GPU=[{}] model with bs={},epochs={},lr={}\n".format( str(multi_gpu), str(batch_size), str(num_epochs), learning_rate)) t_start = time.time() h = model.fit(x_train, y_train, validation_data=(x_val, y_val), batch_size=batch_size, epochs=num_epochs, callbacks=callbacks) t_end = time.time() time_info = { 't_start': t_start, 't_end': t_end, 't_elapsed': (t_end - t_start) } if save: print("\nSaving model") save_model(model, "models/{}.h5".format(model_name)) # return a dictionary with the model instance as well return { 'name': model_name, 'train_info': h, 'time_info': time_info, "model": model }
def NN_GA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) activation = ['relu'] learning_rate = [5, 0.01, 0.1, 1, 2, 3, 4, 7, 10] algorithim = 'genetic_alg' iters = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ] nodes = [128, 128, 128, 128] population = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] mutation = [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1] outcomes = [] max_attempts = [10, 50, 100, 200, 500, 1000] clips = [5, 10, 100, 1000, 10000, 100000] act = 'relu' lr = 5 itera = 100 pop = 1500 mut = 0.1 ma = 100 clip = 5 seed = 1 while 1 == 1: iters_outs = {} for iter_test in iters: start = time.time() print(algorithim, act, lr, iter_test, ' ', pop, mut, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test, algorithm=algorithim, pop_size=pop, mutation_prob=mut, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) train_time = time.time() - start print('Train time', train_time) start = time.time() y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_train_query_time = time.time() - start print('y_train_query_time', y_train_query_time) start = time.time() y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) y_test_query_time = time.time() - start print('y_test_query_time', y_test_query_time) nn_loss = nn_model.loss print('loss', nn_loss) outcome = {} outcome['schedule'] = ' ' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = iter_test outcome['population'] = pop outcome['mutation'] = mut outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome[ 'runtime'] = train_time + y_train_query_time + y_test_query_time outcome['Train time'] = train_time outcome['y_train_query_time'] = y_train_query_time outcome['y_test_query_time'] = y_test_query_time outcome['loss'] = nn_loss outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GA-iteretsets.csv', mode='a', header=False) iters_outs[iter_test] = y_test_roc old_val = itera itera = max(iters_outs, key=iters_outs.get) print('best iter', itera, 'old', old_val) raise SystemExit(0) mut_outs = {} for mut_test in mutation: start = time.time() print(algorithim, act, lr, itera, ' ', pop, mut_test, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, pop_size=pop, mutation_prob=mut_test, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = ' ' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['population'] = pop outcome['mutation'] = mut_test outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False) mut_outs[mut_test] = y_test_roc old_val = mut mut = max(mut_outs, key=mut_outs.get) print('best mut', mut, 'old', old_val) clips_outs = {} for clip_test in clips: start = time.time() print(algorithim, act, lr, itera, ' ', pop, mut, ma, clip_test) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, pop_size=pop, mutation_prob=mut, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip_test, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = ' ' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['population'] = pop outcome['mutation'] = mut outcome['max_attempts'] = ma outcome['clip'] = clip_test outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False) clips_outs[clip_test] = y_test_roc old_val = clip clip = max(clips_outs, key=clips_outs.get) print('best clip', clip, 'old', old_val) maxa_outs = {} for maxa_test in max_attempts: start = time.time() print(algorithim, act, lr, itera, ' ', pop, mut, maxa_test, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, pop_size=pop, mutation_prob=mut, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=maxa_test, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = ' ' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['population'] = pop outcome['mutation'] = mut outcome['max_attempts'] = maxa_test outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False) maxa_outs[maxa_test] = y_test_roc old_val = ma ma = max(maxa_outs, key=maxa_outs.get) print('best ma', ma, 'old', old_val) lr_outs = {} for lr_test in learning_rate: start = time.time() print(algorithim, act, lr_test, itera, ' ', pop, mut, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, pop_size=pop, mutation_prob=mut, bias=True, is_classifier=True, learning_rate=lr_test, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = ' ' outcome['activation'] = act outcome['learning_rate'] = lr_test outcome['max_iters'] = itera outcome['population'] = pop outcome['mutation'] = mut outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False) lr_outs[lr_test] = y_test_roc old_lr = lr lr = max(lr_outs, key=lr_outs.get) print('best lr', lr, 'old', old_lr) pop_outs = {} for pop_test in population: start = time.time() print(algorithim, act, lr, itera, ' ', pop_test, mut, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, pop_size=pop_test, mutation_prob=mut, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = ' ' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['population'] = pop_test outcome['mutation'] = mut outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False) pop_outs[pop_test] = y_test_roc old_val = pop pop = max(pop_outs, key=pop_outs.get) print('best pop', pop, 'old', old_val)
# results save path root = opt.dataset + '_' + opt.save_root + '/' model = opt.dataset + '_' if not os.path.isdir(root): os.mkdir(root) if not os.path.isdir(root + 'Fixed_results'): os.mkdir(root + 'Fixed_results') # data_loader transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) ]) train_loader = util.data_load('data/' + opt.dataset, opt.train_subfolder, transform, opt.batch_size, shuffle=True) test_loader = util.data_load('data/' + opt.dataset, opt.test_subfolder, transform, opt.test_batch_size, shuffle=True) test = test_loader.__iter__().__next__()[0] img_size = test.size()[2] if opt.inverse_order: fixed_y_ = test[:, :, :, 0:img_size] fixed_x_ = test[:, :, :, img_size:] else: fixed_x_ = test[:, :, :, 0:img_size] fixed_y_ = test[:, :, :, img_size:]
def run_sl_algos(filename, result_col, full_param=False, test_all=False, debug=False, rDTree=True, pDTree={}, rknn=True, pknn={}, rSVM=True, pSVM={}, rNN=True, pNN={}, rBTree=True, pBTree={}, numFolds=10, njobs=-1, scalar=1, make_graphs=False, nolegend=False): print(filename, '-', scalar) start = time.time() X_train, X_test, y_train, y_test = util.data_load(filename, result_col, debug, scalar, make_graphs) print('data_load:', time.strftime("%H:%M:%S", time.gmtime(time.time() - start))) min_score = 1 max_score = 0 if rDTree: runTime, train_score, test_score = DTree.train_DTree( filename, X_train, X_test, y_train, y_test, full_param, debug, numFolds, njobs, scalar, make_graphs, pDTree) print('DTree: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, test_score) min_score = min(min_score, test_score) max_score = max(max_score, test_score) if rknn: runTime, train_score, test_score = knn.train_knn( filename, X_train, X_test, y_train, y_test, full_param, debug, numFolds, njobs, scalar, make_graphs, pknn) print('knn: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, test_score) min_score = min(min_score, test_score) max_score = max(max_score, test_score) if rBTree: runTime, train_score, test_score = BTree.train_BTree( filename, X_train, X_test, y_train, y_test, full_param, debug, numFolds, njobs, scalar, make_graphs, pBTree) print('BTree: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, test_score) min_score = min(min_score, test_score) max_score = max(max_score, test_score) if rSVM: L_score, R_Score, S_Score, P_Score = 0, 0, 0, 0 # runTime, train_score, L_score = SVM.train_svm(filename, X_train, X_test, y_train, y_test, 'linear', # full_param, debug, numFolds, njobs, scalar, make_graphs) # print('SVM-L: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, L_score) # In 14 data sets, the most common scoring was rbf > poly > linear > sigmoid if len(pSVM) > 0: runTime, train_score, R_Score = SVM.train_svm( filename, X_train, X_test, y_train, y_test, pSVM['kernel'], full_param, debug, numFolds, njobs, scalar, make_graphs, pSVM) print('SVM: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, R_Score) else: # In 14 data sets, the most common scoring was rbf > poly > linear > sigmoid runTime, train_score, R_Score = SVM.train_svm( filename, X_train, X_test, y_train, y_test, 'rbf', full_param, debug, numFolds, njobs, scalar, make_graphs) print('SVM-R: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, R_Score) if test_all or R_Score < 0.8: runTime, train_score, P_Score = SVM.train_svm( filename, X_train, X_test, y_train, y_test, 'poly', full_param, debug, numFolds, njobs, scalar, make_graphs) print('SVM-P: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, P_Score) if test_all or max(R_Score, P_Score) < 0.9: runTime, train_score, L_score = SVM.train_svm( filename, X_train, X_test, y_train, y_test, 'linear', full_param, debug, numFolds, njobs, scalar, make_graphs) print('SVM-L: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, L_score) if test_all or max(R_Score, P_Score, L_score) < 0.9: runTime, train_score, S_Score = SVM.train_svm( filename, X_train, X_test, y_train, y_test, 'sigmoid', full_param, debug, numFolds, njobs, scalar, make_graphs) print('SVM-S: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, S_Score) overall_score = max(L_score, R_Score, S_Score, P_Score) min_score = min(min_score, overall_score) max_score = max(max_score, overall_score) if rNN: A_score, S_Score = 0, 0 if len(pNN) > 0: runTime, train_score, R_Score = NN.train_NN( filename, X_train, X_test, y_train, y_test, pNN['solver'], full_param, debug, numFolds, njobs, scalar, make_graphs, pNN, nolegend) print('NN: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, R_Score) else: # In 10 out of 14 data sets adam was better than sgd, so testing it first runTime, train_score, S_Score = NN.train_NN( filename, X_train, X_test, y_train, y_test, 'adam', full_param, debug, numFolds, njobs, scalar, make_graphs) print('NN-A: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, S_Score) if test_all or S_Score < 0.6: runTime, train_score, A_score = NN.train_NN( filename, X_train, X_test, y_train, y_test, 'sgd', full_param, debug, numFolds, njobs, scalar, make_graphs) print('NN-S: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, A_score) overall_score = max(A_score, S_Score) min_score = min(min_score, overall_score) max_score = max(max_score, overall_score) print('Overall Variance: ', round(max_score - min_score, 4), '\n')
os.mkdir(root + 'test_results/AtoB') if not os.path.isdir(root + 'test_results/BtoA'): os.mkdir(root + 'test_results/BtoA') if not os.path.isdir(root + 'train_results'): os.mkdir(root + 'train_results') if not os.path.isdir(root + 'train_results/AtoB'): os.mkdir(root + 'train_results/AtoB') if not os.path.isdir(root + 'train_results/BtoA'): os.mkdir(root + 'train_results/BtoA') # data_loader transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) ]) train_loader_A = util.data_load('data/' + opt.dataset, opt.train_subfolder + 'A', transform, opt.batch_size, shuffle=True) train_loader_B = util.data_load('data/' + opt.dataset, opt.train_subfolder + 'B', transform, opt.batch_size, shuffle=True) test_loader_A = util.data_load('data/' + opt.dataset, opt.test_subfolder + 'A', transform, opt.batch_size, shuffle=False) test_loader_B = util.data_load('data/' + opt.dataset, opt.test_subfolder + 'B', transform, opt.batch_size, shuffle=False) # network G_A = network.generator(opt.input_ngc, opt.output_ngc, opt.ngf, opt.nb) G_B = network.generator(opt.input_ngc, opt.output_ngc, opt.ngf, opt.nb) D_A = network.discriminator(opt.input_ndc, opt.output_ndc, opt.ndf) D_B = network.discriminator(opt.input_ndc, opt.output_ndc, opt.ndf) G_A.weight_init(mean=0.0, std=0.02) G_B.weight_init(mean=0.0, std=0.02) D_A.weight_init(mean=0.0, std=0.02) D_B.weight_init(mean=0.0, std=0.02) G_A.cuda() G_B.cuda()
def run_ul_algos(filename, result_col, debug=False, numFolds=10, njobs=-1, scalar=1, make_graphs=False, nolegend=False, verbose=False, random_seed=1, rNN=False, pNN={}): np.random.seed(random_seed) random.seed(random_seed) X_train, X_test, y_train, y_test = util.data_load(filename, result_col, debug, scalar, make_graphs, random_seed) vis.gen_vis(X_train, X_test, random_seed, filename[:-4]) clustering.ul_Kmeans(X_train, y_train, random_seed, filename[:-4], result_col, verbose) clustering.ul_EM(X_train, y_train, random_seed, filename[:-4], result_col, verbose) dimred.ulPCA(X_train, y_train, random_seed, filename[:-4], verbose) dimred.ulICA(X_train, y_train, random_seed, filename[:-4], verbose) dimred.randProj(X_train, y_train, random_seed, filename[:-4], verbose) dimred.ul_LLE(X_train, y_train, random_seed, filename[:-4], verbose) new_Xtrain = copy.deepcopy(X_train) new_ytrain = copy.deepcopy(y_train) dr_cluster.pca_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose) new_Xtrain = copy.deepcopy(X_train) new_ytrain = copy.deepcopy(y_train) dr_cluster.ica_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose) new_Xtrain = copy.deepcopy(X_train) new_ytrain = copy.deepcopy(y_train) dr_cluster.rp_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose) new_Xtrain = copy.deepcopy(X_train) new_ytrain = copy.deepcopy(y_train) dr_cluster.lle_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose) # Run NNs for Dim Reduction if rNN: for n in range(1, 21): print('PCA', n) new_Xtrain = copy.deepcopy(X_train) new_Xtest = copy.deepcopy(X_test) new_ytrain = copy.deepcopy(y_train) new_ytest = copy.deepcopy(y_test) DR_NN.train_NN_PCA(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest, random_seed=random_seed, scalar=scalar, njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend, pNN=pNN, num_dim=n) print('ICA', n) new_Xtrain = copy.deepcopy(X_train) new_Xtest = copy.deepcopy(X_test) new_ytrain = copy.deepcopy(y_train) new_ytest = copy.deepcopy(y_test) DR_NN.train_NN_ICA(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest, random_seed=random_seed, scalar=scalar, njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend, pNN=pNN, num_dim=n) print('RP', n) new_Xtrain = copy.deepcopy(X_train) new_Xtest = copy.deepcopy(X_test) new_ytrain = copy.deepcopy(y_train) new_ytest = copy.deepcopy(y_test) DR_NN.train_NN_RP(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest, random_seed=random_seed, scalar=scalar, njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend, pNN=pNN, num_dim=n) for n in range(1, 21): print('LLE', n) new_Xtrain = copy.deepcopy(X_train) new_Xtest = copy.deepcopy(X_test) new_ytrain = copy.deepcopy(y_train) new_ytest = copy.deepcopy(y_test) DR_NN.train_NN_LLE(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest, random_seed=random_seed, scalar=scalar, njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend, pNN=pNN, num_dim=n) # Run NN for clustering if rNN: for n in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]: print('kmeans', n) new_Xtrain = copy.deepcopy(X_train) new_Xtest = copy.deepcopy(X_test) new_ytrain = copy.deepcopy(y_train) new_ytest = copy.deepcopy(y_test) cluster_NN.train_kmeansNN(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest, random_seed=random_seed, scalar=scalar, debug=verbose, njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend, pNN=pNN, num_clusts=n) print('diag') # cov_types = ['diag', 'tied', 'full', 'spherical'] # for cov in cov_types: for n in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]: print('diag', n) new_Xtrain = copy.deepcopy(X_train) new_Xtest = copy.deepcopy(X_test) new_ytrain = copy.deepcopy(y_train) new_ytest = copy.deepcopy(y_test) cluster_NN.train_EM_NN(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest, random_seed=random_seed, scalar=scalar, njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend, pNN=pNN, num_clusts=n, cov_type='diag')
def NN_SA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) activation = 'relu' learning_rate = [0.01, 0.1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] algorithim = 'simulated_annealing' iters = [ 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000 ] nodes = [128, 128, 128, 128] temperatures = [0.001, 0.01] decay_rates = [0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99] outcomes = [] max_attempts = [10, 50, 100, 200, 500, 1000] clips = [5, 10, 100, 1000, 10000, 100000] csv_file = 'NN_SA-itertests.csv' act = 'relu' lr = 10 itera = 10000 temp = 10000 dec = 0.92 ma = 50 clip = 10 while 1 == 1: iters_outs = {} for iter_test in iters: start = time.time() print(algorithim, act, lr, iter_test, 'GeomDecay', temp, dec, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) train_time = time.time() - start print('Train time', train_time) start = time.time() y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_train_query_time = time.time() - start print('y_train_query_time', y_train_query_time) start = time.time() y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) y_test_query_time = time.time() - start print('y_test_query_time', y_test_query_time) nn_loss = nn_model.loss print('loss', nn_loss) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = iter_test outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome[ 'runtime'] = train_time + y_train_query_time + y_test_query_time outcome['Train time'] = train_time outcome['y_train_query_time'] = y_train_query_time outcome['y_test_query_time'] = y_test_query_time outcome['loss'] = nn_loss outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) iters_outs[iter_test] = y_test_roc old_val = itera itera = max(iters_outs, key=iters_outs.get) print('best iter', itera, 'old', old_val) raise SystemExit(0) temp_outs = {} for temp_test in temperatures: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp_test, dec, ma, clip) nn_model = mlrose.NeuralNetwork( hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay(init_temp=temp_test, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp_test outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) temp_outs[temp_test] = y_test_roc old_temp = temp temp = max(temp_outs, key=temp_outs.get) print('best temp', temp, 'old', old_temp) decay_outs = {} for decay_test in decay_rates: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp, decay_test, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=decay_test), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = decay_test outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) decay_outs[decay_test] = y_test_roc old_val = dec dec = max(decay_outs, key=decay_outs.get) print('best decay', dec, 'old', old_val) clips_outs = {} for clip_test in clips: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp, dec, ma, clip_test) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip_test, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip_test outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) clips_outs[clip_test] = y_test_roc old_val = clip clip = max(clips_outs, key=clips_outs.get) print('best clip', clip, 'old', old_val) maxa_outs = {} for maxa_test in max_attempts: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp, dec, maxa_test, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=maxa_test, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = maxa_test outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) maxa_outs[maxa_test] = y_test_roc old_val = ma ma = max(maxa_outs, key=maxa_outs.get) print('best ma', ma, 'old', old_val) lr_outs = {} for lr_test in learning_rate: start = time.time() print(algorithim, act, lr_test, itera, 'GeomDecay', temp, dec, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr_test, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr_test outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) lr_outs[lr_test] = y_test_roc old_lr = lr lr = max(lr_outs, key=lr_outs.get) print('best lr', lr, 'old', old_lr)
def NN_GA(file_name, classifier_col): # HPs shared by all nodes = [128, 128, 128, 128] act = 'relu' seed = 1 # GD HPs gd_algo = 'gradient_descent' gd_lr = 0.00000009 gd_iter = 10000 gd_ma = 50 gd_clip = 5 # RHC HPs rhc_algo = 'random_hill_climb' rhc_lr = 8 rhc_iter = 10000 rhc_restarts = 10 rhc_ma = 100 rhc_clip = 100 # SA HPs sa_algo = 'simulated_annealing' sa_lr = 10 sa_iter = 10000 sa_temp = 10000 sa_decay = 0.92 sa_ma = 50 sa_clip = 10 # GA HPs ga_algo = 'genetic_alg' ga_lr = 5 ga_iter = 100 ga_pop = 1500 ga_mut = 0.1 ga_ma = 100 ga_clip = 5 X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col, random_seed=seed) # Best GD Algorithm print('Training GD NN') gd_start = time.time() gd_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=gd_algo, max_iters=gd_iter, learning_rate=gd_lr, clip_max=gd_clip, max_attempts=gd_ma) gd_nn_model.fit(X_train, y_train) print('gd loss:', gd_nn_model.loss) gd_train_time = time.time() - gd_start print('gd_train_time:', gd_train_time) start = time.time() gd_y_train_pred = gd_nn_model.predict(X_train) gd_y_train_roc = roc_auc_score(y_train, gd_y_train_pred, multi_class="ovr", average="weighted") gd_y_train_query_time = time.time() - start print('gd_y_train_roc', gd_y_train_roc, 'gd_y_train_roc: ', gd_y_train_query_time) start = time.time() gd_y_test_pred = gd_nn_model.predict(X_test) gd_y_test_roc = roc_auc_score(y_test, gd_y_test_pred, multi_class="ovr", average="weighted") gd_y_test_query_time = time.time() - start print('gd_y_test_roc', gd_y_test_roc, 'gd_y_test_query_time: ', gd_y_test_query_time) # Best RHC Algorithm print('Training RHC NN') rhc_start = time.time() rhc_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=rhc_algo, max_iters=rhc_iter, learning_rate=rhc_lr, clip_max=rhc_clip, max_attempts=rhc_ma, restarts=rhc_restarts) rhc_nn_model.fit(X_train, y_train) print('rhc loss:', rhc_nn_model.loss) rhc_train_time = time.time() - rhc_start print('rhc_train_time:', rhc_train_time) start = time.time() rhc_y_train_pred = rhc_nn_model.predict(X_train) rhc_y_train_roc = roc_auc_score(y_train, rhc_y_train_pred, multi_class="ovr", average="weighted") rhc_y_train_query_time = time.time() - start print('rhc_y_train_roc', rhc_y_train_roc, 'rhc_y_train_roc: ', rhc_y_train_query_time) start = time.time() rhc_y_test_pred = rhc_nn_model.predict(X_test) rhc_y_test_roc = roc_auc_score(y_test, rhc_y_test_pred, multi_class="ovr", average="weighted") rhc_y_test_query_time = time.time() - start print('rhc_y_test_roc', rhc_y_test_roc, 'rhc_y_test_query_time: ', rhc_y_test_query_time) # Best SA Algorithm print('Training SA NN') sa_start = time.time() sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo, max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma, schedule=mlrose.GeomDecay( init_temp=sa_temp, decay=sa_decay)) sa_nn_model.fit(X_train, y_train) print('sa loss:', sa_nn_model.loss) sa_train_time = time.time() - sa_start print('sa_train_time:', sa_train_time) start = time.time() sa_y_train_pred = sa_nn_model.predict(X_train) sa_y_train_roc = roc_auc_score(y_train, sa_y_train_pred, multi_class="ovr", average="weighted") sa_y_train_query_time = time.time() - start print('sa_y_train_roc', sa_y_train_roc, 'sa_y_train_roc: ', sa_y_train_query_time) start = time.time() sa_y_test_pred = sa_nn_model.predict(X_test) sa_y_test_roc = roc_auc_score(y_test, sa_y_test_pred, multi_class="ovr", average="weighted") sa_y_test_query_time = time.time() - start print('sa_y_test_roc', sa_y_test_roc, 'sa_y_test_query_time: ', sa_y_test_query_time) # Best Genetic Algorithm print('Training GA NN') ga_start = time.time() ga_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=ga_algo, max_iters=ga_iter, learning_rate=ga_lr, clip_max=ga_clip, max_attempts=ga_ma, pop_size=ga_pop, mutation_prob=ga_mut) ga_nn_model.fit(X_train, y_train) print('ga loss:', ga_nn_model.loss) ga_train_time = time.time() - ga_start print('ga_train_time:', ga_train_time) start = time.time() ga_y_train_pred = ga_nn_model.predict(X_train) ga_y_train_roc = roc_auc_score(y_train, ga_y_train_pred, multi_class="ovr", average="weighted") ga_y_train_query_time = time.time() - start print('ga_y_train_roc', ga_y_train_roc, 'ga_y_train_roc: ', ga_y_train_query_time) start = time.time() ga_y_test_pred = ga_nn_model.predict(X_test) ga_y_test_roc = roc_auc_score(y_test, ga_y_test_pred, multi_class="ovr", average="weighted") ga_y_test_query_time = time.time() - start print('ga_y_test_roc', ga_y_test_roc, 'ga_y_test_query_time: ', ga_y_test_query_time) # Plot Loss Curves plt.figure() plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve') plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve') plt.plot(gd_nn_model.fitness_curve, label='GD Loss Curve') plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve') plt.title("Neural Network Loss Curves") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.xscale('log') plt.savefig("Images\\Neural Network Loss Curves") plt.show() # Plot Loss Curves (GD Inverted) plt.figure() gd_curve = gd_nn_model.fitness_curve inverted_gd_curve = np.array(gd_curve) * -1 plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve') plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve') plt.plot(inverted_gd_curve, label='GD Loss Curve') plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve') plt.title("Neural Network Loss Curves") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.xscale('log') plt.savefig("Images\\Neural Network Loss Curves-inverted GD") plt.show() # Plot Loss Curves - No GD plt.figure() plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve') plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve') plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve') plt.title("Neural Network Loss Curves") plt.xlabel('Iterations') plt.ylabel('Loss') plt.ylim(bottom=0) plt.grid(True) plt.legend() plt.xscale('log') plt.savefig("Images\\Neural Network Loss Curves - No GD") plt.show()
parser = argparse.ArgumentParser() parser.add_argument('--dataset', required=False, default='maps', help='') parser.add_argument('--test_subfolder', required=False, default='val', help='') parser.add_argument('--ngf', type=int, default=64) parser.add_argument('--input_size', type=int, default=256, help='input size') parser.add_argument('--save_root', required=False, default='results', help='results save path') parser.add_argument('--inverse_order', type=bool, default=True, help='0: [input, target], 1 - [target, input]') opt = parser.parse_args() print(opt) # data_loader transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) ]) test_loader = util.data_load('data/' + opt.dataset, opt.test_subfolder, transform, batch_size=1, shuffle=False) if not os.path.isdir(opt.dataset + '_results/test_results'): os.mkdir(opt.dataset + '_results/test_results') G = network.generator(opt.ngf) G.cuda() G.load_state_dict(torch.load(opt.dataset + '_results/' + opt.dataset + '_generator_param.pkl')) # network n = 0 print('test start!') for x_, _ in test_loader: if opt.inverse_order: y_ = x_[:, :, :, :x_.size()[2]] x_ = x_[:, :, :, x_.size()[2]:]
def NN_RHC(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) activation = ['relu'] learning_rate = [6, 7, 5, 8, 9, 1, 2, 3, 4, 10] algorithim = 'random_hill_climb' iters = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000] nodes = [128, 128, 128, 128] restarts = [100, 1000] outcomes = [] max_attempts = [100, 200, 500, 1000] clips = [5, 10, 100, 1000, 10000, 100000] act = 'relu' lr = 8 itera = 10000 res = 10 ma = 100 clip = 100 seed = 1 while 1 == 1: iters_outs = {} for iter_test in iters: start = time.time() print(algorithim, act, lr, iter_test, 'GeomDecay', res, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test, algorithm=algorithim, restarts=res, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) train_time = time.time() - start print('Train time', train_time) start = time.time() y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_train_query_time = time.time() - start print('y_train_query_time', y_train_query_time) start = time.time() y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) y_test_query_time = time.time() - start print('y_test_query_time', y_test_query_time) nn_loss = nn_model.loss print('loss', nn_loss) outcome = {} outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = iter_test outcome['restarts'] = res outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = train_time + y_train_query_time + y_test_query_time outcome['Train time'] = train_time outcome['y_train_query_time'] = y_train_query_time outcome['y_test_query_time'] = y_test_query_time outcome['loss'] = nn_loss outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN_RHC-itertests.csv', mode='a', header=False) iters_outs[iter_test] = y_test_roc old_val = itera itera = max(iters_outs, key=iters_outs.get) print('best iter', itera, 'old', old_val) raise SystemExit(0) res_outs = {} for res_test in restarts: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', res_test, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, restarts=res_test, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['restarts'] = res_test outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN_RHC.csv', mode='a', header=False) res_outs[res_test] = y_test_roc old_val = res res = max(res_outs, key=res_outs.get) print('best temp', res, 'old', old_val) clips_outs = {} for clip_test in clips: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', res, ma, clip_test) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, restarts=res, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip_test, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['restarts'] = res outcome['max_attempts'] = ma outcome['clip'] = clip_test outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN_RHC.csv', mode='a', header=False) clips_outs[clip_test] = y_test_roc old_val = clip clip = max(clips_outs, key=clips_outs.get) print('best clip', clip, 'old', old_val) maxa_outs = {} for maxa_test in max_attempts: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', res, maxa_test, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, restarts=res, bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=maxa_test, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['restarts'] = res outcome['max_attempts'] = maxa_test outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN_RHC.csv', mode='a', header=False) maxa_outs[maxa_test] = y_test_roc old_val = ma ma = max(maxa_outs, key=maxa_outs.get) print('best ma', ma, 'old', old_val) lr_outs = {} for lr_test in learning_rate: start = time.time() print(algorithim, act, lr_test, itera, 'GeomDecay', res, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, restarts=res, bias=True, is_classifier=True, learning_rate=lr_test, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=seed, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr_test outcome['max_iters'] = itera outcome['restarts'] = res outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv('NN_RHC.csv') lr_outs[lr_test] = y_test_roc old_lr = lr lr = max(lr_outs, key=lr_outs.get) print('best lr', lr, 'old', old_lr)