def NN_SA(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col)
# SA HPs
nodes = [128, 128, 128, 128]
act = 'relu'
seed = 1
sa_algo = 'simulated_annealing'
sa_lr = 10
sa_iter = 10000
sa_temp = 10000
sa_decay = 0.92
sa_ma = 50
sa_clip = 10
temperature = [0.1, 1, 10, 100, 1000, 10000]
plt.figure()
for t in temperature:
print('temperature', t)
sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True,
is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo,
max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma,
schedule=mlrose.GeomDecay(init_temp=t, decay=sa_decay))
sa_nn_model.fit(X_train, y_train)
plt.plot(sa_nn_model.fitness_curve, label='temp =' + str(t))
plt.title("NN SA - Temperature")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid(True)
plt.legend()
plt.savefig("Images\\NN - SA - Temperature")
plt.xscale('log')
plt.savefig("Images\\NN - SA - Temperature - log")
plt.show()
decay_rates = [0.1, 0.2, 0.3, 0.4, 0.5, 0.8, 0.92]
plt.figure()
for dr in decay_rates:
print('decay', dr)
sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True,
is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo,
max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma,
schedule=mlrose.GeomDecay(init_temp=sa_temp, decay=dr))
sa_nn_model.fit(X_train, y_train)
plt.plot(sa_nn_model.fitness_curve, label='decay rate =' + str(dr))
plt.title("NN SA - Decay Rate")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid(True)
plt.legend()
plt.savefig("Images\\NN - SA - Decay Rate")
plt.xscale('log')
plt.savefig("Images\\NN - SA - Decay Rate - log")
plt.show()
def main():
opts = util.parse_args()
X, y = util.data_load(opts.dataset)
fc_nn_model = fc.create_model()
ada_model = AdaBoostClassifier(n_estimators=100, random_state=0)
svm_model = SVC(C=1000, gamma=0.1)
n = opts.upsamplen if opts.upsamplen is not None else 1
start = n if opts.upsamplestart is None else 1
if start > n:
print("unsample range error")
sys.exit()
conf_fc, conf_ada, conf_svm = [], [], []
for t in np.arange(start, n + 1):
needed = util.needed_n(X, y, t)
temp_X, temp_y = util.upsample(X, y, needed)
X_train, X_test, y_train, y_test = train_test_split(temp_X,
temp_y,
test_size=0.3,
random_state=42)
X_train, X_test = util.normalize(X_train, X_test)
train_dset = tf.data.Dataset.from_tensor_slices(
(X_train,
y_train)).batch(64,
drop_remainder=False).shuffle(buffer_size=10000)
test_dset = tf.data.Dataset.from_tensor_slices(
(X_test, y_test)).batch(64)
ada_model.fit(X_train, y_train)
svm_model.fit(X_train, y_train)
fc_nn_model.fit(train_dset, epochs=10)
pred_ada = ada_model.predict(X_test)
pred_svm = svm_model.predict(X_test)
conf_ada.append(confusion_matrix(y_test, pred_ada))
conf_svm.append(confusion_matrix(y_test, pred_svm))
temp = np.zeros((2, 2), dtype=int)
for d, labels in test_dset:
predictions = fc_nn_model(d)
for i in range(len(d)):
temp[labels[i]][np.argmax(predictions[i])] += 1
conf_fc.append(temp)
recall_fc = list(map(lambda x: util.recall(x), conf_fc))
recall_ada = list(map(lambda x: util.recall(x), conf_ada))
recall_svm = list(map(lambda x: util.recall(x), conf_svm))
up_range = np.arange(start, n + 1)
d = {"SVM": recall_svm, "Adaboost": recall_ada, "FC_NN": recall_fc}
legends = ["SVM", "Adaboost", "FC_NN"]
for key in d:
plt.plot(up_range, d[key])
plt.title("Recall Vs Upsample Graph")
plt.xlabel("Upsample rate")
plt.ylabel("Recall")
plt.legend(legends)
plt.show()
def main():
opts = util.parse_args()
X, y = util.data_load(opts.dataset)
model = create_model()
model_layers = create_model_more_layers()
n = opts.upsamplen if opts.upsamplen is not None else 1
start = n if opts.upsamplestart is None else 1
all_conf = []
all_conf_layers = []
if start > n:
print("Upsample start should be larger than end")
sys.exit()
for t in np.arange(start, n + 1):
print("t", t)
needed = util.needed_n(X, y, t)
temp_X, temp_y = util.upsample(X, y, needed)
X_train, X_test, y_train, y_test = train_test_split(temp_X,
temp_y,
test_size=0.3,
random_state=42)
X_train, X_test = util.normalize(X_train, X_test)
train_dset = tf.data.Dataset.from_tensor_slices(
(X_train,
y_train)).batch(64,
drop_remainder=False).shuffle(buffer_size=10000)
test_dset = tf.data.Dataset.from_tensor_slices(
(X_test, y_test)).batch(64)
model.fit(train_dset, epochs=10)
model_layers.fit(train_dset, epochs=10)
conf_mat = np.zeros((2, 2), dtype=int)
conf_mat_layers = np.zeros((2, 2), dtype=int)
for d, labels in test_dset:
predictions = model(d)
predictions_layers = model_layers(d)
for i in range(len(d)):
conf_mat[labels[i]][np.argmax(predictions[i])] += 1
conf_mat_layers[labels[i]][np.argmax(
predictions_layers[i])] += 1
all_conf.append(conf_mat)
all_conf_layers.append(conf_mat_layers)
re_fc, re_fc_layer = list(map(lambda x: util.recall(x), all_conf)), list(
(map(lambda x: util.recall(x), all_conf_layers)))
up_range = np.arange(start, n + 1)
plt.plot(up_range, re_fc)
plt.plot(up_range, re_fc_layer)
plt.title("2-layer NN vs 3-layer NN")
plt.legend(["2-layer", "3-layer"])
plt.xlabel("Upsample rate")
plt.ylabel("Recall")
plt.show()
def NN_GD(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name,
classifier_col)
# GD HPs
nodes = [128, 128, 128, 128]
act = 'relu'
seed = 1
gd_algo = 'gradient_descent'
gd_lr = 0.00000009
gd_iter = 10000
gd_ma = 50
gd_clip = 5
learning_rates = [
0.00000009, 0.00000001, 0.000000001, 0.0000001, 0.000001, 0.00001,
0.0001, 0.001, 0.01, 0.1
]
plt.figure()
for lr in learning_rates:
print('lr', lr)
gd_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
random_state=seed,
bias=True,
is_classifier=True,
early_stopping=True,
curve=True,
algorithm=gd_algo,
max_iters=gd_iter,
learning_rate=lr,
clip_max=gd_clip,
max_attempts=gd_ma)
gd_nn_model.fit(X_train, y_train)
gd_curve = gd_nn_model.fitness_curve
inverted_gd_curve = np.array(gd_curve) * -1
plt.plot(inverted_gd_curve, label='lr =' + str(lr))
plt.title("NN GD - Learning Rates")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid(True)
plt.legend()
plt.savefig("Images\\NN - GD - Learning Rates")
plt.xscale('log')
plt.savefig("Images\\NN - GD - Learning Rates - log")
plt.show()
def NN_GA(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col)
activation = ['relu']
learning_rate = [5, 0.01, 0.1, 1, 2, 3, 4, 7, 10]
algorithim = 'genetic_alg'
iters = [1000, 10000, 50000, 100000]
nodes = [128, 128, 128, 128]
population = [2000, 2100,2200,2300]
mutation = [ 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1]
outcomes = []
max_attempts = [10, 50, 100, 200, 500, 1000]
clips = [5, 10, 100, 1000, 10000, 100000]
# GA HPs
nodes = [128, 128, 128, 128]
act = 'relu'
seed = 1
ga_algo = 'genetic_alg'
ga_lr = 5
ga_iter = 100
ga_pop = 1500
ga_mut = 0.1
ga_ma = 100
ga_clip = 5
Population = [100, 200, 300, 400, 500, 750, 1000, 1240, 1500]
plt.figure()
for p in Population:
print('Population', p)
ga_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=ga_iter,
algorithm=ga_algo, pop_size=p, mutation_prob=ga_mut,
bias=True, is_classifier=True, learning_rate=ga_lr,
early_stopping=True, clip_max=ga_clip, max_attempts=ga_ma,
random_state=seed, curve=True)
ga_nn_model.fit(X_train, y_train)
plt.plot(ga_nn_model.fitness_curve, label='pop =' + str(p))
plt.title("NN GA - Population")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid(True)
plt.legend()
plt.savefig("Images\\NN - GA - Population")
plt.xscale('log')
plt.savefig("Images\\NN - GA - Population - log")
plt.show()
def main():
opts = util.parse_args()
X, y = util.data_load(opts.dataset)
#set upsample range
n = opts.upsamplen if opts.upsamplen is not None else 1
start = n if opts.upsamplestart is None else 1
if start > n:
print("unsample range error")
sys.exit()
for i in np.arange(start, n + 1):
print("i", i)
needed = util.needed_n(X, y, i)
print("needed", needed)
print("pre-upsampled len x", len(X))
upsampled_X, upsampled_y = util.upsample(X, y, needed)
print("length of upsampled_X", len(upsampled_X))
#use a good param to improve speed
X_train, X_test, y_train, y_test = train_test_split(upsampled_X,
upsampled_y,
test_size=0.3,
random_state=42)
X_train, X_test = util.normalize(X_train, X_test)
model = SVC(C=1000, gamma=0.1)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
cm = confusion_matrix(y_test, predictions)
report = classification_report(y_test, predictions)
#precision-recall curve
y_score = model.decision_function(X_test)
average_precision = average_precision_score(y_test, y_score)
disp = plot_precision_recall_curve(model, X_test, y_test)
disp.ax_.set_title('Precision-Recall curve with 10x upsampling: '
'AP={0:0.2f}'.format(average_precision))
plt.show()
#roc curve
# svc_roc=plot_roc_curve(model,X_test,y_test)
# svc_roc.ax_.set_title('SVM ROC Curve with 10x upsampling')
# plt.show()
print(cm)
print(report)
'''
def main():
opts = util.parse_args()
X, y = util.data_load(opts.dataset)
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
for train_index, test_index in skf.split(X, y):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
X_train, X_test = util.normalize(X_train, X_test)
clf = AdaBoostClassifier(n_estimators=100, random_state=0)
clf.fit(X_train, y_train)
"""
conf_mat = np.zeros((2, 2))
for i in range(len(X_test)):
pred = clf.predict(X_test[i])
true = y_test[i]
conf_mat[true][pred] += 1
print(conf_mat)
"""
predictions = clf.predict(X_test)
conf_mat = confusion_matrix(y_test, predictions)
print(conf_mat)
def main():
opts = util.parse_args()
X, y = util.data_load(opts.dataset)
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
model = create_model()
for train_index, test_index in skf.split(X, y):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
X_train, X_test = util.normalize(X_train, X_test)
train_dset = tf.data.Dataset.from_tensor_slices(
(X_train,
y_train)).batch(64,
drop_remainder=False).shuffle(buffer_size=10000)
test_dset = tf.data.Dataset.from_tensor_slices(
(X_test, y_test)).batch(64)
model.fit(train_dset, epochs=10)
conf_mat = np.zeros((2, 2), dtype=int)
for d, labels in test_dset:
predictions = model(d)
for i in range(len(d)):
conf_mat[labels[i]][np.argmax(predictions[i])] += 1
print(conf_mat)
def main():
opts = util.parse_args()
X, y = util.data_load(opts.dataset)
n = opts.upsamplen if opts.upsamplen is not None else 1
start = n if opts.upsamplestart is None else 1
if start > n:
print("Upsample start should be larger than end")
sys.exit()
thresh = opts.threshold if opts.threshold is not None and opts.threshold >= 0.40 else None
for t in np.arange(start, n + 1):
needed = util.needed_n(X, y, t)
temp_X, temp_y = util.upsample(X, y, needed)
X_train, X_test, y_train, y_test = train_test_split(temp_X,
temp_y,
test_size=0.3,
random_state=42)
X_train, X_test = util.normalize(X_train, X_test)
clf = AdaBoostClassifier(n_estimators=100, random_state=0)
clf.fit(X_train, y_train)
conf_upsample = []
if thresh is None:
predictions = clf.predict(X_test)
conf_mat = confusion_matrix(y_test, predictions)
conf_upsample.append(conf_mat)
print(conf_mat)
else:
conf_thresh = []
for i in np.arange(0.4, thresh + 0.01, 0.005):
predictions = (clf.predict_proba(X_test)[:, 1] >=
i).astype(int)
conf_mat = confusion_matrix(y_test, predictions)
conf_thresh.append(conf_mat)
print(i)
print(conf_mat)
util.get_roc_curve(conf_thresh, "Adaboost", "threshold")
plt.show()
def NN_GA(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col)
activation = ['relu']
learning_rate = [0.00000002, 0.00000003, 0.00000004, 0.00000005, 0.00000006, 0.00000007, 0.00000008, 0.00000009]
algorithim = 'gradient_descent'
iters = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000]
nodes = [128, 128, 128, 128]
outcomes = []
max_attempts = [10, 50, 100, 200, 500, 1000]
clips = [5, 10, 100, 1000, 10000, 100000]
clips = [4, 5]
act = 'relu'
lr = 0.00000009
itera = 10000
ma = 100
clip = 5
seed = 1
while 1 == 1:
iters_outs = {}
for iter_test in iters:
start = time.time()
print(algorithim, act, lr, iter_test, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test,
algorithm=algorithim,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
train_time = time.time() - start
print('Train time', train_time)
start = time.time()
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_train_query_time = time.time() - start
print('y_train_query_time', y_train_query_time)
start = time.time()
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
y_test_query_time = time.time() - start
print('y_test_query_time', y_test_query_time)
nn_loss = nn_model.loss
print('loss', nn_loss)
outcome = {}
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = iter_test
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = train_time + y_train_query_time + y_test_query_time
outcome['Train time'] = train_time
outcome['y_train_query_time'] = y_train_query_time
outcome['y_test_query_time'] = y_test_query_time
outcome['loss'] = nn_loss
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GD-itertests.csv')
iters_outs[iter_test] = y_test_roc
old_val = itera
itera = max(iters_outs, key=iters_outs.get)
print('best iter', itera, 'old', old_val)
raise SystemExit(0)
clips_outs = {}
for clip_test in clips:
start = time.time()
print(algorithim, act, lr, itera, ma, clip_test)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip_test, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['max_attempts'] = ma
outcome['clip'] = clip_test
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GD.csv')
clips_outs[clip_test] = y_test_roc
old_val = clip
clip = max(clips_outs, key=clips_outs.get)
print('best clip', clip, 'old', old_val)
maxa_outs = {}
for maxa_test in max_attempts:
start = time.time()
print(algorithim, act, lr, itera, maxa_test, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip, max_attempts=maxa_test,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['max_attempts'] = maxa_test
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GD.csv')
maxa_outs[maxa_test] = y_test_roc
old_val = ma
ma = max(maxa_outs, key=maxa_outs.get)
print('best ma', ma, 'old', old_val)
lr_outs = {}
for lr_test in learning_rate:
start = time.time()
print(algorithim, act, lr_test, itera, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim,
bias=True, is_classifier=True, learning_rate=lr_test,
early_stopping=True, clip_max=clip, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['activation'] = act
outcome['learning_rate'] = lr_test
outcome['max_iters'] = itera
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GD.csv')
lr_outs[lr_test] = y_test_roc
old_lr = lr
lr = max(lr_outs, key=lr_outs.get)
print('best lr', lr, 'old', old_lr)
def run_model(data_fraction=0.1,
batch_size=1,
num_epochs=10,
multi_gpu=False,
data=False,
learning_rate=0.001,
learning_rate_decay=False,
dropout=False,
tensorboard=False,
save=False,
describe=False,
model_id=None):
callbacks = None
if tensorboard:
print("\nUsing tensorboard\n")
import datetime
log_dir = "logs/fit/" + datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
callbacks = [tensorboard_callback]
# load the data
if (data):
# data was provided
print("\nUsing provided data\n")
x_train, y_train, x_val, y_val, x_test, y_test = data
else:
x_train, y_train, x_val, y_val, x_test, y_test = u.data_load(
f=data_fraction)
if (model_id == 'baseline'):
model_name = "v0_t" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
# get the model
_model = get_baseline_conv_model_1()
elif (model_id == 'baseline_vgg'):
model_name = "vBVGG_t" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
# get the model
_model = get_baseline_vgg_model_1(dropout=dropout)
elif (model_id == 'baseline_vgg_block3'):
model_name = "vBVGG3_t" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
# get the model
_model = get_baseline_vgg_model_3(dropout=dropout)
elif (model_id == 'baseline_vgg_block2'):
model_name = "vBVGG2_t" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
# get the model
_model = get_baseline_vgg_model_2(dropout=dropout)
elif (model_id == 'vgg_co_baseline'):
model_name = "vVCB" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
# get the model
_model = vgg_co_baseline_model(dropout=dropout)
elif (model_id == 'baseline_vgg_block4tr'):
model_name = "vBVGG4tr_t" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
_model = get_baseline_vgg_model_4tr(dropout=dropout)
elif (model_id == 'baseline_vgg_no_pool'):
model_name = "vBVNP_t" + str(len(x_train)) + "_e" + str(
num_epochs) + "_b" + str(batch_size) + "_lr" + str(learning_rate)
_model = get_baseline_vgg_model_no_pool(dropout=dropout)
else:
raise Exception("Must specify model id!")
if (learning_rate_decay):
model_name += "_lrd"
if (dropout):
model_name = "{}_d{}".format(model_name, dropout)
if (multi_gpu):
print("Creating multi GPU model")
model = keras.utils.multi_gpu_model(_model, gpus=2)
else:
model = _model
print("\nRuning model:: {}\n".format(model_name))
# compile model
if learning_rate_decay:
print("\nUsing rate decay\n")
model.compile(optimizer=tf.keras.optimizers.Adam(
learning_rate=learning_rate,
decay=learning_rate / num_epochs,
beta_1=0.9,
beta_2=0.999),
loss='mean_squared_error',
metrics=[IoU])
else:
model.compile(optimizer=tf.keras.optimizers.Adam(
learning_rate=learning_rate, beta_1=0.9, beta_2=0.999),
loss='mean_squared_error',
metrics=[IoU])
# describe the model
if (describe):
describe_model(model)
# fit model
print("\nFitting multi_GPU=[{}] model with bs={},epochs={},lr={}\n".format(
str(multi_gpu), str(batch_size), str(num_epochs), learning_rate))
t_start = time.time()
h = model.fit(x_train,
y_train,
validation_data=(x_val, y_val),
batch_size=batch_size,
epochs=num_epochs,
callbacks=callbacks)
t_end = time.time()
time_info = {
't_start': t_start,
't_end': t_end,
't_elapsed': (t_end - t_start)
}
if save:
print("\nSaving model")
save_model(model, "models/{}.h5".format(model_name))
# return a dictionary with the model instance as well
return {
'name': model_name,
'train_info': h,
'time_info': time_info,
"model": model
}
def NN_GA(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name,
classifier_col)
activation = ['relu']
learning_rate = [5, 0.01, 0.1, 1, 2, 3, 4, 7, 10]
algorithim = 'genetic_alg'
iters = [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100
]
nodes = [128, 128, 128, 128]
population = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]
mutation = [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1]
outcomes = []
max_attempts = [10, 50, 100, 200, 500, 1000]
clips = [5, 10, 100, 1000, 10000, 100000]
act = 'relu'
lr = 5
itera = 100
pop = 1500
mut = 0.1
ma = 100
clip = 5
seed = 1
while 1 == 1:
iters_outs = {}
for iter_test in iters:
start = time.time()
print(algorithim, act, lr, iter_test, ' ', pop, mut, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=iter_test,
algorithm=algorithim,
pop_size=pop,
mutation_prob=mut,
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=seed,
curve=True)
nn_model.fit(X_train, y_train)
train_time = time.time() - start
print('Train time', train_time)
start = time.time()
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_train_query_time = time.time() - start
print('y_train_query_time', y_train_query_time)
start = time.time()
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
y_test_query_time = time.time() - start
print('y_test_query_time', y_test_query_time)
nn_loss = nn_model.loss
print('loss', nn_loss)
outcome = {}
outcome['schedule'] = ' '
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = iter_test
outcome['population'] = pop
outcome['mutation'] = mut
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome[
'runtime'] = train_time + y_train_query_time + y_test_query_time
outcome['Train time'] = train_time
outcome['y_train_query_time'] = y_train_query_time
outcome['y_test_query_time'] = y_test_query_time
outcome['loss'] = nn_loss
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GA-iteretsets.csv',
mode='a',
header=False)
iters_outs[iter_test] = y_test_roc
old_val = itera
itera = max(iters_outs, key=iters_outs.get)
print('best iter', itera, 'old', old_val)
raise SystemExit(0)
mut_outs = {}
for mut_test in mutation:
start = time.time()
print(algorithim, act, lr, itera, ' ', pop, mut_test, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
pop_size=pop,
mutation_prob=mut_test,
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=seed,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = ' '
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['population'] = pop
outcome['mutation'] = mut_test
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False)
mut_outs[mut_test] = y_test_roc
old_val = mut
mut = max(mut_outs, key=mut_outs.get)
print('best mut', mut, 'old', old_val)
clips_outs = {}
for clip_test in clips:
start = time.time()
print(algorithim, act, lr, itera, ' ', pop, mut, ma, clip_test)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
pop_size=pop,
mutation_prob=mut,
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip_test,
max_attempts=ma,
random_state=seed,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = ' '
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['population'] = pop
outcome['mutation'] = mut
outcome['max_attempts'] = ma
outcome['clip'] = clip_test
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False)
clips_outs[clip_test] = y_test_roc
old_val = clip
clip = max(clips_outs, key=clips_outs.get)
print('best clip', clip, 'old', old_val)
maxa_outs = {}
for maxa_test in max_attempts:
start = time.time()
print(algorithim, act, lr, itera, ' ', pop, mut, maxa_test, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
pop_size=pop,
mutation_prob=mut,
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=maxa_test,
random_state=seed,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = ' '
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['population'] = pop
outcome['mutation'] = mut
outcome['max_attempts'] = maxa_test
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False)
maxa_outs[maxa_test] = y_test_roc
old_val = ma
ma = max(maxa_outs, key=maxa_outs.get)
print('best ma', ma, 'old', old_val)
lr_outs = {}
for lr_test in learning_rate:
start = time.time()
print(algorithim, act, lr_test, itera, ' ', pop, mut, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
pop_size=pop,
mutation_prob=mut,
bias=True,
is_classifier=True,
learning_rate=lr_test,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=seed,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = ' '
outcome['activation'] = act
outcome['learning_rate'] = lr_test
outcome['max_iters'] = itera
outcome['population'] = pop
outcome['mutation'] = mut
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False)
lr_outs[lr_test] = y_test_roc
old_lr = lr
lr = max(lr_outs, key=lr_outs.get)
print('best lr', lr, 'old', old_lr)
pop_outs = {}
for pop_test in population:
start = time.time()
print(algorithim, act, lr, itera, ' ', pop_test, mut, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
pop_size=pop_test,
mutation_prob=mut,
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=seed,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = ' '
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['population'] = pop_test
outcome['mutation'] = mut
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN-GA.csv', mode='a', header=False)
pop_outs[pop_test] = y_test_roc
old_val = pop
pop = max(pop_outs, key=pop_outs.get)
print('best pop', pop, 'old', old_val)
# results save path
root = opt.dataset + '_' + opt.save_root + '/'
model = opt.dataset + '_'
if not os.path.isdir(root):
os.mkdir(root)
if not os.path.isdir(root + 'Fixed_results'):
os.mkdir(root + 'Fixed_results')
# data_loader
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
train_loader = util.data_load('data/' + opt.dataset,
opt.train_subfolder,
transform,
opt.batch_size,
shuffle=True)
test_loader = util.data_load('data/' + opt.dataset,
opt.test_subfolder,
transform,
opt.test_batch_size,
shuffle=True)
test = test_loader.__iter__().__next__()[0]
img_size = test.size()[2]
if opt.inverse_order:
fixed_y_ = test[:, :, :, 0:img_size]
fixed_x_ = test[:, :, :, img_size:]
else:
fixed_x_ = test[:, :, :, 0:img_size]
fixed_y_ = test[:, :, :, img_size:]
def run_sl_algos(filename,
result_col,
full_param=False,
test_all=False,
debug=False,
rDTree=True,
pDTree={},
rknn=True,
pknn={},
rSVM=True,
pSVM={},
rNN=True,
pNN={},
rBTree=True,
pBTree={},
numFolds=10,
njobs=-1,
scalar=1,
make_graphs=False,
nolegend=False):
print(filename, '-', scalar)
start = time.time()
X_train, X_test, y_train, y_test = util.data_load(filename, result_col,
debug, scalar,
make_graphs)
print('data_load:',
time.strftime("%H:%M:%S", time.gmtime(time.time() - start)))
min_score = 1
max_score = 0
if rDTree:
runTime, train_score, test_score = DTree.train_DTree(
filename, X_train, X_test, y_train, y_test, full_param, debug,
numFolds, njobs, scalar, make_graphs, pDTree)
print('DTree: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, test_score)
min_score = min(min_score, test_score)
max_score = max(max_score, test_score)
if rknn:
runTime, train_score, test_score = knn.train_knn(
filename, X_train, X_test, y_train, y_test, full_param, debug,
numFolds, njobs, scalar, make_graphs, pknn)
print('knn: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, test_score)
min_score = min(min_score, test_score)
max_score = max(max_score, test_score)
if rBTree:
runTime, train_score, test_score = BTree.train_BTree(
filename, X_train, X_test, y_train, y_test, full_param, debug,
numFolds, njobs, scalar, make_graphs, pBTree)
print('BTree: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, test_score)
min_score = min(min_score, test_score)
max_score = max(max_score, test_score)
if rSVM:
L_score, R_Score, S_Score, P_Score = 0, 0, 0, 0
# runTime, train_score, L_score = SVM.train_svm(filename, X_train, X_test, y_train, y_test, 'linear',
# full_param, debug, numFolds, njobs, scalar, make_graphs)
# print('SVM-L: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, L_score)
# In 14 data sets, the most common scoring was rbf > poly > linear > sigmoid
if len(pSVM) > 0:
runTime, train_score, R_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, pSVM['kernel'],
full_param, debug, numFolds, njobs, scalar, make_graphs, pSVM)
print('SVM: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, R_Score)
else:
# In 14 data sets, the most common scoring was rbf > poly > linear > sigmoid
runTime, train_score, R_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'rbf', full_param,
debug, numFolds, njobs, scalar, make_graphs)
print('SVM-R: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, R_Score)
if test_all or R_Score < 0.8:
runTime, train_score, P_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'poly',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('SVM-P: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, P_Score)
if test_all or max(R_Score, P_Score) < 0.9:
runTime, train_score, L_score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'linear',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('SVM-L: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, L_score)
if test_all or max(R_Score, P_Score, L_score) < 0.9:
runTime, train_score, S_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'sigmoid',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('SVM-S: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, S_Score)
overall_score = max(L_score, R_Score, S_Score, P_Score)
min_score = min(min_score, overall_score)
max_score = max(max_score, overall_score)
if rNN:
A_score, S_Score = 0, 0
if len(pNN) > 0:
runTime, train_score, R_Score = NN.train_NN(
filename, X_train, X_test, y_train, y_test, pNN['solver'],
full_param, debug, numFolds, njobs, scalar, make_graphs, pNN,
nolegend)
print('NN: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, R_Score)
else:
# In 10 out of 14 data sets adam was better than sgd, so testing it first
runTime, train_score, S_Score = NN.train_NN(
filename, X_train, X_test, y_train, y_test, 'adam', full_param,
debug, numFolds, njobs, scalar, make_graphs)
print('NN-A: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, S_Score)
if test_all or S_Score < 0.6:
runTime, train_score, A_score = NN.train_NN(
filename, X_train, X_test, y_train, y_test, 'sgd',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('NN-S: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, A_score)
overall_score = max(A_score, S_Score)
min_score = min(min_score, overall_score)
max_score = max(max_score, overall_score)
print('Overall Variance: ', round(max_score - min_score, 4), '\n')
os.mkdir(root + 'test_results/AtoB')
if not os.path.isdir(root + 'test_results/BtoA'):
os.mkdir(root + 'test_results/BtoA')
if not os.path.isdir(root + 'train_results'):
os.mkdir(root + 'train_results')
if not os.path.isdir(root + 'train_results/AtoB'):
os.mkdir(root + 'train_results/AtoB')
if not os.path.isdir(root + 'train_results/BtoA'):
os.mkdir(root + 'train_results/BtoA')
# data_loader
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
train_loader_A = util.data_load('data/' + opt.dataset, opt.train_subfolder + 'A', transform, opt.batch_size, shuffle=True)
train_loader_B = util.data_load('data/' + opt.dataset, opt.train_subfolder + 'B', transform, opt.batch_size, shuffle=True)
test_loader_A = util.data_load('data/' + opt.dataset, opt.test_subfolder + 'A', transform, opt.batch_size, shuffle=False)
test_loader_B = util.data_load('data/' + opt.dataset, opt.test_subfolder + 'B', transform, opt.batch_size, shuffle=False)
# network
G_A = network.generator(opt.input_ngc, opt.output_ngc, opt.ngf, opt.nb)
G_B = network.generator(opt.input_ngc, opt.output_ngc, opt.ngf, opt.nb)
D_A = network.discriminator(opt.input_ndc, opt.output_ndc, opt.ndf)
D_B = network.discriminator(opt.input_ndc, opt.output_ndc, opt.ndf)
G_A.weight_init(mean=0.0, std=0.02)
G_B.weight_init(mean=0.0, std=0.02)
D_A.weight_init(mean=0.0, std=0.02)
D_B.weight_init(mean=0.0, std=0.02)
G_A.cuda()
G_B.cuda()
def run_ul_algos(filename, result_col, debug=False, numFolds=10, njobs=-1, scalar=1, make_graphs=False, nolegend=False,
verbose=False, random_seed=1, rNN=False, pNN={}):
np.random.seed(random_seed)
random.seed(random_seed)
X_train, X_test, y_train, y_test = util.data_load(filename, result_col, debug, scalar, make_graphs, random_seed)
vis.gen_vis(X_train, X_test, random_seed, filename[:-4])
clustering.ul_Kmeans(X_train, y_train, random_seed, filename[:-4], result_col, verbose)
clustering.ul_EM(X_train, y_train, random_seed, filename[:-4], result_col, verbose)
dimred.ulPCA(X_train, y_train, random_seed, filename[:-4], verbose)
dimred.ulICA(X_train, y_train, random_seed, filename[:-4], verbose)
dimred.randProj(X_train, y_train, random_seed, filename[:-4], verbose)
dimred.ul_LLE(X_train, y_train, random_seed, filename[:-4], verbose)
new_Xtrain = copy.deepcopy(X_train)
new_ytrain = copy.deepcopy(y_train)
dr_cluster.pca_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose)
new_Xtrain = copy.deepcopy(X_train)
new_ytrain = copy.deepcopy(y_train)
dr_cluster.ica_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose)
new_Xtrain = copy.deepcopy(X_train)
new_ytrain = copy.deepcopy(y_train)
dr_cluster.rp_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose)
new_Xtrain = copy.deepcopy(X_train)
new_ytrain = copy.deepcopy(y_train)
dr_cluster.lle_clust(new_Xtrain, new_ytrain, random_seed, filename[:-4], result_col, verbose)
# Run NNs for Dim Reduction
if rNN:
for n in range(1, 21):
print('PCA', n)
new_Xtrain = copy.deepcopy(X_train)
new_Xtest = copy.deepcopy(X_test)
new_ytrain = copy.deepcopy(y_train)
new_ytest = copy.deepcopy(y_test)
DR_NN.train_NN_PCA(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest,
random_seed=random_seed, scalar=scalar,
njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend,
pNN=pNN, num_dim=n)
print('ICA', n)
new_Xtrain = copy.deepcopy(X_train)
new_Xtest = copy.deepcopy(X_test)
new_ytrain = copy.deepcopy(y_train)
new_ytest = copy.deepcopy(y_test)
DR_NN.train_NN_ICA(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest,
random_seed=random_seed, scalar=scalar,
njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend,
pNN=pNN, num_dim=n)
print('RP', n)
new_Xtrain = copy.deepcopy(X_train)
new_Xtest = copy.deepcopy(X_test)
new_ytrain = copy.deepcopy(y_train)
new_ytest = copy.deepcopy(y_test)
DR_NN.train_NN_RP(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest,
random_seed=random_seed, scalar=scalar,
njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend,
pNN=pNN, num_dim=n)
for n in range(1, 21):
print('LLE', n)
new_Xtrain = copy.deepcopy(X_train)
new_Xtest = copy.deepcopy(X_test)
new_ytrain = copy.deepcopy(y_train)
new_ytest = copy.deepcopy(y_test)
DR_NN.train_NN_LLE(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest,
random_seed=random_seed, scalar=scalar,
njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend,
pNN=pNN, num_dim=n)
# Run NN for clustering
if rNN:
for n in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]:
print('kmeans', n)
new_Xtrain = copy.deepcopy(X_train)
new_Xtest = copy.deepcopy(X_test)
new_ytrain = copy.deepcopy(y_train)
new_ytest = copy.deepcopy(y_test)
cluster_NN.train_kmeansNN(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest,
random_seed=random_seed, scalar=scalar, debug=verbose,
njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend,
pNN=pNN, num_clusts=n)
print('diag')
# cov_types = ['diag', 'tied', 'full', 'spherical']
# for cov in cov_types:
for n in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]:
print('diag', n)
new_Xtrain = copy.deepcopy(X_train)
new_Xtest = copy.deepcopy(X_test)
new_ytrain = copy.deepcopy(y_train)
new_ytest = copy.deepcopy(y_test)
cluster_NN.train_EM_NN(filename[:-4], new_Xtrain, new_Xtest, new_ytrain, new_ytest,
random_seed=random_seed, scalar=scalar,
njobs=njobs, numFolds=numFolds, make_graphs=make_graphs, nolegend=nolegend,
pNN=pNN, num_clusts=n, cov_type='diag')
def NN_SA(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name,
classifier_col)
activation = 'relu'
learning_rate = [0.01, 0.1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
algorithim = 'simulated_annealing'
iters = [
100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000,
5000, 6000, 7000, 8000, 9000, 10000
]
nodes = [128, 128, 128, 128]
temperatures = [0.001, 0.01]
decay_rates = [0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]
outcomes = []
max_attempts = [10, 50, 100, 200, 500, 1000]
clips = [5, 10, 100, 1000, 10000, 100000]
csv_file = 'NN_SA-itertests.csv'
act = 'relu'
lr = 10
itera = 10000
temp = 10000
dec = 0.92
ma = 50
clip = 10
while 1 == 1:
iters_outs = {}
for iter_test in iters:
start = time.time()
print(algorithim, act, lr, iter_test, 'GeomDecay', temp, dec, ma,
clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=iter_test,
algorithm=algorithim,
schedule=mlrose.GeomDecay(
init_temp=temp, decay=dec),
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=1,
curve=True)
nn_model.fit(X_train, y_train)
train_time = time.time() - start
print('Train time', train_time)
start = time.time()
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_train_query_time = time.time() - start
print('y_train_query_time', y_train_query_time)
start = time.time()
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
y_test_query_time = time.time() - start
print('y_test_query_time', y_test_query_time)
nn_loss = nn_model.loss
print('loss', nn_loss)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = iter_test
outcome['temperatures'] = temp
outcome['decay_rates'] = dec
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome[
'runtime'] = train_time + y_train_query_time + y_test_query_time
outcome['Train time'] = train_time
outcome['y_train_query_time'] = y_train_query_time
outcome['y_test_query_time'] = y_test_query_time
outcome['loss'] = nn_loss
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv(csv_file)
iters_outs[iter_test] = y_test_roc
old_val = itera
itera = max(iters_outs, key=iters_outs.get)
print('best iter', itera, 'old', old_val)
raise SystemExit(0)
temp_outs = {}
for temp_test in temperatures:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', temp_test, dec, ma,
clip)
nn_model = mlrose.NeuralNetwork(
hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
schedule=mlrose.GeomDecay(init_temp=temp_test, decay=dec),
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=1,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['temperatures'] = temp_test
outcome['decay_rates'] = dec
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv(csv_file)
temp_outs[temp_test] = y_test_roc
old_temp = temp
temp = max(temp_outs, key=temp_outs.get)
print('best temp', temp, 'old', old_temp)
decay_outs = {}
for decay_test in decay_rates:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', temp, decay_test,
ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
schedule=mlrose.GeomDecay(
init_temp=temp,
decay=decay_test),
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=1,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['temperatures'] = temp
outcome['decay_rates'] = decay_test
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv(csv_file)
decay_outs[decay_test] = y_test_roc
old_val = dec
dec = max(decay_outs, key=decay_outs.get)
print('best decay', dec, 'old', old_val)
clips_outs = {}
for clip_test in clips:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', temp, dec, ma,
clip_test)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
schedule=mlrose.GeomDecay(
init_temp=temp, decay=dec),
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip_test,
max_attempts=ma,
random_state=1,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['temperatures'] = temp
outcome['decay_rates'] = dec
outcome['max_attempts'] = ma
outcome['clip'] = clip_test
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv(csv_file)
clips_outs[clip_test] = y_test_roc
old_val = clip
clip = max(clips_outs, key=clips_outs.get)
print('best clip', clip, 'old', old_val)
maxa_outs = {}
for maxa_test in max_attempts:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', temp, dec,
maxa_test, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
schedule=mlrose.GeomDecay(
init_temp=temp, decay=dec),
bias=True,
is_classifier=True,
learning_rate=lr,
early_stopping=True,
clip_max=clip,
max_attempts=maxa_test,
random_state=1,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['temperatures'] = temp
outcome['decay_rates'] = dec
outcome['max_attempts'] = maxa_test
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv(csv_file)
maxa_outs[maxa_test] = y_test_roc
old_val = ma
ma = max(maxa_outs, key=maxa_outs.get)
print('best ma', ma, 'old', old_val)
lr_outs = {}
for lr_test in learning_rate:
start = time.time()
print(algorithim, act, lr_test, itera, 'GeomDecay', temp, dec, ma,
clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
max_iters=itera,
algorithm=algorithim,
schedule=mlrose.GeomDecay(
init_temp=temp, decay=dec),
bias=True,
is_classifier=True,
learning_rate=lr_test,
early_stopping=True,
clip_max=clip,
max_attempts=ma,
random_state=1,
curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train,
y_train_pred,
multi_class="ovr",
average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test,
y_test_pred,
multi_class="ovr",
average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr_test
outcome['max_iters'] = itera
outcome['temperatures'] = temp
outcome['decay_rates'] = dec
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv(csv_file)
lr_outs[lr_test] = y_test_roc
old_lr = lr
lr = max(lr_outs, key=lr_outs.get)
print('best lr', lr, 'old', old_lr)
def NN_GA(file_name, classifier_col):
# HPs shared by all
nodes = [128, 128, 128, 128]
act = 'relu'
seed = 1
# GD HPs
gd_algo = 'gradient_descent'
gd_lr = 0.00000009
gd_iter = 10000
gd_ma = 50
gd_clip = 5
# RHC HPs
rhc_algo = 'random_hill_climb'
rhc_lr = 8
rhc_iter = 10000
rhc_restarts = 10
rhc_ma = 100
rhc_clip = 100
# SA HPs
sa_algo = 'simulated_annealing'
sa_lr = 10
sa_iter = 10000
sa_temp = 10000
sa_decay = 0.92
sa_ma = 50
sa_clip = 10
# GA HPs
ga_algo = 'genetic_alg'
ga_lr = 5
ga_iter = 100
ga_pop = 1500
ga_mut = 0.1
ga_ma = 100
ga_clip = 5
X_train, X_test, y_train, y_test = util.data_load(file_name,
classifier_col,
random_seed=seed)
# Best GD Algorithm
print('Training GD NN')
gd_start = time.time()
gd_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
random_state=seed,
bias=True,
is_classifier=True,
early_stopping=True,
curve=True,
algorithm=gd_algo,
max_iters=gd_iter,
learning_rate=gd_lr,
clip_max=gd_clip,
max_attempts=gd_ma)
gd_nn_model.fit(X_train, y_train)
print('gd loss:', gd_nn_model.loss)
gd_train_time = time.time() - gd_start
print('gd_train_time:', gd_train_time)
start = time.time()
gd_y_train_pred = gd_nn_model.predict(X_train)
gd_y_train_roc = roc_auc_score(y_train,
gd_y_train_pred,
multi_class="ovr",
average="weighted")
gd_y_train_query_time = time.time() - start
print('gd_y_train_roc', gd_y_train_roc, 'gd_y_train_roc: ',
gd_y_train_query_time)
start = time.time()
gd_y_test_pred = gd_nn_model.predict(X_test)
gd_y_test_roc = roc_auc_score(y_test,
gd_y_test_pred,
multi_class="ovr",
average="weighted")
gd_y_test_query_time = time.time() - start
print('gd_y_test_roc', gd_y_test_roc, 'gd_y_test_query_time: ',
gd_y_test_query_time)
# Best RHC Algorithm
print('Training RHC NN')
rhc_start = time.time()
rhc_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
random_state=seed,
bias=True,
is_classifier=True,
early_stopping=True,
curve=True,
algorithm=rhc_algo,
max_iters=rhc_iter,
learning_rate=rhc_lr,
clip_max=rhc_clip,
max_attempts=rhc_ma,
restarts=rhc_restarts)
rhc_nn_model.fit(X_train, y_train)
print('rhc loss:', rhc_nn_model.loss)
rhc_train_time = time.time() - rhc_start
print('rhc_train_time:', rhc_train_time)
start = time.time()
rhc_y_train_pred = rhc_nn_model.predict(X_train)
rhc_y_train_roc = roc_auc_score(y_train,
rhc_y_train_pred,
multi_class="ovr",
average="weighted")
rhc_y_train_query_time = time.time() - start
print('rhc_y_train_roc', rhc_y_train_roc, 'rhc_y_train_roc: ',
rhc_y_train_query_time)
start = time.time()
rhc_y_test_pred = rhc_nn_model.predict(X_test)
rhc_y_test_roc = roc_auc_score(y_test,
rhc_y_test_pred,
multi_class="ovr",
average="weighted")
rhc_y_test_query_time = time.time() - start
print('rhc_y_test_roc', rhc_y_test_roc, 'rhc_y_test_query_time: ',
rhc_y_test_query_time)
# Best SA Algorithm
print('Training SA NN')
sa_start = time.time()
sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
random_state=seed,
bias=True,
is_classifier=True,
early_stopping=True,
curve=True,
algorithm=sa_algo,
max_iters=sa_iter,
learning_rate=sa_lr,
clip_max=sa_clip,
max_attempts=sa_ma,
schedule=mlrose.GeomDecay(
init_temp=sa_temp, decay=sa_decay))
sa_nn_model.fit(X_train, y_train)
print('sa loss:', sa_nn_model.loss)
sa_train_time = time.time() - sa_start
print('sa_train_time:', sa_train_time)
start = time.time()
sa_y_train_pred = sa_nn_model.predict(X_train)
sa_y_train_roc = roc_auc_score(y_train,
sa_y_train_pred,
multi_class="ovr",
average="weighted")
sa_y_train_query_time = time.time() - start
print('sa_y_train_roc', sa_y_train_roc, 'sa_y_train_roc: ',
sa_y_train_query_time)
start = time.time()
sa_y_test_pred = sa_nn_model.predict(X_test)
sa_y_test_roc = roc_auc_score(y_test,
sa_y_test_pred,
multi_class="ovr",
average="weighted")
sa_y_test_query_time = time.time() - start
print('sa_y_test_roc', sa_y_test_roc, 'sa_y_test_query_time: ',
sa_y_test_query_time)
# Best Genetic Algorithm
print('Training GA NN')
ga_start = time.time()
ga_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes,
activation=act,
random_state=seed,
bias=True,
is_classifier=True,
early_stopping=True,
curve=True,
algorithm=ga_algo,
max_iters=ga_iter,
learning_rate=ga_lr,
clip_max=ga_clip,
max_attempts=ga_ma,
pop_size=ga_pop,
mutation_prob=ga_mut)
ga_nn_model.fit(X_train, y_train)
print('ga loss:', ga_nn_model.loss)
ga_train_time = time.time() - ga_start
print('ga_train_time:', ga_train_time)
start = time.time()
ga_y_train_pred = ga_nn_model.predict(X_train)
ga_y_train_roc = roc_auc_score(y_train,
ga_y_train_pred,
multi_class="ovr",
average="weighted")
ga_y_train_query_time = time.time() - start
print('ga_y_train_roc', ga_y_train_roc, 'ga_y_train_roc: ',
ga_y_train_query_time)
start = time.time()
ga_y_test_pred = ga_nn_model.predict(X_test)
ga_y_test_roc = roc_auc_score(y_test,
ga_y_test_pred,
multi_class="ovr",
average="weighted")
ga_y_test_query_time = time.time() - start
print('ga_y_test_roc', ga_y_test_roc, 'ga_y_test_query_time: ',
ga_y_test_query_time)
# Plot Loss Curves
plt.figure()
plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve')
plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve')
plt.plot(gd_nn_model.fitness_curve, label='GD Loss Curve')
plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve')
plt.title("Neural Network Loss Curves")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid(True)
plt.legend()
plt.xscale('log')
plt.savefig("Images\\Neural Network Loss Curves")
plt.show()
# Plot Loss Curves (GD Inverted)
plt.figure()
gd_curve = gd_nn_model.fitness_curve
inverted_gd_curve = np.array(gd_curve) * -1
plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve')
plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve')
plt.plot(inverted_gd_curve, label='GD Loss Curve')
plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve')
plt.title("Neural Network Loss Curves")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid(True)
plt.legend()
plt.xscale('log')
plt.savefig("Images\\Neural Network Loss Curves-inverted GD")
plt.show()
# Plot Loss Curves - No GD
plt.figure()
plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve')
plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve')
plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve')
plt.title("Neural Network Loss Curves")
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.ylim(bottom=0)
plt.grid(True)
plt.legend()
plt.xscale('log')
plt.savefig("Images\\Neural Network Loss Curves - No GD")
plt.show()
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', required=False, default='maps', help='')
parser.add_argument('--test_subfolder', required=False, default='val', help='')
parser.add_argument('--ngf', type=int, default=64)
parser.add_argument('--input_size', type=int, default=256, help='input size')
parser.add_argument('--save_root', required=False, default='results', help='results save path')
parser.add_argument('--inverse_order', type=bool, default=True, help='0: [input, target], 1 - [target, input]')
opt = parser.parse_args()
print(opt)
# data_loader
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
test_loader = util.data_load('data/' + opt.dataset, opt.test_subfolder, transform, batch_size=1, shuffle=False)
if not os.path.isdir(opt.dataset + '_results/test_results'):
os.mkdir(opt.dataset + '_results/test_results')
G = network.generator(opt.ngf)
G.cuda()
G.load_state_dict(torch.load(opt.dataset + '_results/' + opt.dataset + '_generator_param.pkl'))
# network
n = 0
print('test start!')
for x_, _ in test_loader:
if opt.inverse_order:
y_ = x_[:, :, :, :x_.size()[2]]
x_ = x_[:, :, :, x_.size()[2]:]
def NN_RHC(file_name, classifier_col):
X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col)
activation = ['relu']
learning_rate = [6, 7, 5, 8, 9, 1, 2, 3, 4, 10]
algorithim = 'random_hill_climb'
iters = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000]
nodes = [128, 128, 128, 128]
restarts = [100, 1000]
outcomes = []
max_attempts = [100, 200, 500, 1000]
clips = [5, 10, 100, 1000, 10000, 100000]
act = 'relu'
lr = 8
itera = 10000
res = 10
ma = 100
clip = 100
seed = 1
while 1 == 1:
iters_outs = {}
for iter_test in iters:
start = time.time()
print(algorithim, act, lr, iter_test, 'GeomDecay', res, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test,
algorithm=algorithim, restarts=res,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
train_time = time.time() - start
print('Train time', train_time)
start = time.time()
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_train_query_time = time.time() - start
print('y_train_query_time', y_train_query_time)
start = time.time()
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
y_test_query_time = time.time() - start
print('y_test_query_time', y_test_query_time)
nn_loss = nn_model.loss
print('loss', nn_loss)
outcome = {}
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = iter_test
outcome['restarts'] = res
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = train_time + y_train_query_time + y_test_query_time
outcome['Train time'] = train_time
outcome['y_train_query_time'] = y_train_query_time
outcome['y_test_query_time'] = y_test_query_time
outcome['loss'] = nn_loss
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN_RHC-itertests.csv', mode='a', header=False)
iters_outs[iter_test] = y_test_roc
old_val = itera
itera = max(iters_outs, key=iters_outs.get)
print('best iter', itera, 'old', old_val)
raise SystemExit(0)
res_outs = {}
for res_test in restarts:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', res_test, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim, restarts=res_test,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['restarts'] = res_test
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN_RHC.csv', mode='a', header=False)
res_outs[res_test] = y_test_roc
old_val = res
res = max(res_outs, key=res_outs.get)
print('best temp', res, 'old', old_val)
clips_outs = {}
for clip_test in clips:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', res, ma, clip_test)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim, restarts=res,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip_test, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['restarts'] = res
outcome['max_attempts'] = ma
outcome['clip'] = clip_test
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN_RHC.csv', mode='a', header=False)
clips_outs[clip_test] = y_test_roc
old_val = clip
clip = max(clips_outs, key=clips_outs.get)
print('best clip', clip, 'old', old_val)
maxa_outs = {}
for maxa_test in max_attempts:
start = time.time()
print(algorithim, act, lr, itera, 'GeomDecay', res, maxa_test, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim, restarts=res,
bias=True, is_classifier=True, learning_rate=lr,
early_stopping=True, clip_max=clip, max_attempts=maxa_test,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr
outcome['max_iters'] = itera
outcome['restarts'] = res
outcome['max_attempts'] = maxa_test
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN_RHC.csv', mode='a', header=False)
maxa_outs[maxa_test] = y_test_roc
old_val = ma
ma = max(maxa_outs, key=maxa_outs.get)
print('best ma', ma, 'old', old_val)
lr_outs = {}
for lr_test in learning_rate:
start = time.time()
print(algorithim, act, lr_test, itera, 'GeomDecay', res, ma, clip)
nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera,
algorithm=algorithim, restarts=res,
bias=True, is_classifier=True, learning_rate=lr_test,
early_stopping=True, clip_max=clip, max_attempts=ma,
random_state=seed, curve=True)
nn_model.fit(X_train, y_train)
y_train_pred = nn_model.predict(X_train)
y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted")
print('y_train_roc', y_train_roc)
y_test_pred = nn_model.predict(X_test)
y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted")
print('y_test_roc', y_test_roc)
runtime = time.time() - start
print('curr run time', time.time() - start)
outcome = {}
outcome['schedule'] = 'GeomDecay'
outcome['activation'] = act
outcome['learning_rate'] = lr_test
outcome['max_iters'] = itera
outcome['restarts'] = res
outcome['max_attempts'] = ma
outcome['clip'] = clip
outcome['y_train_roc'] = y_train_roc
outcome['y_test_roc'] = y_test_roc
outcome['runtime'] = runtime
outcomes.append(outcome)
pd.DataFrame(outcomes).to_csv('NN_RHC.csv')
lr_outs[lr_test] = y_test_roc
old_lr = lr
lr = max(lr_outs, key=lr_outs.get)
print('best lr', lr, 'old', old_lr)
