print(f"ETA is {eta}")
results[n_hidden_nodes][eta] = {}
accumulated_metrics = {}
for m in metrics:
accumulated_metrics[m] = []
name = f"MLP{i:05}_d-{dataset_idx}_b-{batch_size}_h-{n_hidden_nodes}_eta-{eta}".replace(".", ",")
i += 1
print(name)
best = {"best_acc": 0, "best_loss": np.inf}
for _ in range(loops):
print(".", end="")
net = MLP(train[0], train[1], n_hidden_nodes, momentum=0)
train_losses, valid_losses, train_accuracies, valid_accuracies, pocket_epoch = net.train(
train[0], train[1], valid[0], valid[1], eta, epochs,
early_stop_count=early_stop, shuffle=shuffle, batch_size=batch_size
)
cm_train = net.confmat(train[0], train[1])
cm_valid = net.confmat(valid[0], valid[1])
print("cm_train", cm_train)
print("cm_valid", cm_valid)
print("pocket epoch", pocket_epoch)
train_loss, valid_loss = train_losses[pocket_epoch], valid_losses[pocket_epoch]
train_acc, train_sens, train_spec, _, _ = two_class_conf_mat_metrics(
net.confmat(train[0], train[1]))
valid_acc, valid_sens, valid_spec, _, _ = two_class_conf_mat_metrics(
net.confmat(valid[0], valid[1]))
if debug_best_and_plot:
if valid_acc > best["best_acc"] or (
".", ",")
i += 1
print(f"ETA is {eta}")
print(name)
net = MLP(dataset[0],
dataset[1],
n_hidden_nodes,
momentum=momentum)
train_losses, _, train_accuracies, _, pocket_epoch = net.train(
dataset[0],
dataset[1],
dataset[0],
dataset[1],
eta,
epochs,
early_stop_count=early_stop,
shuffle=shuffle,
batch_size=batch_size,
early_stopping_threshold=early_stopping_threshold)
train_accuracies = np.array(train_accuracies) * 100 / n
net.forward(dataset[0])
print("pocket epoch", pocket_epoch)
print(
f"w1:\n{net.weights1}\nw2:{net.weights2}\nnet.hidden:\n{net.hidden}\noutputs:{net.outputs}\n"
)
save_prefix = os.path.join(save_folder, name)
fig, ax1 = plt.subplots()
def run_cv(seed,
fold,
X,
Y,
R,
y_strat,
val_size=0,
pretrain_set=None,
batch_size=32,
k=-1,
learning_rate=0.01,
lr_decay=0.0,
dropout=0.5,
n_epochs=100,
momentum=0.9,
L1_reg=0.001,
L2_reg=0.001,
hiddenLayers=[128, 64]):
X_w = pretrain_set.get_value(
borrow=True) if k > 0 and pretrain_set else None
m = X.shape[1] if k < 0 else k
columns = list(range(m))
columns.extend(['scr', 'R', 'Y'])
df = pd.DataFrame(columns=columns)
kf = StratifiedKFold(n_splits=fold, shuffle=True, random_state=seed)
for train_index, test_index in kf.split(X, y_strat):
X_train, X_test = X[train_index], X[test_index]
Y_train, Y_test = Y[train_index], Y[test_index]
R_train, R_test = R[train_index], R[test_index]
strat_train, strat_test = y_strat[train_index], y_strat[test_index]
if k > 0:
k_best = SelectKBest(f_classif, k=k)
k_best.fit(X_train, Y_train)
X_train, X_test = k_best.transform(X_train), k_best.transform(
X_test)
if pretrain_set:
X_base = k_best.transform(X_w)
pretrain_set = theano.shared(X_base,
name='pretrain_set',
borrow=True)
valid_data = None
if val_size:
X_train, X_val, Y_train, Y_val = train_test_split(
X_train,
Y_train,
test_size=val_size,
random_state=0,
stratify=strat_train)
valid_data = (X_val, Y_val)
train_data = (X_train, Y_train)
n_in = X_train.shape[1]
classifier = MLP(n_in=n_in,
learning_rate=learning_rate,
lr_decay=lr_decay,
dropout=dropout,
L1_reg=L1_reg,
L2_reg=L2_reg,
hidden_layers_sizes=hiddenLayers,
momentum=momentum)
if pretrain_set:
pretrain_config = {
'pt_batchsize': 32,
'pt_lr': 0.01,
'pt_epochs': 500,
'corruption_level': 0.3
}
classifier.pretrain(pretrain_set=pretrain_set,
pretrain_config=pretrain_config)
classifier.tune(train_data,
valid_data=valid_data,
batch_size=batch_size,
n_epochs=n_epochs)
else:
classifier.train(train_data,
valid_data=valid_data,
batch_size=batch_size,
n_epochs=n_epochs)
X_scr = classifier.get_score(X_test)
array1 = np.column_stack((X_test, X_scr[:, 1], R_test, Y_test))
df_temp1 = pd.DataFrame(array1,
index=list(test_index),
columns=columns)
df = df.append(df_temp1)
return df
def run_supervised_transfer_cv(seed,
dataset,
fold=3,
val_size=0,
k=-1,
batch_size=32,
groups=('WHITE', 'BLACK'),
learning_rate=0.01,
lr_decay=0.0,
dropout=0.5,
tune_epoch=200,
tune_lr=0.002,
train_epoch=1000,
L1_reg=0.001,
L2_reg=0.001,
hiddenLayers=[128, 64],
tune_batch=10):
X, Y, R, y_sub, y_strat = dataset
idx = R == groups[1]
X_b, y_b, R_b, y_strat_b = X[idx], Y[idx], R[idx], y_strat[idx]
idx = R == groups[0]
X_w, y_w, R_w, y_strat_w = X[idx], Y[idx], R[idx], y_strat[idx]
pretrain_set = (X_w, y_w)
df = pd.DataFrame(columns=['scr', 'R', 'Y'])
kf = StratifiedKFold(n_splits=fold, shuffle=True, random_state=seed)
for train_index, test_index in kf.split(X_b, y_strat_b):
X_train, X_test = X_b[train_index], X_b[test_index]
Y_train, Y_test = y_b[train_index], y_b[test_index]
R_train, R_test = R_b[train_index], R_b[test_index]
strat_train, strat_test = y_strat_b[train_index], y_strat_b[test_index]
if k > 0:
k_best = SelectKBest(f_classif, k=k)
k_best.fit(X_train, Y_train)
X_train, X_test = k_best.transform(X_train), k_best.transform(
X_test)
X_base = k_best.transform(X_w)
pretrain_set = (X_base, y_w)
valid_data = None
if val_size:
X_train, X_val, Y_train, Y_val = train_test_split(
X_train,
Y_train,
test_size=val_size,
random_state=0,
stratify=strat_train)
valid_data = (X_val, Y_val)
train_data = (X_train, Y_train)
n_in = X_train.shape[1]
classifier = MLP(n_in=n_in,
learning_rate=learning_rate,
lr_decay=lr_decay,
dropout=dropout,
L1_reg=L1_reg,
L2_reg=L2_reg,
hidden_layers_sizes=hiddenLayers)
classifier.train(pretrain_set,
n_epochs=train_epoch,
batch_size=batch_size)
classifier.learning_rate = tune_lr
classifier.tune(train_data,
valid_data=valid_data,
batch_size=tune_batch,
n_epochs=tune_epoch)
scr = classifier.get_score(X_test)
array = np.column_stack((scr[:, 1], R_test, Y_test))
df_temp = pd.DataFrame(array,
index=list(test_index),
columns=['scr', 'R', 'Y'])
df = df.append(df_temp)
y_test, y_scr = list(df['Y'].values), list(df['scr'].values)
A_CI = roc_auc_score(y_test, y_scr)
res = {'folds': fold, 'TL_Auc': A_CI}
df = pd.DataFrame(res, index=[seed])
return df
def demo_run(t):
########################################
#### MLP - Function approximation #####
########################################
n_hidden = t
split = 0.6
# hidden_nodes = [1, 2, 3, 5, 8, 10, 15, 20, 22, 25, 100]
# hidden_nodes = list(range(1, 25 + 1)) + [100]
hidden_nodes = [n_hidden]
# eta_values = [0.002, 0.005, 0.01, 0.05, 0.1, 0.25, 1, 10]
eta_values = [0.25, 0.5]
epochs = 1000 # max number of epochs
early_stop = 400000000 # no early stopping
batch = True
shuffle = False
loops = 4
debug = True
momentum = 0
show_plots = False
step = 0.25
np.random.seed(72)
x = np.arange(-5, 5, step)
y = np.arange(-5, 5, step)
x, y = np.meshgrid(x, y)
z = np.exp(-(x**2 + y**2) / 10) - 0.5
xy = np.concatenate((x.reshape(-1, 1), y.reshape(-1, 1)), axis=1)
dataset = xy.copy(), z.reshape(-1, 1).copy()
n = dataset[0].shape[0]
shuffler = np.random.permutation(n)
shuffled_dataset = dataset[0][shuffler], dataset[1][shuffler]
train = shuffled_dataset[0][0:int(n * split
)], shuffled_dataset[1][0:int(n * split)]
# valid = shuffled_dataset[0][int(n * split):], shuffled_dataset[1][int(n * split):]
valid = shuffled_dataset
metrics = ["pocket_epoch", "train_loss", "valid_loss"]
i = 100
batch_size = train[0].shape[0] if batch else 1
results = {} # results[n_hidden][eta][metric]
for n_hidden_nodes in hidden_nodes:
print(f"x-----> {n_hidden_nodes} hidden nodes <-----x")
results[n_hidden_nodes] = {}
for eta in reversed(eta_values):
print(f"ETA is {eta}")
results[n_hidden_nodes][eta] = {}
accumulated_metrics = {}
for m in metrics:
accumulated_metrics[m] = []
print(f"Loop starting. {loops} loops left to go.")
for loop_idx in range(loops):
name = f"MLP{i:05}_GAUSS" \
f"_h-{n_hidden_nodes}" \
f"_eta-{eta}" \
f"_b-{batch_size}{'' if momentum == 0 else 'momentum:' + str(momentum)}".replace(".", ",")
i += 1
print(name)
net = MLP(train[0],
train[1],
n_hidden_nodes,
momentum=momentum,
outtype="linear")
train_losses, valid_losses, _, _, pocket_epoch = net.train(
train[0],
train[1],
valid[0],
valid[1],
eta,
epochs,
early_stop_count=early_stop,
shuffle=shuffle,
batch_size=batch_size)
train_loss, valid_loss = train_losses[
pocket_epoch], valid_losses[pocket_epoch]
print(f"pocket epoch: {pocket_epoch}\n"
f"train_loss:{train_loss}\n"
f"valid_loss:{valid_loss}")
for m in metrics:
accumulated_metrics[m].append(locals()[m])
if debug:
save_prefix = os.path.join(save_folder,
str(n_hidden_nodes))
ensure_dir(save_prefix)
save_prefix = os.path.join(save_prefix, str(eta))
ensure_dir(save_prefix)
save_prefix = os.path.join(save_prefix, str(batch_size))
ensure_dir(save_prefix)
save_prefix = os.path.join(save_prefix, name)
fig, ax1 = plt.subplots()
plt.title(name + " Learning curve")
x_values = np.arange(len(train_losses))
ax1.set_xlabel('Epoch')
ax1.set_ylabel('Log MSE loss')
ax1.plot(x_values,
np.log(train_losses),
color='tab:red',
label="Training loss",
linewidth=2)
ax1.plot(x_values,
np.log(valid_losses),
color='tab:orange',
label="Validation loss")
ax1.scatter(pocket_epoch,
np.log(valid_loss),
color="green")
ax1.tick_params(axis='y')
fig.legend(bbox_to_anchor=(0.85, 0.7), loc="upper right")
plt.savefig(f"{save_prefix}_learning_curve.png", dpi=300)
if show_plots:
plt.show()
plt.close()
def f(inputs):
net.forward(inputs)
n = np.int(np.sqrt(inputs.shape[0]))
return net.outputs.reshape((n, n))
fig = plt.figure()
ax = fig.gca(projection='3d')
surf = ax.plot_surface(x,
y,
f(xy),
cmap=cm.coolwarm,
linewidth=0,
antialiased=False,
alpha=0.8)
# Customize the z axis.
ax.set_zlim(-1.01, 1.01)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
# Add a color bar which maps values to colors.
fig.colorbar(surf, shrink=0.5, aspect=5)
plt.savefig(f"{save_prefix}_function_approx.png", dpi=300)
if show_plots:
plt.show()
plt.close()
print("\n\n")
for m in metrics:
results[n_hidden_nodes][eta][m] = np.array(
accumulated_metrics[m]).mean(), np.array(
accumulated_metrics[m]).std()
results_filename = f"results m:{momentum}" \
f" etas:{eta_values}" \
f" nh:{hidden_nodes}" \
f" time:{datetime.datetime.now()}" \
f".txt"
with open(os.path.join(save_folder, results_filename), "w") as f:
print(
os.path.join(save_folder,
"results " + str(datetime.datetime.now()) + ".txt"))
print(results)
f.write(f"{results_filename}\n{results}")
print(results)
print(".\n.\n.")
sep = ","
for metric in metrics:
for metric_idx in range(2): # 0-->mean 1-->std
# print table name
print(f"Metric: {metric} {'mean' if metric_idx == 0 else 'std'}")
# first row - table header
print("eta", end="")
for eta in eta_values:
print(sep + str(eta), end="")
print()
# other rows - one for each number of hidden nodes
for nh in hidden_nodes:
print(nh, end="")
for eta in eta_values:
print(sep + str(results[nh][eta][metric][metric_idx]),
end="")
print()
print()
print()
print()