def test_init_manager():
nn = init_manger(2, 5, 10, 2, "mean_euclidean_error", "tanh", "linear")
task = "cup"
eta = 0.006
alpha = 0.2
lambd = 0.0001
epochs = 100
verbose = True
grid_search = False
training_cup = "dataset/blindcup/training_set.csv"
folds = 1
nn_to_plot = []
train_folded, val_folded = k_fold(get_dataset(training_cup), folds)
for i in range(len(train_folded)):
print(
f"KFOLD {i + 1} of {folds} _______________________________________"
)
tr = train_folded[i]
tval = val_folded[i]
nn.train(task, tr, tval, epochs, eta, alpha, lambd, verbose,
grid_search)
nn_to_plot.append(nn)
plot_multinetwork(nn_to_plot, eta, 0, folds, 0, "test")
def run_monk_folded():
task = "monk"
eta = 0.03
alpha = 0.3
lambd = 0.00
eta_decay = 0.0
epochs = 25
verbose = True
grid_search = False
training_set = "dataset/monk3/monk3train_onehot.csv"
test_set = "dataset/monk3/monk3test_onehot.csv"
folds = 4
train_folded, val_folded = k_fold(get_dataset(training_set), folds)
nn_to_plot = []
# architecture --> [hidden layer, nueroni x hidden, eta, alpha, lambd, act hidden layer]
architecture = [1, 3, eta, alpha, lambd, "sigmoid"]
for i in range(len(train_folded)):
print(
f"KFOLD {i + 1} of {folds} _______________________________________"
)
nn = NeuralNet("mean_squared_error")
nn.init_input_layer(17)
nn.init_layer(3, 17, "sigmoid")
nn.init_layer(1, 3, "sigmoid")
tr = train_folded[i]
tval = val_folded[i]
nn.train(task, tr, tval, epochs, eta, alpha, lambd, eta_decay, verbose,
grid_search)
nn_to_plot.append(nn)
plot_multinetwork(nn_to_plot, eta, alpha, lambd, folds, architecture)
def run_model_cup_kfold():
task = "cup"
eta = 0.001
alpha = 0.2
lambd = 0.0000
eta_decay = 0.00
epochs = 250
verbose = True
grid_search = False
training_cup = "dataset/blindcup/training_set2.csv"
folds = 1
train_folded, val_folded = k_fold(get_dataset(training_cup), folds)
nn_to_plot = []
# architecture --> [hidden layer, nueroni x hidden, eta, alpha, lambd, act hidden layer]
architecture = [1, 40, eta, alpha, lambd, "tanh"]
for i in range(len(train_folded)):
print(
f"\nKFOLD {i + 1} of {folds} _______________________________________"
)
# initializing network
nn = NeuralNet("mean_euclidean_error")
# initializing input layer
nn.init_input_layer(10)
# adding layers
nn.init_layer(40, 10, "tanh")
nn.init_layer(2, 40, "linear")
# setting weights xavier init
nn.set_weights_pre_training()
tr = train_folded[i]
tval = val_folded[i]
nn.train(task, tr, tval, epochs, eta, alpha, lambd, eta_decay, verbose,
grid_search)
nn_to_plot.append(nn)
plot_multinetwork(nn_to_plot, eta, alpha, lambd, folds, architecture)
error_kfold_tr = 0
error_kfold_val = 0
for network in nn_to_plot:
error_kfold_tr += network.error_list[-1][1]
error_kfold_val += network.validation_error_list[-1][1]
error_kfold_tr = round(error_kfold_tr / len(train_folded), 5)
error_kfold_val = round(error_kfold_val / len(train_folded), 5)
print(
f"\nNN Architecture: Layers: {len(nn_to_plot[0].layer_list)}, Units x hidden layer: {len(nn_to_plot[0].layer_list[1].net)}"
f"\nHyperparameters: eta: {eta}, alpha: {alpha}, lambda: {lambd}, eta decay: {eta_decay}"
f"\nAverage final error on training set: {error_kfold_tr}"
f"\nAverage final error on validat. set: {error_kfold_val}\n")
def run_kfold_monk():
training_set = "dataset/monk2/monk2train_onehot.csv"
folds = 4
train_folded, val_folded = k_fold(get_dataset(training_set), folds)
nn_to_plot = []
for i in range(len(train_folded)):
model = run_monk_folded(train_folded[i], val_folded[i])
nn_to_plot.append(model)
# NB: ATM eta (in the plot title) is manually set to 0.3
plot_multinetwork(nn_to_plot, 0.3, 0)
def train():
data = read_csv(csv_file=GROUND_TRUTH_PATH,
delimiter=';',
transpose=False,
skip_header=True)
x_data, y_data = prepare_image_data(data=data,
prediction_labels=PREDICTION_LABELS,
training_path=MEDIANS_IMAGE_PATH,
x_shape=IMAGE_SHAPE)
for fold_index, (x_train, x_test, y_train,
y_test) in enumerate(k_fold(x_data, y_data, K_FOLDS)):
model = ECGModel2D(input_shape=IMAGE_SHAPE,
output_size=len(PREDICTION_LABELS))
model.compile(loss=LOSS_FUNCTION, optimizer='nadam', metrics=METRICS)
history = model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
callbacks=MODEL_CALLBACKS,
validation_data=(x_test, y_test),
shuffle=True)
plot_loss(history, PLOT_FILE)
model_path = f'{ os.path.splitext(MODEL_PATH)[0] }_{ fold_index }.h5'
model.save(model_path)
if Experiment is not None:
experiment.add_artifact(PLOT_FILE)
experiment.add_artifact(model_path)
def train_linear(config):
net = LinearNet(config)
features, labels = load_data_for_linear(config)
k_fold(features, labels, net, config)
from keras.models import load_model
from utils import k_fold, prepare_image_data, read_csv
import keras.backend as K
import numpy as np
import os
from settings import PREDICTION_LABELS, IMAGE_SHAPE, K_FOLDS
from settings import GROUND_TRUTH_PATH, MEDIANS_IMAGE_PATH, MODEL_PATH
GROUND_TRUTH = read_csv(csv_file=GROUND_TRUTH_PATH,
delimiter=';',
transpose=False,
skip_header=True)
x_data, y_data = prepare_image_data(data=GROUND_TRUTH,
prediction_labels=PREDICTION_LABELS,
training_path=MEDIANS_IMAGE_PATH,
x_shape=IMAGE_SHAPE)
for fold_index, (x_train, x_test, y_train,
y_test) in enumerate(k_fold(x_data, y_data, K_FOLDS)):
model = load_model(
f'{ os.path.splitext(MODEL_PATH)[0] }_{ fold_index }.h5')
for row_index, row in enumerate(x_test):
prediction = np.round(
model.predict(row.reshape((1, *row.shape))).flatten())
print(prediction, y_test[row_index])
def train():
print('Prediction label(s): ' + str(config.PREDICTION_LABELS))
print('Covariates: None!\n')
print('K folds: ' + str(config.K_FOLDS))
print('Loss function: ' + str(config.LOSS_FUNCTION))
print('Model path: ' + str(config.MODEL_PATH))
print('Model name: ' + str(config.TIMESTAMP))
print('Optimizer: ' + str(config.OPTIMIZER.__class__.__name__) + ', parameters: ' + str(config.OPTIMIZER.get_config()))
print('Batch size: ' + str(config.BATCH_SIZE))
print('Epochs: ' + str(config.EPOCHS))
print('Data set: ' + str(config.DATASET))
print('Data type: Raw ' + config.INPUT_TYPE + '\n')
data, header = read_csv(
csv_file=config.GROUND_TRUTH_PATH,
delimiter=';',
transpose=False,
skip_header=True
)
x_data, y_data = prepare_csv_data(
data=data,
prediction_labels=config.PREDICTION_LABELS,
training_path=config.ECG_PATH,
x_shape=config.X_TRANSPOSE,
data_labels = header
)
try:
y_data = np.array(y_data).astype(np.int16)
except:
y_data = np.round(np.array(y_data).astype(np.float32)).astype(np.int16)
for fold_index, (x_train, x_test, y_train, y_test) in enumerate(k_fold(x_data, y_data, config.K_FOLDS)):
default_stdout.write('\n\n' + datetime.datetime.now().strftime("%H%M") + ': Running fold ' + str(fold_index+1) + ' of ' + str(config.K_FOLDS) + '\n')
print('\n\n' + datetime.datetime.now().strftime("%H%M") + ': Running fold ' + str(fold_index+1) + ' of ' + str(config.K_FOLDS) + '\n')
xshape = x_data[0].shape
MODEL_CALLBACKS = [
ModelCheckpoint(save_best_only=True,
filepath = f'{os.path.splitext(config.MODEL_PATH)[0] }_{ fold_index }_best.h5')
]
with config.STRATEGY.scope():
model = KanResWide_X(input_shape=xshape,output_size=len(config.PREDICTION_LABELS))
model.compile(
optimizer=config.OPTIMIZER,
loss=config.LOSS_FUNCTION,
metrics=config.METRICS)
print('Model type: ' + model.__class__.__name__ + '\n')
history = model.fit(x_train, y_train,
epochs=config.EPOCHS,
batch_size= config.BATCH_SIZE,
verbose=2,
callbacks=MODEL_CALLBACKS,
validation_data=(x_test, y_test),
shuffle=True)
print('\nModel name: ' + model.__class__.__name__ + '\n')
print('Training mean absolute error (last 20 epochs). Mean: ' + str(np.mean(history.history['mae'][-20:])) + ', SD: ' + str(np.std(history.history['mae'][-20:])) + ', median: ' + str(np.median(history.history['mae'][-20:])) + '.\n')
print('Training MSE (last 20 epochs). Mean: ' + str(np.mean(history.history['mse'][-20:])) + ', SD: ' + str(np.std(history.history['mse'][-20:])) + ', median: ' + str(np.median(history.history['mse'][-20:])) + '\n')
print('\nValidation mean absolute error (last 20 epochs). Mean: ' + str(np.mean(history.history['val_mae'][-20:])) + ', SD: ' + str(np.std(history.history['val_mae'][-20:])) + ', median: ' + str(np.median(history.history['val_mae'][-20:])) + '.\n')
print('Validation MSE (last 20 epochs). Mean: ' + str(np.mean(history.history['val_mse'][-20:])) + ', SD: ' + str(np.std(history.history['val_mse'][-20:])) + ', median: ' + str(np.median(history.history['val_mse'][-20:])) + '\n')
bestEpoch = np.argmin(history.history['val_loss'])
print('\nBest validation loss at epoch (1-based): ' + str(bestEpoch+1))
print('Stats at best epoch: Training MAE: ' + str(history.history['mae'][bestEpoch]) + ', Validation MAE: '
+ str(history.history['val_mae'][bestEpoch]) + ', Validation MSE: ' + str(history.history['val_mse'][bestEpoch]) + '.\n')
default_stdout.write('\nModel name: ' + model.__class__.__name__ + '\n')
default_stdout.write('\nValidation mean absolute error (last 20 epochs). Mean: ' + str(np.mean(history.history['val_mae'][-20:])) + ', SD: ' + str(np.std(history.history['val_mae'][-20:])) + ', median: ' + str(np.median(history.history['val_mae'][-20:])) + '.\n')
default_stdout.write('\nValidation MSE (last 20 epochs). Mean: ' + str(np.mean(history.history['val_mse'][-20:])) + ', SD: ' + str(np.std(history.history['val_mse'][-20:])) + ', median: ' + str(np.median(history.history['val_mse'][-20:])) + '\n')
default_stdout.write('Stats at best epoch (' + str(bestEpoch+1) + '): Training MAE: ' + str(history.history['mae'][bestEpoch]) + ', Validation MAE: '
+ str(history.history['val_mae'][bestEpoch]) + ', Validation MSE: ' + str(history.history['val_mse'][bestEpoch]) + '.\n')
try:
if plt is not None:
plt.plot(history.history['val_mae'][20:])
plt.plot(history.history['mae'][20:])
plt.legend(['Validation','Training'])
plt.title(str(config.TIMESTAMP) + '_' + str(fold_index) + ' - learning ' + config.PREDICTION_LABELS[0])
plt.xlabel('Epoch')
plt.ylabel('Mean absolute error')
plt.savefig(os.path.join(config.ROOT_DIRECTORY,'models', config.TIMESTAMP + f'_mean_abs_error_{ fold_index }.png'))
plt.gcf().clear()
except Exception as e:
default_stdout.write('Error plot could not be made: ' + str(e) + '\n')
print('Error plot could not be made: ' + str(e) + '\n')
default_stdout.flush()
plot_path = f'{ os.path.splitext(config.PLOT_FILE)[0] }_{ fold_index }.png'
model_path = f'{ os.path.splitext(config.MODEL_PATH)[0] }_{ fold_index }.h5'
history_path = f'{ os.path.splitext(config.HISTORY_PATH)[0] }_{ fold_index }.npy'
if config.EPOCHS > 5:
try:
plot_loss(history, plot_path)
except Exception as e:
default_stdout.write('Loss plot could not be made: ' + str(e) + '\n')
print('Loss plot could not be made: ' + str(e) + '\n')
model.save(model_path)
np.save(history_path, history.history)
if Experiment is not None:
experiment.add_artifact(config.PLOT_FILE)
experiment.add_artifact(model_path)
def run_grid_search():
epochs = 500
n_input = [10]
n_neurons_layer = [10, 40]
n_hidden_layers = [1]
n_output_neurons = [2]
act_function_hidden = ["tanh"]
act_function_output = ["linear"]
error_function = ["mean_euclidean_error"]
eta_deep_gs = [
0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008, 0.0009
]
alpha_deep_gs = [0.2, 0.6, 0.8]
lambd_deep_gs = [0.0001]
hp_architecture = [
n_hidden_layers, n_neurons_layer, n_input, n_output_neurons,
error_function, act_function_hidden, act_function_output
]
hp_hyperparam = [eta_deep_gs, alpha_deep_gs, lambd_deep_gs]
best_val = 100
for j, architecture in enumerate(itertools.product(*hp_architecture)):
for i, comb in enumerate(itertools.product(*hp_hyperparam)):
print(
f"Architecture "
f"{j + 1} of {len(list(itertools.product(*hp_architecture)))} with "
f"Combination of hyperparameters "
f"{i + 1} of {len(list(itertools.product(*hp_hyperparam)))}____________________________\n"
)
task = "cup"
eta = comb[0]
alpha = comb[1]
lambd = comb[2]
eta_decay = 0.00
verbose = False
grid_search = True
folds = 10
nn_to_plot = []
training_cup_path = "dataset/blindcup/training_set2.csv"
training_cup = get_dataset(training_cup_path)
train_folded, val_folded = k_fold(training_cup, folds)
for k in range(len(train_folded)):
#print(f"KFOLD {k + 1} of {folds} _______________________________________")
nn = init_manger(*architecture)
tr = train_folded[k]
tval = val_folded[k]
nn.train(task, tr, tval, epochs, eta, alpha, lambd, eta_decay,
verbose, grid_search)
nn_to_plot.append(nn)
plot_multinetwork(nn_to_plot, eta, alpha, lambd, folds,
architecture)
error_kfold_tr = 0
error_kfold_val = 0
for network in nn_to_plot:
error_kfold_tr += network.error_list[-1][1]
error_kfold_val += network.validation_error_list[-1][1]
error_kfold_tr = round(error_kfold_tr / len(train_folded), 5)
error_kfold_val = round(error_kfold_val / len(train_folded), 5)
print(
f"NN Architecture: Layers: {len(nn_to_plot[0].layer_list)}, Units x layer: {len(nn_to_plot[0].layer_list[1].net)}"
f"\nHyperparameters: eta: {eta}, alpha: {alpha}, lambda: {lambd}, eta decay: {eta_decay}"
f"\nAverage final error on training set: {error_kfold_tr}"
f"\nAverage final error on validat. set: {error_kfold_val}\n")
if error_kfold_val < best_val:
best_val = error_kfold_val
best_comb = comb
best_arch = architecture
print(
f"Best model with validation error: {best_val} has eta: {best_comb[0]}, alpha: {best_comb[1]}, lambd: {best_comb[2]}, "
f"# hidden layers: {best_arch[0]}, # units: {best_arch[1]}\n")
from keras.models import load_model
from utils import k_fold, prepare_csv_data, read_csv
import keras.backend as K
import numpy as np
import os
from settings import PREDICTION_LABELS, X_TRANSPOSE, K_FOLDS
from settings import GROUND_TRUTH_PATH, MEDIANS_PATH, MODEL_PATH
GROUND_TRUTH = read_csv(
csv_file=GROUND_TRUTH_PATH,
delimiter=';',
transpose=False,
skip_header=True)
x_data, y_data = prepare_csv_data(
data=GROUND_TRUTH,
prediction_labels=PREDICTION_LABELS,
training_path=MEDIANS_PATH,
x_shape=X_TRANSPOSE)
for fold_index, (x_train, x_test, y_train, y_test) in enumerate(k_fold(x_data, y_data, K_FOLDS)):
model = load_model(f'{ os.path.splitext(MODEL_PATH)[0] }_{ fold_index }.h5')
for row_index, row in enumerate(x_test):
prediction = np.round(model.predict(row.reshape((1, *row.shape))).flatten())
print(prediction, y_test[row_index])