def f1_score(self, y_true, y_pred):
"""
evaluate the f1_score of the model, print the score,
act as a sanity check
Parameters:
-----------
y_true: 2d arrays
true labels
y_pred: 2d arrays
predicted labels
"""
score_val = evaluate(y_true, y_pred,
self.binarizer, self.G_tags,
l_print_errors=False, l_deduplication = True)
print('Test score: {0:.2f}'.format(score_val))
def evaluateAndShowAttention(input_sentence):
output_words, attentions = evaluate(encoder1, attn_decoder1,
input_sentence)
print('input =', input_sentence)
print('output =', ' '.join(output_words))
def run_training(cfg):
# Read the configuration file to be able to save it later if desired
if cfg["save_model"]:
with open(cfg["source_path"] + "/tconv_arch/main.py", "r") as f:
cfg_file = f.read()
# Print some information to console
print("Architecture name:", cfg["architecture_name"])
print("Model name:", cfg["model_name"])
# Specify the paths for this script
data_src_path = cfg["source_path"] + "/data/"
model_src_path = cfg["source_path"] + "/tconv_arch/saved_models/"
# Set device on GPU if specified so in the main file, else CPU
device = helpers.determine_device()
# Set up the parameter and tensor classes
params = net_variables.NetworkParameters(cfg=cfg, device=device)
tensors = net_variables.NetworkTensors(params=params)
# Initialize and set up the network model
net = net_model.Model(params=params, tensors=tensors)
# Count number of trainable parameters
pytorch_total_params = sum(p.numel() for p in net.parameters()
if p.requires_grad)
print("Trainable model parameters:", pytorch_total_params)
#
# Set up the optimizer and the criterion (loss)
optimizer = th.optim.Adam(net.parameters(), lr=cfg["learning_rate"])
criterion = nn.MSELoss()
#
# Set up lists to save and store the epoch errors
epoch_errors_train = []
epoch_errors_val = []
best_train = np.infty
best_val = np.infty
#
# Get the training and validation file names
train_data_filenames = np.sort(glob.glob(data_src_path + 'train/*'))
val_data_filenames = np.sort(glob.glob(data_src_path + 'val/*'))
# If desired, restore the network by loading the weights saved in the .pt
# file
if cfg["continue_training"]:
print('Restoring model (that is the network\'s weights) from file...')
net.load_state_dict(
th.load(model_src_path + "/" + cfg["model_name"] + "/" +
cfg["model_name"] + ".pt"))
net.eval()
"""
TRAINING
"""
a = time.time()
#
# Start the training and iterate over all epochs
for epoch in range(cfg["epochs"]):
epoch_start_time = time.time()
# Shuffle the training_data_filenames at the beginning of each epoch to
# have variable (stochastic) training data orders
np.random.shuffle(train_data_filenames)
sequence_errors = []
# Iterate over all training iterations and evaluate the network
for train_iter in range(len(train_data_filenames)):
# Evaluate and train the network for the given training data
mse, _, _, _ = helpers.evaluate(
net=net,
data_filenames=train_data_filenames,
params=params,
tensors=tensors,
criterion=criterion,
optimizer=optimizer,
_iter=train_iter)
sequence_errors.append(mse.item())
epoch_errors_train.append(np.mean(sequence_errors))
# Create a plus or minus sign for the training error
train_sign = "(-)"
if epoch_errors_train[-1] < best_train:
train_sign = "(+)"
best_train = epoch_errors_train[-1]
#
# Compute validation error
# Evaluate and validate the network for the given validation data
mse, _, _, _ = helpers.evaluate(net=net,
data_filenames=val_data_filenames,
params=params,
tensors=tensors,
criterion=criterion,
_iter=0)
epoch_errors_val.append(mse.item())
# Save the model to file (if desired)
if cfg["save_model"] and mse.item() < best_val:
# Start a separate thread to save the model
thread = Thread(target=helpers.save_model_to_file(
model_src_path=model_src_path,
cfg_file=cfg_file,
epoch=epoch,
epoch_errors_train=epoch_errors_train,
epoch_errors_val=epoch_errors_val,
net=net,
params=params))
thread.start()
# Create a plus or minus sign for the validation error
val_sign = "(-)"
if epoch_errors_val[-1] < best_val:
best_val = epoch_errors_val[-1]
val_sign = "(+)"
#
# Print progress to the console
print('Epoch ' +
str(epoch + 1).zfill(int(np.log10(cfg["epochs"])) + 1) + '/' +
str(cfg["epochs"]) + ' took ' +
str(np.round(time.time() - epoch_start_time, 2)).ljust(5, '0') +
' seconds.\t\tAverage epoch training error: ' + train_sign +
str(np.round(epoch_errors_train[-1], 10)).ljust(12, ' ') +
'\t\tValidation error: ' + val_sign +
str(np.round(epoch_errors_val[-1], 10)).ljust(12, ' '))
b = time.time()
print('\nTraining took ' + str(np.round(b - a, 2)) + ' seconds.\n\n')
"""
BRIEF TESTING
"""
plt.style.use('dark_background')
#
# Set up the feed dictionary for the test iteration
test_data_filenames = glob.glob(data_src_path + 'test/*')
x = u'1'
curr_idx = 0
while x == u'1':
_, net_input, net_label, net_outputs = helpers.evaluate(
net=net,
data_filenames=test_data_filenames,
params=params,
tensors=tensors,
criterion=criterion,
_iter=curr_idx)
net_outputs = net_outputs.cpu().detach().numpy()
fig, axes = plt.subplots(2, 2, figsize=[10, 8])
for i in range(2):
for j in range(2):
make_legend = True if (i == 0 and j == 0) else False
helpers.plot_kernel_activity(ax=axes[i, j],
label=net_label,
net_out=net_outputs,
net_in=net_input,
params=params,
make_legend=make_legend)
fig.suptitle('Optimization', fontsize=12)
plt.show()
x = input("Press 1 to see another example, anything else to quit.")
curr_idx += 1
print('Done')
def run_testing(cfg):
# Specify the paths for this script
data_src_path = cfg["source_path"] + "/data/"
model_src_path = cfg["source_path"] + "/tconv_arch/saved_models/"
# setting device on GPU if available, else CPU
device = helpers.determine_device()
# Set up the parameter and tensor classes
params = net_variables.NetworkParameters(cfg=cfg, device=device)
# tensors1 = kernel_variables.KernelTensors(_params=params)
tensors = net_variables.NetworkTensors(params=params)
# Initialize and set up the kernel network
net = net_model.Model(params=params, tensors=tensors)
# Restore the network by loading the weights saved in the .pt file
print('Restoring model (that is the network\'s weights) from file...')
net.load_state_dict(
th.load(model_src_path + "/" + cfg["model_name"] + "/" +
cfg["model_name"] + ".pt",
map_location=params.device))
net.eval()
# Count number of trainable parameters
pytorch_total_params = sum(p.numel() for p in net.parameters()
if p.requires_grad)
print("Trainable model parameters:", pytorch_total_params)
"""
TESTING
"""
plt.style.use('dark_background')
#
# Set up the feed dictionary for the test iteration
test_data_filenames = glob.glob(data_src_path + 'test/*')
x = u'1'
curr_idx = 0
while x == u'1':
error = 0.0
std = 0.0
for idx in range(10):
print("Sequence %s" % (idx + 1))
time_start = time.time()
# Evaluate the network for the given test data
_, net_input, net_label, net_outputs = helpers.evaluate(
net=net,
data_filenames=test_data_filenames,
params=params,
tensors=tensors,
_iter=curr_idx,
testing=True)
forward_pass_duration = time.time() - time_start
print("Forward pass took:", forward_pass_duration, "seconds.")
net_outputs = net_outputs.detach().numpy()
# Plot the wave activity
fig, axes = plt.subplots(2, 2, figsize=[10, 8], sharex="all")
for i in range(2):
for j in range(2):
make_legend = True if (i == 0 and j == 0) else False
helpers.plot_kernel_activity(ax=axes[i, j],
label=net_label,
net_out=net_outputs,
params=params,
make_legend=make_legend)
fig.suptitle('Model ' + cfg["model_name"], fontsize=12)
plt.show()
# Visualize and animate the propagation of the 2d wave
anim = helpers.animate_2d_wave(net_label, net_outputs, params)
# plt.show()
# Compute the error for only the closed loop steps
diff = np.square(net_outputs[:, :, 15:, :, :] -
net_label[:, :, 15:, :, :])
mse = np.mean(diff)
print("Error: %s" % (mse))
error += mse
std += np.std(diff)
curr_idx += 1
print("Average error over 10 sequences: %s" % (error / 10))
print("Average Standard deviation over 10 sequences: %s" % (std / 10))
# Retrieve user input to continue or quit the testing
x = input("Press 1 to see another example, anything else to quit.")
print('Done')