def train_Spectrogram_fine_tuning(examples_datasets_train, labels_datasets_train, num_kernels, filter_size=(4, 10),
number_of_cycle_for_first_training=2, number_of_cycles_rest_of_training=2,
path_weight_to_save_to=
"../weights_SPECTROGRAMS_TWO_CYCLES_normal_training_fine_tuning",
gestures_to_remove=None, number_of_classes=11, batch_size=512,
spectrogram_model=True, feature_vector_input_length=None, learning_rate=0.001316):
participants_train, participants_validation, _ = load_dataloaders_training_sessions_spectrogram(
examples_datasets_train, labels_datasets_train, batch_size=batch_size,
number_of_cycle_for_first_training=number_of_cycle_for_first_training,
number_of_cycles_rest_of_training=number_of_cycles_rest_of_training, gestures_to_remove=gestures_to_remove,
ignore_first=True)
for participant_i in range(len(participants_train)):
print("Participant: ", participant_i)
for session_j in range(0, len(participants_train[participant_i])):
print("Session: ", session_j)
# Define Model
if spectrogram_model:
model = SpectrogramConvNet(number_of_class=number_of_classes, num_kernels=num_kernels,
kernel_size=filter_size).cuda()
else:
model = TSD_Network(number_of_class=number_of_classes, num_neurons=num_kernels,
feature_vector_input_length=feature_vector_input_length).cuda()
# Define Loss functions
cross_entropy_loss_classes = nn.CrossEntropyLoss(reduction='mean').cuda()
# Define Optimizer
print(model.parameters())
optimizer = optim.Adam(model.parameters(), lr=learning_rate, betas=(0.5, 0.999))
# Define Scheduler
precision = 1e-8
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=.2, patience=5,
verbose=True, eps=precision)
if session_j > 0:
# Fine-tune from the previous training
model, _, _, start_epoch = load_checkpoint(
model=model, optimizer=None, scheduler=None,
filename=path_weight_to_save_to +
"/participant_%d/best_state_%d.pt" %
(participant_i, session_j - 1))
best_state = train_model_standard(model=model, criterion=cross_entropy_loss_classes, optimizer=optimizer,
scheduler=scheduler,
dataloaders={"train": participants_train[participant_i][session_j],
"val": participants_validation[participant_i][session_j]},
precision=precision, patience=10, patience_increase=10)
if not os.path.exists(path_weight_to_save_to + "/participant_%d" % participant_i):
os.makedirs(path_weight_to_save_to + "/participant_%d" % participant_i)
torch.save(best_state, f=path_weight_to_save_to +
"/participant_%d/best_state_%d.pt"
% (participant_i, session_j))
def train_AdaBN_spectrograms(examples_datasets_train, labels_datasets_train, num_kernels, filter_size=(4, 10),
algo_name="AdaBN", path_weights_to_save_to="Weights_TSD/weights_", batch_size=512,
patience_increment=10,
path_weights_fine_tuning="../weights_TWO_CYCLES_normal_training_fine_tuning",
number_of_cycle_for_first_training=3, number_of_cycles_rest_of_training=3,
gestures_to_remove=None, number_of_classes=11, spectrogram_model=True,
feature_vector_input_length=None):
participants_train, participants_validation, participants_test = load_dataloaders_training_sessions_spectrogram(
examples_datasets_train, labels_datasets_train, batch_size=batch_size,
number_of_cycle_for_first_training=number_of_cycle_for_first_training, get_validation_set=True,
number_of_cycles_rest_of_training=number_of_cycles_rest_of_training, gestures_to_remove=gestures_to_remove,
ignore_first=True)
for participant_i in range(len(participants_train)):
print("SHAPE SESSIONS: ", np.shape(participants_train[participant_i]))
# Skip the first session as it will be identical to normal training
for session_j in range(1, len(participants_train[participant_i])):
print(np.shape(participants_train[participant_i][session_j]))
# Classifier and discriminator
if spectrogram_model:
gesture_classification = SpectrogramConvNet(number_of_class=number_of_classes, num_kernels=num_kernels,
kernel_size=filter_size, dropout=0.5).cuda()
else:
gesture_classification = TSD_Network(number_of_class=number_of_classes, num_neurons=num_kernels,
feature_vector_input_length=feature_vector_input_length).cuda()
# optimizer
precision = 1e-8
learning_rate = 0.001316
optimizer_classifier = optim.Adam(gesture_classification.parameters(), lr=learning_rate, betas=(0.5, 0.999))
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer_classifier, mode='min', factor=.2,
patience=5, verbose=True, eps=precision)
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename=path_weights_fine_tuning +
"/participant_%d/best_state_%d.pt" %
(participant_i, 0))
# Freeze all the weights except those associated with the BN statistics
gesture_classification.freeze_all_except_BN()
best_weights = AdaBN_adaptation(model=gesture_classification, scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
dataloader=participants_train[participant_i][session_j])
if not os.path.exists(path_weights_to_save_to + algo_name + "/participant_%d" % participant_i):
os.makedirs(path_weights_to_save_to + algo_name + "/participant_%d" % participant_i)
torch.save(best_weights, f=path_weights_to_save_to + algo_name +
"/participant_%d/best_state_%d.pt" %
(participant_i, session_j))
def test_network_DA_algorithm(examples_datasets_train, labels_datasets_train, num_kernels,
path_weights_normal='../weights', path_weights_DA='../weights_DANN',
filter_size=(4, 10), algo_name="DANN", cycle_to_test=None,
gestures_to_remove=None, number_of_classes=11):
participant_train, _, participants_test = load_dataloaders_training_sessions(examples_datasets_train,
labels_datasets_train, batch_size=512,
cycle_for_test=cycle_to_test,
gestures_to_remove=gestures_to_remove)
model_outputs = []
predictions = []
ground_truths = []
accuracies = []
for participant_index, dataset_test in enumerate(participants_test):
model_outputs_participant = []
predictions_participant = []
ground_truth_participant = []
accuracies_participant = []
model = SpectrogramConvNet(number_of_class=number_of_classes, num_kernels=num_kernels,
kernel_size=filter_size).cuda()
print(np.shape(dataset_test))
for session_index, training_session_test_data in enumerate(dataset_test):
if session_index == 0:
best_state = torch.load(
path_weights_normal + "/participant_%d/best_state_%d.pt" %
(participant_index, 0))
else:
best_state = torch.load(
path_weights_DA + "/participant_%d/best_state_%d.pt" %
(participant_index, session_index)) # There is 2 evaluation sessions per training
best_weights = best_state['state_dict']
model.load_state_dict(best_weights)
model_outputs_session = []
predictions_training_session = []
ground_truth_training_sesssion = []
with torch.no_grad():
model.eval()
for inputs, labels in training_session_test_data:
inputs = inputs.cuda()
output = model(inputs)
_, predicted = torch.max(output.data, 1)
model_outputs_session.extend(torch.softmax(output, dim=1).cpu().numpy())
predictions_training_session.extend(predicted.cpu().numpy())
ground_truth_training_sesssion.extend(labels.numpy())
print("Participant ID: ", participant_index, " Session ID: ", session_index, " Accuracy: ",
np.mean(np.array(predictions_training_session) == np.array(ground_truth_training_sesssion)))
predictions_participant.append(predictions_training_session)
model_outputs_participant.append(model_outputs_session)
ground_truth_participant.append(ground_truth_training_sesssion)
accuracies_participant.append(np.mean(np.array(predictions_training_session) ==
np.array(ground_truth_training_sesssion)))
accuracies.append(np.array(accuracies_participant))
predictions.append(predictions_participant)
model_outputs.append(model_outputs_participant)
ground_truths.append(ground_truth_participant)
print("ACCURACY PARTICIPANT: ", accuracies_participant)
print(np.array(accuracies).flatten())
accuracies_to_display = []
for accuracies_from_participant in np.array(accuracies).flatten():
accuracies_to_display.extend(accuracies_from_participant)
print(accuracies_to_display)
print("OVERALL ACCURACY: " + str(np.mean(accuracies_to_display)))
file_to_open = "../../results/test_accuracy_on_training_sessions_" + algo_name + "_" + str(
filter_size[1]) + ".txt"
np.save("../../results/predictions_training_session_" + algo_name, (ground_truths, predictions, model_outputs))
with open(file_to_open, "a") as myfile:
myfile.write("Predictions: \n")
myfile.write(str(predictions) + '\n')
myfile.write("Ground Truth: \n")
myfile.write(str(ground_truths) + '\n')
myfile.write("ACCURACIES: \n")
myfile.write(str(accuracies) + '\n')
myfile.write("OVERALL ACCURACY: " + str(np.mean(accuracies_to_display)))
def train_DA_spectrograms_evaluation(examples_datasets_evaluations, labels_datasets_evaluation,
examples_datasets_train, labels_datasets_train, algo_name,
num_kernels, filter_size, path_weights_to_load_from, path_weights_DA,
batch_size=512, patience_increment=10, use_recalibration_data=False,
number_of_cycle_for_first_training=4, number_of_cycles_rest_of_training=4,
spectrogram_model=True, feature_vector_input_length=None, learning_rate=0.001316):
# Get the data to use as the SOURCE from the training sessions
participants_train, participants_validation, _ = load_dataloaders_training_sessions_spectrogram(
examples_datasets_train, labels_datasets_train, batch_size=batch_size,
number_of_cycle_for_first_training=number_of_cycle_for_first_training, get_validation_set=True,
number_of_cycles_rest_of_training=number_of_cycles_rest_of_training, gestures_to_remove=None,
ignore_first=True, shuffle=True)
# Get the data to use as the TARGET from the evaluation sessions
participants_evaluation_dataloader = load_dataloaders_test_sessions(
examples_datasets_evaluation=examples_datasets_evaluations,
labels_datasets_evaluation=labels_datasets_evaluation, batch_size=batch_size, shuffle=True, drop_last=True)
for participant_i in range(len(participants_evaluation_dataloader)):
print("SHAPE SESSIONS: ", np.shape(participants_evaluation_dataloader[participant_i]))
for session_j in range(0, len(participants_evaluation_dataloader[participant_i])):
# There is two evaluation session for every training session. We train on the first one
if session_j % 2 == 0:
# Get the weights trained
corresponding_training_session_index = 0 if use_recalibration_data is False else int(session_j / 2)
# Classifier and discriminator
if spectrogram_model:
gesture_classification = SpectrogramConvNet(number_of_class=11, num_kernels=num_kernels,
kernel_size=filter_size, dropout=0.5).cuda()
else:
gesture_classification = TSD_Network(number_of_class=11, num_neurons=num_kernels,
feature_vector_input_length=feature_vector_input_length).cuda()
# loss functions
crossEntropyLoss = nn.CrossEntropyLoss().cuda()
# optimizer
precision = 1e-8
optimizer_classifier = optim.Adam(gesture_classification.parameters(), lr=learning_rate,
betas=(0.5, 0.999))
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer_classifier, mode='min', factor=.2,
patience=5, verbose=True, eps=precision)
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename=path_weights_to_load_from +
"/participant_%d/best_state_%d.pt" %
(participant_i, corresponding_training_session_index))
best_state = None
if "DANN" in algo_name:
best_state = DANN_BN_Training(gesture_classifier=gesture_classification, scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
train_dataset_source=participants_train[participant_i][
corresponding_training_session_index],
train_dataset_target=participants_evaluation_dataloader[
participant_i][session_j],
validation_dataset_source=participants_validation[participant_i][
corresponding_training_session_index],
crossEntropyLoss=crossEntropyLoss,
patience_increment=patience_increment,
domain_loss_weight=1e-1)
elif "VADA" in algo_name:
# VADA need Conditional Entropy loss and Virtual Adversarial Training loss too
conditionalEntropy = ConditionalEntropyLoss().cuda()
vatLoss = VATLoss(gesture_classification).cuda()
best_state = vada_BN_Training(gesture_classifier=gesture_classification,
conditionalEntropyLoss=conditionalEntropy,
crossEntropyLoss=crossEntropyLoss, vatLoss=vatLoss,
scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
train_dataset_source=participants_train[participant_i][
corresponding_training_session_index],
train_dataset_target=
participants_evaluation_dataloader[participant_i][session_j],
validation_dataset_source=participants_validation[participant_i][
corresponding_training_session_index],
patience_increment=patience_increment)
elif "DirtT" in algo_name:
# Dirt T need Conditional Entropy loss and Virtual Adversarial Training loss too
conditionalEntropy = ConditionalEntropyLoss().cuda()
vatLoss = VATLoss(gesture_classification).cuda()
# Classifier and discriminator
if spectrogram_model:
gesture_classification = SpectrogramConvNet(number_of_class=11, num_kernels=num_kernels,
kernel_size=filter_size, dropout=0.5).cuda()
else:
gesture_classification = TSD_Network(number_of_class=11, num_neurons=num_kernels,
feature_vector_input_length=feature_vector_input_length).cuda()
# loss functions
crossEntropyLoss = nn.CrossEntropyLoss().cuda()
# optimizer
precision = 1e-8
optimizer_classifier = optim.Adam(gesture_classification.parameters(), lr=learning_rate,
betas=(0.5, 0.999))
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer_classifier, mode='min',
factor=.2,
patience=5, verbose=True, eps=precision)
if use_recalibration_data:
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename=path_weights_to_load_from +
"/participant_%d/best_state_WITH_recalibration%d.pt" %
(participant_i, session_j))
else:
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename=path_weights_to_load_from +
"/participant_%d/best_state_NO_recalibration%d.pt" %
(participant_i, session_j))
best_state = dirt_T_training(gesture_classifier=gesture_classification,
conditionalEntropyLoss=conditionalEntropy,
crossEntropyLoss=crossEntropyLoss, vatLoss=vatLoss,
scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
train_dataset_source=participants_evaluation_dataloader[participant_i][
session_j],
patience_increment=patience_increment, batch_size=batch_size)
if use_recalibration_data:
if not os.path.exists(path_weights_DA + algo_name + "/participant_%d" % participant_i):
os.makedirs(path_weights_DA + algo_name + "/participant_%d" % participant_i)
torch.save(best_state, f=path_weights_DA + algo_name +
"/participant_%d/best_state_WITH_recalibration%d.pt" %
(participant_i, session_j))
else:
if not os.path.exists(path_weights_DA + algo_name + "/participant_%d" % participant_i):
os.makedirs(path_weights_DA + algo_name + "/participant_%d" % participant_i)
print(os.listdir(path_weights_DA + algo_name))
torch.save(best_state, f=path_weights_DA + algo_name +
"/participant_%d/best_state_NO_recalibration%d.pt" % (
participant_i, session_j))
def train_DA_spectrograms(examples_datasets_train, labels_datasets_train, num_kernels, filter_size=(4, 10),
path_weights_to_load_from_for_dirtT='../weights_VADA_TWO_Cycles', algo_name="DANN",
path_weights_to_save_to="../Weights/weights_", batch_size=512, patience_increment=10,
path_weights_fine_tuning="../weights_TWO_CYCLES_normal_training_fine_tuning",
number_of_cycle_for_first_training=3, number_of_cycles_rest_of_training=3,
gestures_to_remove=None, number_of_classes=11, spectrogram_model=True,
feature_vector_input_length=None, learning_rate=0.001316):
participants_train, participants_validation, participants_test = load_dataloaders_training_sessions_spectrogram(
examples_datasets_train, labels_datasets_train, batch_size=batch_size,
number_of_cycle_for_first_training=number_of_cycle_for_first_training, get_validation_set=True,
number_of_cycles_rest_of_training=number_of_cycles_rest_of_training, gestures_to_remove=gestures_to_remove,
ignore_first=True)
for participant_i in range(len(participants_train)):
print("SHAPE SESSIONS: ", np.shape(participants_train[participant_i]))
# Skip the first session as it will be identical to normal training
for session_j in range(1, len(participants_train[participant_i])):
print(np.shape(participants_train[participant_i][session_j]))
# Classifier and discriminator
if spectrogram_model:
gesture_classification = SpectrogramConvNet(number_of_class=number_of_classes, num_kernels=num_kernels,
kernel_size=filter_size, dropout=0.5).cuda()
else:
gesture_classification = TSD_Network(number_of_class=number_of_classes, num_neurons=num_kernels,
feature_vector_input_length=feature_vector_input_length).cuda()
# loss functions
crossEntropyLoss = nn.CrossEntropyLoss().cuda()
# optimizer
precision = 1e-8
optimizer_classifier = optim.Adam(gesture_classification.parameters(), lr=learning_rate, betas=(0.5, 0.999))
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer_classifier, mode='min', factor=.2,
patience=5, verbose=True, eps=precision)
# Fine-tune from the previous training
'''
if session_j == 0:
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename=path_weights_fine_tuning +
"/participant_%d/best_state_%d.pt" %
(participant_i, 0))
else:
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename="../weights_Reduced_Spectrograms_SLADANN" +
"/participant_%d/best_state_%d.pt" %
(participant_i, session_j))
'''
gesture_classification, optimizer_classifier, scheduler, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=scheduler,
filename=path_weights_fine_tuning +
"/participant_%d/best_state_%d.pt" %
(participant_i, 0))
if "DANN" in algo_name:
best_weights = DANN_BN_Training(gesture_classifier=gesture_classification, scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
train_dataset_source=participants_train[participant_i][0],
train_dataset_target=participants_train[participant_i][session_j],
validation_dataset_source=participants_validation[participant_i][0],
crossEntropyLoss=crossEntropyLoss,
patience_increment=patience_increment,
domain_loss_weight=1e-1)
elif "VADA" in algo_name:
# VADA need Conditional Entropy loss and Virtual Adversarial Training loss too
conditionalEntropy = ConditionalEntropyLoss().cuda()
vatLoss = VATLoss(gesture_classification).cuda()
best_weights = vada_BN_Training(gesture_classifier=gesture_classification,
conditionalEntropyLoss=conditionalEntropy,
crossEntropyLoss=crossEntropyLoss, vatLoss=vatLoss, scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
train_dataset_source=participants_train[participant_i][0],
train_dataset_target=participants_train[participant_i][session_j],
validation_dataset_source=participants_validation[participant_i][0],
patience_increment=patience_increment)
elif "Dirt_T" in algo_name:
optimizer_classifier = optim.Adam(gesture_classification.parameters(), lr=learning_rate,
betas=(0.5, 0.999))
# Dirt T need Conditional Entropy loss and Virtual Adversarial Training loss too
conditionalEntropy = ConditionalEntropyLoss().cuda()
vatLoss = VATLoss(gesture_classification).cuda()
gesture_classification, optimizer_classifier, _, start_epoch = load_checkpoint(
model=gesture_classification, optimizer=optimizer_classifier, scheduler=None,
filename=path_weights_to_load_from_for_dirtT +
"/participant_%d/best_state_%d.pt" %
(participant_i, session_j))
best_weights = dirt_T_training(gesture_classifier=gesture_classification,
conditionalEntropyLoss=conditionalEntropy,
crossEntropyLoss=crossEntropyLoss, vatLoss=vatLoss, scheduler=scheduler,
optimizer_classifier=optimizer_classifier,
train_dataset_source=participants_train[participant_i][session_j],
patience_increment=patience_increment, batch_size=batch_size)
if not os.path.exists(path_weights_to_save_to + algo_name + "/participant_%d" % participant_i):
os.makedirs(path_weights_to_save_to + algo_name + "/participant_%d" % participant_i)
torch.save(best_weights, f=path_weights_to_save_to + algo_name +
"/participant_%d/best_state_%d.pt" %
(participant_i, session_j))