def train(self):
self.model = model_setter.get_model_params()
self.model.build(input_shape=(None, 96, 160, 3))
self.model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
metrics=['accuracy'])
self.model.summary()
log_dir = f"logs/fit/{self.output_path}"
checkpoint_path = f"{self.output_path}/cp-{{epoch:04d}}.ckpt"
best_checkpoint = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_path,
monitor='val_loss',
verbose=0,
mode='min',
save_best_only=True)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
self.history = self.model.fit(
self.images, [self.directions_labels, self.speeds_labels],
epochs=self.nb_epochs,
validation_split=self.validation_split,
shuffle=True,
verbose=1,
callbacks=[best_checkpoint, tensorboard_callback])
if self.test_split > 0:
validation.evaluate(self)
def main():
args = parseArgs()
function = args.function
input_file = args.input
output_file = args.output
true_file = args.trueout
result_file = args.result
method = args.method
if function == ("Pred" or "Predict"):
if input_file == "":
print("Error - Read inputs")
elif output_file == "":
print("Error - Write output")
else:
prediction(input_file, output_file, method)
elif function == ("Eval" or "Evaluate"):
if input_file == "" or true_file == "":
print("Error - Read inputs")
elif output_file == "":
print("Error - Write output")
else:
evaluate(input_file, output_file, true_file, result_file)
else:
print("Please choose 'Pred' function or 'Eval' function")
def execute_experiments(dataset, w2v_model, n_splits, model_option, output_csv_path, score_threshold):
results = []
shuffle(dataset)
folds = KFold(n_splits=n_splits, random_state=7, shuffle=False)
splits = [(train_index, test_index) for train_index, test_index in folds.split(dataset)]
prep_dataset = np.array([[row[0], row[1], compute_candidates(row, w2v_model)] for row in dataset])
for train_index, test_index in splits:
train, test = prep_dataset[train_index], prep_dataset[test_index]
model = fit_model(model_option, train, w2v_model)
results.append(evaluate(test, model, score_threshold))
save_results(results, output_csv_path)
def svm_baseline(X, Y, X_test, Y_test, method=None):
setup_seed(20)
clf = SVC(gamma='auto', class_weight='balanced').fit(X, Y)
train_acc = accuracy_score(Y, clf.predict(X))
train_pre = precision_score(Y, clf.predict(X))
train_rec = recall_score(Y, clf.predict(X))
train_fscore = f1_score(Y, clf.predict(X))
train_mcc = matthews_corrcoef(Y, clf.predict(X))
Y_pred = clf.predict(X_test)
precision, recall, fscore, mcc, val_acc = evaluate(Y_test, Y_pred)
print('T_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f' %
(train_acc, train_pre, train_rec, train_fscore, train_mcc))
print('V_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f' %
(val_acc, precision, recall, fscore, mcc))
def nn_baseline(X, Y, X_test, Y_test):
setup_seed(20)
clf = MLPClassifier(random_state=100).fit(X, Y)
train_acc = accuracy_score(Y, clf.predict(X))
train_pre = precision_score(Y, clf.predict(X))
train_rec = recall_score(Y, clf.predict(X))
train_fscore = f1_score(Y, clf.predict(X))
train_mcc = matthews_corrcoef(Y, clf.predict(X))
Y_pred = clf.predict(X_test)
precision, recall, fscore, mcc, val_acc = evaluate(Y_test, Y_pred)
print('T_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f' %
(train_acc, train_pre, train_rec, train_fscore, train_mcc))
print('V_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f' %
(val_acc, precision, recall, fscore, mcc))
def lr_baseline(X, Y, X_test, Y_test, method=None):
setup_seed(20)
clf = linear_model.LogisticRegression().fit(X, Y)
train_acc = accuracy_score(Y, clf.predict(X))
train_pre = precision_score(Y, clf.predict(X))
train_rec = recall_score(Y, clf.predict(X))
train_fscore = f1_score(Y, clf.predict(X))
train_mcc = matthews_corrcoef(Y, clf.predict(X))
Y_pred = clf.predict(X_test)
precision, recall, fscore, mcc, val_acc = evaluate(Y_test, Y_pred)
print('T_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f' %
(train_acc, train_pre, train_rec, train_fscore, train_mcc))
print('V_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f' %
(val_acc, precision, recall, fscore, mcc))
def knn_baseline(X, Y, X_test, Y_test, method=None):
setup_seed(20)
clf = neighbors.KNeighborsClassifier().fit(X, Y)
train_acc = accuracy_score(Y, clf.predict(X))
train_pre = precision_score(Y, clf.predict(X))
train_rec = recall_score(Y, clf.predict(X))
train_fscore = f1_score(Y, clf.predict(X))
train_mcc = matthews_corrcoef(Y, clf.predict(X))
Y_pred = clf.predict(X_test)
precision, recall, fscore, mcc, val_acc = evaluate(Y_test, Y_pred)
print('T_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f' %
(train_acc, train_pre, train_rec, train_fscore, train_mcc))
print('V_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f' %
(val_acc, precision, recall, fscore, mcc))
def execute_experiments(dataset, w2v_model, n_splits, model_option, output_csv_path):
results = []
shuffle(dataset)
folds = KFold(n_splits=n_splits, random_state=7, shuffle=False)
splits = [(train_index, test_index) for train_index, test_index in folds.split(dataset)]
x, y, label_encoder = preprocess_dataset(dataset, w2v_model)
for train_index, test_index in splits:
dataset_split = np.array(dataset)[test_index]
x_train, x_test = x[train_index], x[test_index]
y_train, y_test = y[train_index], [row[1] for row in dataset_split]
l_sizes = [len(row[0]) for row in dataset_split]
model = fit_model(model_option, x_train, y_train, w2v_model)
results.append(evaluate(x_test, y_test, l_sizes, model, label_encoder))
save_results(results, output_csv_path)
def rf_baseline(X, Y, X_test, Y_test):
setup_seed(20)
clf = ensemble.RandomForestClassifier().fit(X, Y)
train_acc = accuracy_score(Y, clf.predict(X))
train_pre = precision_score(Y, clf.predict(X))
train_rec = recall_score(Y, clf.predict(X))
train_fscore = f1_score(Y, clf.predict(X))
train_mcc = matthews_corrcoef(Y, clf.predict(X))
Y_pred = clf.predict(X_test)
precision, recall, fscore, mcc, val_acc = evaluate(Y_test, Y_pred)
outcome = [train_acc, train_pre, train_rec, train_fscore, val_acc, precision, recall, fscore]
return outcome
print('T_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f'
% (train_acc, train_pre, train_rec, train_fscore, train_mcc))
print('V_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f'
% (val_acc, precision, recall, fscore, mcc))
def validate(combined_model, unsupervised_val, retrievable_items):
batches_per_epoch = len(unsupervised_val)
kbar = pkbar.Kbar(target=batches_per_epoch, width=8)
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
combined_model.eval()
image_embeddings = []
text_embeddings = []
running_loss = 0.0
print("Start validation")
for indx, unsupervised_inputs in enumerate(unsupervised_val):
unsupervised_image_inputs = unsupervised_inputs[0].to(device)
unsupervised_text_inputs = unsupervised_inputs[1].to(device)
with torch.set_grad_enabled(False):
text_embeddings_unsupervised, image_embeddings_unsupervised = combined_model(
unsupervised_text_inputs, unsupervised_image_inputs)
unsupervised_loss = criterion_unsupervised(
text_embeddings_unsupervised, image_embeddings_unsupervised)
image_embeddings.append(
image_embeddings_unsupervised.detach().clone())
text_embeddings.append(
text_embeddings_unsupervised.detach().clone())
# statistics
running_loss += unsupervised_loss
kbar.update(indx, values=[("unsupervised_loss", unsupervised_loss)])
recalls = []
for item_count in retrievable_items:
recalls.append((evaluate(text_embeddings, image_embeddings, 5,
item_count), item_count))
epoch_loss = running_loss / len(unsupervised_val)
return epoch_loss, recalls
def crossValidate(data_set):
#80 because size gives total data points not no. of rows
empty_confusion_matrix = np.zeros((4, 4))
empty_metrics = (empty_confusion_matrix, 0.0, 0.0, 0.0, 0.0)
unpruned_results = [[empty_metrics for a in range(9)] for b in range(10)]
pruned_results = [[empty_metrics for a in range(9)] for b in range(10)]
split_size = int(data_set.size / 80)
startTime = time.time()
#split out the testing data
for i in range(10):
split_set = np.split(data_set, [i * split_size, (i + 1) * split_size])
test_set = split_set[1]
set_without_test = np.concatenate((split_set[0], split_set[2]), axis=0)
#split out the validation
for j in range(9):
split_training_set = np.split(
set_without_test, [j * split_size, (j + 1) * split_size])
validation_set = split_training_set[1]
training_set = np.concatenate(
(split_training_set[0], split_training_set[2]), axis=0)
tree = getTree(training_set, 0)
#print("Depth of tree:")
#print(tree[1])
tree = tree[0]
unpruned_results[i][j] = evaluate(test_set, tree)
pruned_tree = prune(tree, validation_set)
pruned_results[i][j] = evaluate(test_set, pruned_tree)
#stuff for printing nicely
percent = (float(i * 9) + float(j + 1)) / 0.9
timeElapsed = time.time() - startTime
timeLeft = timeElapsed / percent * (100 - percent)
print("\r\t",
round(percent, 2),
"%\t Time elapsed: ",
int(timeElapsed / 3600),
":",
int((timeElapsed / 60) % 60),
":",
int(timeElapsed % 60),
"\t Time left: ",
int(timeLeft / 3600),
":",
int((timeLeft / 60) % 60),
":",
int(timeLeft % 60),
end=" ",
sep="")
average_unpruned_results = average_metrics(unpruned_results)
average_pruned_results = average_metrics(pruned_results)
print("Done:")
print()
print("\nResults before pruning:\n")
print_metrics(average_unpruned_results)
print("\nResults after pruning:\n")
print_metrics(average_pruned_results)
def train_cnn(model, epochs, learning_rate, batch_size, X, Y, X_test, Y_test):
"""
Training loop for a model utilizing hidden states.
verify enables sanity checks of the model.
epochs decides the number of training iterations.
learning rate decides how much the weights are updated each iteration.
batch_size decides how many examples are in each mini batch.
show_attention decides if attention weights are plotted.
"""
print_interval = 10
optimizer = torch.optim.SGD(model.parameters(),
lr=learning_rate,
weight_decay=0.01)
criterion = torch.nn.CrossEntropyLoss()
num_of_examples = X.shape[0]
num_of_batches = math.floor(num_of_examples / batch_size)
#if verify:
# verify_model(model, X, Y, batch_size)
all_losses = []
all_val_losses = []
all_accs = []
all_pres = []
all_recs = []
all_fscores = []
all_mccs = []
all_val_accs = []
start_time = time.time()
for epoch in range(epochs):
model.train()
running_loss = 0
running_acc = 0
running_pre = 0
running_pre_total = 0
running_rec = 0
running_rec_total = 0
epoch_fscore = 0
running_mcc_numerator = 0
running_mcc_denominator = 0
running_rec_total = 0
#hidden = model.init_hidden(batch_size)
for count in range(0, num_of_examples - batch_size + 1, batch_size):
#repackage_hidden(hidden)
#X_batch = X[:, count:count+batch_size, :]
X_batch = X[count:count + batch_size, :, :, :]
Y_batch = Y[count:count + batch_size]
scores = model(X_batch)
loss = criterion(scores, Y_batch)
optimizer.zero_grad()
loss.backward()
optimizer.step()
predictions = predictions_from_output(scores)
conf_matrix = get_confusion_matrix(Y_batch, predictions)
TP, FP, FN, TN = conf_matrix[0][0], conf_matrix[0][1], conf_matrix[
1][0], conf_matrix[1][1]
running_acc += TP + TN
running_pre += TP
running_pre_total += TP + FP
running_rec += TP
running_rec_total += TP + FN
running_mcc_numerator += (TP * TN - FP * FN)
if ((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) == 0:
running_mcc_denominator += 0
else:
running_mcc_denominator += math.sqrt(
(TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
running_loss += loss.item()
elapsed_time = time.time() - start_time
epoch_acc = running_acc / Y.shape[0]
all_accs.append(epoch_acc)
if running_pre_total == 0:
epoch_pre = 0
else:
epoch_pre = running_pre / running_pre_total
all_pres.append(epoch_pre)
if running_rec_total == 0:
epoch_rec = 0
else:
epoch_rec = running_rec / running_rec_total
all_recs.append(epoch_rec)
if (epoch_pre + epoch_rec) == 0:
epoch_fscore = 0
else:
epoch_fscore = 2 * epoch_pre * epoch_rec / (epoch_pre + epoch_rec)
all_fscores.append(epoch_fscore)
if running_mcc_denominator == 0:
epoch_mcc = 0
else:
epoch_mcc = running_mcc_numerator / running_mcc_denominator
all_mccs.append(epoch_mcc)
epoch_loss = running_loss / num_of_batches
all_losses.append(epoch_loss)
with torch.no_grad():
model.eval()
test_scores = model(X_test)
predictions = predictions_from_output(test_scores)
predictions = predictions.view_as(Y_test)
precision, recall, fscore, mcc, val_acc = evaluate(
Y_test, predictions)
val_loss = criterion(test_scores, Y_test).item()
all_val_losses.append(val_loss)
all_val_accs.append(val_acc)
if (epoch + 1) % print_interval == 0:
print('Epoch %d Time %s' % (epoch, get_time_string(elapsed_time)))
print(
'T_loss %.3f\tT_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f'
% (epoch_loss, epoch_acc, epoch_pre, epoch_rec, epoch_fscore,
epoch_mcc))
print(
'V_loss %.3f\tV_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f'
% (val_loss, val_acc, precision, recall, fscore, mcc))
def train_cnn(model, epochs, learning_rate, batch_size, X, Y, X_test, Y_test,
subtype):
"""
Training loop for a model utilizing hidden states.
verify enables sanity checks of the model.
epochs decides the number of training iterations.
learning rate decides how much the weights are updated each iteration.
batch_size decides how many examples are in each mini batch.
show_attention decides if attention weights are plotted.
"""
print_interval = 1
if subtype == 'PAX' or subtype == 'M2' or subtype == 'NS2':
Weight_Decay = 0.05
else:
Weight_Decay = 0.001
for i in range(1): # define different optimizers
if i == 0:
optimizer = torch.optim.SGD(model.parameters(),
lr=learning_rate,
weight_decay=Weight_Decay)
op_title = 'SGD'
print("SGD")
Color = 'r'
elif i == 1:
optimizer = torch.optim.Adam(model.parameters(),
lr=0.0001,
weight_decay=Weight_Decay)
op_title = 'Adam'
print("Adam")
Color = 'b'
elif i == 2:
optimizer = torch.optim.RMSprop(model.parameters(),
lr=0.0001,
weight_decay=Weight_Decay)
op_title = 'RMSprop'
print("RMSprop")
Color = 'g'
elif i == 3:
optimizer = torch.optim.Adadelta(model.parameters(),
lr=0.005,
weight_decay=Weight_Decay)
op_title = 'Adadelta'
print("Adadelta")
Color = 'c'
elif i == 4:
optimizer = torch.optim.Adagrad(model.parameters(),
lr=0.005,
weight_decay=Weight_Decay)
op_title = 'Adagrad'
print("Adagrad")
Color = 'y'
criterion = torch.nn.CrossEntropyLoss()
num_of_examples = X.shape[0]
num_of_batches = math.floor(num_of_examples / batch_size)
# if verify:
# verify_model(model, X, Y, batch_size)
all_losses = []
all_val_losses = []
all_accs = []
all_pres = []
all_recs = []
all_fscores = []
all_mccs = []
all_val_accs = []
best_acc = 0
best_loss = 10
start_time = time.time()
for epoch in range(epochs):
model.train()
running_loss = 0
running_acc = 0
running_pre = 0
running_pre_total = 0
running_rec = 0
running_rec_total = 0
epoch_fscore = 0
running_mcc_numerator = 0
running_mcc_denominator = 0
running_rec_total = 0
# hidden = model.init_hidden(batch_size)
for count in range(0, num_of_examples - batch_size + 1,
batch_size):
# repackage_hidden(hidden)
# X_batch = X[:, count:count+batch_size, :]
X_batch = X[count:count + batch_size, :, :, :]
Y_batch = Y[count:count + batch_size]
scores = model(X_batch)
loss = criterion(scores, Y_batch)
optimizer.zero_grad()
loss.backward()
optimizer.step()
predictions = predictions_from_output(scores)
conf_matrix = get_confusion_matrix(Y_batch, predictions)
TP, FP, FN, TN = conf_matrix[0][0], conf_matrix[0][
1], conf_matrix[1][0], conf_matrix[1][1]
running_acc += TP + TN
running_pre += TP
running_pre_total += TP + FP
running_rec += TP
running_rec_total += TP + FN
running_mcc_numerator += (TP * TN - FP * FN)
if ((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) == 0:
running_mcc_denominator += 0
else:
running_mcc_denominator += math.sqrt(
(TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
running_loss += loss.item()
elapsed_time = time.time() - start_time
epoch_acc = running_acc / Y.shape[0]
all_accs.append(epoch_acc)
if running_pre_total == 0:
epoch_pre = 0
else:
epoch_pre = running_pre / running_pre_total
all_pres.append(epoch_pre)
if running_rec_total == 0:
epoch_rec = 0
else:
epoch_rec = running_rec / running_rec_total
all_recs.append(epoch_rec)
if (epoch_pre + epoch_rec) == 0:
epoch_fscore = 0
else:
epoch_fscore = 2 * epoch_pre * epoch_rec / (epoch_pre +
epoch_rec)
all_fscores.append(epoch_fscore)
if running_mcc_denominator == 0:
epoch_mcc = 0
else:
epoch_mcc = running_mcc_numerator / running_mcc_denominator
all_mccs.append(epoch_mcc)
epoch_loss = running_loss / num_of_batches
all_losses.append(epoch_loss)
with torch.no_grad():
model.eval()
test_scores = model(X_test)
predictions = predictions_from_output(test_scores)
predictions = predictions.view_as(Y_test)
pred_prob = calculate_prob(test_scores)
precision, recall, fscore, mcc, val_acc = evaluate(
Y_test, predictions)
val_loss = criterion(test_scores, Y_test).item()
all_val_losses.append(val_loss)
all_val_accs.append(val_acc)
if val_acc > best_acc and (val_acc < epoch_acc + 0.01):
torch.save(model.state_dict(),
str(subtype) + '_params.pkl')
print("Higher accuracy, New best ", subtype,
" model saved.")
best_epoch = epoch
best_pred_prob = pred_prob
best_acc = val_acc
best_loss = val_loss
best_outcome = [
epoch_acc, epoch_pre, epoch_rec, epoch_fscore, val_acc,
precision, recall, fscore
]
elif (val_acc == best_acc) and (val_loss < best_loss) and (
val_acc < epoch_acc + 0.01):
torch.save(model.state_dict(),
str(subtype) + '_params.pkl')
print("Lower loss, New best ", subtype, " model saved.")
best_epoch = epoch
best_pred_prob = pred_prob
best_acc = val_acc
best_loss = val_loss
best_outcome = [
epoch_acc, epoch_pre, epoch_rec, epoch_fscore, val_acc,
precision, recall, fscore
]
if (epoch + 1) % print_interval == 0:
#print("Epoch: ", epoch)
print('Epoch %d Time %s' %
(epoch, get_time_string(elapsed_time)))
print(
'T_loss %.3f\tT_acc %.3f\tT_pre %.3f\tT_rec %.3f\tT_fscore %.3f\tT_mcc %.3f'
% (epoch_loss, epoch_acc, epoch_pre, epoch_rec,
epoch_fscore, epoch_mcc))
print(
'V_loss %.3f\tV_acc %.3f\tV_pre %.3f\tV_rec %.3f\tV_fscore %.3f\tV_mcc %.3f'
% (val_loss, val_acc, precision, recall, fscore, mcc))
############################### save ROC figure ################################
print(best_epoch)
fpr_cnn, tpr_cnn, _ = roc_curve(Y_test.cpu(), best_pred_prob.cpu())
print(subtype, "/'s auc:", auc(fpr_cnn, tpr_cnn))
'''
plt.figure(1)
# plt.xlim(0, 0.8)
plt.ylim(0, 1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_cnn, tpr_cnn, label = "ResNeXt-" + subtype + "(" + str(auc(fpr_cnn, tpr_cnn)).split('.')[0] + '.' + str(auc(fpr_cnn, tpr_cnn)).split('.')[1][:3] + ")")
'''
############################# save optimizer figure #########################
plt.plot(np.arange(epochs),
all_val_accs,
lw=0.8,
color=Color,
label=op_title)
print(op_title, " is done!")
plt.xlabel('Epochs')
plt.ylabel('Validation Accuracy')
#plt.title("")
plt.legend()
plt.ylim((0, 0.75))
plt.grid()
filename = "test_" + str(subtype) + ".eps"
plt.savefig(filename, format="eps", dpi=300)
print(str(subtype), " is done!")
plt.clf()
#plt.show()
'''
plt.legend(loc='best')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.savefig("ROC.eps", format="eps", dpi=300)
'''
return best_outcome
def get_evaluation_metric(set, tree):
metrics = evaluate(set, tree)
return metrics[4]
def main(config, resume):
# parameters
batch_size = config.get('batch_size', 32)
start_epoch = config['epoch']['start']
max_epoch = config['epoch']['max']
lr = config.get('lr', 0.0005)
use_conf = config.get('use_conf', False)
## path
save_path = config['save_path']
timestamp = datetime.now().strftime(r"%Y-%m-%d_%H-%M-%S")
save_path = os.path.join(save_path, timestamp)
result_path = os.path.join(save_path, 'result')
if not os.path.exists(result_path):
os.makedirs(result_path)
model_path = os.path.join(save_path, 'model')
if not os.path.exists(model_path):
os.makedirs(model_path)
dest = shutil.copy('train.py', save_path)
print("save to: ", dest)
## cuda or cpu
if config['n_gpu'] == 0 or not torch.cuda.is_available():
device = torch.device("cpu")
print("using CPU")
else:
device = torch.device("cuda:0")
## dataloader
dataset = Dataset(phase='train', do_augmentations=False)
data_loader = DataLoader(
dataset,
batch_size=int(batch_size),
num_workers=1,
shuffle=True,
drop_last=True,
pin_memory=True,
# **loader_kwargs,
)
val_dataset = Dataset(phase='val', do_augmentations=False)
val_data_loader = DataLoader(
val_dataset,
batch_size=int(batch_size),
num_workers=1,
shuffle=True,
drop_last=True,
pin_memory=True,
# **loader_kwargs,
)
## few shot
do_few_shot = True
if do_few_shot:
fs_dataset = Dataset(
phase='train',
do_augmentations=False,
metafile_path='metadata/detection_train_images.json')
fs_data_loader = DataLoader(
fs_dataset,
batch_size=int(128),
num_workers=1,
shuffle=True,
pin_memory=True,
# **loader_kwargs,
)
## CNN model
output_dim = 3
model = MyNet(output_dim)
model = model.to(device)
model.train()
print(model)
## loss
criterion = nn.CrossEntropyLoss(reduction='none')
## optimizer
params = list(filter(lambda p: p.requires_grad, model.parameters()))
optim_params = {
'lr': lr,
'weight_decay': 0,
'amsgrad': False,
}
optimizer = torch.optim.Adam(params, **optim_params)
lr_params = {
'milestones': [10],
'gamma': 0.1,
}
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, **lr_params)
loss_avg = AverageMeter()
acc_avg = AverageMeter()
fs_loss_avg = AverageMeter()
fs_acc_avg = AverageMeter()
logger = SimpleLogger(['train_loss', 'train_acc', 'val_loss', 'val_acc'])
## loop
for epoch in range(start_epoch, max_epoch):
loss_avg.reset()
for batch_idx, batch in tqdm(
enumerate(data_loader),
total=len(data_loader),
ncols=80,
desc=f'training epoch {epoch}',
):
data = batch[0].to(device)
gt_lbls = batch[1].to(device)
gt_gt_lbls = batch[2].to(device)
## set zerograd
optimizer.zero_grad()
## run forward pass
out = model(data) ## logits: [B, NC]; conf: [B, 1]
preds = torch.max(out, dim=-1)[1]
# print("out shape: ", out.shape)
weights = model.compute_entropy_weight(out)
# print("weights shape: ", weights.shape)
## compute loss
class_loss = criterion(out, gt_lbls) ## [B, 1]
# print("class_loss shape: ", class_loss.shape)
if use_conf:
loss = (class_loss * (weights**2) + (1 - weights)**2).mean()
else:
loss = class_loss.mean()
## record
loss_avg.update(loss.item(), batch_size)
positive = ((gt_lbls == preds) + (gt_gt_lbls > 2)).sum()
batch_acc = positive.to(torch.float) / batch_size
acc_avg.update(batch_acc.item(), batch_size)
## run backward pass
loss.backward()
optimizer.step() ## update
## each epoch
logger.update(loss_avg.avg, 'train_loss')
logger.update(acc_avg.avg, 'train_acc')
print("train loss: ", loss_avg.avg)
print("train acc: ", acc_avg.avg)
if do_few_shot and fs_data_loader is not None:
for batch_idx, batch in tqdm(
enumerate(fs_data_loader),
total=len(fs_data_loader),
ncols=80,
desc=f'training epoch {epoch}',
):
data = batch[0].to(device)
gt_lbls = batch[1].to(device)
gt_gt_lbls = batch[2].to(device)
## set zerograd
optimizer.zero_grad()
## run forward pass
out = model(data) ## logits: [B, NC]; conf: [B, 1]
preds = torch.max(out, dim=-1)[1]
# print("out shape: ", out.shape)
weights = model.compute_entropy_weight(out)
# print("weights shape: ", weights.shape)
## compute loss
class_loss = criterion(out, gt_lbls) ## [B, 1]
# print("class_loss shape: ", class_loss.shape)
if use_conf:
loss = (class_loss * (weights**2) +
(1 - weights)**2).mean()
else:
loss = class_loss.mean()
## record
positive = ((gt_lbls == preds) + (gt_gt_lbls > 2)).sum()
batch_acc = positive.to(torch.float) / data.shape[0]
fs_loss_avg.update(loss.item(), data.shape[0])
fs_acc_avg.update(batch_acc.item(), data.shape[0])
## run backward pass
loss = loss * 1.0
loss.backward()
optimizer.step() ## update
# print(f"\nfew-shot: {preds}, {gt_gt_lbls}")
## each epoch
print("fs train loss: ", fs_loss_avg.avg)
print("fs train acc: ", fs_acc_avg.avg)
if val_data_loader is not None:
log = evaluate(model.eval(),
val_data_loader,
device,
use_conf=use_conf)
model.train()
logger.update(log['loss'], 'val_loss')
logger.update(log['acc'], 'val_acc')
print("val loss: ", log['loss'])
print("val acc: ", log['acc'])
best_idx = logger.get_best('val_acc', best='max')
if best_idx == epoch:
print('save ckpt')
## save ckpt
_save_checkpoint(model_path, epoch, model)
lr_scheduler.step()
print()
## save final model
_save_checkpoint(model_path, epoch, model)
f.write("**** NOISY TREE -> pruned) ****\n")
f.write(visualPrunedNoisy)
f.close()
print("\t------------------------------------")
print("\tTrees correctly printed to file.")
except IOError:
print("\t------------------------------------")
print(
"\t*** IOError: Could not write to file! Trees visualization NOT printed! ***"
)
# STEP 3 - EVALUATION
print("\n------- 3 - EVALUATION -------")
print("\tEvaluating cleanTree on cleanTest...")
metrics = evaluate(cleanTest, cleanTree)
print_metrics(metrics)
print("\t------------------------------------")
print("\tEvaluating cleanTree on noisyTest...")
metrics = evaluate(noisyTest, cleanTree)
print_metrics(metrics)
print("\t------------------------------------")
print("\tEvaluating noisyTree on cleanTest...")
metrics = evaluate(cleanTest, noisyTree)
print_metrics(metrics)
print("\t------------------------------------")
print("\tEvaluating noisyTree on noisyTest...")
metrics = evaluate(noisyTest, noisyTree)
print_metrics(metrics)
#STEP 4 - PRUNING (AND EVALUATION AGAIN)
def train(combined_model, supervised, unsupervised, optimizer, mmd_weight):
batches_per_epoch = len(supervised)
kbar = pkbar.Kbar(target=batches_per_epoch, width=8)
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
combined_model.train()
images_supervised = []
text_supervised = []
images_unsupervised = []
text_unsupervised = []
running_loss = 0.0
running_loss_supervised = 0.0
running_loss_unsupervised = 0.0
print("Start training")
for indx, (supervised_inputs,
unsupervised_inputs) in enumerate(zip(supervised,
unsupervised)):
img_inputs = supervised_inputs[0].to(device)
text_inputs = supervised_inputs[1].to(device)
unsupervised_image_inputs = unsupervised_inputs[0].to(device)
unsupervised_text_inputs = unsupervised_inputs[1].to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(True):
text_embeddings_supervised, image_embeddings_supervised = combined_model(
text_inputs, img_inputs)
# unsupervised output
text_embeddings_unsupervised, image_embeddings_unsupervised = combined_model(
unsupervised_text_inputs, unsupervised_image_inputs)
supervised_loss = criterion_supervised(
text_embeddings_supervised, image_embeddings_supervised)
unsupervised_loss = criterion_unsupervised(
text_embeddings_unsupervised, image_embeddings_unsupervised)
# Scale unsupervised loss with batch size
unsupervised_loss = unsupervised_loss * mmd_weight
loss = supervised_loss + unsupervised_loss
loss.backward()
optimizer.step()
images_supervised.append(
image_embeddings_supervised.detach().clone())
text_supervised.append(text_embeddings_supervised.detach().clone())
images_unsupervised.append(
image_embeddings_unsupervised.detach().clone())
text_unsupervised.append(
text_embeddings_unsupervised.detach().clone())
# statistics
running_loss += loss.item()
running_loss_supervised += supervised_loss
running_loss_unsupervised += unsupervised_loss
kbar.update(indx,
values=[("loss", loss),
("supervised_loss", supervised_loss),
("unsupervised_loss", unsupervised_loss)])
recall_supervised = evaluate(text_supervised, images_supervised)
recall_unsupervised = evaluate(text_unsupervised, images_unsupervised)
epoch_loss = (running_loss / len(supervised),
running_loss_supervised / len(supervised),
running_loss_unsupervised / len(supervised))
return epoch_loss, recall_supervised, recall_unsupervised
def main():
# load and split raw data in 3 different file for training purposes
# init_clean(spot_check=20,src="data/deu.txt", dest="data/english-german")
split_data(dest="data/english-german", data_size=10000)
train_model(base_filename="data/english-german",model_filename="model/model")
evaluate(filename="data/english-german", modelname="model/model")
