def val_epoch(epoch, data_loader, model, criterion, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
precisions = AverageMeter() #
recalls = AverageMeter()
printer = 50 if opt.dataset == 'jester' else 5
end_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
with torch.no_grad():
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
precision = calculate_precision(outputs, targets) #
recall = calculate_recall(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
if i % printer == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t'
'Data {data_time.val:.5f} ({data_time.avg:.5f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'precision': precisions.avg,
'recall': recalls.avg
})
return losses.avg, accuracies.avg
def score_claim(claim):
cleaned_claim = claim.replace("/", " ")
choices = utils.query_customized_lucene(cleaned_claim,
k=max(k),
jar_name=args.jar)
retrieved = utils.process_lucene_output(choices)
relevant = claim_to_article[claim]
mAP = {}
for i in k:
precision = utils.calculate_precision(retrieved=retrieved,
relevant=relevant,
k=i)
recall = utils.calculate_recall(retrieved=retrieved,
relevant=relevant,
k=i)
mAP[i] = {}
mAP[i]['precision'] = precision
mAP[i]['recall'] = recall
return mAP
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt,
epoch_logger, batch_logger):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
precisions = AverageMeter() #
recalls = AverageMeter()
end_time = time.time()
# i, (inputs, targets) = next(iter(enumerate(data_loader)))
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
#targets = targets.cuda(non_blocking=True)
targets = targets.to('cuda:0', non_blocking=True)
inputs = inputs.to('cuda:0', non_blocking=True)
#print("NO CUDA")
inputs = Variable(inputs)
targets = Variable(targets)
#pdb.set_trace()
#print(model)
#print(inputs)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
precision = calculate_precision(outputs, targets) #
recall = calculate_recall(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'precision': precisions.val,
'recall': recalls.val,
'lr': optimizer.param_groups[0]['lr']
})
if i % 10 == 0:
print('Epoch: [{0}][{1}/{2}]\t lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})\t'
'Precision {precision.val:.3f}({precision.avg:.3f})\t'
'Recall {recall.val:.3f}({recall.avg:.3f})'.format(
epoch,
i,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
lr=optimizer.param_groups[0]['lr'],
acc=accuracies,
precision=precisions,
recall=recalls))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'precision': precisions.avg,
'recall': recalls.avg,
'lr': optimizer.param_groups[0]['lr']
})
def __init__(self, words_per_document, num_classes, vocabulary_size,
embedding_size, filter_sizes, num_filters, l2_reg_lambda,
train_size, batch_size):
# Placeholders
self.emb_place_holder = tf.placeholder(tf.float32,
[None, embedding_size],
name="emb_place_holder")
self.x_place_holder = tf.placeholder(tf.int32,
[None, words_per_document],
name="x")
self.y_place_holder = tf.placeholder(tf.float32, [None, num_classes],
name="labels")
self.dropout_keep_prob = tf.placeholder(tf.float32,
name="dropout_keep_prob")
self.learning_rate = tf.placeholder(tf.float32, name="learning_rate")
self.decay_rate = tf.placeholder(tf.float32, name="decay_rate")
l2_loss = tf.constant(0.0, name="l2_loss")
# First Layer Embeddings [MiniBatchSize, Largest Size, embSize]
with tf.device('/cpu:0'):
self.W = tf.Variable(tf.constant(
0.0, shape=[vocabulary_size, embedding_size]),
trainable=True,
name="W")
embedding_init = self.W.assign(value=self.emb_place_holder)
self.embedded_chars = tf.nn.embedding_lookup(
embedding_init, self.x_place_holder)
self.embedded_chars_expanded = tf.expand_dims(
self.embedded_chars, -1)
# Second Layer Convolutional filter size [3, 128] [4, 128] [5, 128]
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1),
name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]),
name="b")
conv = tf.nn.conv2d(self.embedded_chars_expanded,
W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="h")
# Maxpooling layer
pooled = tf.nn.max_pool(
h,
ksize=[1, words_per_document - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding="VALID",
name="pool")
tf.summary.histogram("weights", W)
tf.summary.histogram("biases", b)
tf.summary.histogram("activations", h)
pooled_outputs.append(pooled)
num_filters_total = num_filters * len(filter_sizes)
self.h_pool = tf.concat(pooled_outputs, 3)
self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])
# Dropout
with tf.name_scope("dropout"):
self.h_drop = tf.nn.dropout(self.h_pool_flat,
self.dropout_keep_prob)
tf.summary.histogram("dropout", self.h_drop)
# Last layer # xw_plus_b: compute matmul(x, weights) + biases
with tf.name_scope("output"):
W = tf.Variable(tf.truncated_normal(
[num_filters_total, num_classes], stddev=0.1),
name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b")
self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
self.predictions = tf.argmax(self.scores, 1, name="predictions")
l2_loss += tf.nn.l2_loss(W, name="l2_loss")
l2_loss += tf.nn.l2_loss(b, name="l2_loss")
tf.summary.histogram("l2", l2_loss)
tf.summary.histogram("weigths", W)
tf.summary.histogram("biases", b)
# Loss function # cross entropy between what we got and our labels
with tf.name_scope("loss"):
cross_entropy_r = tf.nn.softmax_cross_entropy_with_logits(
logits=self.scores, labels=self.y_place_holder)
self.cross_entropy = tf.reduce_mean(
cross_entropy_r,
name="cross_entropy") + (l2_reg_lambda * l2_loss)
self.stream_loss, self.stream_loss_update = tf.contrib.metrics.streaming_mean(
self.cross_entropy)
tf.summary.scalar("loss_tr", self.stream_loss)
# tf.summary.scalar("loss_tr", self.cross_entropy)
# Train Step # using learning rate 0.0004
# minimize is it the same as compute_gradients and apply gradients
with tf.name_scope("train"):
self.global_step_ = tf.Variable(0,
name="global_step",
trainable=False)
lr = tf.train.exponential_decay(self.learning_rate,
self.global_step_ * batch_size,
train_size, self.decay_rate)
self.train_step = tf.train.AdamOptimizer(lr).minimize(
self.cross_entropy,
global_step=self.global_step_,
name="train_oper")
with tf.name_scope("accuracy"):
correct_prediction = tf.equal(self.predictions,
tf.argmax(self.y_place_holder, 1),
name="correct_prediction")
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction,
tf.float32),
name="accuracy")
self.stream_accuracy, self.stream_accuracy_update = tf.contrib.metrics.streaming_mean(
self.accuracy)
tf.summary.scalar("accuracy", self.stream_accuracy)
# tf.summary.scalar("accuracy", self.accuracy)
with tf.name_scope("confussion_matrix"):
labels_max = tf.argmax(self.y_place_holder, 1, name="label_max")
# self.matrix = tf.contrib.metrics.confusion_matrix(labels_max, self.predictions, num_classes=2, name="matrix")
self.matrix = tf.confusion_matrix(labels_max,
self.predictions,
num_classes=2,
name="matrix")
true_positive = self.matrix[1, 1]
true_negative = self.matrix[0, 0]
false_positive = self.matrix[0, 1]
false_negative = self.matrix[1, 0]
self.precision_mini_batch = utils.calculate_precision(
true_positive, false_positive)
self.recall_mini_batch = utils.calculate_recall(
true_positive, false_negative)
self.f1_score_min_batch = utils.calculate_f1(
self.precision_mini_batch, self.recall_mini_batch)
self.stream_precision, self.stream_precision_update = tf.contrib.metrics.streaming_mean(
self.precision_mini_batch)
self.stream_recall, self.stream_recall_update = tf.contrib.metrics.streaming_mean(
self.recall_mini_batch)
self.stream_f1, self.stream_f1_update = tf.contrib.metrics.streaming_mean(
self.f1_score_min_batch)
tf.summary.scalar("Precision", self.stream_precision)
tf.summary.scalar("Recall", self.stream_recall)
tf.summary.scalar("F1", self.stream_f1)
# tf.summary.scalar("Precision", self.precision_mini_batch)
# tf.summary.scalar("Recall", self.recall_mini_batch)
# tf.summary.scalar("F1", self.f1_score_min_batch)
# if should_load:
# log("Data processing ok load network...")
# saver = tf.train.Saver()
# try:
# saver.restore(sess, checkpoint_file_path)
# except Exception as e:
# log("Not able to load file")
# summaries
self.summary = tf.summary.merge_all()
self.accuracy_validation_placeholder = tf.placeholder(
tf.float32, name="acc_val_placeholder")
self.loss_validation_placeholder = tf.placeholder(
tf.float32, name="loss_val_placeholder")
self.precision_validation_placeholder = tf.placeholder(
tf.float32, name="prec_val_placeholder")
self.recall_validation_placeholder = tf.placeholder(
tf.float32, name="recall_val_placeholder")
self.f1_validation_placeholder = tf.placeholder(
tf.float32, name="f1_val_placeholder")
with tf.name_scope("validation"):
self.acc_validation_mean = tf.reduce_mean(
self.accuracy_validation_placeholder)
self.loss_validation_mean = tf.reduce_mean(
self.loss_validation_placeholder)
self.prec_validation_mean = tf.reduce_mean(
self.precision_validation_placeholder)
self.recall_validation_mean = tf.reduce_mean(
self.recall_validation_placeholder)
self.f1_validation_mean = tf.reduce_mean(
self.f1_validation_placeholder)
loss_val = tf.summary.scalar("loss_val", self.loss_validation_mean)
accuracy_val = tf.summary.scalar("accuracy_val",
self.acc_validation_mean)
precission_val = tf.summary.scalar("Precision_val",
self.prec_validation_mean)
recall_val = tf.summary.scalar("Recall_val",
self.recall_validation_mean)
f1_val = tf.summary.scalar("F1_val", self.f1_validation_mean)
self.summary_val = tf.summary.merge(
[loss_val, accuracy_val, precission_val, recall_val, f1_val])
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
if not opt.no_softmax_in_test:
outputs = F.softmax(outputs)
recorder.append(outputs.data.cpu().numpy().copy())
y_true.extend(targets.cpu().numpy().tolist())
y_pred.extend(outputs.argmax(1).cpu().numpy().tolist())
#outputs = torch.unsqueeze(torch.mean(outputs, 0), 0)
#pdb.set_trace()
# print(outputs.shape, targets.shape)
if outputs.size(1) <= 4:
prec1 = calculate_accuracy(outputs, targets, topk=(1, ))
precision = calculate_precision(outputs, targets) #
recall = calculate_recall(outputs, targets)
top1.update(prec1[0], inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('[{0}/{1}]\t'
'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t'
'prec@1 {top1.avg:.5f} \t'
'precision {precision.val:.5f} ({precision.avg:.5f})\t'
'recall {recall.val:.5f} ({recall.avg:.5f})'.format(
i + 1,
def test_calculate_precision(self):
precision_value = calculate_precision(self.relevant_entries, self.retrieved_entries)
assert precision_value >= 0.85 and precision_value <= 0.858
def testing_step(checkpoint_dir, batch_size, x_test, y_test, vocab_inv_emb_dset, LOG_FILE, parameters, early_stop_log):
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# all_vars = graph.get_operations()
x_placeholder = graph.get_operation_by_name("x").outputs[0]
y_placeholder = graph.get_operation_by_name("labels").outputs[0]
embedding_placeholder = graph.get_operation_by_name("emb_place_holder").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
accuracies = graph.get_operation_by_name("accuracy/accuracy").outputs[0]
loss = graph.get_operation_by_name("loss/cross_entropy").outputs[0]
label_match = graph.get_operation_by_name("confussion_matrix/label_max").outputs[0]
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
l2_loss = graph.get_operation_by_name("l2_loss").outputs[0]
testing_data = (Dataset.from_tensor_slices((x_test, y_test))
.shuffle(buffer_size=10)
.batch(parameters.batch_size)
.make_initializable_iterator())
next_element_testing = testing_data.get_next()
epochs = 1
step = 0
accuracies_test = []
losses_test = []
precisions_test = []
recalls_test = []
f1_scores_test = []
for actual_epoch in range(epochs):
sess.run(testing_data.initializer)
while True:
try:
batch_testing = sess.run(next_element_testing)
feed_dict = {x_placeholder: batch_testing[0], y_placeholder: batch_testing[1],
embedding_placeholder: vocab_inv_emb_dset, dropout_keep_prob: 1.0}
acc_batch, loss_batch, label_op, pred_op, l2_loss_op = \
sess.run([accuracies, loss, label_match, predictions, l2_loss], feed_dict)
matrix_batch = confusion_matrix(label_op, pred_op)
true_positive = matrix_batch[1, 1]
true_negative = matrix_batch[0, 0]
false_positive = matrix_batch[0, 1]
false_negative = matrix_batch[1, 0]
precision_mini_batch = utils.calculate_precision(true_positive, false_positive)
recall_mini_batch = utils.calculate_recall(true_positive, false_negative)
f1_score_min_batch = utils.calculate_f1(precision_mini_batch, recall_mini_batch)
accuracies_test.append(acc_batch)
losses_test.append(loss_batch)
precisions_test.append(precision_mini_batch)
recalls_test.append(recall_mini_batch)
f1_scores_test.append(f1_score_min_batch)
log("Step " + str(step) + "(epoch " + str(epochs) + ")" + "Test accuracy: " + str(
acc_batch) +
" test loss: " + str(loss_batch) + " test precission: " + str(precision_mini_batch) +
" test recall: " + str(recall_mini_batch) + "test F1: " + str(f1_score_min_batch), LOG_FILE)
except tf.errors.OutOfRangeError:
avg_accuracy = np.mean(accuracies_test)
avg_losses = np.mean(losses_test)
avg_precision = np.mean(precisions_test)
avg_recall = np.mean(recalls_test)
avg_f1 = np.mean(f1_scores_test)
log(str(parameters), LOG_FILE)
log("Final results, test accuracy: " + str(avg_accuracy) + " test loss: " + str(avg_losses) +
" test precission: " + str(avg_precision) + " test recall: " + str(
avg_recall) + " test f1: "
+ str(avg_f1), LOG_FILE)
log("End training dataset epoch: " + str(early_stop_log), LOG_FILE)
break
def train_epoch(epoch,
data_loader,
model,
criterion,
optimizer,
opt,
epoch_logger,
batch_logger,
global_step,
tsboard_writer=None):
print('train at epoch {}'.format(epoch))
sys.stdout.flush()
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
precisions = AverageMeter() #
recalls = AverageMeter()
printer = 100 if opt.dataset == 'jester' else 10
end_time = time.time()
# i, (inputs, targets) = next(iter(enumerate(data_loader)))
for i, (inputs, gravity_position, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(non_blocking=True)
inputs = Variable(inputs)
positions = Variable(gravity_position)
targets = Variable(targets)
#pdb.set_trace()
outputs = model(inputs, positions)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
precision = calculate_precision(outputs, targets) #
recall = calculate_recall(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'precision': precisions.val,
'recall': recalls.val,
'lr': optimizer.param_groups[0]['lr']
})
if i % printer == 0:
print('Epoch: [{0}][{1}/{2}]\t lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})\t'
'Precision {precision.val:.3f}({precision.avg:.3f})\t'
'Recall {recall.val:.3f}({recall.avg:.3f})'.format(
epoch,
i,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
lr=optimizer.param_groups[0]['lr'],
acc=accuracies,
precision=precisions,
recall=recalls))
sys.stdout.flush()
if tsboard_writer is not None:
tsboard_writer.add_scalar("training loss", losses.val, global_step)
tsboard_writer.add_scalar("training acc", accuracies.val,
global_step)
tsboard_writer.add_scalar("training precision", precisions.val,
global_step)
tsboard_writer.add_scalar("training recall", recalls.val,
global_step)
global_step += 1
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'precision': precisions.avg,
'recall': recalls.avg,
'lr': optimizer.param_groups[0]['lr']
})
return global_step
def main(**kwargs):
fp16 = kwargs['fp16']
batch_size = kwargs['batch_size']
num_workers = kwargs['num_workers']
pin_memory = kwargs['pin_memory']
patience_limit = kwargs['patience']
initial_lr = kwargs['learning_rate']
gradient_accumulation_steps = kwargs['grad_accumulation_steps']
device = kwargs['device']
epochs = kwargs['epochs']
freeze_bert = kwargs['freeze_bert_layers']
slots = kwargs['slots']
if fp16:
scaler = GradScaler()
tokenizer = BertTokenizer.from_pretrained(
'bert-base-uncased', model_max_length=128
) # for TM_1, out of 303066 samples, 5 are above 128 tokens
if kwargs['dataset'] == "TM":
train_data, val_data = load_taskmaster_datasets(
utils.datasets,
tokenizer,
train_percent=0.9,
for_testing_purposes=kwargs['testing_for_bugs'])
if kwargs['dataset'] == "MW":
train_data = load_multiwoz_dataset("multi-woz/train_dials.json",
tokenizer, slots,
kwargs['testing_for_bugs'])
val_data = load_multiwoz_dataset("multi-woz/dev_dials.json", tokenizer,
slots, kwargs['testing_for_bugs'])
if kwargs['dataset'] == "MW22":
train_data = load_MW_22_dataset_training(tokenizer, slots,
kwargs['testing_for_bugs'])
val_data = load_MW_22_dataset_validation(tokenizer, slots,
kwargs['testing_for_bugs'])
train_dataset = VE_dataset(train_data)
val_dataset = VE_dataset(val_data)
collator = collate_class(tokenizer.pad_token_id)
train_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collator,
num_workers=num_workers,
pin_memory=pin_memory)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=int(batch_size /
2),
shuffle=True,
collate_fn=collator,
num_workers=num_workers,
pin_memory=pin_memory)
# BertForValueExtraction
if kwargs['model_path'] and os.path.isdir(kwargs['model_path']):
from_pretrained = kwargs['model_path']
else:
from_pretrained = 'bert-base-uncased'
model = BertForValueExtraction(num_labels=len(label2id.keys()),
from_pretrained=from_pretrained,
freeze_bert=freeze_bert)
model.to(device)
model.train()
optimizer = torch.optim.Adam(params=model.parameters(), lr=initial_lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
'max',
factor=0.5,
patience=1,
min_lr=initial_lr /
100,
verbose=True)
best_acc, count = 0, 0
for epoch in range(epochs):
# train loop
total_loss = 0
optimizer.zero_grad()
pbar = tqdm(enumerate(train_dataloader), total=len(train_dataloader))
for i, batch in pbar:
input_ids = batch['input_ids'].to(device)
attention_mask = batch['attention_mask'].to(device)
token_type_ids = batch['token_type_ids'].to(device)
labels = batch['labels'].to(device)
text = batch['text']
# print(f"input_ids: {input_ids.shape} - {input_ids}")
# print(f"attention_mask: {attention_mask.shape} - {attention_mask}")
# print(f"token_type_ids: {token_type_ids.shape} - {token_type_ids}")
# print(f"labels: {labels.shape} - {labels}")
if fp16:
with autocast():
loss = model.calculate_loss(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
labels=labels)
total_loss += loss.item()
loss = loss / gradient_accumulation_steps
scaler.scale(loss).backward()
else:
loss = model.calculate_loss(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
labels=labels)
loss.backward()
total_loss += loss.item()
if ((i + 1) % gradient_accumulation_steps) == 0:
if fp16:
scaler.step(optimizer)
scaler.update()
else:
optimizer.step()
optimizer.zero_grad()
batch_num = ((i + 1) / gradient_accumulation_steps)
pbar.set_description(f"Loss: {total_loss/batch_num:.4f}")
# validation loop
model.eval()
with torch.no_grad():
TP, FP, FN, TN = model.evaluate(val_dataloader, device)
pr = utils.calculate_precision(TP, FP)
re = utils.calculate_recall(TP, FN)
F1 = utils.calculate_F1(TP, FP, FN)
acc = utils.calculate_accuracy(TP, FP, FN, TN)
balanced_acc = utils.calculate_balanced_accuracy(TP, FP, FN, TN)
print(
f"Validation: pr {pr:.4f} - re {re:.4f} - F1 {F1:.4f} - acc {acc:.4f} - balanced acc {balanced_acc:.4f}"
)
scheduler.step(balanced_acc)
if balanced_acc > best_acc:
best_acc = balanced_acc
count = 0
if kwargs['model_path']:
model.save_(f"{kwargs['model_path']}-ACC{best_acc:.4f}")
else:
count += 1
if count == patience_limit:
print("ran out of patience stopping early")
break
model.train()
def val_epoch_true(epoch,
data_loader,
model,
criterion,
opt,
logger,
global_step,
tsboard_writer=None):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
precisions = AverageMeter() #
recalls = AverageMeter()
end_time = time.time()
vid_ids = data_loader.dataset.ids
vid_ids.append(-1)
out_queue = []
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(non_blocking=True)
with torch.no_grad():
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
outputs = F.softmax(outputs, dim=1)
outputs = outputs.cpu().numpy()[0].reshape(-1, )
out_queue.append(outputs)
if vid_ids[i + 1] != vid_ids[i]:
output = np.mean(out_queue, 0)
outputs = torch.from_numpy(output).float().unsqueeze_(0).cuda()
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
precision = calculate_precision(outputs, targets)
recall = calculate_recall(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
out_queue = []
if i % 100 == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t'
'Data {data_time.val:.5f} ({data_time.avg:.5f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'precision': precisions.avg,
'recall': recalls.avg
})
if tsboard_writer is not None:
tsboard_writer.add_scalar("validation loss", losses.avg, global_step)
tsboard_writer.add_scalar("validation acc", accuracies.avg,
global_step)
tsboard_writer.add_scalar("validation precision", precisions.avg,
global_step)
tsboard_writer.add_scalar("validation recall", recalls.avg,
global_step)
return losses.avg, accuracies.avg
def test(data_loader, model, opt, class_names):
print('test')
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
end_time = time.time()
output_buffer = []
previous_video_id = ''
test_results = {'results': {}}
total_output_buffer = []
total_target_buffer = []
for i, (inputs, positions, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
with torch.no_grad():
inputs = Variable(inputs)
positions = Variable(positions)
targets = Variable(targets).cuda()
outputs = model(inputs, positions)
if not opt.no_softmax_in_test:
outputs = F.softmax(outputs, dim=1)
total_output_buffer.append(outputs)
total_target_buffer.append(targets)
for j in range(outputs.size(0)):
if not (i == 0
and j == 0) and targets[j].item() != previous_video_id:
calculate_video_results(output_buffer, previous_video_id,
test_results, class_names)
output_buffer = []
output_buffer.append(outputs[j].data.cpu())
previous_video_id = targets[j].item()
if (i % 100) == 0:
with open(
os.path.join(opt.result_path,
'{}.json'.format(opt.test_subset)), 'w') as f:
json.dump(test_results, f)
batch_time.update(time.time() - end_time)
end_time = time.time()
print('[{}/{}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format(
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time))
total_output = torch.cat(total_output_buffer, dim=0)
total_target = torch.cat(total_target_buffer, dim=0)
acc = calculate_accuracy(total_output, total_target)
precision = calculate_precision(total_output, total_target)
recall = calculate_recall(total_output, total_target)
print("Overall:\t acc: {}, precision: {}, recall: {}".format(
acc, precision, recall))
with open(os.path.join(opt.result_path, '{}.json'.format(opt.test_subset)),
'w') as f:
json.dump(test_results, f)
text = batch['text']
outputs = model(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids)
logits = outputs.logits
preds = torch.max(logits, dim=2)[1]
tp, fp, fn, tn = model.evaluate(preds.tolist(), labels.tolist(), attention_mask.tolist())
TP += tp
FP += fp
FN += fn
TN += tn
pr = utils.calculate_precision(TP, FP)
re = utils.calculate_recall(TP, FN)
F1 = utils.calculate_F1(TP, FP, FN)
acc = utils.calculate_accuracy(TP, FP, FN, TN)
balanced_acc = utils.calculate_balanced_accuracy(TP, FP, FN, TN)
print(f"Validation: pr {pr:.4f} - re {re:.4f} - F1 {F1:.4f} - acc {acc:.4f} - balanced acc {balanced_acc:.4f}")
scheduler.step(balanced_acc)
if balanced_acc > best_acc:
best_acc = balanced_acc
count = 0
else:
count += 1
print(count)
if count == patience_limit:
def test(self, annotation_path='', video_path=''):
opt = self.opt
if annotation_path != '':
opt.annotation_path = annotation_path
if opt.root_path != '':
opt.annotation_path = os.path.join(opt.root_path,
opt.annotation_path)
# if video_path != '':
# opt.video_path = video_path
# if opt.root_path != '':
# opt.video_path = os.path.join(opt.root_path, opt.video_path)
if not os.path.exists(opt.result_path):
os.makedirs(opt.result_path)
with open(os.path.join(opt.result_path, 'opts.json'), 'w') as opt_file:
json.dump(vars(opt), opt_file)
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
# original
spatial_transform = Compose([
#Scale(opt.sample_size),
Scale(112),
CenterCrop(112),
ToTensor(opt.norm_value),
norm_method
])
temporal_transform = TemporalCenterCrop(opt.sample_duration)
target_transform = ClassLabel()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
test_logger = Logger(os.path.join(opt.result_path, 'test.log'),
['top1', 'precision', 'recall'])
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
assert opt.arch == checkpoint['arch']
opt.begin_epoch = checkpoint['epoch']
self.model.load_state_dict(checkpoint['state_dict'])
recorder = []
self.model.eval()
batch_time = AverageMeter()
top1 = AverageMeter()
precisions = AverageMeter()
recalls = AverageMeter()
y_true = []
y_pred = []
end_time = time.time()
for i, (inputs, targets) in enumerate(test_loader):
if not opt.no_cuda:
targets = targets.cuda(non_blocking=True)
#inputs = Variable(torch.squeeze(inputs), volatile=True)
with torch.no_grad():
inputs = Variable(inputs)
targets = Variable(targets)
outputs = self.model(inputs)
if not opt.no_softmax_in_test:
outputs = F.softmax(outputs, dim=1)
recorder.append(outputs.data.cpu().numpy().copy())
y_true.extend(targets.cpu().numpy().tolist())
y_pred.extend(outputs.argmax(1).cpu().numpy().tolist())
_cls = outputs.argmax(1).cpu().numpy().tolist()[0]
prec1 = self.calculate_accuracy(outputs, targets, topk=(1, ))
precision = calculate_precision(outputs, targets)
recall = calculate_recall(outputs, targets)
top1.update(prec1[0], inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
test_logger.log({
'top1': top1.avg,
'precision': precisions.avg,
'recall': recalls.avg
})
print('-----Evaluation is finished------')
print('Overall Prec@1 {:.05f}%'.format(top1.avg * 100))
return y_pred, y_true, test_data
if __name__ == "__main__":
orig_settings = termios.tcgetattr(sys.stdin)
tty.setcbreak(sys.stdin)
# load and init data
data = pd.read_csv("data/ex2data1.txt", ",")
X = data.iloc[:, :-1].to_numpy()
y = data.iloc[:, -1].to_numpy()
X = utils.map_feature(X[:, 0], X[:, 1], 2)
weights = zeros(X.shape[1])
print(
'Menu:\n1) Press G for gradient descent.\n2) Press O for optimized algorithm.\n3) Press Q for quit.'
)
key = sys.stdin.read(1)[0]
if key == 'g' or key == 'G':
print("Run gradient descent")
weights = gradient_descent(weights, X, y, 500000, 0.00101, 0.003)
utils.calculate_precision(weights, X, y)
elif key == 'o' or key == 'O':
print("Run optimized algorithm")
weights, _, _ = opt.fmin_tnc(func=cost_function_regularized,
x0=weights,
args=(X, y, 0.1),
messages=0)
utils.calculate_precision(weights, X, y)
elif key == 'q' or key == 'Q':
exit()
