def val_epoch(epoch, data_loader, model, criterion, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
inputs = Variable(inputs, volatile=True)
targets = Variable(targets, volatile=True)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
return losses.avg
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt,
epoch_logger, batch_logger):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
print("len: ", len(data_loader))
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt,
epoch_logger, batch_logger):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
write_embedding = True
writer = None
embedding_log = 20
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
if write_embedding and epoch % embedding_log == 0:
if writer is None:
writer = SummaryWriter(comment='_embedding_train_' + str(i))
n_iter = (epoch * len(data_loader)) + i
middle_frame = math.floor(inputs.data.shape[2] / 2)
writer.add_embedding(outputs.data,
metadata=targets.data,
label_img=torch.squeeze(
inputs.data[:, :, middle_frame, :, :], 2),
global_step=n_iter)
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
return epoch, losses.avg, accuracies.avg
def train_epoch(epoch, data_loader, model, model_clone, feature_v_dict,
feature_a_dict, nowidx, criterion, optimizer, opt,
epoch_logger, batch_logger):
print('train at epoch {}'.format(epoch))
model.train()
model_clone.train()
cuda = torch.device("cuda")
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (video, audio) in enumerate(data_loader):
data_time.update(time.time() - end_time)
video = video.to(device=cuda)
audio = audio.to(device=cuda)
bs = video.size(0)
device = video.device
feature_v, feature_a = model(video, audio)
target = torch.arange(bs).to(device=device)
feature_a_all = torch.cat([feature_a.detach(), feature_a_dict], dim=0)
feature_v_all = torch.cat([feature_v.detach(), feature_v_dict], dim=0)
cosv2a = torch.mm(feature_a_all, feature_v.t()).t()
cosa2v = torch.mm(feature_v_all, feature_a.t()).t()
loss1 = criterion(cosv2a, target)
loss2 = criterion(cosa2v, target)
loss = loss1 + loss2
tmp = cosv2a.argmax(dim=1)
print(tmp)
acc = calculate_accuracy(cosv2a, target)
losses.update(loss.data.item(), video.size(0))
accuracies.update(acc, video.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
momentum_update(model, model_clone, 0.9)
with torch.no_grad():
feature_v, feature_a = model_clone(video, audio)
feature_v_dict[nowidx:nowidx + bs] = feature_v.detach()
feature_a_dict[nowidx:nowidx + bs] = feature_a.detach()
nowidx += bs
if nowidx + bs > feature_v_dict.shape[0]:
nowidx = 0
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
return feature_v_dict, feature_a_dict, nowidx
def train_epoch(epoch, data_loader, model, criterion, optimizer, args, device,
epoch_logger, batch_logger):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top3 = AverageMeter()
# switch to train mode
model.train()
end_time = time.time()
for i, (input, target) in enumerate(data_loader):
# measure data loading time
data_time.update(time.time() - end_time)
input = input.to(device)
target = target.to(device)
# compute output and loss
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec3 = calculate_accuracy(output, target, topk=(1, 3))
losses.update(loss.item(), input.size(0))
# prec1[0]: convert torch.Size([1]) to torch.Size([])
top1.update(prec1[0].item(), input.size(0))
top3.update(prec3[0].item(), input.size(0))
"""
a = np.array([1, 2, 3])
b = torch.from_numpy(a) # tensor([ 1, 2, 3])
c = b.sum() # tensor(6)
d = b.sum(0) # tensor(6)
e = b.sum(0, keepdim=True) # tensor([ 6]), torch.Size([1])
e[0] # tensor(6), torch.Size([])
e.item() # 6
"""
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if (i + 1) % args.log_interval == 0:
print('Train Epoch [{0}/{1}]([{2}/{3}])\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@3 {top3.val:.3f} ({top3.avg:.3f})\t'
'LR {lr:f}\t'
'Batch Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data Time {data_time.val:.3f} ({data_time.avg:.3f})'.format(
epoch,
args.epochs,
i + 1,
len(data_loader),
loss=losses,
top1=top1,
top3=top3,
lr=optimizer.param_groups[0]['lr'],
batch_time=batch_time,
data_time=data_time))
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'top1': top1.val,
'top3': top3.val,
'lr': optimizer.param_groups[0]['lr']
})
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'top1': top1.avg,
'top3': top3.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % args.checkpoint_interval == 0:
save_file_path = os.path.join(
args.checkpoint_path, 'save_{}_{}.pth'.format(args.arch, epoch))
states = {
'epoch': epoch,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
vis.plot('Train loss', losses.avg)
vis.plot('Train accu', top1.avg)
vis.log("epoch:{epoch}, lr:{lr}, loss:{loss}, accu:{accu}".format(
epoch=epoch,
lr=optimizer.param_groups[0]['lr'],
loss=losses.avg,
accu=top1.avg))
def run_iter(inputs, labels, iteration, train=True, log_fn=None):
prefix = 'train' if train else 'val'
loss_eq = args.loss if train else args.val_loss
if log_fn is None:
def log_fn(tag, value):
if isinstance(value, float) or len(value.size()) == 0:
writer.add_scalar(f'{prefix}/{tag}', value, iteration)
elif len(value.size()) == 3 and iteration % 100 == 0:
writer.add_image(f'{prefix}/{tag}', value, iteration)
elif len(value.size()) == 4 and iteration % 100 == 0:
writer.add_images(f'{prefix}/{tag}', value, iteration)
model.eval() # set model to eval to generate adversarial examples
if torch.cuda.is_available():
inputs = inputs.cuda()
labels = labels.cuda()
non_batch_dims = inputs.size()[1:]
inputs_to_compute: List[torch.Tensor] = []
computed_logits: List[torch.Tensor] = []
computed_grads: List[torch.Tensor] = []
loss_objects: Dict[str, losses.Loss] = {}
for loss_name, Loss in LOSS_FUNCTIONS.items():
loss_objects[loss_name] = Loss(
model,
normalizer,
inputs,
labels,
args.dataset,
log_fn,
train,
inputs_to_compute,
computed_logits,
)
# DETERMINE INPUTS TO RUN MODEL OVER
inputs_to_compute.append(inputs)
first_eval_locals: DefaultDict[str, Any] = \
defaultdict(lambda: math.nan)
first_eval_locals.update(loss_objects)
first_eval_locals.update(globals())
first_eval_locals.update(builtins.__dict__)
eval(loss_eq, None, first_eval_locals)
# FORWARD PASS
if train:
optimizer.zero_grad()
model.train() # now we set the model to train mode
for inputs in inputs_to_compute:
inputs.requires_grad = True
all_inputs = torch.cat(inputs_to_compute)
all_logits = model(normalizer(all_inputs))
# CALCULATE GRADIENT
base_loss = utils.calculate_base_loss(
all_logits,
torch.cat([labels] * len(inputs_to_compute)),
)
all_grads, = torch.autograd.grad(base_loss.sum(),
all_inputs,
create_graph=True,
retain_graph=True)
for i in range(len(inputs_to_compute)):
sl = slice(i * len(inputs), (i + 1) * len(inputs))
computed_logits.append(all_logits[sl])
computed_grads.append(all_grads[sl])
# CONSTRUCT LOSS
logits = computed_logits[0]
grads = computed_grads[0]
base_loss = utils.calculate_base_loss(logits, labels)
grads_magnitude = (grads ** 2) \
.sum(dim=list(range(1, len(grads.size())))) \
.sqrt()
l_base = torch.mean(base_loss)
l_ce = nn.CrossEntropyLoss()(logits, labels)
if 'diff' in args.base_loss:
l_diff = l_base
l_grad = torch.mean(grads_magnitude)
l_grad_l1 = grads.abs().sum(
dim=list(range(1, len(grads.size())))).mean()
for loss_object in loss_objects.values():
loss_object.switch_to_eval()
second_eval_locals = dict(locals())
second_eval_locals.update(loss_objects)
second_eval_locals.update(builtins.__dict__)
loss = eval(loss_eq, None, second_eval_locals)
# LOGGING
accuracy = utils.calculate_accuracy(logits, labels)
log_fn('loss/loss', loss.item())
log_fn('loss/ce', l_ce.item())
log_fn('loss/base', l_base.item())
if 'diff' in args.base_loss:
log_fn('loss/diff', l_diff.item())
log_fn('loss/grad', l_grad.item())
log_fn('loss/grad_l1', l_grad_l1.item())
log_fn('accuracy', accuracy.item())
if train:
print(f'ITER {iteration:06d}',
f'accuracy: {accuracy.item() * 100:5.1f}%',
f'loss: {loss.item():.2f}',
sep='\t')
if train and iteration % 1000 == 0:
for name, param in model.named_parameters():
writer.add_histogram(f'train/model/{name}',
param.clone().cpu().data.numpy(),
iteration)
# OPTIMIZATION
if train:
loss.backward()
# log gradient norm
param_grads = torch.cat(
[param.grad.data.view(-1) for param in model.parameters()])
log_fn('param_grad_l2', param_grads.norm(p=2).item())
log_fn('param_grad_linf', param_grads.norm(p=math.inf).item())
# clip gradients and optimize
nn.utils.clip_grad_value_(model.parameters(), args.clip_grad)
optimizer.step()
def test_epoch(epoch, model, data_loader, criterion, logger, num_classes=200, cas_num=3):
model.eval()
epoch_losses_cas = np.zeros([cas_num, 3])
epoch_acc_cas = np.zeros([cas_num])
epoch_off_cas = np.zeros([cas_num])
for n_iter, (feat, start_unit, end_unit, gt_start_unit, gt_end_unit, label) in enumerate(data_loader):
mini_batch_size = len(feat)
# In BMVC paper, we used prob multiplication to calculate final prob,
# but later experiments showed that weighted average gives more stable results.
# final_action_prob = torch.ones([mini_batch_size, num_classes])
final_action_prob = torch.zeros([mini_batch_size, num_classes])
for k in range(cas_num):
# compute input and target
inputs = []
targets = []
for b in range(mini_batch_size):
inputs.append(data_loader.dataset.generate_input(feat[b], start_unit[b], end_unit[b]))
targets.append(data_loader.dataset.generate_target(label[b], start_unit[b], end_unit[b],
gt_start_unit[b], gt_end_unit[b]))
inputs = torch.stack(inputs, 0)
targets = torch.stack(targets, 0)
targets = targets.cuda()
# run model for current cascade
outputs = model(inputs)
losses = criterion(outputs, targets)
epoch_losses_cas[k, 0] += losses['loss'].cpu().detach().numpy()
epoch_losses_cas[k, 1] += losses['cls_loss'].cpu().detach().numpy()
epoch_losses_cas[k, 2] += losses['reg_loss'].cpu().detach().numpy()
epoch_acc_cas[k] += calculate_accuracy(outputs, targets)
epoch_off_cas[k] += calculate_offset(outputs, targets)
# update the boundaries of current proposals
outputs = outputs.cpu().detach()
action_score = outputs[:, 1:1+num_classes]
action_prob = torch.softmax(action_score, 1)
# In BMVC paper, we used prob multiplication to calculate final prob,
# but later experiments showed that weighted average gives more stable results.
# final_action_prob *= action_prob
final_action_prob = final_action_prob + prob_weights[k] * action_prob
if k == cas_num - 1:
break
pred_action = torch.argmax(final_action_prob, 1) + 1 # (b,), 1 ~ 200
start_unit = [start_unit[i] + outputs[i, 1+num_classes+pred_action[i]].item() for i in range(mini_batch_size)]
end_unit = [end_unit[i] + outputs[i, 2*(1+num_classes)+pred_action[i]].item() for i in range(mini_batch_size)]
for i in range(mini_batch_size):
if start_unit[i] > end_unit[i]:
new_su = (start_unit[i]+end_unit[i]) / 2.0 - 0.2
new_eu = (start_unit[i]+end_unit[i]) / 2.0 + 0.2
start_unit[i] = new_su
end_unit[i] = new_eu
epoch_losses_cas /= (n_iter + 1)
epoch_acc_cas /= (n_iter + 1)
epoch_off_cas /= (n_iter + 1)
for k in range(cas_num):
logger.log({
'epoch': epoch,
'cas': k,
'loss': epoch_losses_cas[k, 0],
'cls_loss': epoch_losses_cas[k, 1],
'reg_loss': epoch_losses_cas[k, 2],
'acc': epoch_acc_cas[k],
'off': epoch_off_cas[k]
})
print('Epoch {} validation (cascade 0) loss: {} cls_loss: {}, reg_loss: {} acc: {} off: {}'.format(
epoch, epoch_losses_cas[0, 0], epoch_losses_cas[0, 1], epoch_losses_cas[0, 2], epoch_acc_cas[0], epoch_off_cas[0]))
for k in range(1, cas_num):
print(' (cascade {}) loss: {} cls_loss: {}, reg_loss: {} acc: {} off: {}'.format(
k, epoch_losses_cas[k, 0], epoch_losses_cas[k, 1], epoch_losses_cas[k, 2], epoch_acc_cas[k], epoch_off_cas[k]))
return epoch_losses_cas[cas_num-1, 0], epoch_acc_cas[cas_num-1]
def train_epoch(epoch, data_loader, model, teacher_model, optimizer, opt,
epoch_logger, batch_logger):
print('train at epoch {}'.format(epoch))
model.train()
teacher_model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
# Get teacher model outputs
# if not _teacher_outputs_dict:
# print("Executing teacher model...")
# for i, (inputs, targets, indices) in enumerate(data_loader):
# if not opt.no_cuda:
# inputs, targets = inputs.cuda(async=True), targets.cuda(async=True)
# inputs = Variable(inputs)
# targets = Variable(targets)
# teacher_output = teacher_model(inputs).data.cpu().numpy()
# print("Teacher output shape", teacher_output.shape)
# teacher_outputs.append(teacher_output)
# for i in range(len(indices)):
# _teacher_outputs_dict[indices[i].item()] = teacher_output[i]
#else:
# print("Skipping teacher model execution...")
# Check if it is teacher output
# print("teacher output", _teacher_outputs_dict.keys())
end_time = time.time()
for i, (inputs, targets, indices) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
inputs = inputs.cuda(async=True)
targets = targets.cuda(async=True)
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
# loss = criterion(outputs, targets)
# teacher_outputs = []
# for idx in indices:
# print("idx, teacher output", idx, _teacher_outputs_dict[idx.item()])
# teacher_outputs.append(_teacher_outputs_dict[idx.item()])
#teacher_outputs = np.asarray(teacher_outputs)
# teacher_output = torch.from_numpy(teacher_outputs)
#if not opt.no_cuda:
# teacher_output = teacher_output.cuda(async=True)
# teacher_output = Variable(teacher_output, requires_grad=False)
# get teacher output.
teacher_output = teacher_model(inputs)
loss = loss_kd(outputs, teacher_output, targets)
acc = calculate_accuracy(outputs, targets)
# losses.update(loss.data[0], inputs.size(0))
losses.update(loss.data.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
if not opt.no_cuda:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': state_dict,
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
def val_epoch(epoch,
data_loader,
model,
criterion,
output_directory,
logger=None,
write_to_file=True):
print('validation at epoch {}'.format(epoch))
fieldnames = ['video', 'label']
if write_to_file:
filename = 'test-' + str(epoch) + '.csv'
test_csv = os.path.join(output_directory, filename)
with open(test_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets, paths) in enumerate(data_loader):
data_time.update(time.time() - end_time)
# if not opt.no_cuda:
# targets = targets.cuda(async=True)
# inputs = Variable(inputs, volatile=True)
# targets = Variable(targets, volatile=True)
if torch.cuda.is_available():
inputs = inputs.cuda()
targets = targets.cuda()
torch.cuda.synchronize()
outputs = model(inputs)
loss = criterion(outputs, targets)
torch.cuda.synchronize()
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Test: Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
_, pred = outputs.topk(1, 1, True)
pred = pred.t()
print('pred: ', pred.item())
if write_to_file:
with open(test_csv, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({'video': paths[0], 'label': pred.item() + 1})
# logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
if logger is not None:
logger.add_scalar('Test/loss', losses.avg, epoch)
logger.add_scalar('Test/Acc', accuracies.avg, epoch)
return losses.avg
def classify_video_offline(class_names, model, opt, video_path_formatter=lambda root_path, label, video_id: Path(
join(root_path, label, video_id))):
device = torch.device('cpu' if opt.no_cuda else 'cuda')
data_loader = create_dataset_offline(opt, video_path_formatter)
accuracies = AverageMeter()
class_size = len(class_names)
class_idx = list(range(0, class_size))
ground_truth_labels = []
predicted_labels = []
executions_times = []
all_video_results = []
all_execution_times = []
print('Starting prediction phase')
with torch.no_grad():
for (inputs, targets, segments, video_name) in data_loader:
targets = targets.to(device, non_blocking=True)
# One video at a time
video_name = video_name[0]
segments = segments[0]
print(f'Giving input {video_name} to the NN...')
start_time = time.time()
outputs = model(inputs)
end_time = time.time()
execution_time = (end_time - start_time)
# print(len(inputs))
# print(len(targets))
# print(f'Execution time: {execution_time}')
executions_times.append(execution_time)
acc = calculate_accuracy(outputs, targets)
ground_truth, predictions = ground_truth_and_predictions(outputs, targets)
predictions = flatten(predictions)
# print(ground_truth)
# print(predictions)
ground_truth_labels.extend(ground_truth)
predicted_labels.extend(predictions)
accuracies.update(acc, inputs.size(0))
video_outputs = outputs.cpu().data
exec_times_with_segments = []
for i in range(len(segments)):
segment = segments[i]
# TODO this is not totally correct, but i have no choice here
# This is because now, the input frame batches are processed only once and together, so i do not have the
# execution time for a single batch. So, this outputs the time the NN takes to process all the batches of
# a single video
# To fix this, you should adapt the code in the 'live' settings here, because there the prediction time
# is per batch. See `classify_video_online` or just slice the input tensor
exec_time = execution_time
exec_times_with_segments.append((segment, exec_time))
executions_times_with_video_name = {
video_name: exec_times_with_segments
}
single_video_result = {
'video': video_name,
'clips': []
}
for i in range(len(predictions)):
clip_results = {
'segment': segments[i]
}
if opt.mode == 'score':
clip_results['label'] = class_names[predictions[i]]
clip_results['scores'] = video_outputs[i].tolist()
elif opt.mode == 'feature':
clip_results['features'] = video_outputs[i].tolist()
single_video_result['clips'].append(clip_results)
all_video_results.append(single_video_result)
all_execution_times.append(executions_times_with_video_name)
accuracies_avg = accuracies.avg
precision, recall, fscore, _ = \
precision_recall_fscore_support(ground_truth_labels,
predicted_labels,
labels=class_idx)
mean_exec_times = np.mean(executions_times)
std_exec_times = np.std(executions_times)
# print(f'Acc:{accuracies_avg}')
# print(f'prec: {precision}')
# print(f'rec: {recall}')
# print(f'f-score: {fscore}')
# print(mean_exec_times)
# print(std_exec_times)
with open(opt.output_csv, 'w+') as csv_file:
writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
header = ["VIDEO_NAME", "MEAN_PREDICTION_TIME", "STANDARD_DEV_PREDICTION_TIME", "ACCURACY"]
create_column_metric_csv_header("PRECISION", class_names, header)
create_column_metric_csv_header("RECALL", class_names, header)
create_column_metric_csv_header("F-SCORE", class_names, header)
writer.writerow(header)
final_row = ['Metrics_overall_dataset', mean_exec_times, std_exec_times, accuracies_avg]
create_column_metric_csv_content(precision, final_row)
create_column_metric_csv_content(recall, final_row)
create_column_metric_csv_content(fscore, final_row)
writer.writerow(final_row)
return all_video_results, all_execution_times
def fine_tune(load_dir,
save_dir,
filename,
dataloader_train,
dataloader_val,
ContinuousTrain=False):
'''
load pretrained model
resnet-18-kinetics.pth: --model resnet --model_depth 18 --resnet_shortcut A
resnet-34-kinetics.pth: --model resnet --model_depth 34 --resnet_shortcut A
resnet-34-kinetics-cpu.pth: CPU ver. of resnet-34-kinetics.pth
resnet-50-kinetics.pth: --model resnet --model_depth 50 --resnet_shortcut B
resnet-101-kinetics.pth: --model resnet --model_depth 101 --resnet_shortcut B
resnet-152-kinetics.pth: --model resnet --model_depth 152 --resnet_shortcut B
resnet-200-kinetics.pth: --model resnet --model_depth 200 --resnet_shortcut B
preresnet-200-kinetics.pth: --model preresnet --model_depth 200 --resnet_shortcut B
wideresnet-50-kinetics.pth: --model wideresnet --model_depth 50 --resnet_shortcut B --wide_resnet_k 2
resnext-101-kinetics.pth: --model resnext --model_depth 101 --resnet_shortcut B --resnext_cardinality 32
densenet-121-kinetics.pth: --model densenet --model_depth 121
densenet-201-kinetics.pth: --model densenet --model_depth 201
'''
num_epochs = 100
step = 0
if not ContinuousTrain:
model = resnet.resnet152(sample_size=112,
sample_duration=args.n_frames_per_clip,
shortcut_type='B',
num_classes=83)
checkpoint = utils.load_checkpoint(load_dir, filename)
model = nn.DataParallel(model, device_ids=[1])
state_dict = deepcopy(model.state_dict())
feature_state_dict = {
key: value
for key, value in checkpoint['state_dict'].items()
if key not in ['module.fc.weight', 'module.fc.bias']
}
state_dict.update(feature_state_dict)
model.load_state_dict(state_dict)
# set fine tune parameters: Conv5_x and fc layer from original paper
for param in model.module.parameters():
param.requires_grad = False
for named_child in model.module.named_children():
if named_child[0] == 'fc' or named_child[0] == 'layer4':
# if named_child[0] == 'fc':
for param in named_child[1].parameters():
param.requires_grad = True
# pdb.set_trace()
else:
print('Recover model from {}/resnet18.pth......'.format(save_dir))
checkpoint = utils.load_checkpoint(save_dir, 'resnet18.pth')
model = checkpoint['model']
state_dict = checkpoint['state_dict']
model.load_state_dict(state_dict)
step = checkpoint['step']
model.to(device)
summary(model, (3, args.n_frames_per_clip, 112, 112))
# pdb.set_trace()
model.train()
# determine optimizer
criterion = nn.CrossEntropyLoss()
fc_lr_layers = list(map(id, model.module.fc.parameters()))
pretrained_lr_layers = [
p for p in model.parameters()
if id(p) not in fc_lr_layers and p.requires_grad == True
]
# pretrained_lr_layers = filter(lambda p:
# id(p) not in fc_lr_layers, model.parameters())
optimizer = torch.optim.SGD([{
"params": model.module.fc.parameters()
}, {
"params": pretrained_lr_layers,
"lr": 1e-4,
'weight_decay': 1e-3
}],
lr=1e-3,
momentum=0.9,
weight_decay=1e-2)
train_logger = utils.Logger(
os.path.join('output', 'R3D-fine-tune-all.log'), [
'step', 'train_loss', 'train_acc', 'val_loss', 'val_acc',
'lr_feature', 'lr_fc'
])
scheduler = lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)
# scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=2)
train_loss = utils.AverageMeter()
train_acc = utils.AverageMeter()
val_loss = utils.AverageMeter()
val_acc = utils.AverageMeter()
for epoch in trange(num_epochs): # loop over the dataset multiple times
train_loss.reset()
train_acc.reset()
for data in dataloader_train:
inputs, masks, labels = data
inputs, labels = inputs.to(device,
non_blocking=True).float(), labels.to(
device, non_blocking=True).long()
optimizer.zero_grad()
outputs = model(inputs)
loss_ = criterion(outputs, labels)
loss_.backward()
optimizer.step()
train_loss.update(loss_.item())
train_acc.update(utils.calculate_accuracy(outputs, labels))
if step % 50 == 0:
val_loss.reset()
val_acc.reset()
model.eval()
for data_val in dataloader_val:
inputs_val, masks_val, labels_val = data_val
inputs_val, labels_val = inputs_val.to(device, non_blocking=True).float(), \
labels_val.to(device, non_blocking=True).long()
outputs_val = model(inputs_val)
val_loss_ = criterion(outputs_val, labels_val)
val_loss.update(val_loss_.item())
val_acc.update(
utils.calculate_accuracy(outputs_val, labels_val))
model.train()
print(
'epoch{}/{} train_acc:{:.3f} train_loss:{:.3f} val_acc:{:.3f} val_loss:{:.3f}'
.format(epoch + 1, num_epochs, train_acc.val,
train_loss.val, val_acc.avg, val_loss.avg))
train_logger.log({
'step': step,
'train_loss': train_loss.val,
'train_acc': train_acc.val,
'val_loss': val_loss.avg,
'val_acc': val_acc.avg,
# 'lr_feature': optimizer.param_groups[1]['lr'],
'lr_feature': 0,
'lr_fc': optimizer.param_groups[0]['lr']
})
if step % 200 == 0:
utils.save_checkpoint(model, optimizer, step, save_dir,
'resnet18.pth')
step += 1
scheduler.step()
def train_epoch(epoch: int, data_loader: DataLoader, model: torch.nn.Module,
criterion: CrossEntropyLoss, optimizer: torch_optimizer,
opt: Namespace, epoch_logger: Logger, result_dir_name: str,
device: torch.device):
print(f'train at epoch {epoch}')
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
if opt.show_top5:
accuracies5 = AverageMeter()
else:
accuracies5 = None
end_time = time.time()
epoch_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
targets = targets.to(device, non_blocking=True)
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
loss = criterion(outputs, targets)
losses.update(loss.item(), inputs.size(0))
if opt.show_top5:
acc1, acc5 = calculate_accuracy_1_5(outputs, targets)
accuracies.update(acc1, inputs.size(0))
accuracies5.update(acc5, inputs.size(0))
else:
acc1 = calculate_accuracy(outputs, targets)
accuracies.update(acc1, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
if opt.show_top5:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc-Top1 {acc.val:.3f} ({acc.avg:.3f})\t'
'Acc-Top5 {acc5.val:.3f} ({acc5.avg:.3f))\t'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies,
acc5=accuracies5))
else:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc-Top1 {acc.val:.3f} ({acc.avg:.3f})\t'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_time = time.time() - epoch_time
if opt.show_top5:
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc-top1': accuracies.avg,
'acc-top5': accuracies5.avg,
'lr': optimizer.param_groups[0]['lr'],
'batch': opt.batch_size,
'batch-time': batch_time.avg,
'epoch-time': epoch_time
})
else:
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc-top1': accuracies.avg,
'lr': optimizer.param_groups[0]['lr'],
'batch': opt.batch_size,
'batch-time': batch_time.avg,
'epoch-time': epoch_time
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(result_dir_name,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
def main():
'''
Train function.
'''
os.environ[
'TORCH_HOME'] = t_cfg.MODEL_PATH #set the env variable so the model is downloaded inside this folder.
########################################################## Model Initialization & Loading ##########################################################
model_instance = model.Model(model_download_path=t_cfg.MODEL_PATH,
new_model_name=t_cfg.MODEL_NAME,
input_feature_size=t_cfg.FEATURE_INPUT_SIZE,
num_class=t_cfg.NUM_CLASSES)
vgg_model = model_instance()
optimizer = Adam(vgg_model.parameters(),
lr=t_cfg.LEARNING_RATE) #optimizer
lr_decay = lr_scheduler.ExponentialLR(
optimizer, gamma=t_cfg.LR_DECAY_RATE
) #scheduler is used to lower the learning rate during training later.
loss_criterion = torch.nn.CrossEntropyLoss() #loss function.
vgg_model = vgg_model.to(
t_cfg.DEVICE) #move the network to GPU if available.
print("--- Model Architecture ---")
print(vgg_model)
########################################################## Data Initialization & Loading ##########################################################
#Initialize the training data class.
training_data = LoadDataset(
resized_image_size=t_cfg.RESIZED_IMAGE_SIZE,
total_images=t_cfg.TOTAL_DATA,
classes=t_cfg.CLASSES,
data_list=t_cfg.IMG_LABEL_LIST,
transform=transforms.Compose([
RandomRotate(angle_range=t_cfg.ROTATION_RANGE,
prob=t_cfg.ROTATION_PROB),
RandomShear(shear_range=t_cfg.SHEAR_RANGE, prob=t_cfg.SHEAR_PROB),
RandomHorizontalFlip(prob=t_cfg.HFLIP_PROB),
RandomVerticalFlip(prob=t_cfg.VFLIP_PROB),
RandomNoise(mode=t_cfg.NOISE_MODE, prob=t_cfg.NOISE_PROB),
ToTensor(mode='training')
]))
dataloader = DataLoader(training_data,
batch_size=t_cfg.BATCH_SIZE,
shuffle=t_cfg.DATA_SHUFFLE,
num_workers=t_cfg.NUM_WORKERS)
########################################################## Model Training & Saving ##########################################################
best_accuracy = 0
entire_loss_list = []
entire_accuracy_list = []
for epoch_idx in range(t_cfg.EPOCH):
print("Training for epoch %d has started!" % (epoch_idx + 1))
epoch_training_loss = []
epoch_accuracy = []
i = 0
for i, sample in tqdm(enumerate(dataloader)):
batch_x, batch_y = sample['image'].to(
t_cfg.DEVICE), sample['label'].to(t_cfg.DEVICE)
optimizer.zero_grad(
) #clear the gradients in the optimizer between every batch.
net_output = vgg_model(batch_x) #output from the network.
total_loss = loss_criterion(input=net_output, target=batch_y)
epoch_training_loss.append(
total_loss.item()) #append the loss of every batch.
total_loss.backward() #calculate the gradients.
optimizer.step()
batch_acc = calculate_accuracy(network_output=net_output,
target=batch_y)
epoch_accuracy.append(batch_acc.cpu().numpy())
lr_decay.step() #decay rate update
curr_accuracy = sum(epoch_accuracy) / i
curr_loss = sum(epoch_training_loss)
print("The accuracy at epoch %d is %g" % (epoch_idx, curr_accuracy))
print("The loss at epoch %d is %g" % (epoch_idx, curr_loss))
entire_accuracy_list.append(curr_accuracy)
entire_loss_list.append(curr_loss)
if curr_accuracy > best_accuracy:
torch.save(vgg_model.state_dict(), t_cfg.SAVE_PATH)
best_accuracy = curr_accuracy
print("Model is saved !")
########################################################## Graphs ##########################################################
if t_cfg.PLOT_GRAPH:
plot_graph(t_cfg.EPOCH, "Epoch", "Training Loss",
"Training Loss for %d epoch" % (t_cfg.EPOCH), "./loss.png",
[entire_loss_list, 'r--', "Loss"])
plot_graph(t_cfg.EPOCH, "Epoch", "Training Accuracy",
"Training Accuracy for %d epoch" % (t_cfg.EPOCH),
"./accuracy.png", [entire_accuracy_list, 'b--', "Accuracy"])
def train_epoch(epoch, fold_id, data_loader, model, criterion, optimizer, opt,
epoch_logger, batch_logger, writer):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
writer = writer
end_time = time.time()
for i, (inputs, labels) in enumerate(data_loader):
data_time.update(time.time() - end_time)
labels = list(map(int, labels))
inputs = torch.unsqueeze(inputs, 1) #在 1 的位置加一个维度
inputs = inputs.type(torch.FloatTensor)
if not opt.no_cuda:
labels = torch.LongTensor(labels).cuda(async=True)
inputs = Variable(inputs)
labels = Variable(labels)
outputs = model(inputs)
loss = criterion(outputs, labels)
acc = calculate_accuracy(outputs, labels)
losses.update(loss.data, inputs.size(0))
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i - 1),
'loss': round(losses.val.item(), 4),
'acc': round(accuracies.val.item(), 4),
'lr': optimizer.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_logger.log({
'epoch': epoch,
'loss': round(losses.avg.item(), 4),
'acc': round(accuracies.avg.item(), 4),
'lr': optimizer.param_groups[0]['lr']
})
writer.add_scalar('train/loss', losses.avg, epoch)
writer.add_scalar('train/accuracy', accuracies.avg, epoch)
if opt.save_weight:
if epoch % opt.checkpoint == 0:
save_dir = OsJoin(opt.result_path, opt.data_type,
opt.model_name + str(opt.model_depth),
'weights_fold%s' % fold_id)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = OsJoin(
save_dir, '{}{}_weights_fold{}_epoch{}.pth'.format(
opt.model_name, opt.model_depth, fold_id, epoch))
states = {
'fold': fold_id,
'epoch': epoch,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_path)
def val_epoch(epoch, data_loader, model, criterion, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
cls_losses = AverageMeter()
op_losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, proposal, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
# if not opt.no_cuda:
# targets = targets.cuda(async=True)
with torch.no_grad():
inputs = Variable(inputs)
proposal = Variable(proposal.float())
outputs, op_loss = model(inputs, proposal, training=False)
# targets = Variable(targets, volatile=True)
# targets = targets.data.cpu()
outputs = outputs.data.cpu()
cls_loss = criterion(outputs, targets)
op_loss = op_loss.data.cpu()
loss = cls_loss + opt.op_loss_weight * op_loss.mean()
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
cls_losses.update(cls_loss.data.cpu()[0], inputs.size(0))
op_losses.update(op_loss.data.cpu()[0], inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
if (i + 1) % opt.log_step == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'CLSLoss {cls_loss.val:.4f} ({cls_loss.avg:.4f})\t'
'OPLoss {op_loss.val:.4f} ({op_loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
cls_loss=cls_losses,
op_loss=op_losses,
acc=accuracies),
flush=True)
logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
return losses.avg
def val_epoch_true(epoch, data_loader, model, criterion, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
precisions = AverageMeter() #
recalls = AverageMeter()
end_time = time.time()
vid_ids = data_loader.dataset.ids
vid_ids.append(-1)
out_queue = []
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
with torch.no_grad():
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
outputs = F.softmax(outputs, dim=1)
outputs = outputs.cpu().numpy()[0].reshape(-1, )
out_queue.append(outputs)
if vid_ids[i + 1] != vid_ids[i]:
output = np.mean(out_queue, 0)
outputs = torch.from_numpy(output).float().unsqueeze_(0).cuda()
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
precision = calculate_precision(outputs, targets)
recall = calculate_recall(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
precisions.update(precision, inputs.size(0))
recalls.update(recall, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
out_queue = []
if i % 100 == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.5f} ({batch_time.avg:.5f})\t'
'Data {data_time.val:.5f} ({data_time.avg:.5f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'precision': precisions.avg,
'recall': recalls.avg
})
return losses.avg, accuracies.avg
def train_epoch(epoch, data_loader, model, criterion, optimizer, logger=None):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if torch.cuda.is_available():
targets = targets.cuda(async=True)
inputs = inputs.cuda()
targets = targets.cuda()
# inputs = Variable(inputs)
# targets = Variable(targets)
torch.cuda.synchronize()
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
torch.cuda.synchronize()
batch_time.update(time.time() - end_time)
end_time = time.time()
# batch_logger.log({
# 'epoch': epoch,
# 'batch': i + 1,
# 'iter': (epoch - 1) * len(data_loader) + (i + 1),
# 'loss': losses.val,
# 'acc': accuracies.val,
# 'lr': optimizer.param_groups[0]['lr']
# })
print('Train: Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
if logger is not None:
current_step = epoch * len(data_loader) + i
logger.add_scalar('Train/Loss', losses.avg, current_step)
logger.add_scalar('Train/Acc', accuracies.avg, current_step)
def test(model, load_dir, ExemplarSet_file, checkpoint_file, dataloader, class_id1, class_id2):
in_features = model.module.fc.in_features
out_features = model.module.fc.out_features
# model.module.fc = nn.Linear(in_features, len(class_id1)+len(class_id2), bias=False)
model.module.fc = nn.Linear(in_features, len(class_id1)+len(class_id2))
model.to(device)
print('Start testing for class {}.....'.format(range(class_id1[0], class_id2[-1])))
print('Model {} and exemplar sets {} loaded'.format(checkpoint_file, ExemplarSet_file))
ExemplarSet_file = os.path.join(load_dir, ExemplarSet_file)
checkpoint_file = os.path.join(load_dir, checkpoint_file)
with open(ExemplarSet_file, 'rb') as f:
exemplar_sets = pickle.load(f)
checkpoint = torch.load(checkpoint_file)
# model = checkpoint['model']
model.load_state_dict(checkpoint['state_dict'])
model.eval()
print('Computing exemplar means for {} classes........'.format(len(exemplar_sets)))
exemplar_means = compute_mean(model, exemplar_sets)
acc = utils.AverageMeter()
acc_class = utils.AverageMeter()
acc_class_cache = []
for class_i, dataloader_i in enumerate(dataloader):
acc_class.reset()
for data in dataloader_i:
inputs, labels = data
inputs = inputs.to(device, non_blocking=True).float()
preds = classfier(model, exemplar_means, inputs)
acc.update(utils.calculate_accuracy_ForIcarl(preds, labels))
acc_class.update(utils.calculate_accuracy_ForIcarl(preds, labels))
acc_class_cache.append(acc_class.avg)
print('Accuracy for old classes:')
print(acc_class_cache[0])
print('Accuracy for new classes:')
print(acc_class_cache[1])
print('Mean accuracy')
print(acc.avg)
acc.reset()
acc_class.reset()
acc_class_cache = []
for class_i, dataloader_i in enumerate(dataloader):
acc_class.reset()
for data in dataloader_i:
inputs, labels = data
inputs = inputs.to(device, non_blocking=True).float()
labels = labels.to(device, non_blocking=True)
probs, logits = model(inputs)
acc.update(utils.calculate_accuracy(probs, labels))
acc_class.update(utils.calculate_accuracy(probs, labels))
acc_class_cache.append(acc_class.avg)
print('\n')
print('Accuracy for old classes:')
print(acc_class_cache[0])
print('Accuracy for new classes:')
print(acc_class_cache[1])
print('Mean accuracy')
print(acc.avg)
del inputs, labels, probs, logits, model
torch.cuda.empty_cache()
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt,
epoch_logger, batch_logger):
print('train at epoch {}'.format(epoch))
model.train()
cuda = torch.device("cuda")
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (video, audio) in enumerate(data_loader):
data_time.update(time.time() - end_time)
video = video.to(device=cuda)
audio = audio.to(device=cuda)
bs = video.size(0)
device = video.device
#feature_v, feature_a = model(video, audio)
#target = torch.arange(bs).to(device=device)
#cosv2a = torch.mm(feature_a, feature_v.t()).t()
#cosa2v = torch.mm(feature_v, feature_a.t()).t()
#loss1 = criterion(cosv2a, target)
#loss2 = criterion(cosa2v, target)
#loss = loss1 + loss2
#tmp = cosv2a.argmax(dim=1)
#acc = calculate_accuracy(cosv2a, target)
#fc_correct, fc_wrong = model(video, audio)
dot_va, dot_av, label_va, label_av = model(video, audio)
#fc = torch.cat([fc_correct, fc_wrong], dim=0)
#target = torch.cat([torch.ones((bs, ), dtype=torch.long), torch.zeros((bs, ), dtype=torch.long)], dim=0)
#target = target.to(device=device)
#loss = criterion(fc, target)
loss = criterion(dot_va, label_va) + criterion(dot_av, label_av)
acc = calculate_accuracy(dot_va, label_va)
losses.update(loss.data.item(), video.size(0))
accuracies.update(acc, video.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
loss = utils.cross_entropy_loss(output, label)
train = tf.train.AdamOptimizer(0.001).minimize(loss)
global_vars_init_op = tf.global_variables_initializer()
local_vars_init_op = tf.local_variables_initializer()
combined_op = tf.group(local_vars_init_op, global_vars_init_op)
model_variables = slim.get_model_variables()
saver = tf.train.Saver(model_variables)
with tf.Session() as sess:
sess.run(combined_op)
# saver.restore(sess, '/home/kris/PycharmProjects/traffic_sign_recognition/lenet_parameters.ckpt')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(50000):
ignore, pred, error, images, truth = sess.run([train, output, loss, image_batch, label_batch])
#for i in range(len(images)):
# cv2.imshow("image" + str(i), cv2.resize(images[i], (256, 256)));
# print(truth[i])
#cv2.waitKey(0)
accuracy = utils.calculate_accuracy(pred, truth)
print("%d round loss = %f, accuracy = %f" % (i, error, accuracy))
if i % 199 == 0:
saver.save(sess, '/home/kris/PycharmProjects/traffic_sign_recognition/lenet_parameters.ckpt')
coord.request_stop()
coord.join(threads)
saver.save(sess, '/home/kris/PycharmProjects/traffic_sign_recognition/lenet_parameters.ckpt')
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt,
epoch_logger, batch_logger, writer):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
dataset = "ucf"
if opt.dataset == "hmdb51":
dataset = "hmdb"
end_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
acc = 0
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
if opt.dataset == "hmdb51":
targets -= 1
outputs = model(inputs.cuda())
loss = criterion(outputs, targets)
losses.update(loss.item(), inputs.size(0))
acc = calculate_accuracy(outputs, targets)
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
writer.add_scalar('%s/train_loss' % dataset, losses.val, (epoch - 1) * len(data_loader) + (i + 1))
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
def val_epoch(epoch, data_loader, model, criterion, args, device,
epoch_logger):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top3 = AverageMeter()
# switch to evaluate mode
model.eval()
end_time = time.time()
for i, (input, target) in enumerate(data_loader):
# measure data loading time
data_time.update(time.time() - end_time)
input = input.to(device)
target = target.to(device)
# compute output and loss
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec3 = calculate_accuracy(output, target, topk=(1, 3))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0].item(), input.size(0))
top3.update(prec3[0].item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end_time)
end_time = time.time()
if (i + 1) % args.log_interval == 0:
print('Valid Epoch [{0}/{1}]([{2}/{3}])\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@3 {top3.val:.3f} ({top3.avg:.3f})\t'
'Batch Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data Time {data_time.val:.3f} ({data_time.avg:.3f})'.format(
epoch,
args.epochs,
i + 1,
len(data_loader),
loss=losses,
top1=top1,
top3=top3,
batch_time=batch_time,
data_time=data_time))
print(' * Prec@1 {top1.avg:.2f}% | Prec@3 {top3.avg:.2f}%'.format(
top1=top1, top3=top3))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'top1': top1.avg,
'top3': top3.avg
})
vis.plot('Val loss', losses.avg)
vis.plot('Val accu', top1.avg)
return losses.avg, top1.avg
def run_epoch_pass(mode, dataloader, model, criterion, optimizer):
"""Perform one train or test pass through the data (epoch)
"""
batch_time = AverageMeter("Batch Time")
data_time = AverageMeter("Data Time")
losses = AverageMeter("Losses")
top1 = AverageMeter("Top 1 Accuracy")
top5 = AverageMeter("Top 5 Accuracy")
end = time()
if mode == "Train":
model.train()
elif mode == "Test":
model.eval()
else:
assert mode in ("Train", "Test"), f"Unsupported mode {mode}"
bar = Bar(mode, max=len(dataloader))
for batch_idx, (inputs, targets) in enumerate(dataloader):
# measure data loading time
data_time.update(time() - end)
if USE_CUDA:
inputs, targets = inputs.cuda(), targets.cuda()
# compute output
outputs = model(inputs)
loss = criterion(outputs, targets)
# measure accuracy and record loss
# pylint: disable=unbalanced-tuple-unpacking
prec1, prec5 = calculate_accuracy(outputs.data,
targets.data,
topk=(1, 5))
losses.update(loss.data.item(), inputs.size(0))
top1.update(prec1.item(), inputs.size(0))
top5.update(prec5.item(), inputs.size(0))
if mode == "Train":
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time() - end)
end = time()
# plot progress
bar.suffix = "({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}".format(
batch=batch_idx + 1,
size=len(dataloader),
data=data_time.avg,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
top1=top1.avg,
top5=top5.avg,
)
bar.next()
bar.finish()
return (losses.avg, top1.avg)
def val_epoch(epoch, data_loader, model, criterion, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
######### temp line, needs to be removed##################################
file = 'epoch_'+ str(epoch)+'_validation_matrix.csv'
confusion_matrix = np.zeros((opt.n_classes,opt.n_classes))
confidence_for_each_validation = {}
###########################################################################
#pdb.set_trace()
for i, (inputs, targets, paths) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(non_blocking=True)
with torch.no_grad():
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
######### temp line, needs to be removed##################################
for j in range(len(targets)):
confidence_for_each_validation[paths[j]] = [x.item() for x in outputs[j]]
rows = [int(x) for x in targets]
columns = [int(x) for x in torch.argmax(outputs, 1)]
assert len(rows) == len(columns)
for idx in range(len(rows)):
confusion_matrix[rows[idx]][columns[idx]] += 1
###########################################################################
losses.update(loss.data, inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
######### temp line, needs to be removed##################################
print(confusion_matrix)
confusion_matrix = pd.DataFrame(confusion_matrix)
confusion_matrix.to_csv(opt.result_path + 'ConfusionMatrix_' + str(epoch) + '.csv')
confidence_matrix = pd.DataFrame.from_dict(confidence_for_each_validation, orient='index')
confidence_matrix.to_csv(opt.result_path + 'ConfidenceMatrix.csv')
######### temp line, needs to be removed##################################
logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
return losses.avg
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt,
epoch_logger):
print('train at epoch {}'.format(epoch))
model.train()
batch_time = AverageMeter() #このepochにおける１iterにかかる時間
losses = AverageMeter() #このepochにおける1iterのlossとlossの平均を保持
accuracies = AverageMeter() #このepochにおける1iterのaccuraciesとaccuraciesの平均を保持
device = torch.device("cuda") #マルチGPUを使う場合の呪文
end_time = time.time()
#以下学習フロー
for i, (inputs, targets, video_ids) in enumerate(data_loader):
model.zero_grad() #モデルの勾配をゼロに
inputs = inputs.to(
device) #マルチGPUの場合の手続き，これをしないと各GPUに分けてデータをのせることができない
targets = targets.squeeze(1) #tensor(10,1)からtensor(10)の１次元テンソルへ変換
targets = targets.cuda() #cudaへ
outputs = model(
inputs
) #inputs：（バッチサイズ， チャンネル数（３）, フレーム数(16), width(114), height(114)）をモデルに入力し，outputs：(バッチサイズ, 各クラス尤度)を返すforward計算
loss = criterion(
outputs, targets
) #outputsとtargetsのLOSSを計算，クラスエントロピーなので正解次元の推定尤度のlogにマイナスを乗じた値のバッチごとの平均がLOSSになる
acc = calculate_accuracy(outputs,
targets) #outputsとtargetsからバッチの中でどれだけ正解したか出力
losses.update(loss.item(), inputs.size(
0)) #このiterにおけるlossとこのepochにおけるこれまでのiterにおけるlossの平均を計算し保持
accuracies.update(acc, inputs.size(
0)) #このiterにおけるaccとこのepochにおけるこれまでのiterにおけるaccの平均を計算し保持
optimizer.zero_grad() #optimizerの勾配をゼロに
loss.backward() #backward計算で勾配を算出
optimizer.step() #算出した勾配から重みを更新
batch_time.update(
time.time() -
end_time) #このiterでかかった時間を算出しこのepochにおけるこれまでのiterでかかった時間の平均を計算し保持
end_time = time.time() #時間を更新
#各種情報を表示
"""
print('Epoch: [{0}][{1}/{2}]\t'.format(
epoch,
i + 1,
len(data_loader)
))
"""
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
loss=losses,
acc=accuracies))
#epoch数，このepochにおけるlossの平均，このepochにおけるacc平均，このepochにおける学習率をtrain.logへ書き込む
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join("./result/learned_weight_day1-3_1-6_rgb",
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
def val_epoch(epoch, data_set, model, criterion, optimizer, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
losses = AverageMeter()
accuracies = AverageMeter()
data_set.file_open()
valildation_loader = torch.utils.data.DataLoader(
dataset=data_set,
batch_size=opt["validation_batch_size"],
shuffle=False,
pin_memory=True)
val_process = tqdm(valildation_loader)
start_time = time.time()
for i, (inputs, targets) in enumerate(val_process):
if i > 0:
val_process.set_description("Loss: %.4f, Acc: %.4f" %
(losses.avg, accuracies.avg))
if opt["cuda_devices"] is not None:
#targets = targets.cuda(async=True)
inputs = inputs.type(torch.FloatTensor)
inputs = inputs.cuda()
targets = targets.type(torch.FloatTensor)
targets = targets.cuda()
with torch.no_grad():
if opt["VAE_enable"]:
outputs, distr = model(inputs)
loss = criterion(outputs, targets, distr)
else:
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs.cpu(), targets.cpu())
losses.update(loss.cpu(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
epoch_time = time.time() - start_time
data_set.file_close()
print("validation: epoch:{0}\t seg_acc:{1:.4f} \t using:{2:.3f} minutes".
format(epoch, accuracies.avg, epoch_time / 60))
logger.log(phase="val",
values={
'epoch': epoch,
'loss': format(losses.avg.item(), '.4f'),
'acc': format(accuracies.avg.item(), '.4f'),
'lr': optimizer.param_groups[0]['lr']
})
with open("runs/val_log/log.txt", "a+") as file:
file.write({
'epoch': epoch,
'loss': format(losses.avg.item(), '.4f'),
'acc': format(accuracies.avg.item(), '.4f'),
'lr': optimizer.param_groups[0]['lr']
}.__str__() + "\n")
return losses.avg, accuracies.avg
def test(model, test_loader, test_batch_logger, test_logger, badcase_npy):
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
accuracies_chexing = AverageMeter()
accuracies_ori = AverageMeter()
accuracies = AverageMeter()
print('----------开始测试----------')
badcase_keys = []
with torch.no_grad():
for i, (inputs, targets, keys) in enumerate(test_loader):
data_time.update(time.time() - end_time)
targets = targets.numpy().tolist()
targets = [label_2_gt[str(target)] for target in targets]
labels_chexing = [
int(gt_2_chexing[str(target)]) for target in targets
]
labels_ori = [int(gt_2_ori[str(target)]) for target in targets]
labels_chexing = torch.tensor(labels_chexing)
labels_ori = torch.tensor(labels_ori)
out_chexing, out_ori = model(inputs)
# acc = calculate_accuracy(out_chexing, labels_chexing)
acc_chexing = calculate_accuracy(out_chexing, labels_chexing)
acc_ori = calculate_accuracy(out_ori, labels_ori)
acc = calculate_union_accuracy((out_chexing, out_ori),
(labels_chexing, labels_ori))
badcase = find_badcase((out_chexing, out_ori),
(labels_chexing, labels_ori), keys)
badcase_keys.extend(badcase)
accuracies_chexing.update(acc_chexing, inputs.size(0))
accuracies_ori.update(acc_ori, inputs.size(0))
#
# accuracies_midbrand.update(acc_midbrand, inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
# test_batch_logger.log({
# 'batch': i + 1,
# 'acc': accuracies.val,
# })
print('test iter: {}/{}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Acc_chexing {acc_chexing.val:.3f} ({acc_chexing.avg:.3f})\t'
'Acc_ori {acc_ori.val:.3f} ({acc_ori.avg:.3f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
i + 1,
len(test_loader),
batch_time=batch_time,
data_time=data_time,
acc_chexing=accuracies_chexing,
acc_ori=accuracies_ori,
acc=accuracies))
np.save(badcase_npy, badcase_keys)
def val_epoch(epoch: int, data_loader: DataLoader, model: torch.nn.Module,
criterion: CrossEntropyLoss, opt: Namespace,
epoch_logger: Logger, device: torch.device):
print(f'validation at epoch {epoch}')
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
if opt.show_top5:
accuracies5 = AverageMeter()
else:
accuracies5 = None
end_time = time.time()
epoch_time = time.time()
with torch.no_grad():
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
targets = targets.to(device, non_blocking=True)
outputs = model(inputs)
loss = criterion(outputs, targets)
losses.update(loss.item(), inputs.size(0))
if opt.show_top5:
acc1, acc5 = calculate_accuracy_1_5(outputs, targets)
accuracies.update(acc1, inputs.size(0))
accuracies5.update(acc5, inputs.size(0))
else:
acc1 = calculate_accuracy(outputs, targets)
accuracies.update(acc1, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
if opt.show_top5:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc-Top1 {acc.val:.3f} ({acc.avg:.3f})\t'
'Acc-Top5 {acc5.val:.3f} ({acc5.avg:.3f))\t'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies,
acc5=accuracies5))
else:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc-Top1 {acc.val:.3f} ({acc.avg:.3f})\t'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_time = time.time() - epoch_time
if opt.show_top5:
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc-top1': accuracies.avg,
'acc-top5': accuracies5.avg,
'batch': opt.batch_size,
'batch-time': batch_time.avg,
'epoch-time': epoch_time
})
else:
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc-top1': accuracies.avg,
'batch': opt.batch_size,
'batch-time': batch_time.avg,
'epoch-time': epoch_time
})
def val_epoch(epoch,
data_loader,
model,
criterion,
device,
logger,
is_master_node,
tb_writer=None,
distributed=False):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
# losses_chexing = AverageMeter()
# losses_ori = AverageMeter()
losses = AverageMeter()
#
# losses_midbrand = AverageMeter()
accuracies_chexing = AverageMeter()
accuracies_ori = AverageMeter()
#
# accuracies_midbrand = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
with torch.no_grad():
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
inputs = inputs.to(device, non_blocking=True)
targets = targets.to(device, non_blocking=True)
outputs = model(inputs)
loss = criterion(outputs, targets)
acc = calculate_accuracy(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
itera = (epoch - 1) * len(data_loader) + (i + 1)
if is_master_node:
print('Val Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
if distributed:
loss_sum = torch.tensor([losses.sum],
dtype=torch.float32,
device=device)
loss_count = torch.tensor([losses.count],
dtype=torch.float32,
device=device)
acc_sum = torch.tensor([accuracies.sum],
dtype=torch.float32,
device=device)
acc_count = torch.tensor([accuracies.count],
dtype=torch.float32,
device=device)
dist.all_reduce(loss_sum, op=dist.ReduceOp.SUM)
dist.all_reduce(loss_count, op=dist.ReduceOp.SUM)
dist.all_reduce(acc_sum, op=dist.ReduceOp.SUM)
dist.all_reduce(acc_count, op=dist.ReduceOp.SUM)
losses.avg = loss_sum.item() / loss_count.item()
accuracies.avg = acc_sum.item() / acc_count.item()
if logger is not None:
logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
if is_master_node:
if tb_writer is not None:
tb_writer.add_scalar('val/loss', losses.avg, epoch)
tb_writer.add_scalar('val/acc', accuracies.avg, epoch)
return losses.avg
def val_epoch(epoch, data_loader, model, criterion, opt, logger):
print('validation at epoch {}'.format(epoch))
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
accuracies_5 = AverageMeter()
end_time = time.time()
for i, (inputs, targets, video_id) in enumerate(data_loader):
data_time.update(time.time() - end_time)
print('This line has been executed')
# print (inputs.size())
if not opt.no_cuda:
targets = targets.cuda(async=True)
# inputs= Variable(inputs)
# print (inputs.view(-1,3,inputs.size(3), inputs.size(4)).size())
inputs = Variable(inputs.view(-1, 3, inputs.size(3), inputs.size(4)),
volatile=True)
# print (inputs.size())
targets = Variable(targets, volatile=True)
outputs = model(inputs).mean(dim=0, keepdim=True)
print(outputs.size())
# outputs= outputs.view(-1,16)
# print (outputs.size())
# print (targets.size())
loss = criterion(outputs, targets)
acc, acc_5 = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
accuracies.update(acc, inputs.size(0))
accuracies_5.update(acc_5, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
if opt.debug:
if acc_5 == 0:
print(video_id)
print(targets)
print('Validation, Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'
'Acc_5 {acc_5.val:.3f} ({acc_5.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies,
acc_5=accuracies_5))
logger.log({
'epoch': epoch,
'loss': losses.avg.tolist(),
'acc': accuracies.avg,
'acc_5': accuracies_5.avg
})
return losses.avg
ctc_logit_matrix = preprocess_ocr_logit(ctc_logit_matrix,
softmax_theta)
# search
predict = beam_search.search(ctc_logit_matrix)
# post-processing
predict = jaconv.normalize(predict)
lbl = jaconv.normalize(lbl)
# track
mapping_inout_result += [(predict, lbl)]
print("take time:", time.time() - s_time)
"""
3. Calculating accuracy
"""
preds = [elem[0] for elem in mapping_inout_result]
lbls = [elem[1] for elem in mapping_inout_result]
# Debug & Testing
if False:
for input_fn, pred, lbl in zip(list(mapping_inout.keys()), preds,
lbls):
print("input_fn:",
os.path.split(input_fn)[-1], ",pred:", pred, ",lbl:", lbl,
"is_correct:", pred == lbl)
acc_char, em = calculate_accuracy(preds, lbls)
print("acc by char:", acc_char)
print("em:", em)
