def train(train_loader, model, criterion, optimizer, epoch, args, print_func):
batch_time = CNN_utils.AverageMeter()
data_time = CNN_utils.AverageMeter()
losses = CNN_utils.AverageMeter()
top1 = CNN_utils.AverageMeter()
top5 = CNN_utils.AverageMeter()
# switch to train mode
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
# target_idx = target.nonzero() [:,1]
# compute output
output = model(input)
loss = criterion(output, target)
# log_softmax_output = F.log_softmax(output, dim=1)
#
# loss = - torch.sum(log_softmax_output * target) / output.shape[0]
losses.update(loss.item(), input.size(0))
prec1 = CNN_utils.accuracy_multihots(output, target, topk=(1, 3))
top1.update(prec1[0], input.size(0))
# top5.update(prec5[0], input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print_func('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@{kout} {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
kout=5,
top5=top5))
def validate(val_loader, model, criterion, args, print_func):
if val_loader is None:
return 0, 0
batch_time = CNN_utils.AverageMeter()
losses = CNN_utils.AverageMeter()
top1 = CNN_utils.AverageMeter()
# top5 = CNN_utils.AverageMeter()
# switch to evaluate mode
model.eval()
with torch.no_grad():
end = time.time()
for i, (input, target) in enumerate(val_loader):
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
# compute output
output = model(input)
# loss = criterion(output, target)
# log_softmax_output = F.log_softmax(output, dim=1)
#
# loss = - torch.sum(log_softmax_output * target)/ output.shape[0]
# measure accuracy and record loss
loss = criterion(output, target)
prec1 = CNN_utils.accuracy_multihots(output, target, topk=(1, 3))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0], input.size(0))
# top5.update(prec5[0], input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print_func('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1))
print_func(' * Prec@1 {top1.avg:.3f}'.format(top1=top1))
return top1.avg, losses.avg
def train(train_loader, model, text_model, criterion, optimizer, epoch, args, print_func):
batch_time = CNN_utils.AverageMeter()
data_time = CNN_utils.AverageMeter()
losses_cls = CNN_utils.AverageMeter()
losses_ebd = CNN_utils.AverageMeter()
top1 = CNN_utils.AverageMeter()
losses_cls_aux = CNN_utils.AverageMeter()
# switch to train mode
model.train()
text_model.eval()
end = time.time()
mCEL = MclassCrossEntropyLoss()
cos = torch.nn.CosineSimilarity()
for i, (input, target, text_info) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
textAvg = text_info[1]
sentCode = text_info[0]
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
sentCode = sentCode.cuda(args.gpu, non_blocking=True)
textAvg = textAvg.cuda(args.gpu, non_blocking=True)
with torch.no_grad():
sentence_output = text_model(sentCode)
text_cls = F.softmax(sentence_output[0], dim=1)
text_cls[text_cls<0.01] = 0
# text_class_values, text_class_ids = torch.max(text_output[0], dim=1)
# text_class_ids[text_class_values<0.5] = 4
# compute output
output_cls, output_proj = model(input)
new_target = (text_cls + target)/torch.sum(text_cls+target, dim=1, keepdim=True).expand_as(text_cls)
loss_cls = mCEL(output_cls, new_target)
loss_ebd = torch.sum(1 - cos(output_proj, textAvg)) / output_proj.shape[0]
# loss_cls_aux = mCEL(output_cls_basic, text_cls)
losses_cls.update(loss_cls.item(), input.size(0))
losses_ebd.update(loss_ebd.item(), input.size(0))
# losses_cls_aux.update(loss_cls_aux.item(), input.size(0))
loss = loss_cls + args.alpha* loss_ebd
prec1 = CNN_utils.accuracy_multihots(output_cls, target, topk=(1, 3))
top1.update(prec1[0], input.size(0))
# top5.update(prec5[0], input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print_func('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss_cls {loss_cls.val:.4f} ({loss_cls.avg:.4f})\t'
'Loss_ebd {loss_ebd.val:.4f} ({loss_ebd.avg:.4f})\t'
'Loss_cls_aux {loss_cls_aux.val:.4f} ({loss_cls_aux.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'
.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss_cls=losses_cls, loss_ebd=losses_ebd, loss_cls_aux=losses_cls_aux, top1=top1))
def train(train_loader, visual_model, criterion, optimizer, epoch, args, print_func):
batch_time = CNN_utils.AverageMeter()
data_time = CNN_utils.AverageMeter()
losses_cls = CNN_utils.AverageMeter()
losses_ebd = CNN_utils.AverageMeter()
top1 = CNN_utils.AverageMeter()
# = CNN_utils.AverageMeter()
# switch to train mode
visual_model.train()
end = time.time()
cos = torch.nn.CosineSimilarity()
for i, (input, target, text_embedding) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
text_embedding = text_embedding.cuda(args.gpu, non_blocking=True)
# text_embedding = text_model(text)
# compute output
output_cls, output_proj = visual_model(input)
log_softmax_output = F.log_softmax(output_cls, dim=1)
loss_cls = - torch.sum(log_softmax_output * target) / output_cls.shape[0]
loss_ebd = torch.sum(1 - cos(output_proj, text_embedding)) / output_proj.shape[0]
# loss_ebd = (output_proj - text_embedding)**2 / output_proj.shape[0]
losses_cls.update(loss_cls.item(), input.size(0))
losses_ebd.update(loss_ebd.item(), input.size(0))
loss = loss_ebd
prec1 = CNN_utils.accuracy_multihots(output_cls, target, topk=(1, 3))
top1.update(prec1[0], input.size(0))
# top5.update(prec5[0], input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print_func('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss_cls {loss_cls.val:.4f} ({loss_cls.avg:.4f})\t'
'Loss_ebd {loss_ebd.val:.4f} ({loss_ebd.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'
.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss_cls=losses_cls, loss_ebd=losses_ebd, top1=top1))
return losses_ebd.avg
def train(train_loader, model, text_model, criterion, optimizer, epoch, args,
print_func):
batch_time = CNN_utils.AverageMeter()
data_time = CNN_utils.AverageMeter()
losses_cls = CNN_utils.AverageMeter()
losses_ebd = CNN_utils.AverageMeter()
top1 = CNN_utils.AverageMeter()
top5 = CNN_utils.AverageMeter()
# switch to train mode
model.train()
text_model.eval()
end = time.time()
cos = torch.nn.CosineSimilarity()
for i, (input, target, text) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
text = text.cuda(args.gpu, non_blocking=True)
text_embedding, word_features = text_model(text)
# compute output
output_cls, output_proj = model(input)
x_weights = torch.sigmoid(
torch.bmm(word_features, output_proj.unsqueeze(2)))
# x_weighted_global = torch.sum((x_anchors.permute([0, 2, 1]) * x_weights, (1,), keepdim=False)
x_weighted_global = torch.bmm(word_features.permute([0, 2, 1]),
x_weights).squeeze(-1)
log_softmax_output = F.log_softmax(output_cls, dim=1)
loss_cls = -torch.sum(
log_softmax_output * target) / output_cls.shape[0]
loss_ebd = torch.sum(
1 - cos(output_proj, x_weighted_global)) / output_proj.shape[0]
losses_cls.update(loss_cls.item(), input.size(0))
losses_ebd.update(loss_ebd.item(), input.size(0))
loss = loss_cls + args.alpha * loss_ebd
prec1 = CNN_utils.accuracy_multihots(output_cls, target, topk=(1, 3))
top1.update(prec1[0], input.size(0))
# top5.update(prec5[0], input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print_func('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss_cls {loss_cls.val:.4f} ({loss_cls.avg:.4f})\t'
'Loss_ebd {loss_ebd.val:.4f} ({loss_ebd.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss_cls=losses_cls,
loss_ebd=losses_ebd,
top1=top1))
def train(train_loader, visual_model, text_model, text_generator, criterion,
optimizer, epoch, args, print_func):
batch_time = CNN_utils.AverageMeter()
data_time = CNN_utils.AverageMeter()
losses_visual_cls = CNN_utils.AverageMeter()
losses_text_cls = CNN_utils.AverageMeter()
losses_ebd = CNN_utils.AverageMeter()
losses = CNN_utils.AverageMeter()
top1 = CNN_utils.AverageMeter()
top5 = CNN_utils.AverageMeter()
# switch to train mode
visual_model.train()
text_model.train()
text_generator.eval()
end = time.time()
cos = torch.nn.CosineSimilarity()
for i, (visual_input, visual_target, text) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
if args.gpu is not None:
visual_input = visual_input.cuda(args.gpu, non_blocking=True)
visual_target = visual_target.cuda(args.gpu, non_blocking=True)
text = text.cuda(args.gpu, non_blocking=True)
text_target, text_input = text_generator(text)
text_target = F.softmax(text_target, dim=1)
# compute output
visual_cls, visual_proj = visual_model(visual_input)
text_cls, text_proj = text_model(text_input)
log_softmax_text = F.log_softmax(text_cls, dim=1)
loss_text_cls = -torch.sum(
log_softmax_text * text_target) / text_cls.shape[0]
log_softmax_visual = F.log_softmax(visual_cls, dim=1)
loss_visual_cls = -torch.sum(
log_softmax_visual * visual_target) / visual_cls.shape[0]
loss_ebd = torch.sum(
1 - cos(visual_proj, text_proj)) / visual_proj.shape[0]
losses_visual_cls.update(loss_visual_cls.item(), visual_input.size(0))
losses_text_cls.update(loss_text_cls.item(), text_input.size(0))
losses_ebd.update(loss_ebd.item(), visual_input.size(0))
loss = loss_visual_cls + loss_ebd + loss_text_cls
losses.update(loss.item(), visual_input.size(0))
prec1 = CNN_utils.accuracy_multihots(visual_cls,
visual_target,
topk=(1, 3))
top1.update(prec1[0], visual_input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print_func(
'Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss_visual cls {losses_visual_cls.val:.4f} ({losses_visual_cls.avg:.4f})\t'
'Loss_text cls {losses_text_cls.val:.4f} ({losses_text_cls.avg:.4f})\t'
'Loss_ebd {losses_ebd.val:.4f} ({losses_ebd.avg:.4f})\t'
'total loss {losses.val:.4f} ({losses.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
losses_visual_cls=losses_visual_cls,
losses_text_cls=losses_text_cls,
losses_ebd=losses_ebd,
losses=losses,
top1=top1))
