def compute_features(dataloader, model, N):
if args.verbose:
print('Compute features')
batch_time = AverageMeter()
end = time.time()
model.eval()
# discard the label information in the dataloader
for i, (input_tensor, _) in enumerate(dataloader):
input_var = torch.autograd.Variable(input_tensor.cuda(), volatile=True)
aux = model(input_var).data.cpu().numpy()
if i == 0:
features = np.zeros((N, aux.shape[1])).astype('float32')
if i < len(dataloader) - 1:
features[i * args.batch: (i + 1) * args.batch] = aux.astype('float32')
else:
# special treatment for final batch
features[i * args.batch:] = aux.astype('float32')
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.verbose and (i % 200) == 0:
print('{0} / {1}\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})'
.format(i, len(dataloader), batch_time=batch_time))
return features
def train(loader, model, optimizer, criterion, fc6_8, losses, it=0, total_iterations=None, stepsize=None, verbose=True):
# to log
batch_time = AverageMeter()
data_time = AverageMeter()
top1 = AverageMeter()
end = time.time()
current_iteration = it
# use dropout for the MLP
model.train()
# in the batch norms always use global statistics
model.features.eval()
for (input, target) in loader:
# measure data loading time
data_time.update(time.time() - end)
# adjust learning rate
if current_iteration != 0 and current_iteration % stepsize == 0:
for param_group in optimizer.param_groups:
param_group['lr'] = param_group['lr'] * 0.5
print('iter {0} learning rate is {1}'.format(current_iteration, param_group['lr']))
# move input to gpu
input = input.cuda(non_blocking=True)
# forward pass with or without grad computation
output = model(input)
target = target.float().cuda()
mask = (target == 255)
loss = torch.sum(criterion(output, target).masked_fill_(mask, 0)) / target.size(0)
# backward
optimizer.zero_grad()
loss.backward()
# clip gradients
torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
# and weights update
optimizer.step()
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if verbose is True and current_iteration % 25 == 0:
print('Iteration[{0}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
current_iteration, batch_time=batch_time,
data_time=data_time, loss=losses))
current_iteration = current_iteration + 1
if total_iterations is not None and current_iteration == total_iterations:
break
return current_iteration
def train_moco(epoch, train_loader, model, model_ema, contrast, criterion, optimizer, opt):
"""
one epoch training for instance discrimination
"""
model.train()
model_ema.eval()
def set_bn_train(m):
classname = m.__class__.__name__
if classname.find('BatchNorm') != -1:
m.train()
model_ema.apply(set_bn_train)
batch_time = AverageMeter()
data_time = AverageMeter()
loss_meter = AverageMeter()
prob_meter = AverageMeter()
end = time.time()
for idx, (inputs, _, index) in enumerate(train_loader):
data_time.update(time.time() - end)
bsz = inputs.size(0)
inputs = inputs.float()
if opt.gpu is not None:
inputs = inputs.cuda(opt.gpu, non_blocking=True)
else:
inputs = inputs.cuda()
index = index.cuda(opt.gpu, non_blocking=True)
# ===================forward=====================
x1, x2 = torch.split(inputs, [3, 3], dim=1)
# ids for ShuffleBN
shuffle_ids, reverse_ids = get_shuffle_ids(bsz)
feat_q = model(x1)
with torch.no_grad():
x2 = x2[shuffle_ids]
feat_k = model_ema(x2)
feat_k = feat_k[reverse_ids]
out = contrast(feat_q, feat_k)
loss = criterion(out)
prob = out[:, 0].mean()
# ===================backward=====================
optimizer.zero_grad()
if opt.amp:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
# ===================meters=====================
loss_meter.update(loss.item(), bsz)
prob_meter.update(prob.item(), bsz)
moment_update(model, model_ema, opt.alpha)
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
# print info
if (idx + 1) % opt.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'
'prob {prob.val:.3f} ({prob.avg:.3f})'.format(
epoch, idx + 1, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=loss_meter, prob=prob_meter))
print(out.shape)
sys.stdout.flush()
return loss_meter.avg, prob_meter.avg
def train(train_loader, model, criterion, optimizer, epoch, opt):
"""one epoch training"""
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
end = time.time()
for idx, (images, labels) in enumerate(train_loader):
data_time.update(time.time() - end)
images = torch.cat([images[0], images[1]], dim=0)
if torch.cuda.is_available():
images = images.cuda(non_blocking=True)
labels = labels.cuda(non_blocking=True)
bsz = labels.shape[0]
# warm-up learning rate
warmup_learning_rate(opt, epoch, idx, len(train_loader), optimizer)
# compute loss
features = model(images)
f1, f2 = torch.split(features, [bsz, bsz], dim=0)
features = torch.cat([f1.unsqueeze(1), f2.unsqueeze(1)], dim=1)
if opt.method == 'SupCon':
loss = criterion(features, labels)
elif opt.method == 'SimCLR':
loss = criterion(features)
else:
raise ValueError('contrastive method not supported: {}'.format(
opt.method))
# update metric
losses.update(loss.item(), bsz)
# SGD
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# print info
if (idx + 1) % opt.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})'.format(
epoch,
idx + 1,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
sys.stdout.flush()
return losses.avg
def main(args):
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
Net = DDNet(device)
Net.to(device)
train_data_roots = ('/media/nlpr/zwzhou/数据资源/MOT16/train',
'/media/nlpr/zwzhou/数据资源/MOT15/train')
train_subsets = (('MOT16-02', 'MOT16-04', 'MOT16-05', 'MOT16-10',
'MOT16-11', 'MOT16-13'),
('ADL-Rundle-6', 'ETH-Bahnhof', 'ETH-Pedcross2',
'ETH-Sunnyday', 'KITTI-13', 'KITTI-17', 'PETS09-S2L1',
'TUD-Stadtmitte', 'Venice-2'))
mot_dataset = DDDataset(train_data_roots, train_subsets)
mot_dataloader = DataLoader(mot_dataset,
batch_size=config.train_batch_size,
shuffle=True,
pin_memory=True,
num_workers=32,
drop_last=True,
collate_fn=collate_fn)
# resume
if args.ckp is not None:
assert os.path.isfile(args.ckp)
Net.load_state_dict(torch.load(args.ckp, map_location='cpu'))
torch.cuda.empty_cache()
print('Train net from: %s' % args.ckp)
if args.val_data_root is not None:
val_data_roots = ('/media/nlpr/zwzhou/数据资源/MOT16/train',
'/media/nlpr/zwzhou/数据资源/MOT15/train')
val_subsets = (('MOT16-09', ), ('TUD-Campus', 'ADL-Rundle-8'))
val_dataset = DDDataset(val_data_roots, val_subsets, transform=False)
val_dataloader = DataLoader(val_dataset,
batch_size=config.train_batch_size,
shuffle=False,
pin_memory=True,
num_workers=16,
drop_last=False,
collate_fn=collate_fn)
val_avg_loss, val_avg_closs, val_avg_rloss, val_avg_dloss = AverageMeter(
), AverageMeter(), AverageMeter(), AverageMeter()
train_val_plot = SavePlot('debug', 'train_val_plot')
avg_loss, avg_closs, avg_rloss, avg_dloss = AverageMeter(), AverageMeter(
), AverageMeter(), AverageMeter()
print('Training without RAMModule ...')
for epoch in range(config.epoches):
Net.train()
print('Train epoch {}/{}'.format(epoch + 1, config.epoches))
train_loss = []
with tqdm(total=len(mot_dataloader)) as progbar:
for _, batch in enumerate(mot_dataloader):
closs, rloss, dloss, loss = Net.step(batch)
closs_ = closs.cpu().item()
if np.isnan(closs_): sys.exit(0)
avg_closs.update(closs.cpu().item())
avg_rloss.update(rloss.cpu().item())
avg_dloss.update(dloss.cpu().item())
avg_loss.update(loss.cpu().item())
progbar.set_postfix(closs='{:05.5f}'.format(avg_closs.avg),
rloss='{:05.5f}'.format(avg_rloss.avg),
sloss='{:05.5f}'.format(avg_dloss.avg),
tloss='{:05.3f}'.format(avg_loss.avg))
progbar.update()
train_loss.append(avg_loss.avg)
train_loss = np.mean(train_loss)
if args.val_data_root is not None:
# 验证集
Net.eval()
val_loss = []
print('Validation epoch {}/{}'.format(epoch + 1, config.epoches))
with tqdm(total=len(val_dataloader)) as progbar:
for _, batch in enumerate(val_dataloader):
closs, rloss, dloss, loss = Net.step(batch, train=False)
closs_ = closs.cpu().item()
if np.isnan(closs_): sys.exit(0)
val_avg_closs.update(closs.cpu().item())
val_avg_rloss.update(rloss.cpu().item())
val_avg_dloss.update(dloss.cpu().item())
val_avg_loss.update(loss.cpu().item())
progbar.set_postfix(
closs='{:05.5f}'.format(val_avg_closs.avg),
rloss='{:05.5f}'.format(val_avg_rloss.avg),
sloss='{:05.5f}'.format(val_avg_dloss.avg),
tloss='{:05.3f}'.format(val_avg_loss.avg))
progbar.update()
val_loss.append(val_avg_loss.avg)
val_loss = np.mean(val_loss)
train_val_plot.update(train_loss, val_loss)
print('Train loss: {}, val loss: {}'.format(train_loss, val_loss))
'''save model'''
Net.save(Net, 'weights', 100)
def test(test_loader, net, criterion, testF):
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
net.eval()
end = time.time()
for idx, (img, target) in enumerate(test_loader, start=1):
if args.cuda:
img = img.cuda()
target = target.cuda()
with torch.no_grad():
_, _, _, _, output = net(img)
loss = criterion(output, target)
prec1, prec5 = accuracy(output, target, topk=(1,5))
losses.update(loss.item(), img.size(0))
top1.update(prec1.item(), img.size(0))
top5.update(prec5.item(), img.size(0))
f_l = [losses.avg, top1.avg, top5.avg]
print('Loss: {:.4f}, Prec@1: {:.2f}, Prec@5: {:.2f}'.format(*f_l))
testF.write('{},{},{}\n'.format(epoch, losses.avg, top1.avg))
testF.flush()
return top1.avg, top5.avg
def train(train_loader, model, criterion, optimizer, epoch):
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
running_metric_text = runningScore(2)
running_metric_kernel = runningScore(2)
end = time.time()
for batch_idx, (imgs, gt_texts, gt_kernels,
training_masks) in enumerate(train_loader):
data_time.update(time.time() - end)
imgs = Variable(imgs.cuda())
gt_texts = Variable(gt_texts.cuda())
gt_kernels = Variable(gt_kernels.cuda())
training_masks = Variable(training_masks.cuda())
outputs = model(imgs)
texts = outputs[:, 0, :, :]
kernels = outputs[:, 1:, :, :]
selected_masks = ohem_batch(texts, gt_texts, training_masks)
selected_masks = Variable(selected_masks.cuda())
loss_text = criterion(texts, gt_texts, selected_masks)
loss_kernels = []
mask0 = torch.sigmoid(texts).data.cpu().numpy()
mask1 = training_masks.data.cpu().numpy()
selected_masks = ((mask0 > 0.5) & (mask1 > 0.5)).astype('float32')
selected_masks = torch.from_numpy(selected_masks).float()
selected_masks = Variable(selected_masks.cuda())
for i in range(6):
kernel_i = kernels[:, i, :, :]
gt_kernel_i = gt_kernels[:, i, :, :]
loss_kernel_i = criterion(kernel_i, gt_kernel_i, selected_masks)
loss_kernels.append(loss_kernel_i)
loss_kernel = sum(loss_kernels) / len(loss_kernels)
loss = 0.7 * loss_text + 0.3 * loss_kernel
losses.update(loss.item(), imgs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
score_text = cal_text_score(texts, gt_texts, training_masks,
running_metric_text)
score_kernel = cal_kernel_score(kernels, gt_kernels, gt_texts,
training_masks, running_metric_kernel)
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % 20 == 0:
output_log = '({batch}/{size}) Batch: {bt:.3f}s | TOTAL: {total:.0f}min | ETA: {eta:.0f}min | Loss: {loss:.4f} | Acc_t: {acc: .4f} | IOU_t: {iou_t: .4f} | IOU_k: {iou_k: .4f}'.format(
batch=batch_idx + 1,
size=len(train_loader),
bt=batch_time.avg,
total=batch_time.avg * batch_idx / 60.0,
eta=batch_time.avg * (len(train_loader) - batch_idx) / 60.0,
loss=losses.avg,
acc=score_text['Mean Acc'],
iou_t=score_text['Mean IoU'],
iou_k=score_kernel['Mean IoU'])
print(output_log)
sys.stdout.flush()
return (losses.avg, score_text['Mean Acc'], score_kernel['Mean Acc'],
score_text['Mean IoU'], score_kernel['Mean IoU'])
def train(train_loader, model, classifier, criterion, optimizer, epoch, opt):
"""one epoch training"""
model.eval()
classifier.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
end = time.time()
for idx, (images, labels) in enumerate(train_loader):
data_time.update(time.time() - end)
if type(images) == list:
views = len(images)
images = torch.cat(images, dim=0)
labels = labels.repeat(views)
images = images.cuda(non_blocking=True)
labels = labels.cuda(non_blocking=True)
bsz = labels.shape[0]
# warm-up learning rate
warmup_learning_rate(opt, epoch, idx, len(train_loader), optimizer)
# compute loss
with torch.no_grad():
features = model.encoder(images)
output = classifier(features.detach())
loss = criterion(output, labels)
# update metric
losses.update(loss.item(), bsz)
acc1, acc5 = accuracy(output, labels, topk=(1, 5))
top1.update(acc1[0], bsz)
# SGD
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# print info
if (idx + 1) % opt.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
epoch, idx + 1, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, top1=top1))
sys.stdout.flush()
return losses.avg, top1.avg
def train(train_loader, model, criterions, optimizer, epoch):
"""training the model"""
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
loss_protest = AverageMeter()
loss_v = AverageMeter()
protest_acc = AverageMeter()
violence_mse = AverageMeter()
visattr_acc = AverageMeter()
end = time.time()
loss_history = []
for i, sample in enumerate(train_loader):
# measure data loading batch_time
input, target = sample['image'], sample['label']
data_time.update(time.time() - end)
if args.cuda:
input = input.cuda()
for k, v in target.items():
target[k] = v.cuda()
target_var = {}
for k,v in target.items():
target_var[k] = Variable(v)
input_var = Variable(input)
output = model(input_var)
losses, scores, N_protest = calculate_loss(output, target_var, criterions)
optimizer.zero_grad()
loss = 0
for l in losses:
loss += l
# back prop
loss.backward()
optimizer.step()
if N_protest:
loss_protest.update(losses[0].data[0], input.size(0))
loss_v.update(loss.data[0] - losses[0].data[0], N_protest)
else:
# when there is no protest image in the batch
loss_protest.update(losses[0].data[0], input.size(0))
loss_history.append(loss.data[0])
protest_acc.update(scores['protest_acc'], input.size(0))
violence_mse.update(scores['violence_mse'], N_protest)
visattr_acc.update(scores['visattr_acc'], N_protest)
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}] '
'Time {batch_time.val:.2f} ({batch_time.avg:.2f}) '
'Data {data_time.val:.2f} ({data_time.avg:.2f}) '
'Loss {loss_val:.3f} ({loss_avg:.3f}) '
'Protest {protest_acc.val:.3f} ({protest_acc.avg:.3f}) '
'Violence {violence_mse.val:.5f} ({violence_mse.avg:.5f}) '
'Vis Attr {visattr_acc.val:.3f} ({visattr_acc.avg:.3f})'
.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time,
loss_val=loss_protest.val + loss_v.val,
loss_avg = loss_protest.avg + loss_v.avg,
protest_acc = protest_acc, violence_mse = violence_mse,
visattr_acc = visattr_acc))
return loss_history
def main():
args, log = get_train_args()
log.info('[Program starts. Loading data...]')
train, dev, dev_y, embedding, opt = load_data(vars(args))
device, args.gpu_ids = util.get_available_devices()
log.info('[Data loaded.]')
if args.save_dawn_logs:
dawn_start = datetime.now()
log.info('dawn_entry: epoch\tf1Score\thours')
model = DRQA(opt, embedding = embedding)
model = nn.DataParallel(model, args.gpu_ids)
model = model.to(device)
epoch_0 = 0
best_val_score = 0.0
# ema = util.EMA(model, args.ema_decay)
## get optimizer and scheduler
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = optim.Adamax(parameters, weight_decay = opt['weight_decay'])
scheduler = sched.LambdaLR(optimizer, lambda s: 1.)
train_loss = AverageMeter()
for epoch in range(epoch_0, epoch_0 + args.epochs):
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
updates = 0
model.train()
for i, batch in enumerate(batches):
# Transfer to GPU
inputs = [e.to(device) for e in batch[:7]]
target_s = batch[7].to(device)
target_e = batch[8].to(device)
optimizer.zero_grad()
# Forward
score_s,score_e = model(*inputs)
loss = F.nll_loss(score_s, target_s) + F.nll_loss(score_e, target_e)
train_loss.update(loss.item())
# Backward
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),opt['grad_clipping'])
optimizer.step()
updates +=1
# Clip gradients
if i % args.log_per_updates == 0:
log.info('> epoch [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}]'.format(
epoch,updates, train_loss.value,
str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0]))
log.debug('\n')
# eval
batches = BatchGen(dev, batch_size=args.batch_size, evaluation=True, gpu=args.cuda)
predictions = []
for i, batch in enumerate(batches):
model.eval()
inputs = [e.to(device) for e in batch[:7]]
# Run forward
with torch.no_grad():
score_s, score_e = model(*inputs)
# Get argmax test spans
text = batch[-2]
spans = batch[-1]
pred = []
max_len = opt['max_len'] or score_s.size(1)
for i in range(score_s.size(0)):
scores = torch.ger(score_s[i], score_e[i])
scores.triu_().tril_(max_len - 1)
scores = scores.cpu().clone().numpy()
s_idx, e_idx = np.unravel_index(np.argmax(scores), scores.shape)
s_offset, e_offset = spans[i][s_idx][0], spans[i][e_idx][1]
pred.append(text[i][s_offset:e_offset])
predictions.extend(pred)
log.debug('> evaluating [{}/{}]'.format(i, len(batches)))
em, f1 = score(predictions, dev_y)
log.warning("dev EM: {} F1: {}".format(em, f1))
if args.save_dawn_logs:
time_diff = datetime.now() - dawn_start
log.warning("dawn_entry: {}\t{}\t{}".format(epoch, f1/100.0, float(time_diff.total_seconds() / 3600.0)))
# save
if not args.save_last_only or epoch == epoch_0 + args.epochs - 1:
model_file = os.path.join(args.model_dir, 'checkpoint_epoch_{}.pt'.format(epoch))
params = {
'state_dict': {
'network': model.state_dict(),
'optimizer': optimizer.state_dict(),
'updates': updates,
'loss': train_loss.state_dict()
},
'config': opt,
'epoch': epoch,
'em': em,
'f1': f1,
'best_eval': best_val_score,
'random_state': random.getstate(),
'torch_state': torch.random.get_rng_state(),
#'torch_cuda_state': torch.cuda.get_rng_state()
}
torch.save(params, model_file)
log.info('model saved to {}'.format(model_file))
if f1 > best_val_score:
best_val_score = f1
copyfile(
model_file,
os.path.join(args.model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
def main():
'''parameter initialization'''
args = parser.parse_args()
exp_name_dir = util.experiment_name_dir(args.experiment_name)
'''model on gpu'''
model = TrackerSiamRPN()
'''setup train data loader'''
name = 'GOT-10k'
assert name in ['VID', 'GOT-10k', 'All']
if name == 'GOT-10k':
root_dir = args.train_path
seq_dataset = GOT10k(root_dir, subset='train')
elif name == 'VID':
root_dir = '../data/ILSVRC'
seq_dataset = ImageNetVID(root_dir, subset=('train'))
elif name == 'All':
root_dir_vid = '../data/ILSVRC'
seq_datasetVID = ImageNetVID(root_dir_vid, subset=('train'))
root_dir_got = args.train_path
seq_datasetGOT = GOT10k(root_dir_got, subset='train')
seq_dataset = util.data_split(seq_datasetVID, seq_datasetGOT)
print('seq_dataset', len(seq_dataset))
train_z_transforms = transforms.Compose([ToTensor()])
train_x_transforms = transforms.Compose([ToTensor()])
train_data = TrainDataLoader(seq_dataset, train_z_transforms,
train_x_transforms, name)
anchors = train_data.anchors
train_loader = DataLoader(dataset=train_data,
batch_size=config.train_batch_size,
shuffle=True,
num_workers=config.train_num_workers,
pin_memory=True)
'''setup val data loader'''
name = 'GOT-10k'
assert name in ['VID', 'GOT-10k', 'All']
if name == 'GOT-10k':
root_dir = args.train_path
seq_dataset_val = GOT10k(root_dir, subset='val')
elif name == 'VID':
root_dir = '../data/ILSVRC'
seq_dataset_val = ImageNetVID(root_dir, subset=('val'))
elif name == 'All':
root_dir_vid = '../data/ILSVRC'
seq_datasetVID = ImageNetVID(root_dir_vid, subset=('val'))
root_dir_got = args.train_path
seq_datasetGOT = GOT10k(root_dir_got, subset='val')
seq_dataset_val = util.data_split(seq_datasetVID, seq_datasetGOT)
print('seq_dataset_val', len(seq_dataset_val))
valid_z_transforms = transforms.Compose([ToTensor()])
valid_x_transforms = transforms.Compose([ToTensor()])
val_data = TrainDataLoader(seq_dataset_val, valid_z_transforms,
valid_x_transforms, name)
val_loader = DataLoader(dataset=val_data,
batch_size=config.valid_batch_size,
shuffle=False,
num_workers=config.valid_num_workers,
pin_memory=True)
'''load weights'''
if not args.checkpoint_path == None:
assert os.path.isfile(
args.checkpoint_path), '{} is not valid checkpoint_path'.format(
args.checkpoint_path)
checkpoint = torch.load(args.checkpoint_path, map_location='cpu')
if 'model' in checkpoint.keys():
model.net.load_state_dict(
torch.load(args.checkpoint_path, map_location='cpu')['model'])
else:
model.net.load_state_dict(
torch.load(args.checkpoint_path, map_location='cpu'))
torch.cuda.empty_cache()
print('You are loading the model.load_state_dict')
elif config.pretrained_model:
checkpoint = torch.load(config.pretrained_model)
# change name and load parameters
checkpoint = {
k.replace('features.features', 'featureExtract'): v
for k, v in checkpoint.items()
}
model_dict = model.net.state_dict()
model_dict.update(checkpoint)
model.net.load_state_dict(model_dict)
#torch.cuda.empty_cache()
print('You are loading the pretrained model')
'''train phase'''
train_closses, train_rlosses, train_tlosses = AverageMeter(), AverageMeter(
), AverageMeter()
val_closses, val_rlosses, val_tlosses = AverageMeter(), AverageMeter(
), AverageMeter()
train_val_plot = SavePlot(exp_name_dir, 'train_val_plot')
model.adjust_lr(args.epoch_i)
for epoch in range(args.epoch_i, config.epoches):
model.net.train()
if config.fix_former_3_layers:
util.freeze_layers(model.net)
print('Train epoch {}/{}'.format(epoch + 1, config.epoches))
train_loss = []
with tqdm(total=config.train_epoch_size) as progbar:
for i, dataset in enumerate(train_loader):
closs, rloss, loss = model.step(epoch,
dataset,
anchors,
i,
train=True)
closs_ = closs.cpu().item()
if np.isnan(closs_):
sys.exit(0)
train_closses.update(closs.cpu().item())
train_rlosses.update(rloss.cpu().item())
train_tlosses.update(loss.cpu().item())
progbar.set_postfix(closs='{:05.3f}'.format(train_closses.avg),
rloss='{:05.5f}'.format(train_rlosses.avg),
tloss='{:05.3f}'.format(train_tlosses.avg))
progbar.update()
train_loss.append(train_tlosses.avg)
if i >= config.train_epoch_size - 1:
'''save model'''
model.save(model, exp_name_dir, epoch)
break
train_loss = np.mean(train_loss)
'''val phase'''
val_loss = []
with tqdm(total=config.val_epoch_size) as progbar:
print('Val epoch {}/{}'.format(epoch + 1, config.epoches))
for i, dataset in enumerate(val_loader):
val_closs, val_rloss, val_tloss = model.step(epoch,
dataset,
anchors,
train=False)
closs_ = val_closs.cpu().item()
if np.isnan(closs_):
sys.exit(0)
val_closses.update(val_closs.cpu().item())
val_rlosses.update(val_rloss.cpu().item())
val_tlosses.update(val_tloss.cpu().item())
progbar.set_postfix(closs='{:05.3f}'.format(val_closses.avg),
rloss='{:05.5f}'.format(val_rlosses.avg),
tloss='{:05.3f}'.format(val_tlosses.avg))
progbar.update()
val_loss.append(val_tlosses.avg)
if i >= config.val_epoch_size - 1:
break
val_loss = np.mean(val_loss)
train_val_plot.update(train_loss, val_loss)
print('Train loss: {}, val loss: {}'.format(train_loss, val_loss))
def train(train_loader, model, criterions, optimizer, epoch):
"""training the model"""
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
loss_protest = AverageMeter()
loss_v = AverageMeter()
protest_acc = AverageMeter()
violence_mse = AverageMeter()
visattr_acc = AverageMeter()
end = time.time()
loss_history = []
for i, sample in enumerate(train_loader):
# measure data loading batch_time
input, target = sample['image'], sample['label']
data_time.update(time.time() - end)
if args.cuda:
input = input.cuda()
for k, v in target.items():
target[k] = v.cuda()
target_var = {}
for k, v in target.items():
target_var[k] = Variable(v)
input_var = Variable(input)
output = model(input_var)
losses, scores, N_protest = calculate_loss(output, target_var,
criterions)
optimizer.zero_grad()
loss = 0
for l in losses:
loss += l
# back prop
loss.backward()
optimizer.step()
if N_protest:
loss_protest.update(losses[0].data[0], input.size(0))
loss_v.update(loss.data[0] - losses[0].data[0], N_protest)
else:
# when there is no protest image in the batch
loss_protest.update(losses[0].data[0], input.size(0))
loss_history.append(loss.data[0])
protest_acc.update(scores['protest_acc'], input.size(0))
violence_mse.update(scores['violence_mse'], N_protest)
visattr_acc.update(scores['visattr_acc'], N_protest)
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}] '
'Time {batch_time.val:.2f} ({batch_time.avg:.2f}) '
'Data {data_time.val:.2f} ({data_time.avg:.2f}) '
'Loss {loss_val:.3f} ({loss_avg:.3f}) '
'Protest {protest_acc.val:.3f} ({protest_acc.avg:.3f}) '
'Violence {violence_mse.val:.5f} ({violence_mse.avg:.5f}) '
'Vis Attr {visattr_acc.val:.3f} ({visattr_acc.avg:.3f})'.
format(epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss_val=loss_protest.val + loss_v.val,
loss_avg=loss_protest.avg + loss_v.avg,
protest_acc=protest_acc,
violence_mse=violence_mse,
visattr_acc=visattr_acc))
return loss_history
def train(loader, model, crit, opt, epoch):
"""Training of the CNN.
Args:
loader (torch.utils.data.DataLoader): Data loader
model (nn.Module): CNN
crit (torch.nn): loss
opt (torch.optim.SGD): optimizer for every parameters with True
requires_grad in model except top layer
epoch (int)
"""
batch_time = AverageMeter()
losses = AverageMeter()
data_time = AverageMeter()
forward_time = AverageMeter()
backward_time = AverageMeter()
# switch to train mode
model.train()
# create an optimizer for the last fc layer
optimizer_tl = torch.optim.SGD(
model.top_layer.parameters(),
lr=args.lr,
weight_decay=10**args.wd,
)
end = time.time()
for i, (input_tensor, target) in enumerate(loader):
data_time.update(time.time() - end)
# save checkpoint
n = len(loader) * epoch + i
if n % args.checkpoints == 0:
path = os.path.join(
args.exp,
'checkpoints',
'checkpoint_' + str(n / args.checkpoints) + '.pth.tar',
)
if args.verbose:
print('Save checkpoint at: {0}'.format(path))
torch.save({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer' : opt.state_dict()
}, path)
target = target.cuda(async=True)
input_var = torch.autograd.Variable(input_tensor.cuda())
target_var = torch.autograd.Variable(target)
output = model(input_var)
loss = crit(output, target_var)
# record loss
losses.update(loss.data[0], input_tensor.size(0))
# compute gradient and do SGD step
opt.zero_grad()
optimizer_tl.zero_grad()
loss.backward()
opt.step()
optimizer_tl.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.verbose and (i % 200) == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data: {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss: {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, i, len(loader), batch_time=batch_time,
data_time=data_time, loss=losses))
return losses.avg
def train(train_loader, model, criterion, optimizer, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
# switch to train mode
model.train()
end = time.time()
for i, (input, target, seq_lengths) in enumerate(train_loader):
#torch.Size([128, 334677])
# measure data2 loading time
data_time.update(time.time() - end)
if args.cuda:
input = input.cuda()
target = target.cuda()
# compute output
output = model(input, seq_lengths)
loss = criterion(output, target)
# measure accuracy and record loss
prec1 = accuracy(output.data, target, topk=(1, ))
losses.update(loss.data, input.size(0))
top1.update(prec1[0][0], input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i != 0 and i % args.print_freq == 0:
print(
'Epoch: [{0}][{1}/{2}] Time {batch_time.val:.3f} ({batch_time.avg:.3f}) '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) Loss {loss.val:.4f} ({loss.avg:.4f}) '
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1))
gc.collect()
def train(train_loader, model, criterion, optimizer, epoch, summary_writer):
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
running_metric_text = runningScore(2)
L1_loss = torch.nn.L1Loss()
end = time.time()
for batch_idx, (imgs, gt_texts, training_masks, ori_imgs, border_map,
geo_map_gt, densebox_gt) in enumerate(train_loader):
if ori_imgs.cpu().numpy().mean() == 0:
continue
data_time.update(time.time() - end)
imgs = Variable(imgs.cuda())
gt_texts = Variable(gt_texts[:, ::4, ::4].cuda())
training_masks = Variable(training_masks[:, ::4, ::4].cuda())
# geo_map_gt = Variable(geo_map_gt[:, :, ::4, ::4].cuda())
border_map = Variable(border_map[:, ::4, ::4].cuda())
densebox_map = Variable(densebox_gt[:, :, ::4, ::4].cuda())
outputs = model(imgs)
gaussian_large, border_map_pred = outputs
large_map_loss, select_masks = gaussian_map_loss(gaussian_large,
gt_texts,
training_masks,
criterion,
text_thres=0.2,
center_thres=0.7)
# densebox regression
# densebox_weights = torch.where(torch.abs(densebox_map) > 256, torch.ones_like(densebox_map) * 0.1, torch.ones_like(densebox_map))
# region_gt = torch.where(gt_texts > 0.3, torch.ones_like(gt_texts), torch.zeros_like(gt_texts))
# densebox_loss = smooth_l1_loss(densebox_pred, densebox_map)
# densebox_mask = region_gt[:, None, :, :] * training_masks[:, None, :, :] * densebox_weights
# pos_num = densebox_mask.sum()
# densebox_loss = densebox_loss * densebox_mask
# densebox_loss = 0 if pos_num == 0 else densebox_loss.sum() / (1.0 * pos_num * 8.)
# loss for border_map
select_border_masks = ohem_batch(torch.sigmoid(border_map_pred),
border_map, training_masks,
0.5).cuda()
loss_border = criterion(border_map_pred, border_map,
select_border_masks + training_masks)
loss = large_map_loss + loss_border
losses.update(loss.item(), imgs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
score_center = cal_text_score(gaussian_large, gt_texts, training_masks,
running_metric_text, 0, 0.7)
score_region = cal_text_score(border_map_pred, border_map,
training_masks, running_metric_text, 0,
0.2)
batch_time.update(time.time() - end)
end = time.time()
if batch_idx % 40 == 0:
# visualization
global_step = epoch * len(train_loader) + batch_idx
maps = torch.sigmoid(gaussian_large[0:1])
center_map = torch.where(maps > 0.8, maps, torch.zeros_like(maps))
text_map = torch.where(maps > 0.2, maps, torch.zeros_like(maps))
summary_writer.add_images('gt/img',
ori_imgs[0:1],
global_step=global_step)
summary_writer.add_images('gt/score_map0',
torch.unsqueeze(gt_texts[0:1], 1),
global_step=global_step)
summary_writer.add_images('gt/training_mask',
torch.unsqueeze(training_masks[0:1], 1),
global_step=global_step)
summary_writer.add_images('gt/selected_mask',
torch.unsqueeze(select_masks[0:1], 1),
global_step=global_step)
summary_writer.add_images('gt/selected_border_mask',
torch.unsqueeze(select_border_masks[0:1],
1),
global_step=global_step)
# summary_writer.add_images('gt/score_map1', torch.unsqueeze(gt_texts[0:1, :, :, 1], 1), global_step=global_step)
# summary_writer.add_images('gt/score_map2', torch.unsqueeze(gt_texts[0:1, :, :, 2], 1), global_step=global_step)
# summary_writer.add_images('gt/center_map', torch.unsqueeze(center_gt[0:1], 1), global_step=global_step)
# summary_writer.add_images('gt/region_map', torch.unsqueeze(region_gt[0:1], 1), global_step=global_step)
# summary_writer.add_images('gt/geo_map', torch.unsqueeze(geo_map_gt[0:1, 0, :, :], 1), global_step=global_step)
summary_writer.add_images('gt/border_map',
torch.unsqueeze(border_map[0:1], 1),
global_step=global_step)
summary_writer.add_images('predicition/score_map',
torch.sigmoid(gaussian_large[0:1]),
global_step=global_step)
# summary_writer.add_images('predicition/score_map_large', torch.sigmoid(gaussian_large[0:1]), global_step=global_step)
# summary_writer.add_images('predicition/score_map_small', torch.sigmoid(gaussian_small[0:1]), global_step=global_step)
summary_writer.add_images('predicition/center_map',
torch.sigmoid(center_map[0:1]),
global_step=global_step)
summary_writer.add_images('predicition/region_map',
torch.sigmoid(text_map[0:1]),
global_step=global_step)
summary_writer.add_images('predicition/border_map',
torch.sigmoid(
border_map_pred[0:1, :, :]),
global_step=global_step)
# summary_writer.add_scalar('loss/reg_loss', weighted_mse_loss, global_step=global_step)
# summary_writer.add_scalar('loss/reg_center_loss', loss_center, global_step=global_step)
# summary_writer.add_scalar('loss/center_dice_loss', loss_center_dice, global_step=global_step)
# summary_writer.add_scalar('loss/region_dice_loss', loss_region_dice, global_step=global_step)
summary_writer.add_scalar('loss/border_loss',
loss_border,
global_step=global_step)
summary_writer.add_scalar('loss/large_loss',
large_map_loss,
global_step=global_step)
# summary_writer.add_scalar('loss/densebox_loss', densebox_loss, global_step=global_step)
# summary_writer.add_scalar('loss/mid_loss', mid_map_loss, global_step=global_step)
# summary_writer.add_scalar('loss/small_loss', small_map_loss, global_step=global_step)
# summary_writer.add_scalar('loss/text_region_loss', mse_region_loss, global_step=global_step)
summary_writer.add_scalar('metric/acc_c',
score_center['Mean Acc'],
global_step=global_step)
summary_writer.add_scalar('metric/iou_c',
score_center['Mean IoU'],
global_step=global_step)
summary_writer.add_scalar('metric/acc_t',
score_region['Mean Acc'],
global_step=global_step)
summary_writer.add_scalar('metric/iou_t',
score_region['Mean IoU'],
global_step=global_step)
output_log = '({batch}/{size}) Batch: {bt:.3f}s | TOTAL: {total:.0f}min | ETA: {eta:.0f}min | Loss: {loss:.4f} | Acc_c: {acc_c: .4f} | IOU_c: {iou_c: .4f} | Acc_t: {acc_t: .4f} | IOU_t: {iou_t: .4f} '.format(
batch=batch_idx + 1,
size=len(train_loader),
bt=batch_time.avg,
total=batch_time.avg * batch_idx / 60.0,
eta=batch_time.avg * (len(train_loader) - batch_idx) / 60.0,
loss=losses.avg,
acc_c=score_region['Mean Acc'],
iou_c=score_region['Mean IoU'],
acc_t=score_center['Mean Acc'],
iou_t=score_center['Mean IoU'],
)
print(output_log)
sys.stdout.flush()
return (losses.avg, score_center['Mean Acc'], score_center['Mean IoU'])
def train(epoch, train_loader, model, classifier, criterion, optimizer, opt):
"""
one epoch training
"""
model.eval()
classifier.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
end = time.time()
for idx, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
input = input.float()
if opt.gpu is not None:
input = input.cuda(opt.gpu, non_blocking=True)
target = target.cuda(opt.gpu, non_blocking=True)
# ===================forward=====================
with torch.no_grad():
feat_l, feat_ab = model(input, opt.layer)
feat = torch.cat((feat_l.detach(), feat_ab.detach()), dim=1)
output = classifier(feat)
##### This may be not neccessary
target = target.long()
#####
loss = criterion(output, target)
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
top5.update(acc5[0], input.size(0))
# ===================backward=====================
optimizer.zero_grad()
loss.backward()
optimizer.step()
# ===================meters=====================
batch_time.update(time.time() - end)
end = time.time()
# print info
if idx % opt.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
idx,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5))
sys.stdout.flush()
return top1.avg, top5.avg, losses.avg
def train(epoch, train_loader, module_list, criterion_list, optimizer, opt):
"""One epoch training"""
# set modules as train()
for module in module_list:
module.train()
# set teacher as eval()
module_list[-1].eval()
criterion_cls = criterion_list[0]
criterion_div = criterion_list[1]
criterion_kd = criterion_list[2]
model_s = module_list[0]
model_t = module_list[-1]
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
end = time.time()
for idx, data in enumerate(train_loader):
if opt.distill in ['contrast']:
input, target, index, contrast_idx = data
else:
input, target, index = data
data_time.update(time.time() - end)
input = input.float()
if torch.cuda.is_available():
input = input.cuda()
target = target.cuda()
index = index.cuda()
if opt.distill in ['contrast']:
contrast_idx = contrast_idx.cuda()
# ===================forward=====================
preact = False
if opt.distill in ['abound', 'overhaul']:
preact = True
feat_s, logit_s = model_s(input, is_feat=True)
with torch.no_grad():
feat_t, logit_t = model_t(input, is_feat=True)
feat_t = [f.detach() for f in feat_t]
# cls + kl div
loss_cls = criterion_cls(logit_s, target)
loss_div = criterion_div(logit_s, logit_t)
# other kd beyond KL divergence
if opt.distill == 'kd':
loss_kd = 0
elif opt.distill == 'contrast':
f_s = module_list[1](feat_s[-1])
f_t = module_list[2](feat_t[-1])
loss_kd = criterion_kd(f_s, f_t, index, contrast_idx)
elif opt.distill == 'hint':
f_s = feat_s[-1]
f_t = feat_t[-1]
loss_kd = criterion_kd(f_s, f_t)
elif opt.distill == 'attention':
g_s = feat_s[1:-1]
g_t = feat_t[1:-1]
loss_group = criterion_kd(g_s, g_t)
loss_kd = sum(loss_group)
else:
raise NotImplementedError(opt.distill)
loss = opt.gamma * loss_cls + opt.alpha * loss_div + opt.beta * loss_kd
acc1, acc5 = accuracy(logit_s, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
top5.update(acc5[0], input.size(0))
# ===================backward=====================
optimizer.zero_grad()
loss.backward()
optimizer.step()
# ===================meters=====================
batch_time.update(time.time() - end)
end = time.time()
# print info
if idx % opt.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
idx,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5))
sys.stdout.flush()
print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'.format(top1=top1,
top5=top5))
return top1.avg, losses.avg
def validate(args, model, classifier, val_loader, criterion, epoch):
# switch to evaluate mode
model.eval()
classifier.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
with torch.no_grad():
end = time.time()
for batch_idx, (input1, input2, input3, target) in enumerate(tqdm(val_loader, disable=False)):
# Get inputs and target
input1, input2, input3, target = input1.float(), input2.float(), input3.float(), target.long()
# Reshape augmented tensors (B x C x H x W)
input1, input2, input3, target = input1.reshape(-1, 3, args.tile_h, args.tile_w), input2.reshape(-1, 3, args.tile_h, args.tile_w), input3.reshape(-1, 3, args.tile_h, args.tile_w), target.view(-1, 1).reshape(-1, )
# Move the variables to Cuda
input1, input2, input3, target = input1.cuda(), input2.cuda(), input3.cuda(), target.cuda()
# compute output ###############################
feats = model(input1, input2, input3)
output = classifier(feats)
loss = criterion(output, target)
# compute loss and accuracy ####################
batch_size = target.size(0)
losses.update(loss.item(), batch_size)
pred = torch.argmax(output, dim=1)
acc.update(torch.sum(target == pred).item() / batch_size, batch_size)
# measure elapsed time
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
# print statistics and write summary every N batch
if (batch_idx + 1) % args.print_freq == 0:
print('Val: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'
'acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch, batch_idx + 1, len(val_loader), batch_time=batch_time, data_time=data_time, loss=losses,
acc=acc))
return losses.avg, acc.avg
def train_para(train_loader, nets, optimizer_para, criterionPara, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
para_losses = AverageMeter()
tnet = nets['tnet']
paraphraser = nets['paraphraser']
paraphraser.train()
end = time.time()
for idx, (img, _) in enumerate(train_loader, start=1):
data_time.update(time.time() - end)
if args.cuda:
img = img.cuda()
_, _, _, rb3, _ = tnet(img)
_, rb3_rec = paraphraser(rb3.detach())
para_loss = criterionPara(rb3_rec, rb3.detach())
para_losses.update(para_loss.item(), img.size(0))
optimizer_para.zero_grad()
para_loss.backward()
optimizer_para.step()
batch_time.update(time.time() - end)
end = time.time()
if idx % args.print_freq == 0:
print('Epoch[{0}]:[{1:03}/{2:03}] '
'Time:{batch_time.val:.4f} '
'Data:{data_time.val:.4f} '
'Para:{para_losses.val:.4f}({para_losses.avg:.4f})'.format(
epoch, idx, len(train_loader), batch_time=batch_time, data_time=data_time,
para_losses=para_losses))
def validate(val_loader, model, classifier, criterion, opt):
"""validation"""
model.eval()
classifier.eval()
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
with torch.no_grad():
end = time.time()
for idx, (images, labels) in enumerate(val_loader):
if type(images) == list:
views = len(images)
images = torch.cat(images, dim=0)
labels = labels.repeat(views)
images = images.float().cuda()
labels = labels.cuda()
bsz = labels.shape[0]
# forward
output = classifier(model.encoder(images))
loss = criterion(output, labels)
# update metric
losses.update(loss.item(), bsz)
acc1, acc5 = accuracy(output, labels, topk=(1, 5))
top1.update(acc1[0], bsz)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if idx % opt.print_freq == 0:
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
idx, len(val_loader), batch_time=batch_time,
loss=losses, top1=top1))
print(' * Acc@1 {top1.avg:.3f}'.format(top1=top1))
return losses.avg, top1.avg
def train(train_loader, nets, optimizer, criterions, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
cls_losses = AverageMeter()
ft_losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
snet = nets['snet']
tnet = nets['tnet']
paraphraser = nets['paraphraser']
translator = nets['translator']
criterionCls = criterions['criterionCls']
criterionFT = criterions['criterionFT']
snet.train()
translator.train()
end = time.time()
for idx, (img, target) in enumerate(train_loader, start=1):
data_time.update(time.time() - end)
if args.cuda:
img = img.cuda()
target = target.cuda()
_, _, _, rb3_s, output_s = snet(img)
_, _, _, rb3_t, _ = tnet(img)
factor_s = translator(rb3_s)
factor_s = normalize(factor_s)
factor_t, _ = paraphraser(rb3_t)
factor_t = normalize(factor_t)
cls_loss = criterionCls(output_s, target)
ft_loss = criterionFT(factor_s, factor_t.detach()) * args.lambda_ft
loss = cls_loss + ft_loss
prec1, prec5 = accuracy(output_s, target, topk=(1,5))
cls_losses.update(cls_loss.item(), img.size(0))
ft_losses.update(ft_loss.item(), img.size(0))
top1.update(prec1.item(), img.size(0))
top5.update(prec5.item(), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if idx % args.print_freq == 0:
print('Epoch[{0}]:[{1:03}/{2:03}] '
'Time:{batch_time.val:.4f} '
'Data:{data_time.val:.4f} '
'Cls:{cls_losses.val:.4f}({cls_losses.avg:.4f}) '
'FT:{ft_losses.val:.4f}({ft_losses.avg:.4f}) '
'prec@1:{top1.val:.2f}({top1.avg:.2f}) '
'prec@5:{top5.val:.2f}({top5.avg:.2f})'.format(
epoch, idx, len(train_loader), batch_time=batch_time, data_time=data_time,
cls_losses=cls_losses, ft_losses=ft_losses, top1=top1, top5=top5))
def train(train_loader, net, optimizer, criterion, epoch, trainF):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
net.train()
end = time.time()
for idx, (img, target) in enumerate(train_loader, start=1):
data_time.update(time.time() - end)
if args.cuda:
img = img.cuda()
target = target.cuda()
_, _, _, _, output = net(img)
loss = criterion(output, target)
prec1, prec5 = accuracy(output, target, topk=(1,5))
losses.update(loss.item(), img.size(0))
top1.update(prec1.item(), img.size(0))
top5.update(prec5.item(), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if idx % args.print_freq == 0:
print('Epoch[{0}]:[{1:03}/{2:03}] '
'Time:{batch_time.val:.4f} '
'Data:{data_time.val:.4f} '
'loss:{losses.val:.4f}({losses.avg:.4f}) '
'prec@1:{top1.val:.2f}({top1.avg:.2f}) '
'prec@5:{top5.val:.2f}({top5.avg:.2f})'.format(
epoch, idx, len(train_loader), batch_time=batch_time, data_time=data_time,
losses=losses, top1=top1, top5=top5))
trainF.write('{},{},{}\n'.format(epoch, loss.data[0], prec1))
trainF.flush()
def test(test_loader, nets, criterions):
cls_losses = AverageMeter()
ft_losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
snet = nets['snet']
tnet = nets['tnet']
paraphraser = nets['paraphraser']
translator = nets['translator']
criterionCls = criterions['criterionCls']
criterionFT = criterions['criterionFT']
snet.eval()
translator.eval()
end = time.time()
for idx, (img, target) in enumerate(test_loader, start=1):
if args.cuda:
img = img.cuda()
target = target.cuda()
with torch.no_grad():
_, _, _, rb3_s, output_s = snet(img)
_, _, _, rb3_t, _ = tnet(img)
factor_s = translator(rb3_s)
factor_s = normalize(factor_s)
factor_t, _ = paraphraser(rb3_t)
factor_t = normalize(factor_t)
cls_loss = criterionCls(output_s, target)
ft_loss = criterionFT(factor_s, factor_t.detach()) * args.lambda_ft
prec1, prec5 = accuracy(output_s, target, topk=(1,5))
cls_losses.update(cls_loss.item(), img.size(0))
ft_losses.update(ft_loss.item(), img.size(0))
top1.update(prec1.item(), img.size(0))
top5.update(prec5.item(), img.size(0))
f_l = [cls_losses.avg, ft_losses.avg, top1.avg, top5.avg]
print('Cls: {:.4f}, FT: {:.4f}, Prec@1: {:.2f}, Prec@5: {:.2f}'.format(*f_l))
def train_student(epoch, train_loader, isd, criterion, optimizer, opt):
"""
one epoch training for CompReSS
"""
isd.train()
batch_time = AverageMeter()
data_time = AverageMeter()
loss_meter = AverageMeter()
end = time.time()
for idx, (indices, (im_q, im_k), _) in enumerate(train_loader):
data_time.update(time.time() - end)
im_q = im_q.cuda(non_blocking=True)
im_k = im_k.cuda(non_blocking=True)
# ===================forward=====================
sim_q, sim_k = isd(im_q=im_q, im_k=im_k)
loss = criterion(inputs=sim_q, targets=sim_k)
# ===================backward=====================
optimizer.zero_grad()
loss.backward()
optimizer.step()
# ===================meters=====================
loss_meter.update(loss.item(), im_q.size(0))
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
# print info
if (idx + 1) % opt.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'.format(
epoch,
idx + 1,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=loss_meter))
sys.stdout.flush()
return loss_meter.avg
def train(train_loader, model, reglog, criterion, optimizer, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# freeze also batch norm layers
model.eval()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
#adjust learning rate
learning_rate_decay(optimizer, len(train_loader) * epoch + i, args.lr)
target = target.cuda(non_blocking=True)
input_var = torch.autograd.Variable(input.cuda())
target_var = torch.autograd.Variable(target)
# compute output
output = forward(input_var, model, reglog.conv)
output = reglog(output)
loss = criterion(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0], input.size(0))
top5.update(prec5[0], input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.verbose and i % 100 == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5))
def train(loader, model, crit, opt, epoch):
"""Training of the CNN.
Args:
loader (torch.utils.data.DataLoader): Data loader
model (nn.Module): CNN
crit (torch.nn): loss
opt (torch.optim.SGD): optimizer for every parameters with True
requires_grad in model except top layer
epoch (int)
"""
batch_time = AverageMeter()
losses = AverageMeter()
data_time = AverageMeter()
forward_time = AverageMeter()
backward_time = AverageMeter()
# switch to train mode
model.train()
# create an optimizer for the last fc layer
optimizer_tl = torch.optim.SGD(
model.top_layer.parameters(),
lr=args.lr,
weight_decay=10**args.wd,
)
end = time.time()
for i, (input_tensor, target) in enumerate(loader):
data_time.update(time.time() - end)
# save checkpoint
n = len(loader) * epoch + i
if n % args.checkpoints == 0:
path = os.path.join(
args.exp,
'checkpoints',
'checkpoint_' + str(n / args.checkpoints) + '.pth.tar',
)
if args.verbose:
print('Save checkpoint at: {0}'.format(path))
torch.save(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': opt.state_dict()
}, path)
target = target.cuda()
input_var = torch.autograd.Variable(input_tensor.cuda())
target_var = torch.autograd.Variable(target)
output = model(input_var)
loss = crit(output, target_var)
# record loss
losses.update(loss.data, input_tensor.size(0))
# compute gradient and do SGD step
opt.zero_grad()
optimizer_tl.zero_grad()
loss.backward()
opt.step()
optimizer_tl.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.verbose and (i % 200) == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data: {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss: {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
return losses.avg
def validate(val_loader, model, reglog, criterion):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
softmax = nn.Softmax(dim=1).cuda()
end = time.time()
for i, (input_tensor, target) in enumerate(val_loader):
if args.tencrops:
bs, ncrops, c, h, w = input_tensor.size()
input_tensor = input_tensor.view(-1, c, h, w)
target = target.cuda(non_blocking=True)
input_var = torch.autograd.Variable(input_tensor.cuda(), volatile=True)
target_var = torch.autograd.Variable(target, volatile=True)
output = reglog(forward(input_var, model, reglog.conv))
if args.tencrops:
output_central = output.view(bs, ncrops, -1)[:, ncrops // 2 - 1, :]
output = softmax(output)
output = torch.squeeze(output.view(bs, ncrops, -1).mean(1))
else:
output_central = output
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
top1.update(prec1[0], input_tensor.size(0))
top5.update(prec5[0], input_tensor.size(0))
loss = criterion(output_central, target_var)
losses.update(loss.item(), input_tensor.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.verbose and i % 100 == 0:
print('Validation: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
return top1.avg, top5.avg, losses.avg
def train_ins(epoch, train_loader, model, contrast, criterion, optimizer, opt):
"""
one epoch training for instance discrimination
"""
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
loss_meter = AverageMeter()
prob_meter = AverageMeter()
end = time.time()
for idx, (inputs, _, index) in enumerate(train_loader):
data_time.update(time.time() - end)
bsz = inputs.size(0)
inputs = inputs.float()
if opt.gpu is not None:
inputs = inputs.cuda(opt.gpu, non_blocking=True)
else:
inputs = inputs.cuda()
index = index.cuda(opt.gpu, non_blocking=True)
# ===================forward=====================
feat = model(inputs)
out = contrast(feat, index)
loss = criterion(out)
prob = out[:, 0].mean()
# ===================backward=====================
optimizer.zero_grad()
if opt.amp:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
# ===================meters=====================
loss_meter.update(loss.item(), bsz)
prob_meter.update(prob.item(), bsz)
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
# print info
if (idx + 1) % opt.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'
'prob {prob.val:.3f} ({prob.avg:.3f})'.format(
epoch, idx + 1, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=loss_meter, prob=prob_meter))
print(out.shape)
sys.stdout.flush()
return loss_meter.avg, prob_meter.avg
class Evaluator():
def __init__(self,
data_loader,
logger,
config,
name='Evaluator',
metrics='classfication',
summary_writer=None):
self.data_loader = data_loader
self.logger = logger
self.name = name
self.summary_writer = summary_writer
self.step = 0
self.config = config
self.log_frequency = config.log_frequency
self.loss_meters = AverageMeter()
self.acc_meters = AverageMeter()
self.acc5_meters = AverageMeter()
self.report_metrics = self.classfication_metrics if metrics == 'classfication' else self.regression_metrics
return
def log(self, epoch, GLOBAL_STEP):
display = log_display(epoch=epoch,
global_step=GLOBAL_STEP,
time_elapse=self.time_used,
**self.logger_payload)
self.logger.info(display)
def eval(self, epoch, GLOBAL_STEP, model, criterion):
for i, (images, labels) in enumerate(self.data_loader):
self.eval_batch(x=images,
y=labels,
model=model,
criterion=criterion)
self.log(epoch, GLOBAL_STEP)
return
def eval_batch(self, x, y, model, criterion):
model.eval()
x, y = x.to(device, non_blocking=True), y.to(device, non_blocking=True)
start = time.time()
with torch.no_grad():
pred = model(x)
loss = criterion(pred, y)
end = time.time()
self.time_used = end - start
self.step += 1
self.report_metrics(pred, y, loss)
return
def classfication_metrics(self, x, y, loss):
acc, acc5 = accuracy(x, y, topk=(1, 5))
self.loss_meters.update(loss.item(), y.shape[0])
self.acc_meters.update(acc.item(), y.shape[0])
self.acc5_meters.update(acc5.item(), y.shape[0])
self.logger_payload = {
"acc": acc,
"acc_avg": self.acc_meters.avg,
"top5_acc": acc5,
"top5_acc_avg": self.acc5_meters.avg,
"loss": loss,
"loss_avg": self.loss_meters.avg
}
if self.summary_writer is not None:
self.summary_writer.add_scalar(os.path.join(self.name, 'acc'), acc,
self.step)
self.summary_writer.add_scalar(os.path.join(self.name, 'loss'),
loss, self.step)
def regression_metrics(self, x, y, loss):
diff = abs((x - y).mean().detach().item())
self.loss_meters.update(loss.item(), y.shape[0])
self.acc_meters.update(diff, y.shape[0])
self.logger_payload = {
"|diff|": diff,
"|diff_avg|": self.acc_meters.avg,
"loss": loss,
"loss_avg": self.loss_meters.avg
}
if self.summary_writer is not None:
self.summary_writer.add_scalar(os.path.join(self.name, 'diff'),
diff, self.step)
self.summary_writer.add_scalar(os.path.join(self.name, 'loss'),
loss, self.step)
def _reset_stats(self):
self.loss_meters.reset()
self.acc_meters.reset()
self.acc5_meters.reset()
def train(epoch, train_loader, model, contrast, criterion_l, criterion_ab, optimizer, opt):
"""
one epoch training
"""
model.train()
contrast.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
l_loss_meter = AverageMeter()
ab_loss_meter = AverageMeter()
l_prob_meter = AverageMeter()
ab_prob_meter = AverageMeter()
end = time.time()
for idx, (inputs, _, index) in enumerate(train_loader):
data_time.update(time.time() - end)
bsz = inputs.size(0)
inputs = inputs.float()
if torch.cuda.is_available():
index = index.cuda(async=True)
inputs = inputs.cuda()
# ===================forward=====================
feat_l, feat_ab = model(inputs)
out_l, out_ab = contrast(feat_l, feat_ab, index)
l_loss = criterion_l(out_l)
ab_loss = criterion_ab(out_ab)
l_prob = out_l[:, 0].mean()
ab_prob = out_ab[:, 0].mean()
loss = l_loss + ab_loss
# ===================backward=====================
optimizer.zero_grad()
if opt.amp:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
# ===================meters=====================
losses.update(loss.item(), bsz)
l_loss_meter.update(l_loss.item(), bsz)
l_prob_meter.update(l_prob.item(), bsz)
ab_loss_meter.update(ab_loss.item(), bsz)
ab_prob_meter.update(ab_prob.item(), bsz)
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
# print info
if (idx + 1) % opt.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'
'l_p {lprobs.val:.3f} ({lprobs.avg:.3f})\t'
'ab_p {abprobs.val:.3f} ({abprobs.avg:.3f})'.format(
epoch, idx + 1, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, lprobs=l_prob_meter,
abprobs=ab_prob_meter))
print(out_l.shape)
sys.stdout.flush()
return l_loss_meter.avg, l_prob_meter.avg, ab_loss_meter.avg, ab_prob_meter.avg
def semi_train(loader, semi_loader, model, fd, crit, opt_body, opt_category, epoch, device, args):
batch_time = AverageMeter()
losses = AverageMeter()
semi_losses = AverageMeter()
# switch to train mode
model.train()
end = time.time()
for i, ((input_tensor, label), pseudo_target, imgidx) in enumerate(loader):
input_var = torch.autograd.Variable(input_tensor.to(device))
pseudo_target_var = torch.autograd.Variable(pseudo_target.to(device, non_blocking=True))
output = model(input_var)
loss = crit(output, pseudo_target_var.long())
# record loss
losses.update(loss.item(), input_tensor.size(0))
# compute gradient and do SGD step
opt_body.zero_grad()
loss.backward()
opt_body.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.verbose and (i % args.display_count) == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time: {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'PSEUDO_Loss: {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, i, len(loader), batch_time=batch_time, loss=losses))
'''SUPERVISION with a few labelled dataset'''
model.cluster_layer = None
model.category_layer = nn.Sequential(
nn.Linear(fd, args.nmb_category),
nn.Softmax(dim=1),
)
model.category_layer[0].weight.data.normal_(0, 0.01)
model.category_layer[0].bias.data.zero_()
model.category_layer = model.category_layer.double()
model.category_layer.to(device)
category_save = os.path.join(args.exp, 'category_layer.pth.tar')
if os.path.isfile(category_save):
category_layer_param = torch.load(category_save)
model.category_layer.load_state_dict(category_layer_param)
semi_output_save = []
semi_label_save = []
for i, (input_tensor, label) in enumerate(semi_loader):
input_var = torch.autograd.Variable(input_tensor.to(device))
label_var = torch.autograd.Variable(label.to(device, non_blocking=True))
output = model(input_var)
semi_loss = crit(output, label_var.long())
# compute gradient and do SGD step
opt_category.zero_grad()
opt_body.zero_grad()
semi_loss.backward()
opt_category.step()
opt_body.step()
# record loss
semi_losses.update(semi_loss.item(), input_tensor.size(0))
# Record accuracy
output = torch.argmax(output, axis=1)
semi_output_save.append(output.data.cpu().numpy())
semi_label_save.append(label.data.cpu().numpy())
# measure elapsed time
if args.verbose and (i % args.display_count) == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'SEMI_Loss: {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, i, len(semi_loader), loss=semi_losses))
semi_output_flat = flatten_list(semi_output_save)
semi_label_flat = flatten_list(semi_label_save)
semi_accu_list = [out == lab for (out, lab) in zip(semi_output_flat, semi_label_flat)]
semi_accuracy = sum(semi_accu_list)/len(semi_accu_list)
return losses.avg, semi_losses.avg, semi_accuracy
def test(val_loader, model, criterion):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (input, target, seq_lengths) in enumerate(val_loader):
if args.cuda:
input, target = input.cuda(), target.cuda()
# compute output
output = model(input, seq_lengths)
loss = criterion(output, target)
# measure accuracy and record loss
prec1 = accuracy(output.data, target, topk=(1, ))
losses.update(loss.data, input.size(0))
top1.update(prec1[0][0], input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i != 0 and i % args.print_freq == 0:
print(
'Test: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f}) '
'Loss {loss.val:.4f} ({loss.avg:.4f}) Prec@1 {top1.val:.3f} ({top1.avg:.3f})'
.format(i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1))
gc.collect()
print(' * Prec@1 {top1.avg:.3f}'.format(top1=top1))
return top1.avg
def train(args, model, classifier, train_loader, criterion, optimizer, epoch):
# Switch to train mode
model.train()
classifier.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
total_feats = []
total_targets = []
end = time.time()
for batch_idx, (input1, input2, input3, target) in enumerate(tqdm(train_loader, disable=False)):
# Get inputs and target
input1, input2, input3, target = input1.float(), input2.float(), input3.float(), target.long()
# Reshape augmented tensors (B x C x H x W)
input1, input2, input3, target = input1.reshape(-1, 3, args.tile_h, args.tile_w), input2.reshape(-1, 3, args.tile_h, args.tile_w), input3.reshape(-1, 3, args.tile_h, args.tile_w), target.view(-1, 1).reshape(-1, )
# Move the variables to Cuda
input1, input2, input3, target = input1.cuda(), input2.cuda(), input3.cuda(), target.cuda()
# compute output ###############################
feats = model(input1, input2, input3)
output = classifier(feats)
loss = criterion(output, target)
# compute gradient and do SGD step #############
optimizer.zero_grad()
loss.backward()
optimizer.step()
# compute loss and accuracy ####################
batch_size = target.size(0)
losses.update(loss.item(), batch_size)
pred = torch.argmax(output, dim=1)
acc.update(torch.sum(target == pred).item() / batch_size, batch_size)
# Save features for T-SNE visualization
total_feats.append(feats)
total_targets.append(target)
# measure elapsed time
torch.cuda.synchronize()
batch_time.update(time.time() - end)
end = time.time()
# print statistics and write summary every N batch
if (batch_idx + 1) % args.print_freq == 0:
print('Train: [{0}][{1}/{2}]\t'
'BT {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'DT {data_time.val:.3f} ({data_time.avg:.3f})\t'
'loss {loss.val:.3f} ({loss.avg:.3f})\t'
'acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch, batch_idx + 1, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses,
acc=acc))
final_feats = torch.cat(total_feats).detach()
final_targets = torch.cat(total_targets).detach()
return losses.avg, acc.avg, final_feats, final_targets
def validate(val_loader, model, classifier, criterion, opt):
"""
evaluation
"""
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
classifier.eval()
with torch.no_grad():
end = time.time()
for idx, (input, target) in enumerate(val_loader):
input = input.float()
if opt.gpu is not None:
input = input.cuda(opt.gpu, non_blocking=True)
target = target.cuda(opt.gpu, non_blocking=True)
# compute output
feat_l, feat_ab = model(input, opt.layer)
feat = torch.cat((feat_l.detach(), feat_ab.detach()), dim=1)
output = classifier(feat)
##### This may be not necessary
target = target.long()
#####
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
top5.update(acc5[0], input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if idx % opt.print_freq == 0:
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
idx,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'.format(top1=top1,
top5=top5))
return top1.avg, top5.avg, losses.avg
def validate(val_loader, model, criterions, epoch):
"""Validating"""
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
loss_protest = AverageMeter()
loss_v = AverageMeter()
protest_acc = AverageMeter()
violence_mse = AverageMeter()
visattr_acc = AverageMeter()
end = time.time()
loss_history = []
for i, sample in enumerate(val_loader):
# measure data loading batch_time
input, target = sample['image'], sample['label']
if args.cuda:
input = input.cuda()
for k, v in target.items():
target[k] = v.cuda()
input_var = Variable(input)
target_var = {}
for k,v in target.items():
target_var[k] = Variable(v)
output = model(input_var)
losses, scores, N_protest = calculate_loss(output, target_var, criterions)
loss = 0
for l in losses:
loss += l
if N_protest:
loss_protest.update(losses[0].data[0], input.size(0))
loss_v.update(loss.data[0] - losses[0].data[0], N_protest)
else:
# when no protest images
loss_protest.update(losses[0].data[0], input.size(0))
loss_history.append(loss.data[0])
protest_acc.update(scores['protest_acc'], input.size(0))
violence_mse.update(scores['violence_mse'], N_protest)
visattr_acc.update(scores['visattr_acc'], N_protest)
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.2f} ({batch_time.avg:.2f}) '
'Loss {loss_val:.3f} ({loss_avg:.3f}) '
'Protest Acc {protest_acc.val:.3f} ({protest_acc.avg:.3f}) '
'Violence MSE {violence_mse.val:.5f} ({violence_mse.avg:.5f}) '
'Vis Attr Acc {visattr_acc.val:.3f} ({visattr_acc.avg:.3f})'
.format(
epoch, i, len(val_loader), batch_time=batch_time,
loss_val =loss_protest.val + loss_v.val,
loss_avg = loss_protest.avg + loss_v.avg,
protest_acc = protest_acc,
violence_mse = violence_mse, visattr_acc = visattr_acc))
print(' * Loss {loss_avg:.3f} Protest Acc {protest_acc.avg:.3f} '
'Violence MSE {violence_mse.avg:.5f} '
'Vis Attr Acc {visattr_acc.avg:.3f} '
.format(loss_avg = loss_protest.avg + loss_v.avg,
protest_acc = protest_acc,
violence_mse = violence_mse, visattr_acc = visattr_acc))
return loss_protest.avg + loss_v.avg, loss_history
