def validate(self, loader, model, criterion, epoch, args):
timer = Timer()
losses = AverageMeter()
top1 = AverageMeter()
wtop1 = AverageMeter()
alloutputs = []
metrics = {}
# switch to evaluate mode
model.eval()
def part(x):
return itertools.islice(x, int(len(x) * args.val_size))
for i, x in enumerate(part(loader)):
inputs, target, meta = parse(x)
output, loss, weights = forward(inputs,
target,
model,
criterion,
meta['id'],
train=False)
prec1 = triplet_accuracy(output, target)
wprec1 = triplet_accuracy(output, target, weights)
losses.update(loss.item(), inputs[0].size(0))
top1.update(prec1, inputs[0].size(0))
wtop1.update(wprec1, inputs[0].size(0))
alloutputs.extend(
zip([(x.item(), y.item()) for x, y in zip(*output)], target,
weights))
timer.tic()
if i % args.print_freq == 0:
print('[{name}] Test [{epoch}]: [{0}/{1} ({2})]\t'
'Time {timer.val:.3f} ({timer.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'WAcc@1 {wtop1.val:.3f} ({wtop1.avg:.3f})\t'.format(
i,
int(len(loader) * args.val_size),
len(loader),
name=args.name,
timer=timer,
loss=losses,
top1=top1,
epoch=epoch,
wtop1=wtop1))
metrics.update(triplet_allk(*zip(*alloutputs)))
metrics.update({'top1val': top1.avg, 'wtop1val': wtop1.avg})
print(
' * Acc@1 {top1val:.3f} \t WAcc@1 {wtop1val:.3f}'
'\n topk1: {topk1:.3f} \t topk2: {topk2:.3f} \t '
'topk5: {topk5:.3f} \t topk10: {topk10:.3f} \t topk50: {topk50:.3f}'
.format(**metrics))
return metrics
def train(self, loader, model, criterion, optimizer, epoch, args):
adjust_learning_rate(args.lr, args.lr_decay_rate, optimizer, epoch)
timer = Timer()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
wtop1 = AverageMeter()
metrics = {}
# switch to train mode
model.train()
optimizer.zero_grad()
def part(x):
return itertools.islice(x, int(len(x) * args.train_size))
for i, x in enumerate(part(loader)):
inputs, target, meta = parse(x)
data_time.update(timer.thetime() - timer.end)
output, loss, weights = forward(inputs, target, model, criterion,
meta['id'])
prec1 = triplet_accuracy(output, target)
wprec1 = triplet_accuracy(output, target, weights)
losses.update(loss.item(), inputs[0].size(0))
top1.update(prec1, inputs[0].size(0))
wtop1.update(wprec1, inputs[0].size(0))
loss.backward()
if i % args.accum_grad == args.accum_grad - 1:
print('updating parameters')
optimizer.step()
optimizer.zero_grad()
timer.tic()
if i % args.print_freq == 0:
print('[{name}] Epoch: [{0}][{1}/{2}({3})]\t'
'Time {timer.val:.3f} ({timer.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'WAcc@1 {wtop1.val:.3f} ({wtop1.avg:.3f})\t'.format(
epoch,
i,
int(len(loader) * args.train_size),
len(loader),
name=args.name,
timer=timer,
data_time=data_time,
loss=losses,
top1=top1,
wtop1=wtop1))
metrics.update({'top1': top1.avg, 'wtop1': wtop1.avg})
return metrics
def fine_tune_train_and_val(args, recorder):
# =
global lowest_val_loss, best_prec1
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # close the warning
torch.manual_seed(1)
cudnn.benchmark = True
timer = Timer()
# == dataset config==
num_class, data_length, image_tmpl = ft_data_config(args)
train_transforms, test_transforms, eval_transforms = ft_augmentation_config(
args)
train_data_loader, val_data_loader, _, _, _, _ = ft_data_loader_init(
args, data_length, image_tmpl, train_transforms, test_transforms,
eval_transforms)
# == model config==
model = ft_model_config(args, num_class)
recorder.record_message('a', '=' * 100)
recorder.record_message('a', '-' * 40 + 'finetune' + '-' * 40)
recorder.record_message('a', '=' * 100)
# == optim config==
train_criterion, val_criterion, optimizer = ft_optim_init(args, model)
# == data augmentation(self-supervised) config==
tc = TC(args)
# == train and eval==
print('*' * 70 + 'Step2: fine tune' + '*' * 50)
for epoch in range(args.ft_start_epoch, args.ft_epochs):
timer.tic()
ft_adjust_learning_rate(optimizer, args.ft_lr, epoch, args.ft_lr_steps)
train_prec1, train_loss = train(args, tc, train_data_loader, model,
train_criterion, optimizer, epoch,
recorder)
# train_prec1, train_loss = random.random() * 100, random.random()
recorder.record_ft_train(train_loss / 5.0, train_prec1 / 100.0)
if (epoch + 1) % args.ft_eval_freq == 0:
val_prec1, val_loss = validate(args, tc, val_data_loader, model,
val_criterion, recorder)
# val_prec1, val_loss = random.random() * 100, random.random()
recorder.record_ft_val(val_loss / 5.0, val_prec1 / 100.0)
is_best = val_prec1 > best_prec1
best_prec1 = max(val_prec1, best_prec1)
checkpoint = {
'epoch': epoch + 1,
'arch': "i3d",
'state_dict': model.state_dict(),
'best_prec1': best_prec1
}
recorder.save_ft_model(checkpoint, is_best)
timer.toc()
left_time = timer.average_time * (args.ft_epochs - epoch)
message = "Step2: fine tune best_prec1 is: {} left time is : {} now is : {}".format(
best_prec1, timer.format(left_time), datetime.now())
print(message)
recorder.record_message('a', message)
return recorder.filename
def validate_egovideo(self, loader, model, epoch, args):
""" Run video-level validation on the Charades ego test set"""
timer = Timer()
outputs, gts, ids = [], [], []
outputsw = []
metrics = {}
# switch to evaluate mode
model.eval()
for i, x in enumerate(loader):
inp, target, meta = parse(x)
target = target.long().cuda(async=True)
assert target[0, :].eq(target[1, :]).all(), "val_video not synced"
input_var = torch.autograd.Variable(inp.cuda(), volatile=True)
output, w_x, w_z = model(input_var)
output = torch.nn.Softmax(dim=1)(output)
sw_x = torch.nn.Softmax(dim=0)(w_x) * w_x.shape[0]
sw_x = (sw_x - sw_x.mean()) / sw_x.std()
scale = torch.clamp(1 + (sw_x - 1) * 0.05, 0, 100)
print('scale min: {}\t max: {}\t std: {}'.format(
scale.min().data[0],
scale.max().data[0],
scale.std().data[0]))
scale = torch.clamp(scale, 0, 100)
scale *= scale.shape[0] / scale.sum()
outputw = output * scale.unsqueeze(1)
# store predictions
output_video = output.mean(dim=0)
outputs.append(output_video.data.cpu().numpy())
outputsw.append(outputw.mean(dim=0).data.cpu().numpy())
gts.append(target[0, :])
ids.append(meta['id'][0])
timer.tic()
if i % args.print_freq == 0:
print('Test2: [{0}/{1}]\t'
'Time {timer.val:.3f} ({timer.avg:.3f})'.format(
i, len(loader), timer=timer))
# mAP, _, ap = meanap.map(np.vstack(outputs), np.vstack(gts))
mAP, _, ap = meanap.charades_nanmap(np.vstack(outputs), np.vstack(gts))
mAPw, _, _ = meanap.charades_nanmap(np.vstack(outputsw),
np.vstack(gts))
metrics['mAPego'] = mAP
metrics['mAPegow'] = mAPw
print(ap)
print(' * mAPego {mAPego:.3f} \t mAPegow {mAPegow:.3f}'.format(
**metrics))
submission_file(ids, outputs,
'{}/egoepoch_{:03d}.txt'.format(args.cache, epoch + 1))
return metrics
def validate_video(self, loader, model, epoch, args):
""" Run video-level validation on the Charades test set"""
timer = Timer()
outputs, gts, ids = [], [], []
metrics = {}
# switch to evaluate mode
model.eval()
for i, x in enumerate(loader):
inputs, target, meta = parse(x)
target = target.long().cuda(async=True)
assert target[0, :].eq(target[1, :]).all(), "val_video not synced"
input_vars = [
torch.autograd.Variable(inp.cuda(), volatile=True)
for inp in inputs
]
output = model(
*input_vars)[-1] # classification should be last output
output = torch.nn.Softmax(dim=1)(output)
# store predictions
output_video = output.mean(dim=0)
outputs.append(output_video.data.cpu().numpy())
gts.append(target[0, :])
ids.append(meta['id'][0])
timer.tic()
if i % args.print_freq == 0:
print('Test2: [{0}/{1}]\t'
'Time {timer.val:.3f} ({timer.avg:.3f})'.format(
i, len(loader), timer=timer))
# mAP, _, ap = meanap.map(np.vstack(outputs), np.vstack(gts))
mAP, _, ap = meanap.charades_map(np.vstack(outputs), np.vstack(gts))
metrics['mAP'] = mAP
print(ap)
print(' * mAP {:.3f}'.format(mAP))
submission_file(ids, outputs,
'{}/epoch_{:03d}.txt'.format(args.cache, epoch + 1))
return metrics
def pretext_train(args, recorder):
if args.gpus is not None:
print("Use GPU: {} for pretext training".format(args.gpus))
num_class, data_length, image_tmpl = pt_data_config(args)
# print("tp_length is: ", data_length)
train_transforms, test_transforms, eval_transforms = pt_augmentation_config(
args)
train_loader, val_loader, eval_loader, train_samples, val_samples, eval_samples = pt_data_loader_init(
args, data_length, image_tmpl, train_transforms, test_transforms,
eval_transforms)
n_data = len(train_loader)
model, model_ema = pt_model_config(args, num_class)
# == optim config==
contrast, criterion, optimizer = pt_optim_init(args, model, n_data)
model = model.cuda()
# == load weights ==
model, model_ema = pt_load_weight(args, model, model_ema, optimizer,
contrast)
if args.pt_method in ['dsm', 'moco']:
model_ema = model_ema.cuda()
# copy weights from `model' to `model_ema'
moment_update(model, model_ema, 0)
cudnn.benchmark = True
# optionally resume from a checkpoint
args.start_epoch = 1
# ==================================== our data augmentation method=================================
if args.pt_method in ['dsm', 'dsm_triplet']:
pos_aug = GenPositive()
neg_aug = GenNegative()
# =======================================add message =====================
recorder.record_message('a', '=' * 100)
recorder.record_message('a', '-' * 40 + 'pretrain' + '-' * 40)
recorder.record_message('a', '=' * 100)
# ====================update lr_decay from str to numpy=========
iterations = args.pt_lr_decay_epochs.split(',')
args.pt_lr_decay_epochs = list([])
for it in iterations:
args.pt_lr_decay_epochs.append(int(it))
timer = Timer()
# routine
print('*' * 70 + 'Step1: pretrain' + '*' * 20 + '*' * 50)
for epoch in range(args.pt_start_epoch, args.pt_epochs + 1):
timer.tic()
pt_adjust_learning_rate(epoch, args, optimizer)
print("==> training...")
time1 = time.time()
if args.pt_method == "moco":
loss, prob = train_moco(epoch, train_loader, model, model_ema,
contrast, criterion, optimizer, args,
recorder)
elif args.pt_method == "dsm":
loss, prob = train_dsm(epoch, train_loader, model, model_ema,
contrast, criterion, optimizer, args,
pos_aug, neg_aug, recorder)
# loss, prob = epoch * 0.01, 0.02*epoch
elif args.pt_method == "dsm_triplet":
loss = train_dsm_triplet(epoch, train_loader, model, optimizer,
args, pos_aug, neg_aug, recorder)
else:
Exception("Not support method now!")
recorder.record_pt_train(loss)
time2 = time.time()
print('epoch {}, total time {:.2f}'.format(epoch, time2 - time1))
timer.toc()
left_time = timer.average_time * (args.pt_epochs - epoch)
message = "Step1: pretrain now loss is: {} left time is : {} now is: {}".format(
loss, timer.format(left_time), datetime.now())
print(message)
recorder.record_message('a', message)
state = {
'opt': args,
'model': model.state_dict(),
'contrast': contrast.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch,
}
recorder.save_pt_model(args, state, epoch)
print("finished pretrain, the trained model is record in: {}".format(
recorder.pt_checkpoint))
return recorder.pt_checkpoint
def inference_mean_exemplar(
model,
current_epoch,
current_iter,
local_rank,
data_loader,
dataset_name,
device="cuda",
max_instance=3200,
mute=False,
):
model.train(False)
# convert to a torch.device for efficiency
device = torch.device(device)
if not mute:
logger = logging.getLogger("maskrcnn_benchmark.inference")
logger.info("Start evaluation")
total_timer = Timer()
inference_timer = Timer()
total_timer.tic()
torch.cuda.empty_cache()
if not mute:
pbar = tqdm(total=len(data_loader), desc="Validation in progress")
with torch.no_grad():
all_pred_obj, all_truth_obj, all_pred_attr, all_truth_attr = [], [], [], []
obj_loss_all, attr_loss_all = 0, 0
cnt = 0
for iteration, out_dict in enumerate(data_loader):
if type(max_instance) is int:
if iteration == max_instance // model.cfg.EXTERNAL.BATCH_SIZE:
break
if type(max_instance) is float:
if iteration > max_instance * len(
data_loader) // model.cfg.EXTERNAL.BATCH_SIZE:
break
# print(iteration)
images = torch.stack(out_dict['images'])
obj_labels = torch.cat(out_dict['object_labels'], -1)
attr_labels = torch.cat(out_dict['attribute_labels'], -1)
cropped_image = torch.stack(out_dict['cropped_image'])
images = images.to(device)
obj_labels = obj_labels.to(device)
attr_labels = attr_labels.to(device)
cropped_image = cropped_image.to(device)
# loss_dict = model(images, targets)
pred_obj = model.mean_of_exemplar_classify(cropped_image)
all_pred_obj.extend(to_list(pred_obj))
all_truth_obj.extend(to_list(obj_labels))
cnt += 1
if not mute:
pbar.update(1)
obj_f1 = f1_score(all_truth_obj, all_pred_obj, average='micro')
#attr_f1 = f1_score(all_truth_attr, all_pred_attr, average='micro')
obj_loss_all /= (cnt + 1e-10)
# wait for all processes to complete before measuring the time
total_time = total_timer.toc()
model.train(True)
return obj_f1, 0, len(all_truth_obj)
def inference(model,
current_epoch,
current_iter,
local_rank,
data_loader,
dataset_name,
device="cuda",
max_instance=3200,
mute=False,
verbose_return=False):
model.train(False)
# convert to a torch.device for efficiency
device = torch.device(device)
if not mute:
logger = logging.getLogger("maskrcnn_benchmark.inference")
logger.info("Start evaluation")
total_timer = Timer()
total_timer.tic()
torch.cuda.empty_cache()
if not mute:
pbar = tqdm(total=len(data_loader), desc="Validation in progress")
def to_list(tensor):
return tensor.cpu().numpy().tolist()
with torch.no_grad():
all_pred_obj, all_truth_obj, all_pred_attr, all_truth_attr = [], [], [], []
all_image_ids, all_boxes = [], []
all_pred_attr_prob = []
all_raws = []
obj_loss_all, attr_loss_all = 0, 0
cnt = 0
for iteration, out_dict in enumerate(data_loader):
if type(max_instance) is int:
if iteration == max_instance // model.cfg.EXTERNAL.BATCH_SIZE:
break
if type(max_instance) is float:
if iteration > max_instance * len(
data_loader) // model.cfg.EXTERNAL.BATCH_SIZE:
break
# print(iteration)
if verbose_return:
all_image_ids.extend(out_dict['image_ids'])
all_boxes.extend(out_dict['gt_bboxes'])
all_raws.extend(out_dict['raw'])
ret_dict = inference_step(model, out_dict, device)
loss_attr, loss_obj, attr_score, obj_score = ret_dict.get('attr_loss', None), \
ret_dict.get('obj_loss', None), \
ret_dict.get('attr_score', None), \
ret_dict.get('obj_score', None)
if loss_attr is not None:
attr_loss_all += loss_attr.item()
pred_attr_prob, pred_attr = ret_dict[
'pred_attr_prob'], ret_dict['pred_attr']
all_pred_attr.extend(to_list(pred_attr))
all_truth_attr.extend(to_list(ret_dict['attr_labels']))
all_pred_attr_prob.extend(to_list(pred_attr_prob))
if loss_obj is not None:
obj_loss_all += loss_obj.item()
_, pred_obj = obj_score.max(-1)
all_pred_obj.extend(to_list(pred_obj))
all_truth_obj.extend(to_list(ret_dict['obj_labels']))
cnt += 1
if not mute:
pbar.update(1)
obj_f1 = f1_score(all_truth_obj, all_pred_obj, average='micro')
attr_f1 = f1_score(all_truth_attr, all_pred_attr, average='micro')
obj_loss_all /= (cnt + 1e-10)
attr_loss_all /= (cnt + 1e-10)
if not mute:
logger.info(
'Epoch: {}\tIteration: {}\tObject f1: {}\tAttr f1:{}\tObject loss:{}\tAttr loss:{}'
.format(current_epoch, current_iter, obj_f1, attr_f1,
obj_loss_all, attr_loss_all))
#compute_on_dataset(model, data_loader, local_rank, device, inference_timer, output_file)
# wait for all processes to complete before measuring the time
total_time = total_timer.toc()
model.train(True)
if not verbose_return:
return obj_f1, attr_f1, len(all_truth_attr)
else:
return obj_f1, attr_f1, all_pred_attr, all_truth_attr, all_pred_obj, all_truth_obj, all_image_ids, all_boxes, \
all_pred_attr_prob, all_raws
def train_and_eval(args):
# =
global lowest_val_loss, best_prec1
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # close the warning
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
torch.manual_seed(1)
cudnn.benchmark = True
timer = Timer()
recorder = Record(args)
# == dataset config==
num_class, data_length, image_tmpl = data_config(args)
train_transforms, test_transforms = augmentation_config(args)
train_data_loader, val_data_loader = data_loader_init(
args, data_length, image_tmpl, train_transforms, test_transforms)
# == model config==
models = []
optimizers = []
for i in range(args.mutual_num):
model = model_config(args, num_class)
models.append(model)
recorder.record_message('a', '=' * 100)
recorder.record_message('a', str(model.module))
recorder.record_message('a', '=' * 100)
# == optim config==
for i in range(args.mutual_num):
train_criterion, val_criterion, optimizer = optim_init(args, model)
optimizers.append(optimizer)
# == data augmentation(self-supervised) config==
tc = TC(args)
# == train and eval==
for epoch in range(args.start_epoch, args.epochs):
timer.tic()
for i in range(args.mutual_num):
adjust_learning_rate(optimizers[i], args.lr, epoch, args.lr_steps)
if args.eval_indict == 'acc':
train_prec1, train_loss = train(args, tc, train_data_loader,
models, train_criterion,
optimizers, epoch, recorder)
# train_prec1, train_loss = random.random() * 100, random.random()
recorder.record_train(train_loss / 5.0, train_prec1 / 100.0)
else:
train_loss = train(args, tc, train_data_loader, models,
train_criterion, optimizers, epoch, recorder)
# train_prec1, train_loss = random.random() * 100, random.random()
recorder.record_train(train_loss)
if (epoch + 1) % args.eval_freq == 0:
if args.eval_indict == 'acc':
val_prec1, val_loss = validate(args, tc, val_data_loader,
models, val_criterion, recorder)
# val_prec1, val_loss = random.random() * 100, random.random()
recorder.record_val(val_loss / 5.0, val_prec1 / 100.0)
is_best = val_prec1 > best_prec1
best_prec1 = max(val_prec1, best_prec1)
checkpoint = {
'epoch': epoch + 1,
'arch': "i3d",
'state_dict': model.state_dict(),
'best_prec1': best_prec1
}
else:
val_loss = validate(args, tc, val_data_loader, models,
val_criterion, recorder)
# val_loss = random.random()
# val_prec1, val_loss = random.random() * 100, random.random()
recorder.record_val(val_loss)
is_best = val_loss < lowest_val_loss
lowest_val_loss = min(val_loss, lowest_val_loss)
checkpoint = {
'epoch': epoch + 1,
'arch': "i3d",
'state_dict': model.state_dict(),
'lowest_val': lowest_val_loss
}
recorder.save_model(checkpoint, is_best)
timer.toc()
left_time = timer.average_time * (args.epochs - epoch)
if args.eval_indict == 'acc':
message = "best_prec1 is: {} left time is : {}".format(
best_prec1, timer.format(left_time))
else:
message = "lowest_val_loss is: {} left time is : {}".format(
lowest_val_loss, timer.format(left_time))
print(message)
recorder.record_message('a', message)
# return recorder.best_name
return recorder.filename
