def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None):
batch_time = AverageMeter()
losses = AverageMeter()
aps = AverageMeter()
pre_list = []
target_list = []
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(val_loader):
target = target.cuda().float()
# compute output
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
pre_list.append(output.data.cpu().numpy())
target_list.append(target.cpu().numpy())
losses.update(loss.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time {batch_time.avg:.3f}\t'
'Loss {loss.avg:.4f}'.format(i,
len(val_loader),
batch_time=batch_time,
loss=losses))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
ap_value = map_sklearn(np.vstack(target_list), np.vstack(pre_list))
output = ('Testing Results\t mAP: {}'.format(np.around(ap_value, 3)))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
if tf_writer is not None:
tf_writer.add_scalar('loss/test', losses.avg, epoch)
tf_writer.add_scalar('acc/test_map', ap_value, epoch)
return ap_value
def build_database(model_rgb, model_flow, dataset, threshold):
assert isinstance(threshold, float)
model_rgb.eval()
model_flow.eval()
end = time.time()
data_time = AverageMeter()
database = [[] for _ in range(num_class)]
for i, (input, label, _, _, _) in enumerate(dataset):
with torch.no_grad():
input_var = torch.from_numpy(input).cuda()
target_var = torch.Tensor([label]).type(torch.cuda.LongTensor)
assert isinstance(alpha,
float) and (alpha < 1) and (alpha > 0), alpha
logit_rgb, weight_rgb = model_rgb(input_var[:, :400])
logit_flow, weight_flow = model_flow(input_var[:, 400:])
if isinstance(label, int):
score, score_rgb, weight_rgb, score_flow, weight_flow \
= get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, label, alpha) # [length]
for sind, s in enumerate(score):
if s > threshold:
database[label].append({
'feat_rgb': input[sind, :400],
'feat_flow': input[sind, 400:],
'feat_cat': input[sind, :],
'score': s,
'raw_score_rgb': score_rgb[sind],
'raw_score_flow': score_flow[sind],
'attention_rgb': weight_rgb[sind],
'attention_flow': weight_flow[sind]
})
elif isinstance(label, list):
for lab in label:
assert isinstance(lab, int)
score, score_rgb, weight_rgb, score_flow, weight_flow \
= get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, lab, alpha)
if s > threshold:
database[lab].append({
'feat_rgb': input[sind, :400],
'feat_flow': input[sind, 400:],
'feat_cat': input[sind, :],
'score': s,
'raw_score_rgb': score_rgb[sind],
'raw_score_flow': score_flow[sind],
'attention_rgb': weight_rgb[sind],
'attention_flow': weight_flow[sind]
})
print(
"Database has generated. It has {} classes, and num of element in each class is: {}"
.format(len(database), [len(dc) for dc in database]))
return database
def make_predictions(model, testloader, seq_length, encode_feature_len,
params):
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
end_time = time.time()
iteration = len(testloader)
AA_Attention_Records = np.zeros([len(testloader.dataset), seq_length])
Predict_Array = np.zeros(len(testloader.dataset))
aa_feature_len = encode_feature_len + seq_length #
for batch_idx, data in enumerate(testloader):
aa_feature, index = data
aa_feature = aa_feature.float()
aa_feature = aa_feature.cuda()
batch_size = aa_feature.size(0)
h_0 = np.random.uniform(-0.5,
high=0.5,
size=(batch_size, aa_feature_len))
c_0 = np.random.uniform(-0.5,
high=0.5,
size=(batch_size, aa_feature_len))
h_0 = torch.from_numpy(h_0).float()
c_0 = torch.from_numpy(c_0).float()
h_0 = h_0.cuda()
c_0 = c_0.cuda()
data_time.update(time.time() - end_time, batch_size)
with torch.no_grad():
pred_batch, attention_batch = model(aa_feature, (h_0, c_0),
use_attention=True)
index = index.detach().cpu().numpy()
attention_batch = attention_batch.detach().cpu().numpy()
AA_Attention_Records[index] = attention_batch[:, 0, :]
pred = torch.softmax(pred_batch, dim=1)
pred = pred.detach().cpu().numpy()
Predict_Array[index] = pred[:, 1]
batch_time.update(time.time() - end_time, batch_size)
end_time = time.time()
print('Iter: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format(
batch_idx + 1,
iteration,
batch_time=batch_time,
data_time=data_time,
))
return Predict_Array, AA_Attention_Records
def validate(val_loader, model, criterion, epoch, log, tf_writer):
losses = AverageMeter()
top1 = AverageMeter()
model.eval()
with torch.no_grad():
for input, target in val_loader:
input, target = input.cuda(), target.cuda()
output = model(input)
loss = criterion(output, target)
# accuracy and loss
prec1, = accuracy(output.data, target, topk=(1, ))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
output = ('Validate: Prec@1 {top1.avg:.2f} Loss {loss.avg:.3f}'.format(
top1=top1, loss=losses))
print(output)
log.write(output + '\n')
log.flush()
tf_writer.add_scalar('loss/val', losses.avg, epoch)
tf_writer.add_scalar('acc/val_top1', top1.avg, epoch)
return top1.avg
def train(train_loader, model, criterion, kl_loss, logsoftmax, softmax,
optimizer, epoch, log, tf_writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
loss_kl = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
total = 0
shift = 0
for i, moda in enumerate(args.modality):
tmp = total
if moda == 'RGB':
total += 3
elif moda == 'Flow':
total += 10
elif moda == 'RGBDiff':
total += 18
if i == 0:
shift = total
if i == args.rank and i > 0:
start_ind = tmp - shift
end_ind = total - shift
elif i == args.rank and i == 0:
start_ind = 0
end_ind = total
if args.rank == 0:
inds = []
for x in range(args.num_segments):
inds.extend(list(range(x * total + start_ind,
x * total + end_ind)))
send_inds = []
for x in range(args.num_segments):
send_inds.extend(
list(range(x * total + end_ind, x * total + total)))
else:
inds = []
for x in range(args.num_segments):
inds.extend(
list(
range(x * (total - shift) + start_ind,
x * (total - shift) + end_ind)))
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode5r
model.train()
if args.rank == 0:
iter_through = train_loader
else:
iter_through = range(
int(len([x for x in open(args.train_list[0])]) / args.batch_size))
end = time.time()
for i, data in enumerate(iter_through):
# measure data loading time
data_time.update(time.time() - end)
if args.rank == 0:
input, target = data
target = target.cuda(args.gpus[-1])
input = input.cuda(args.gpus[0])
if args.world_size > 1:
torch.distributed.broadcast(input[:, send_inds].contiguous(),
0)
torch.distributed.broadcast(target, 0)
else:
input = torch.zeros(
(args.batch_size, (total - shift) * args.num_segments, 224,
224)).cuda(args.gpus[0])
target = torch.zeros((args.batch_size, ),
dtype=torch.int64).cuda(args.gpus[-1])
torch.distributed.broadcast(input, 0)
torch.distributed.broadcast(target, 0)
input_var = torch.autograd.Variable(input[:, inds].contiguous())
target_var = torch.autograd.Variable(target)
# compute output
output = model(input_var).cuda(args.gpus[-1])
loss1 = criterion(output, target_var)
if args.world_size > 1:
reduce_output = output.clone().detach()
distr.all_reduce(reduce_output)
reduce_output = (reduce_output -
output.detach()) / (args.world_size - 1)
loss2 = kl_loss(logsoftmax(output),
softmax(reduce_output.detach()))
else:
loss2 = torch.tensor(0.)
loss = loss1 + loss2
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss1.item(), input.size(0))
loss_kl.update(loss2.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# compute gradient and do SGD step
loss.backward()
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(),
args.clip_gradient)
optimizer.step()
optimizer.zero_grad()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss1.val:.4f} ({loss1.avg:.4f})\t'
'LossKL {loss2.val:.4f} ({loss2.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(iter_through),
batch_time=batch_time,
data_time=data_time,
loss1=losses,
loss2=loss_kl,
top1=top1,
top5=top5,
lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
tf_writer.add_scalar('loss/mutual', loss_kl.avg, epoch)
tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
target = target.cuda()
input_var = torch.autograd.Variable(input)
target_var = torch.autograd.Variable(target)
# compute output
output = model(input_var)
loss = criterion(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# compute gradient and do SGD step
loss.backward()
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(),
args.clip_gradient)
optimizer.step()
optimizer.zero_grad()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# if i % 100 == 0:
if i % args.print_freq == 0:
output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5,
lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def validate(val_loader,
model,
decoder,
criterion,
epoch,
log=None,
tf_writer=None,
index2wordDict=None):
batch_time = AverageMeter()
data_time = AverageMeter()
#losses = AverageMeter()
BLEUs = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
with open(os.path.join(args.root_model, args.store_name, "val.txt"),
"a+") as txt:
txt.write("epoch\t" + str(epoch) + "\n")
for i, data in enumerate(val_loader):
loss = 0
data_time.update(time.time() - end)
#if isinstance(data, bool):
# continue
#if not torch.any(data):
# continue
if not data:
continue
try:
[URL, id, label, clips] = data
except Exception as e:
print(e)
print(data)
with open("errr.txt", "a+") as txt:
txt.write(str(data))
input_video = torch.tensor([], dtype=torch.float).cuda()
input_caption = torch.tensor([], dtype=torch.long).cuda()
#cap_nums = []
for clip in range(0, len(clips), max(len(clips) // 50, 1)):
if list(input_video.size()) != [0]:
if input_video.size()[1] > 50:
break
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat((input_video, video.float().cuda()), 1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
#cap_nums.append(clip[2]) #for recording filename in result
input_video = input_video.view(1, -1, 3, crop_size, crop_size)
input_caption = input_caption.view(1, -1,
input_caption.size()[-1], 1)
#print(input_video.size())
#print(input_caption.size())
target = label.cuda()
input_video_var = torch.autograd.Variable(input_video)
input_caption_var = torch.autograd.Variable(input_caption)
target_var = torch.autograd.Variable(target)
# compute output
wExtraLoss = 1 if args.prune == '' else 0.1
encoder_output = model(input_video_var,
input_caption_var) #size=(1, frames, 2048)
t_input = torch.zeros((1, 64, 2048)).cuda()
for idx_e, e in enumerate(encoder_output[0]):
t_input[0, idx_e] = e
#print("encoder_output size", encoder_output.size())
decoder_outputs = []
#decoder_input = torch.tensor([[0]]).cuda() #SOS
#decoder_hidden = decoder.initHidden()
decoder_output = decoder(t_input)
decoder_outputs = decoder_output.argmax(-1)
#decoder_outputs = decoder_output[0,:len(target[0])].argmax(-1)
#print("output size=",output.size())
#print("target)var=",target_var.size())
#print(output)
#print(target_var)
decoder_sentence = [
index2wordDict[index.item()] for index in decoder_outputs
]
print(decoder_sentence)
#print(target.size())
target_sentence = [
index2wordDict[index.item()] for index in target[0]
]
print(target_sentence)
bleu = sentence_bleu(
target_sentence, decoder_sentence
) #default weight is [0.25,0.25,0.25,0.25] = bleu-4
#print("bleu =",bleu)
BLEUs.update(bleu)
#auc = AUC(output.squeeze(), target_var.squeeze())
#print("auc=",auc)
#AUCs.update(auc)
#losses.update(loss.item(), )
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
txtoutput = ('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'BLEU {top5.val:.3f} ({top5.avg:.3f}\t'
"output/target {output}/{target}").format(
i,
len(val_loader),
batch_time=batch_time,
output=decoder_sentence,
target=target_sentence,
top5=BLEUs)
if i % args.print_freq == 0:
print(txtoutput)
if log is not None:
log.write(txtoutput + '\n')
log.flush()
with open(
os.path.join(args.root_model, args.store_name, "val.txt"),
"a+") as txt:
txt.write(txtoutput + "\n")
#break
txtoutput = ('Testing Results: BLEUs {auc.avg:.3f} '.format(auc=BLEUs))
with open(os.path.join(args.root_model, args.store_name, "val.txt"),
"a+") as txt:
txt.write(txtoutput + "\n")
print(txtoutput)
if log is not None:
log.write(txtoutput + '\n')
log.flush()
if tf_writer is not None:
#tf_writer.add_scalar('loss/test', losses.avg, epoch)
#tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
#tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
tf_writer.add_scalar('bleu/test', BLEUs.avg, epoch)
return BLEUs.avg
def validate(val_loader, model, criterion, iter, logger=None):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (r, f, path, label, video_index, seg_index,
verify) in enumerate(val_loader):
with torch.no_grad():
r = r.cuda()
f = f.cuda()
target_var = label.cuda(async=True).type(torch.cuda.LongTensor)
output, _ = model(r, f)
loss = criterion(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output, target_var, topk=(1, 5))
losses.update(loss.item(), r.shape[0])
top1.update(prec1.item(), r.shape[0])
top5.update(prec5.item(), r.shape[0])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print(('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.6f} ({loss.avg:.6f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5)))
# if i == 99: break # Randomly Test 100 batch data (which is in the training data, so this is not a really evaluation).
print((
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses)))
return top1.avg, top5.avg
# feature_length = 800
path_list = []
file_list = []
label_list = []
vid = []
annotation_list = []
# annotation_dict = build_annotation_dict(annotation_root=annotation_root, index_file=index_file)
ant = []
feature = []
last_ind = 0
last_path = ""
last_label = 0
verbose = False
testtime = AverageMeter()
#### build network
# fe = I3DFeatureExtractor()
# fe = load_fe_from_i3d(fe, i3d_model_checkpoint)
# fe = torch.nn.DataParallel(fe, device_ids=[i for i in range(torch.cuda.device_count())]).cuda()
# fe.eval()
####
# clf = Classifier(feature_length, num_class, isbn=True)
# fe = I3DFeatureExtractor()
# I3DClassifier = torch.nn.Sequential(
# fe,
# clf
# )
# ckpt = torch.load(i3d_model_checkpoint)
def validate(val_loader, model, criterion, epoch):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
torch.set_grad_enabled(False)
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
for i, (input, target, name) in enumerate(val_loader):
# discard final batch
if i == len(val_loader) - 1:
break
target = target.cuda(async=True)
# compute output
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if epoch > int(args.epochs * 0.7):
batch_out = torch.argmax(output, dim=1)
batch_reslut = (batch_out != target).tolist()
for i in range(len(batch_reslut)):
if batch_reslut[i] == 1:
print(name[i])
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(output)
output = (
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses))
print(output)
return top1.avg
def validate(val_loader, model, criterion, iter=None, logger=None):
batch_time = AverageMeter()
losses = AverageMeter()
accu = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (r, f, path, label, video_index, seg_index, verify) in enumerate(val_loader):
with torch.no_grad():
r = r.cuda()
f = f.cuda()
target_var = label.cuda(async=True).type(torch.cuda.LongTensor)
output = model([r, f])
loss = criterion(output, target_var)
# measure accuracy and record loss
prec1, = accuracy(output, target_var, topk=(1,))
losses.update(loss.item(), r.shape[0])
accu.update(prec1.item(), r.shape[0])
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print(('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.6f} ({loss.avg:.6f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'.format(
i, len(val_loader), batch_time=batch_time, loss=losses,
top1=accu)))
print(('Testing Results: Prec@1 {top1.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=accu, loss=losses)))
return accu.avg
def validate(model, epoch):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
'''
Make sure that the validation works so I can confirm that there is an improvement/loss in
accuracy when running this model.
Run model
'''
val_loader = get_val_loader(model)
criterion = torch.nn.CrossEntropyLoss().cuda()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(val_loader):
target = target.cuda()
# compute output
output = model.net(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % 100 == 0:
output = (
'Test: [{0}/{1}]\t'
#'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(output)
output = (
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses))
print(output)
return top1.avg
def v_train(train_loader, val_loader, model, v_model, vnet, criterion,
valcriterion, optimizer, v_optimizer, optimizer_vnet, epoch, log,
tf_writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
val_loader_iter = iter(val_loader)
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
v_model.load_state_dict(model.state_dict())
target = target.cuda()
input_var = torch.autograd.Variable(input)
target_var = torch.autograd.Variable(target)
# compute output
output = v_model(input_var)
# loss = criterion(output, target_var)
cost = criterion(output, target_var)
cost_v = torch.reshape(cost, (-1, 1))
v_lambda = vnet(cost_v.data)
l_f_v = torch.sum(cost_v * v_lambda) / len(cost_v)
v_model.zero_grad()
grads = torch.autograd.grad(l_f_v, (v_model.module.params()),
create_graph=True)
# to be modified
v_lr = args.lr * ((0.1**int(epoch >= 80)) * (0.1**int(epoch >= 100)))
v_model.module.update_params(lr_inner=v_lr, source_params=grads)
del grads
# phase 2. pixel weights step
try:
inputs_val, targets_val = next(val_loader_iter)
except StopIteration:
val_loader_iter = iter(val_loader)
inputs_val, targets_val = next(val_loader_iter)
inputs_val, targets_val = inputs_val.cuda(), targets_val.cuda()
y_g_hat = v_model(inputs_val)
l_g_meta = valcriterion(y_g_hat, targets_val) # val loss
optimizer_vnet.zero_grad()
l_g_meta.backward()
optimizer_vnet.step()
# phase 1. network weight step (w)
output = model(input_var)
cost = criterion(output, target)
cost_v = torch.reshape(cost, (-1, 1))
with torch.no_grad():
v_new = vnet(cost_v)
loss = torch.sum(cost_v * v_new) / len(cost_v)
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# compute gradient and do SGD step
# loss.backward()
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(),
args.clip_gradient)
# optimizer.step()
# optimizer.zero_grad()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5,
lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
if i == 20:
os.system("gpustat")
target = target.cuda()
input_var = torch.autograd.Variable(input)
target_var = torch.autograd.Variable(target)
# compute output
output = model(input_var)
loss = criterion(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# compute gradient and do SGD step
loss.backward()
no_grad_cnt = 0
if i % args.iter_size == 0:
# scale down gradients when iter size is functioning
if args.iter_size != 1:
for g in optimizer.param_groups:
for p in g['params']:
if isinstance(p.grad, torch.Tensor):
p.grad /= args.iter_size
else:
no_grad_cnt = no_grad_cnt + 1
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient)
else:
total_norm = 0
optimizer.step()
optimizer.zero_grad()
#if i == 0:
# print("{}\nWARNING: There are {} params without gradient!!!!!\n{}".format("*"*50, no_grad_cnt, "*"*50))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def test(val_loader, model, criterion, epoch, log=None, tf_writer=None):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
nAUDC = 1
pMiss = 1
pred_dict = {}
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
if args.naudc:
naudc_res = []
for i, (input, target, name) in enumerate(val_loader):
batch_size = target.numel()
target = target.cuda()
# compute output
output = model(input)
# if args.dense_sample:
# output = output.reshape(batch_size, 10, -1).mean(1)
# loss = criterion(output, target)
loss = criterion(output.squeeze(-1), target)
# measure accuracy and record loss
if args.loss_type == "bce" or args.loss_type == "wbce":
[prec1, prec5], num = accuracy_bce(output.data,
target,
topk=(1, 5))
losses.update(loss.item(), output.size(0))
if prec1 != -1:
top1.update(prec1.item(), num)
top5.update(prec5.item(), num)
elif args.loss_type == "nll":
[prec1, prec5] = accuracy_nll(output.data, target, topk=(1, 5))
losses.update(loss.item(), output.size(0))
top1.update(prec1.item(), output.size(0))
top5.update(prec5.item(), output.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
rst = torch.softmax(output, dim=1).data.cpu().numpy().copy()
tgt = target.cpu().numpy().copy()
for idk in range(rst.shape[0]):
pred_dict[str(name[idk])] = {
"pred": rst[idk].tolist(),
"gt": tgt[idk].tolist()
}
if args.naudc:
rst = output.data.cpu().numpy().copy()
this_rst_list = []
this_rst_list.append(rst)
ensembled_predict = sum(this_rst_list) / len(this_rst_list)
for p, g in zip(ensembled_predict, target.cpu().numpy()):
naudc_res.append([p[None, ...], g])
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(output)
logging.info(output)
if log is not None:
log.write(output + '\n')
log.flush()
json_str = json.dumps(pred_dict, indent=4)
with open(os.path.join("./pred.json"), 'w') as save_json:
save_json.write(json_str)
if args.naudc:
video_pred_topall = [
np.argsort(np.mean(x[0], axis=0).reshape(-1))[::-1][:args.topk]
for x in naudc_res
]
video_prob_topall = [
np.sort(np.mean(x[0], axis=0).reshape(-1))[::-1][:args.topk]
for x in naudc_res
]
with open(args.val_list) as f:
vid_names = list(json.load(f)["database"].keys())
event_dict = get_eventdict(args)
res_dict, file_dict, eve_dict = getoutput(vid_names, video_pred_topall,
video_prob_topall,
event_dict, args)
json_str = json.dumps(res_dict, indent=4)
with open(os.path.join(args.out_path, "output-mod.json"),
'w') as save_json:
save_json.write(json_str)
json_str = json.dumps(file_dict, indent=4)
with open(os.path.join(args.out_path, "file-index.json"),
'w') as save_json:
save_json.write(json_str)
json_str = json.dumps(eve_dict, indent=4)
with open(os.path.join(args.out_path, "activity-index.json"),
'w') as save_json:
save_json.write(json_str)
call_path = os.path.join(args.call_path, "ActEV_Scorer.py")
s = os.path.join(args.out_path, "output_mod.json")
r = os.path.join(args.out_path, "kitware_eo_s2-test_99.json")
a = os.path.join(args.out_path, "activity-index.json")
f = os.path.join(args.out_path, "file-index.json")
o = args.out_path
print("Start Calculatomg Pmiss and nAUDC")
call = "python {} ActEV_SDL_V2 -s {} -r {} -a {} -f {} -o {} -v -n 36".format(
call_path, s, r, a, f, o)
os.system(call)
csvFile = open(os.path.join(args.out_path, "scores_aggregated.csv"),
"r")
reader = csv.reader(csvFile)
for item in reader:
if "[email protected]" in item[0]:
nAUDC = float(item[0].split("|")[-1])
if "[email protected]" in item[0]:
pMiss = float(item[0].split("|")[-1])
output = (
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f} nAUDC {nAUDC:.5f} pMiss {pMiss:.5f}'
.format(top1=top1, top5=top5, loss=losses, nAUDC=nAUDC, pMiss=pMiss))
print(output)
logging.info(output)
if log is not None:
log.write(output + '\n')
log.flush()
if tf_writer is not None:
tf_writer.add_scalar('loss/test', losses.avg, epoch)
tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
tf_writer.add_scalar('nAUDC/test', nAUDC, epoch)
tf_writer.add_scalar('pmiss/test', pMiss, epoch)
return losses.avg
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
AUCs = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
with open(os.path.join(args.root_model, args.store_name, "val.txt"),
"a+") as txt:
txt.write("epoch\t" + str(epoch) + "\n")
for idx, data in enumerate(val_loader):
totalOutput = torch.tensor([], dtype=torch.float).cuda()
totalTarget = torch.tensor([], dtype=torch.float).cuda()
data_time.update(time.time() - end)
# if isinstance(data, bool):
# continue
#if not torch.any(data):
# continue
if not data:
continue
try:
[URL, id, sift, label, clips] = data
except Exception as e:
print(e)
print(data)
with open("errr.txt", "a+") as txt:
txt.write(str(data))
#cap_nums = []
for i, s in enumerate(sift):
input_video = torch.tensor([], dtype=torch.float).cuda()
input_caption = torch.tensor([], dtype=torch.long).cuda()
#print("i={},s={}".format(i,s))
if i + 1 < len(sift):
if int(sift[i + 1]) - int(s) < 50:
#print( int(sift[i+1])-int(s))
for clip in range(s, sift[i + 1]):
#print(clip)
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(),
video.float().cuda()), 1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0, s:sift[i + 1]].cuda()
else:
for clip in range(s, sift[i + 1],
(int(sift[i + 1]) - int(s)) // 50):
#print(clip)
if (clip - s) / (
(int(sift[i + 1]) - int(s)) // 50) >= 50:
break
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(),
video.float().cuda()), 1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0,
s:sift[i + 1]:(sift[i + 1] - s) //
50].cuda()
target = target[:50]
else:
length = len(clips)
if int(length) - int(s) < 50:
for clip in range(s, length):
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(),
video.float().cuda()), 1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0, s:length].cuda()
else:
for clip in range(s, length,
(int(length) - int(s)) // 50):
if (clip - s) / (
(int(length) - int(s)) // 50) >= 50:
break
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(),
video.float().cuda()), 1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0,
s:length:(length - s) // 50].cuda()
target = target[:50]
#print("input video.size()",input_video.size())
#print("input_caption.size()",input_caption.size())
input_video = input_video.view(1, -1, 3, crop_size, crop_size)
#input_caption = input_caption.view(1,-1,input_caption.size()[-1],1)
input_caption = input_caption.view(1, -1, 64, 1)
input_video_var = torch.autograd.Variable(input_video)
input_caption_var = torch.autograd.Variable(input_caption)
target_var = torch.autograd.Variable(target)
#print("input_video_var.size()",input_video_var.size())
#print("target size",target.size())
# compute output
wExtraLoss = 1 if args.prune == '' else 0.1
output = model(input_video_var, input_caption_var)
#print("output size=",output.size())
#print("target)var=",target_var.size())
#print(output)
#print(target_var)
#print("totalOutput size=",totalOutput.size())
#print("output size=",output.size())
#print("squeeze size=",output.squeeze().size())
#print(totalOutput)
#print("/*"*50)
#print(output)
totalOutput = torch.cat(
(totalOutput.float().cuda(), output.float().cuda()), dim=1)
#print("totalTarget size()",totalTarget.size())
#print("target_Var.size()",target_var.size())
#print("target size",target.size())
#print(target_var)
if len(target_var.size()) != 0:
totalTarget = torch.cat((totalTarget.float().cuda(),
target_var.float().cuda()),
dim=0)
loss = criterion(output.squeeze(), target.squeeze())
#print("loss = ",loss)
#auc = AUC(output.squeeze(), target_var.squeeze())
#print("auc=",auc)
#AUCs.update(auc)
losses.update(loss.item(), )
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
auc = AUC(totalOutput.squeeze(), totalTarget.squeeze())
AUCs.update(auc)
txtoutput = (
'Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
"totalOutput\totalTarget\t{totalOutput}\t{totalTarget}"
'AUC {top5.val:.3f} ({top5.avg:.3f})'.format(
idx,
len(val_loader),
batch_time=batch_time,
loss=losses,
totalOutput=totalOutput,
target=totalTarget,
top5=AUCs))
if idx % args.print_freq == 0:
#output = ('Test: [{0}/{1}]\t'
# 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
# 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
# "output/target {output}/{target}"
# 'AUC {aucs.val:.3f} ({aucs.avg:.3f})'.format(
# i, len(val_loader), batch_time=batch_time, loss=losses,output=output,target=target,
# aucs=AUCs))
print(txtoutput)
if log is not None:
log.write(txtoutput + '\n')
log.flush()
with open(
os.path.join(args.root_model, args.store_name, "val.txt"),
"a+") as txt:
txt.write(txtoutput + "\t" + str(URL) + "\t" + str(id) + "\n")
#break
txtoutput = (
'Testing Results: auc {auc.avg:.3f} Loss {loss.avg:.5f}'.format(
auc=AUCs, loss=losses))
with open(os.path.join(args.root_model, args.store_name, "val.txt"),
"a+") as txt:
txt.write(txtoutput + "\t" + str(URL) + "\t" + str(id) + "\n")
print(txtoutput)
if log is not None:
log.write(txtoutput + '\n')
log.flush()
if tf_writer is not None:
tf_writer.add_scalar('loss/test', losses.avg, epoch)
#tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
#tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
tf_writer.add_scalar('auc/test', AUCs.avg, epoch)
return AUCs.avg
def train_AETAllfast(train_dataloader, valid_dataloader, model, optimizer,
ema_optimizer, mixmatch_criterion, epoch, use_cuda,
aet_model, use_aet, aet_criterion, aet_optimizer,
iteration, Temporature, Alpha, lambda_aet, epoch_logger,
batch_logger, run_type, writer, num_classes, params,
ema_model):
model.train()
ema_model.eval() #do not train,only use for validation
if use_aet:
aet_model1 = aet_model[0]
aet_model2 = aet_model[1]
aet_model3 = aet_model[2]
aet_model4 = aet_model[3]
aet_model5 = aet_model[4]
aet_model1.train()
aet_model2.train()
aet_model3.train()
aet_model4.train()
aet_model5.train()
aet_optimizer1 = aet_optimizer[0]
aet_optimizer2 = aet_optimizer[1]
aet_optimizer3 = aet_optimizer[2]
aet_optimizer4 = aet_optimizer[3]
aet_optimizer5 = aet_optimizer[4]
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
AET_loss1 = AverageMeter()
AET_loss2 = AverageMeter()
AET_loss3 = AverageMeter()
AET_loss4 = AverageMeter()
AET_loss5 = AverageMeter()
Closs = AverageMeter() # loss_x in mixmatch
Enloss = AverageMeter() # loss_u in mixmatch
Entropy_Loss = AverageMeter()
WS = AverageMeter()
accuracies = AverageMeter()
top5 = AverageMeter()
WAET1 = AverageMeter()
WAET2 = AverageMeter()
WAET3 = AverageMeter()
WAET4 = AverageMeter()
WAET5 = AverageMeter()
KL_Loss = AverageMeter()
TeacherKL = AverageMeter()
end_time = time.time()
#labeled_train_iter = iter(train_dataloader)
#unlabeled_train_iter = iter(valid_dataloader)
prefetcher_label = data_prefetcher(train_dataloader)
prefetcher_unlabel = data_prefetcher(valid_dataloader)
labelled_reload = len(train_dataloader)
unlabelled_reload = len(valid_dataloader)
print("one iter for labelled data %d, one iter for unlabeled data %d" %
(labelled_reload, unlabelled_reload))
for batch_idx in range(iteration):
try:
read_data = prefetcher_label.next()
if (batch_idx % labelled_reload == 0
and batch_idx != 0) or read_data is None:
print("reloading labeled dataloader again!!")
prefetcher_label = data_prefetcher(train_dataloader)
(
img1, img1_t
), img2t, img3t, img4t, img5t, img6t, img7t, aff_parat, transform_matrix1t, oper_paramst, target = read_data
except:
#problems happen here, we need to reload the dataloader by our self
print(
"reloading labeled dataloader again!!"
) ##very important to check if the reloading worked in fast mode
prefetcher_label = data_prefetcher(train_dataloader)
(
img1, img1_t
), img2t, img3t, img4t, img5t, img6t, img7t, aff_parat, transform_matrix1t, oper_paramst, target = prefetcher_label.next(
)
#img1 = read_data[0] # original images
#target = read_data[1] # target label
try:
read_data1 = prefetcher_unlabel.next()
if (batch_idx % unlabelled_reload == 0
and batch_idx != 0) or read_data1 is None:
print("reloading unlabeled dataloader again!!")
prefetcher_unlabel = data_prefetcher(valid_dataloader)
(
img1_u, img1_u2
), img2, img3, img4, img5, img6, img7, aff_para, transform_matrix1, oper_params, _ = read_data1
except:
print(
"reloading unlabeled dataloader again!!"
) ##very important to check if the reloading worked in fast mode
prefetcher_unlabel = data_prefetcher(valid_dataloader)
(
img1_u, img1_u2
), img2, img3, img4, img5, img6, img7, aff_para, transform_matrix1, oper_params, _ = prefetcher_unlabel.next(
)
if batch_idx == 0 and epoch % 10 == 0:
if writer != None:
tmp1 = vutils.make_grid(img1_u,
normalize=True,
scale_each=True)
writer.add_image('Image1_u', tmp1, epoch)
tmp1 = vutils.make_grid(img1_u2,
normalize=True,
scale_each=True)
writer.add_image('Image1_u2', tmp1, epoch)
tmp1 = vutils.make_grid(img2, normalize=True, scale_each=True)
writer.add_image('Image_Projective', tmp1, epoch)
tmp1 = vutils.make_grid(img3, normalize=True, scale_each=True)
writer.add_image('Image_Affine', tmp1, epoch)
tmp1 = vutils.make_grid(img4, normalize=True, scale_each=True)
writer.add_image('Image_Similarity', tmp1, epoch)
tmp1 = vutils.make_grid(img5, normalize=True, scale_each=True)
writer.add_image('Image_Euclidean', tmp1, epoch)
tmp1 = vutils.make_grid(img6, normalize=True, scale_each=True)
writer.add_image('Image_CCBS', tmp1, epoch)
tmp1 = vutils.make_grid(img7, normalize=True, scale_each=True)
writer.add_image('Image_Patch', tmp1, epoch)
data_time.update(time.time() - end_time)
batch_size = img1.size(0)
targets_x = torch.zeros(batch_size,
num_classes).scatter_(1, target.view(-1, 1), 1)
if use_cuda:
img1, targets_x = img1.cuda(), targets_x.cuda(non_blocking=True)
img1_t = img1_t.cuda()
target = target.cuda()
img1_u = img1_u.cuda()
img1_u2 = img1_u2.cuda()
img2 = img2.cuda()
img3 = img3.cuda()
img4 = img4.cuda()
img5 = img5.cuda()
img6 = img6.cuda()
img7 = img7.cuda()
# img2t = img2t.cuda()
# img3t = img3t.cuda()
# img4t = img4t.cuda()
# img5t = img5t.cuda()
#img6t = img6t.cuda()
#img7t = img7t.cuda()
if use_aet:
aff_para = aff_para.cuda()
transform_matrix1 = transform_matrix1.cuda()
oper_params = oper_params.cuda()
with torch.no_grad():
# compute guessed labels of unlabel samples
outputs_u = model(img1_u)
outputs_u2 = model(img1_u2)
#outputs_u3 = model(img2)
#outputs_u4 = model(img3)
#outputs_u5= model(img4)
#outputs_u6 = model(img5)
#outputs_u7 = model(img6)
#outputs_u8 = model(img7)
if params['mix_mode'] == 0:
p = (torch.softmax(outputs_u, dim=1) +
torch.softmax(outputs_u2, dim=1)
) / 2 #+ torch.softmax(outputs_u3, dim=1)
# + torch.softmax(outputs_u4, dim=1)+ torch.softmax(outputs_u5, dim=1)+ torch.softmax(outputs_u6, dim=1)
#+ torch.softmax(outputs_u7, dim=1)+ torch.softmax(outputs_u8, dim=1)) / 4 # mean the softmax maatrix
else:
outputs_u8 = model(img7)
p = (torch.softmax(outputs_u, dim=1) + torch.softmax(
outputs_u2, dim=1) + torch.softmax(outputs_u8, dim=1)) / 3
pt = p**(1 / Temporature)
targets_u = pt / pt.sum(dim=1, keepdim=True)
targets_u = targets_u.detach(
) #also serve as p(y|x) with fixed weight(no propagating)
#now add KL divergence part
# outputs_x = model(img1)
# outputs_x2 = model(img1_t)
# p_x=(torch.softmax(outputs_x, dim=1) + torch.softmax(outputs_x2, dim=1))/2
# ptx = p_x ** (1 / Temporature)
# targets_xpred = ptx / ptx.sum(dim=1, keepdim=True)
# targets_xpred = targets_xpred.detach() # also serve as p(y|x) with fixed weight(no propagating)
mixup_count = 3
if params['mix_mode'] == 0:
loss1, loss_ent, Lx, Lu, w = Calculate_CLoss(
model, mixmatch_criterion, [img1, img1_u, img1_u2],
[targets_x, targets_u, targets_u], Alpha, batch_size, 1,
epoch + batch_idx / iteration, params, mixup_count, writer)
else:
loss1, loss_ent, Lx, Lu, w = Calculate_CLoss(
model, mixmatch_criterion, [img1, img1_u, img1_u2, img7],
[targets_x, targets_u, targets_u, targets_u], Alpha,
batch_size, 1, epoch + batch_idx / iteration, params,
mixup_count, writer)
#if batch_idx==0 and epoch==0:
# #write the result of mix up to show
Closs.update(Lx.item(), 1 * batch_size)
Enloss.update(Lu.item(), 1 * batch_size)
WS.update(w, 1 * batch_size)
loss = loss1 #here we do not add any parameter
if use_aet:
use_key = ['Attention']
feature_map1, atten1 = model(img1_u, use_key)
if params['KL_Lambda'] != 0:
#calculate KL mode
use_key1 = use_key + ['classifier']
feature_map2, atten2 = model(img2, use_key1)
pred_img2 = feature_map2[1]
feature_map2 = feature_map2[0]
else:
feature_map2, atten2 = model(img2, use_key)
pred_transform = aet_model1(feature_map1, feature_map2)
transform_matrix1 = transform_matrix1.view(-1, 8)
aet_loss = aet_criterion(pred_transform, transform_matrix1)
loss += aet_loss * min(
lambda_aet * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda'])
WAET1.update(
min(
lambda_aet * np.clip((epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0),
params['max_lambda']), img1.size(0))
if params['KL_Lambda'] != 0:
#calculate KL mode
use_key1 = use_key + ['classifier']
feature_map2, atten2 = model(img3, use_key1)
pred_img3 = feature_map2[1]
feature_map2 = feature_map2[0]
else:
feature_map2, atten2 = model(img3, use_key)
pred_transform2 = aet_model2(feature_map1, feature_map2)
aet_loss2 = aet_criterion(pred_transform2, aff_para)
loss += aet_loss2 * min(
params['lambda1'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda1'])
WAET2.update(
min(
params['lambda1'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda1']),
img1.size(0))
if params['KL_Lambda'] != 0:
#calculate KL mode
use_key1 = use_key + ['classifier']
feature_map2, atten2 = model(img4, use_key1)
pred_img4 = feature_map2[1]
feature_map2 = feature_map2[0]
else:
feature_map2, atten2 = model(img4, use_key)
pred_transform3 = aet_model3(feature_map1, feature_map2)
aet_loss3 = aet_criterion(pred_transform3, aff_para[:, :5])
loss += aet_loss3 * min(
params['lambda2'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda2'])
WAET3.update(
min(
params['lambda2'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda2']),
img1.size(0))
if params['KL_Lambda'] != 0:
#calculate KL mode
use_key1 = use_key + ['classifier']
feature_map2, atten2 = model(img5, use_key1)
pred_img5 = feature_map2[1]
feature_map2 = feature_map2[0]
else:
feature_map2, atten2 = model(img5, use_key)
pred_transform4 = aet_model4(feature_map1, feature_map2)
aet_loss4 = aet_criterion(pred_transform4, aff_para[:, :4])
loss += aet_loss4 * min(
params['lambda3'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda3'])
WAET4.update(
min(
params['lambda3'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda3']),
img1.size(0))
if params['KL_Lambda'] != 0:
#calculate KL mode
use_key1 = use_key + ['classifier']
feature_map2, atten2 = model(img6, use_key1)
pred_img6 = feature_map2[1]
feature_map2 = feature_map2[0]
else:
feature_map2, atten2 = model(img6, use_key)
pred_transform5 = aet_model5(feature_map1, feature_map2)
aet_loss5 = aet_criterion(pred_transform5, oper_params)
loss += aet_loss5 * min(
params['lambda4'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda4'])
WAET5.update(
min(
params['lambda4'] * np.clip(
(epoch * iteration + batch_idx) /
(1024 * iteration), 0.0, 1.0), params['max_lambda4']),
img1.size(0))
# Add KL divergence now for all the other augmentations
if params['KL_Lambda'] != 0:
#we do not have so much gpu memory, save everywhere, though it makes code massy
KLloss = 0
count_KL = 0
if params['max_lambda'] != 0: #designed for ablation study
KLloss += Calculate_KLloss_Result(pred_img2, targets_u)
count_KL += 1
if params['max_lambda1'] != 0:
KLloss += Calculate_KLloss_Result(pred_img3, targets_u)
count_KL += 1
if params['max_lambda2'] != 0:
KLloss += Calculate_KLloss_Result(pred_img4, targets_u)
count_KL += 1
if params['max_lambda3'] != 0:
KLloss += Calculate_KLloss_Result(pred_img5, targets_u)
count_KL += 1
if params['max_lambda4'] != 0:
KLloss += Calculate_KLloss_Result(pred_img6, targets_u)
count_KL += 1
if params['mix_mode'] != 0:
pred_img7 = model(
img7
) #again used for the final KL loss in agreement with the Teacher model
KLloss += Calculate_KLloss_Result(pred_img7, targets_u)
count_KL += 1
if count_KL != 0:
KLloss = KLloss / count_KL
KL_Loss.update(KLloss.item(), batch_size)
if params['KL_Lambda'] != 0:
loss += KLloss * params['KL_Lambda']
losses.update(loss.item(), img1.size(0))
if use_aet:
AET_loss1.update(aet_loss.item(), img1.size(0))
AET_loss2.update(aet_loss2.item(), img1.size(0))
AET_loss3.update(aet_loss3.item(), img1.size(0))
AET_loss4.update(aet_loss4.item(), img1.size(0))
AET_loss5.update(aet_loss5.item(), img1.size(0))
aet_optimizer1.zero_grad()
aet_optimizer2.zero_grad()
aet_optimizer3.zero_grad()
aet_optimizer4.zero_grad()
aet_optimizer5.zero_grad()
optimizer.zero_grad()
loss.backward()
optimizer.step()
if params['use_ema']:
ema_optimizer.step()
if use_aet:
aet_optimizer1.step()
aet_optimizer2.step()
aet_optimizer3.step()
aet_optimizer4.step()
aet_optimizer5.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
with torch.no_grad(): #save gpu space
classify_output = model(img1)
prec1, prec5 = Calculate_top_accuracy(classify_output.data,
target.data,
topk=(1, 5))
accuracies.update(prec1.item(), img1.size(0))
top5.update(prec5.item(), img1.size(0))
if use_aet:
batch_logger.log({
'epoch': epoch,
'batch': batch_idx + 1,
'iter': (epoch) * iteration + (batch_idx + 1),
'loss': losses.val,
'AET1': AET_loss1.val,
'AET2': AET_loss2.val,
'AET3': AET_loss3.val,
'AET4': AET_loss4.val,
'AET5': AET_loss5.val,
'Closs': Closs.val,
'Eloss': Enloss.val,
'KL': KL_Loss.val,
'wd': WS.val,
'waet': WAET1.val,
'top1': accuracies.val,
'top5': top5.val,
'lr1': optimizer.param_groups[0]['lr'],
'lr2': aet_optimizer1.param_groups[0]['lr']
})
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'AET1 {AET1.val:.4f} ({AET1.avg:.4f})\t'
'AET2 {AET2.val:.4f} ({AET2.avg:.4f})\t'
'AET3 {AET3.val:.4f} ({AET3.avg:.4f})\t'
'AET4 {AET4.val:.4f} ({AET4.avg:.4f})\t'
'AET5 {AET5.val:.4f} ({AET5.avg:.4f})\t'
'Closs {Closs.val:.4f} ({Closs.avg:.4f})\t'
'Enloss {Enloss.val:.4f} ({Enloss.avg:.4f})\t'
'KL {KL.val:.4f} ({KL.avg:.4f})\t'
'Top1 {acc.val:.3f} ({acc.avg:.3f})\t'
'Top5 {acc1.val:.3f} ({acc1.avg:.3f})\t'.format(
epoch,
batch_idx + 1,
iteration,
batch_time=batch_time,
data_time=data_time,
AET1=AET_loss1,
AET2=AET_loss2,
AET3=AET_loss3,
AET4=AET_loss4,
AET5=AET_loss5,
Closs=Closs,
Enloss=Enloss,
KL=KL_Loss,
loss=losses,
acc=accuracies,
acc1=top5))
if use_aet:
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'AET1': AET_loss1.avg,
'AET2': AET_loss2.avg,
'AET3': AET_loss3.avg,
'AET4': AET_loss4.avg,
'AET5': AET_loss5.avg,
'Closs': Closs.avg,
'Eloss': Enloss.avg,
'KL': KL_Loss.avg,
'wd': WS.avg,
'waet': WAET1.avg,
'top1': accuracies.avg,
'top5': top5.avg,
'lr1': optimizer.param_groups[0]['lr'],
'lr2': aet_optimizer1.param_groups[0]['lr']
})
if writer != None:
writer.add_scalars('Data/Loss', {'train_loss': losses.avg}, epoch)
writer.add_scalars('Data/AET_Loss1', {'train_loss': AET_loss1.avg},
epoch)
writer.add_scalars('Data/AET_Loss2', {'train_loss': AET_loss2.avg},
epoch)
writer.add_scalars('Data/AET_Loss3', {'train_loss': AET_loss3.avg},
epoch)
writer.add_scalars('Data/AET_Loss4', {'train_loss': AET_loss4.avg},
epoch)
writer.add_scalars('Data/AET_Loss5', {'train_loss': AET_loss5.avg},
epoch)
writer.add_scalars('Data/CLoss', {'train_loss': Closs.avg}, epoch)
writer.add_scalars('Data/ELoss', {'train_loss': Enloss.avg}, epoch)
#writer.add_scalars('Data/Entropy', {'train_loss': Entropy_Loss.avg}, epoch)
writer.add_scalars('Data/KLloss', {'train_loss': KL_Loss.avg},
epoch)
#writer.add_scalars('Data/KL_Teacher_loss', {'train_loss': TeacherKL.avg}, epoch)
writer.add_scalars('Accuracy/top1', {'train_accu': accuracies.avg},
epoch)
writer.add_scalars('Accuracy/top5', {'train_accu': top5.avg},
epoch)
writer.add_scalar('Weight/Lambda1', WAET1.avg, epoch)
writer.add_scalar('Weight/Lambda2', WAET2.avg, epoch)
writer.add_scalar('Weight/Lambda3', WAET3.avg, epoch)
writer.add_scalar('Weight/Lambda4', WAET4.avg, epoch)
writer.add_scalar('Weight/Lambda5', WAET5.avg, epoch)
writer.add_scalar('Weight/Beta', WS.avg, epoch)
writer.add_scalar('LR/LR1', optimizer.param_groups[0]['lr'], epoch)
writer.add_scalar('LR/LR2', aet_optimizer1.param_groups[0]['lr'],
epoch)
if epoch % 10 == 0:
for name, param in model.named_parameters():
writer.add_histogram('CLF' + name,
param.clone().cpu().data.numpy(),
epoch)
for name, param in aet_model1.named_parameters():
writer.add_histogram('AET1' + name,
param.clone().cpu().data.numpy(),
epoch)
for name, param in aet_model2.named_parameters():
writer.add_histogram('AET2' + name,
param.clone().cpu().data.numpy(),
epoch)
for name, param in aet_model3.named_parameters():
writer.add_histogram('AET3' + name,
param.clone().cpu().data.numpy(),
epoch)
for name, param in aet_model4.named_parameters():
writer.add_histogram('AET4' + name,
param.clone().cpu().data.numpy(),
epoch)
for name, param in aet_model5.named_parameters():
writer.add_histogram('AET5' + name,
param.clone().cpu().data.numpy(),
epoch)
if params['use_ema']:
ema_optimizer.step(bn=True)
return losses.avg, Closs.avg, Enloss.avg
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer, args, rank):
torch.cuda.empty_cache()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode
model.train()
end = time.time()
model.zero_grad()
loss_tmp = []
acc_tmp = []
for i, (input, target) in enumerate(train_loader):
adjust_learning_rate(optimizer, epoch, args.lr_type, args.lr_steps, args, (epoch-1) + float(i) / len(train_loader))
i += 1
# if (i+1) % args.batch_multiplier == 0:
# optimizer.step()
# optimizer.zero_grad()
# measure data loading time
data_time.update(time.time() - end)
if args.gpu is not None:
input = input.cuda(args.gpu, non_blocking=True)
target = target.cuda(args.gpu, non_blocking=True)
input_var = input
target_var = target
# compute output
output = model(input_var)
loss = criterion(output, target_var) / args.batch_multiplier # divide batch_multiplier as grad accumulation
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item()*args.batch_multiplier, input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
loss_tmp.append(loss.item()*args.batch_multiplier)
acc_tmp.append(prec1.item())
# compute gradient and do SGD step
if i % args.batch_multiplier != 0:
if args.multiprocessing_distributed:
with model.no_sync():
loss.backward()
else:
loss.backward()
else:
loss.backward()
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient)
optimizer.step()
# optimizer.zero_grad()
if rank == 0:
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % (args.print_freq*args.batch_multiplier*5) == 0:
tf_writer.add_scalar('loss/step', np.mean(loss_tmp), ((epoch-1)*len(train_loader)+i)/args.batch_multiplier)
tf_writer.add_scalar('acc/step', np.mean(acc_tmp), ((epoch-1)*len(train_loader)+i)/args.batch_multiplier)
loss_tmp = []
acc_tmp = []
if i % (args.print_freq * args.batch_multiplier) == 0:
output = ('Epoch: [{0:3d}][{1:4d}/{2:4d}], lr: {lr:.5f}\t'
'Time {time:.1f}\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch, int(i/args.batch_multiplier), int(len(train_loader)/args.batch_multiplier), time=(time.time()-global_time)/60.,
data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
if i % (args.print_freq*args.batch_multiplier*10) == 0:
for tag, value in model.named_parameters():
tag = tag.replace('.', '/')
tf_writer.add_histogram('weights/'+tag, value.detach(), (epoch*len(train_loader)+i)/args.batch_multiplier)
tf_writer.add_histogram('grads/'+tag, value.grad.detach().abs().mean(), (epoch*len(train_loader)+i)/args.batch_multiplier)
if i % args.batch_multiplier == 0:
optimizer.zero_grad()
if rank == 0:
tf_writer.add_scalar('loss/train', losses.avg, epoch-1)
tf_writer.add_scalar('acc/train_top1', top1.avg, epoch-1)
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
if args.no_partialbn:
try:
model.module.partialBN(False)
except:
model.partialBN(False)
else:
try:
model.module.partialBN(True)
except:
model.partialBN(True)
model.train()
end = time.time()
for idx, (input, target) in enumerate(train_loader):
data_time.update(time.time() - end)
input, target = input.cuda(), target.cuda()
output = model(input)
loss = criterion(output, target)
# accuracy and loss
prec1, = accuracy(output.data, target, topk=(1, ))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
# gradient and optimizer
loss.backward()
if (idx + 1) % args.update_weight == 0:
optimizer.step()
optimizer.zero_grad()
# time
batch_time.update(time.time() - end)
end = time.time()
if (idx + 1) % args.print_freq == 0:
output = ('Train: epoch-{0} ({1}/{2})\t'
'batch_time {batch_time.avg:.2f}\t\t'
'data_time {data_time.avg:.2f}\t\t'
'loss {loss.avg:.3f}\t'
'prec@1 {top1.avg:.2f}\t'.format(epoch,
idx + 1,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1))
batch_time.reset()
data_time.reset()
losses.reset()
top1.reset()
print(output)
log.write(output + '\n')
log.flush()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(train_loader,
model,
criterion,
optimizer,
epoch,
logger=None,
scheduler=None):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
data_time.update(time.time() - end)
target = target.cuda()
input_var = input.cuda()
target_var = target
output = model(input_var)
loss = criterion(output, target_var)
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
optimizer.zero_grad()
loss.backward()
if args.clip_gradient is not None:
clip_grad_norm_(model.parameters(), args.clip_gradient)
optimizer.step()
scheduler.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
logger.info(('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5,
lr=optimizer.param_groups[-1]['lr']))) # TODO
return losses.avg, top1.avg, top5.avg
def train(train_loader, model, criterion, optimizer, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to train mode
model.train()
end = time.time()
for i, (r, f, path, label, video_index, seg_index,
verify) in enumerate(train_loader):
r = r.cuda()
f = f.cuda()
output, _ = model(r, f)
target_var = label.cuda(async=True).type(torch.cuda.LongTensor)
loss = criterion(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output, target_var, topk=(1, 5))
losses.update(loss.item(), r.shape[0])
top1.update(prec1.item(), r.shape[0])
top5.update(prec5.item(), r.shape[0])
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
if args.clip_gradient is not None:
total_norm = torch.nn.utils.clip_grad_norm(model.parameters(),
args.clip_gradient)
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print(('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
top1=top1,
top5=top5,
lr=optimizer.param_groups[-1]['lr'])))
def validate(val_loader, model, criterion, logger=None):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
model.eval()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(val_loader):
target = target.cuda()
output = model(input)
loss = criterion(output, target)
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
loss = reduce_tensor(loss)
prec1 = reduce_tensor(prec1)
prec5 = reduce_tensor(prec5)
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
logger.info(
('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5)))
logger.info((
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses)))
return top1.avg, top5.avg, losses.avg
def validate(val_loader,
model,
criterion,
epoch,
log=None,
tf_writer=None,
scheduler=None):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
each_class_output_buffer = torch.Tensor()
each_class_target_buffer = torch.Tensor()
for i, (input, target, path) in enumerate(val_loader):
target = target.cuda()
# compute output
output = model(input)
loss = criterion(output, target)
# tmp = torch.nn.functional.softmax(output[0]).cpu().detach().tolist()
# for idx in range(3):
# print("predict label: {}, conf: {:.2f}".format(idx2label[idx], float(tmp[idx])))
# measure accuracy and record loss
# with open("/home/huyihui/workspace/tsm-huawei/predict_result3.txt", "a+", encoding="utf-8") as f:
# f.write(f"{path[0]},{tmp[0]},{tmp[1]}\n")
prec1, prec5 = accuracy(output.data, target, topk=(1, 2))
each_class_output_buffer = torch.cat(
(each_class_output_buffer, output.detach().cpu().float()), 0)
each_class_target_buffer = torch.cat(
(each_class_target_buffer, target.detach().cpu().float()), 0)
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
if scheduler:
scheduler.step(loss.avg)
output = (
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses))
# acc = calculate_accuracy_each_class(each_class_output_buffer, each_class_target_buffer, 2)
print(output)
# print("-------------------\n", acc)
if log is not None:
log.write(output + '\n')
log.flush()
if tf_writer is not None:
tf_writer.add_scalar('loss/test', losses.avg, epoch)
tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
return top1.avg
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
all_preds = []
all_target = []
with torch.no_grad():
for i, (input, target) in enumerate(val_loader):
target = target.cuda()
# compute output
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
all_target.append(target.cpu().numpy())
all_preds.append(
output.data.topk(5, 1, True, True)[1].cpu().numpy())
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
output = (
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
#import ipdb; ipdb.set_trace()
columns = [
'idx', 'target', 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'
]
lines = []
with open('log/val_results_epoch{}.txt'.format(str(epoch)), 'w') as f:
for b in range(len(all_target)):
for i in range(len(all_target[b])):
preds_formatted = [
s.rjust(6) for s in all_preds[b][i].astype(str).tolist()
]
lines.append(
str(all_target[b][i]).rjust(6) + '\t' +
'\t'.join(preds_formatted))
header = '\t'.join([s.rjust(6) for s in columns])
f.write(header + '\n')
for i in range(len(lines)):
f.write(str(i).rjust(6) + '\t' + lines[i] + '\n')
if tf_writer is not None:
tf_writer.add_scalar('loss/test', losses.avg, epoch)
tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
return top1.avg
def train(train_loader, model, decoder, criterion, optimizer, epoch, log,
tf_writer, index2wordDict):
#global trainDataloader, valDataloader
#print(len(train_loader))
#print(trainDataloader._getMode())
#print(valDataloader._getMode())
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
BLEUs = AverageMeter()
teacher_forcing_ratio = 0.5
bptt = 64
#losses_extra = AverageMeter()
#top1 = AverageMeter()
#top5 = AverageMeter()
#top1_extra = AverageMeter()
#top5_extra = AverageMeter()
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode
model.train()
#print("308")
end = time.time()
#print("in train")
with open(os.path.join(args.root_model, args.store_name, "train.txt"),
"a+") as txt:
txt.write("epoch\t" + str(epoch) + "\n")
for i, data in enumerate(train_loader):
loss = 0
#print("data fetch finish ",i)
#print(data)
#if isinstance(data, bool): #img DNE
# continue
if not data:
continue
data_time.update(time.time() - end)
#print(data[0])
#print(data[1])
[URL, id, label, clips] = data
input_video = torch.tensor([], dtype=torch.float).cuda()
input_caption = torch.tensor([], dtype=torch.long).cuda()
#cap_nums = []
for clip in range(0, len(clips), max(len(clips) // 50, 1)):
#print(type(input_video.size()))
#print(input_video.size())
if list(input_video.size()) != [0]:
if input_video.size()[1] > 50:
break
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat((input_video, video.float().cuda()), 1)
input_caption = torch.cat((input_caption, caption.long().cuda()),
0)
#cap_nums.append(clip[2])
#print(input_video.size())
#print(input_caption.size())
input_video = input_video.view(1, -1, 3, crop_size, crop_size)
input_caption = input_caption.view(1, -1, input_caption.size()[-1], 1)
#print(input_video.size())
#print(input_caption.size())
target = label.cuda()
input_video_var = torch.autograd.Variable(input_video)
input_caption_var = torch.autograd.Variable(input_caption)
target_var = torch.autograd.Variable(target)
# compute output
wExtraLoss = 1 if args.prune == '' else 0.1
#print("input video size=",input_video.size())
encoder_output = model(input_video_var,
input_caption_var) #size=(1, frames, 2048)
print("encoder_output size=", encoder_output.size())
t_input = torch.zeros((1, 64, 2048)).cuda()
for idx_e, e in enumerate(encoder_output[0]):
t_input[0, idx_e] = e
print("transformer input size(),", t_input.size())
decoder_outputs = []
src_mask = decoder.generate_square_subsequent_mask(
target.size(1)).cuda()
#print("target size=,",target.size())
#print("target= ", target)
decoder_output = decoder(t_input, tar=target, mask=src_mask)
#print("tar = ",tar)
#print("tar size = ",tar.size())
decoder_outputs = decoder_output.argmax(-1)
#print("decoder_outputs=",decoder_outputs)
#for idx_t, tar in enumerate(target[0]):
# decoder_output_word = t_input[0,idx_t].argmax().cpu()
# decoder_outputs.append(decoder_output_word)
# print(tar)
# print(decoder_output_word)
#loss += criterion(t_input[idx_t], tar)
#print(decoder_output.transpose(0,1).size())
#print(target.size())
#print(criterion)
loss = criterion(
decoder_output.transpose(0, 1).squeeze(), target.squeeze())
#print("decoder_output_word=",decoder_output_word)
#ttar = torch.zeros(17469, dtype=torch.long).unsqueeze(0).cuda()
#ttar[0,tar.data] = torch.tensor(1, dtype=torch.long).cuda()
#print("ttar size={}, decoder_output size={}".format(ttar.size(), decoder_output.size()))
#print(decoder_output)
#print(args.loss_type)
#print(criterion)
print("{0:*^50}".format("loss " + str(loss)))
#exit()
#print("target)var=",target_var.size())
#print(output)
#print(target_var)
#print("label size=",label.size())
#print(output.size())
#print(target_var.size())
#loss_main = criterion(output.squeeze(), target_var.squeeze())
#loss_main = 0
#lloss = torch.sub(output, target_var).squeeze()
#for l in lloss:
# loss_main += l**2
#extra_loss = criterion(extra, target_var)*wExtraLoss
#loss = loss_main + extra_loss
#loss = loss_main
#print("loss=",loss)
losses.update(loss.item(), )
#print(decoder_outputs)
#print(decoder_outputs[0].item())
decoder_sentence = [
index2wordDict[index.item()] for index in decoder_outputs
]
print(decoder_sentence)
#print(target.size())
target_sentence = [index2wordDict[index.item()] for index in target[0]]
print(target_sentence)
bleu = sentence_bleu([
target_sentence
], decoder_sentence) #default weight is [0.25,0.25,0.25,0.25] = bleu-4
print("bleu =", bleu)
BLEUs.update(bleu)
# compute gradient and do SGD step
st = time.time()
loss.backward()
#print("{0:*^50}".format("after backward\t"+str(time.time()-st)))
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(),
args.clip_gradient)
st = time.time()
optimizer.step()
#print("{0:*^50}".format("after step\t"+str(time.time()-st)))
st = time.time()
optimizer.zero_grad()
#print("{0:*^50}".format("after zero_grad\t"+str(time.time()-st)))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
txtoutput = ('Epoch: [{0}][{1}/{2}], lr: {lr:.7f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
"output/target{output}/{target}"
'bleu {bleu.val:.4f} ({bleu.avg:.3f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
output=decoder_sentence,
target=target_sentence,
data_time=data_time,
loss=losses,
bleu=BLEUs,
lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
if i % args.print_freq == 0:
print(txtoutput)
log.write(txtoutput + '\n')
log.flush()
with open(os.path.join(args.root_model, args.store_name, "train.txt"),
"a+") as txt:
txt.write(txtoutput + "\n")
print("**" * 50)
#break
#exit()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
#tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
#tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
tf_writer.add_scalar('auc/train', BLEUs.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(labeled_trainloader, unlabeled_trainloader, model, criterion,
optimizer, epoch, log):
batch_time = AverageMeter()
data_time = AverageMeter()
total_losses = AverageMeter()
supervised_losses = AverageMeter()
contrastive_losses = AverageMeter()
group_contrastive_losses = AverageMeter()
pl_losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
model = model.cuda()
if args.no_partialbn:
model.module.partialBN(False)
else:
model.module.partialBN(True)
# switch to train mode
model.train()
if epoch >= args.sup_thresh or (args.use_finetuning
and epoch >= args.finetune_start_epoch):
data_loader = zip(labeled_trainloader, unlabeled_trainloader)
else:
data_loader = labeled_trainloader
end = time.time()
for i, data in enumerate(data_loader):
# measure data loading time
data_time.update(time.time() - end)
#reseting losses
contrastive_loss = torch.tensor(0.0).cuda()
pl_loss = torch.tensor(0.0).cuda()
loss = torch.tensor(0.0).cuda()
group_contrastive_loss = torch.tensor(0.0).cuda()
if epoch >= args.sup_thresh or (args.use_finetuning and
epoch >= args.finetune_start_epoch):
(labeled_data, unlabeled_data) = data
images_fast, images_slow = unlabeled_data
images_slow = images_slow.cuda()
images_fast = images_fast.cuda()
images_slow = torch.autograd.Variable(images_slow)
images_fast = torch.autograd.Variable(images_fast)
# contrastive_loss
output_fast = model(images_fast)
if not args.use_finetuning or epoch < args.finetune_start_epoch:
output_slow = model(images_slow, unlabeled=True)
output_fast_detach = output_fast.detach()
if epoch >= args.sup_thresh and epoch < args.finetune_start_epoch:
contrastive_loss = simclr_loss(
torch.softmax(output_fast_detach, dim=1),
torch.softmax(output_slow, dim=1))
if args.use_group_contrastive:
grp_unlabeled_8seg = get_group(output_fast_detach)
grp_unlabeled_4seg = get_group(output_slow)
group_contrastive_loss = compute_group_contrastive_loss(
grp_unlabeled_8seg, grp_unlabeled_4seg)
elif args.use_finetuning and epoch >= args.finetune_start_epoch:
pseudo_label = torch.softmax(output_fast_detach, dim=-1)
max_probs, targets_pl = torch.max(pseudo_label, dim=-1)
mask = max_probs.ge(args.threshold).float()
targets_pl = torch.autograd.Variable(targets_pl)
pl_loss = (F.cross_entropy(
output_fast, targets_pl, reduction='none') * mask).mean()
else:
labeled_data = data
input, target = labeled_data
target = target.cuda()
input = input.cuda()
input = torch.autograd.Variable(input)
target_var = torch.autograd.Variable(target)
output = model(input)
loss = criterion(output, target_var)
total_loss = loss + args.gamma * contrastive_loss + group_contrastive_loss + args.gamma_finetune * pl_loss
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
if epoch >= args.sup_thresh:
total_losses.update(total_loss.item(),
input.size(0) + args.mu * input.size(0))
else:
total_losses.update(total_loss.item(), input.size(0))
supervised_losses.update(loss.item(), input.size(0))
contrastive_losses.update(contrastive_loss.item(),
input.size(0) + args.mu * input.size(0))
group_contrastive_losses.update(
group_contrastive_loss.item(),
input.size(0) + args.mu * input.size(0))
pl_losses.update(pl_loss.item(),
input.size(0) + args.mu * input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# compute gradient and do SGD step
total_loss.backward()
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(),
args.clip_gradient)
optimizer.step()
optimizer.zero_grad()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = (
'Epoch: [{0}][{1}], lr: {lr:.5f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'TotalLoss {total_loss.val:.4f} ({total_loss.avg:.4f})\t'
'Supervised Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Contrastive_Loss {contrastive_loss.val:.4f} ({contrastive_loss.avg:.4f})\t'
'Group_contrastive_Loss {group_contrastive_loss.val:.4f} ({group_contrastive_loss.avg:.4f})\t'
'Pseudo_Loss {pl_loss.val:.4f} ({pl_loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
batch_time=batch_time,
data_time=data_time,
total_loss=total_losses,
loss=supervised_losses,
contrastive_loss=contrastive_losses,
group_contrastive_loss=group_contrastive_losses,
pl_loss=pl_losses,
top1=top1,
top5=top5,
lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
def build_database(model_rgb, model_flow, dataset, threshold):
assert isinstance(threshold, float)
model_rgb.eval()
model_flow.eval()
end = time.time()
data_time = AverageMeter()
database = [[] for _ in range(num_class)]
for ind, (gen, label, _, _, length) in dataset:
# gen(), label, rgb_record.path, video_index, self.segment_num
logit_rgb = []
logit_flow = []
weight_rgb = []
weight_flow = []
feat_rgb = []
feat_flow = []
for seg in gen:
data_time.update(time.time() - end)
r, f = seg
with torch.no_grad():
r = r.cuda()
f = f.cuda()
lr, wr, fr = model_rgb(r)
lf, wf, ff = model_flow(f)
logit_rgb.append(lr)
logit_flow.append(lf)
weight_rgb.append(wr)
weight_flow.append(wf)
feat_rgb.append(fr)
feat_flow.append(f)
logit_rgb = torch.cat(logit_rgb)
logit_flow = torch.cat(logit_flow)
weight_rgb = torch.cat(weight_rgb)
weight_flow = torch.cat(weight_flow)
feat_rgb = torch.cat(feat_rgb)
feat_flow = torch.cat(feat_flow)
if isinstance(label, int):
score, score_rgb, weight_rgb, score_flow, weight_flow \
= get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, label, alpha) # [length]
for sind, s in enumerate(score):
if s > threshold:
database[label].append({'feat_rgb': feat_rgb[sind],
'feat_flow': feat_flow[sind],
'feat_cat': torch.cat([feat_rgb[sind], feat_flow[sind]], 1),
'score': s,
'raw_score_rgb': score_rgb[sind],
'raw_score_flow': score_flow[sind],
'attention_rgb': weight_rgb[sind],
'attention_flow': weight_flow[sind]})
elif isinstance(label, list):
for lab in label:
score, score_rgb, weight_rgb, score_flow, weight_flow \
= get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, lab, alpha)
if s > threshold:
database[lab].append({'feat_rgb': feat_rgb[sind],
'feat_flow': feat_flow[sind],
'feat_cat': torch.cat([feat_rgb[sind], feat_flow[sind]], 1),
'score': s,
'raw_score_rgb': score_rgb[sind],
'raw_score_flow': score_flow[sind],
'attention_rgb': weight_rgb[sind],
'attention_flow': weight_flow[sind]})
print("Database has generated. It has {} classes, and num of element in each class is: {}".format(len(database),
[len(dc) for dc in
database]))
return database
def get_average_meters(number):
return [AverageMeter() for _ in range(number)]
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None):
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(val_loader):
target = target.cuda()
# compute output
output = model(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1.item(), input.size(0))
top5.update(prec5.item(), input.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
output = ('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
output = (
'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}'
.format(top1=top1, top5=top5, loss=losses))
print(output)
if log is not None:
log.write(output + '\n')
log.flush()
if tf_writer is not None:
tf_writer.add_scalar('loss/test', losses.avg, epoch)
tf_writer.add_scalar('acc/test_top1', top1.avg, epoch)
tf_writer.add_scalar('acc/test_top5', top5.avg, epoch)
return top1.avg
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer):
#global trainDataloader, valDataloader
#print(len(train_loader))
#print(trainDataloader._getMode())
#print(valDataloader._getMode())
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
AUCs = AverageMeter()
#losses_extra = AverageMeter()
#top1 = AverageMeter()
#top5 = AverageMeter()
#top1_extra = AverageMeter()
#top5_extra = AverageMeter()
#if args.no_partialbn:
# model.module.partialBN(False)
#else:
# model.module.partialBN(True)
# switch to train mode
model.train()
#print("308")
end = time.time()
#print("in train")
with open(os.path.join(args.root_model, args.store_name, "train.txt"),
"a+") as txt:
txt.write("epoch\t" + str(epoch) + "\n")
for idx, data in enumerate(train_loader):
#print("data fetch finish ",i)
#print(data)
#if isinstance(data, bool): #img DNE
# continue
if not data:
continue
data_time.update(time.time() - end)
#print(data[0])
#print(data[1])
[URL, id, sift, label, clips] = data
#cap_nums = []
#print("sift=",sift)
for i, s in enumerate(sift):
input_video = torch.tensor([], dtype=torch.float).cuda()
input_caption = torch.tensor([], dtype=torch.long).cuda()
#print("i={},s={},lensift={},idx={},epoch={}".format(i, s, len(sift), idx, epoch))
if i + 1 < len(sift):
#print(sift[i+1])
#print(s)
#print(int(sift[i+1])-int(s))
if int(sift[i + 1]) - int(s) < 50:
for clip in range(s, sift[i + 1]):
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(), video.float().cuda()),
1)
#print(input_video.size())
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
#print(input_caption.size())
#print(sift[i+1])
#print(s)
#print(label[0,s: sift[i+1]])
target = label[0, s:sift[i + 1]].cuda()
else:
for clip in range(s, sift[i + 1],
(int(sift[i + 1]) - int(s)) // 50):
#print("clip={},s={},sift[i+1]={}".format(clip, s, sift[i+1]))
if (clip - s) / (
(int(sift[i + 1]) - int(s)) // 50) >= 50:
break
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(), video.float().cuda()),
1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0, s:sift[i + 1]:(
(int(sift[i + 1]) - int(s)) // 50)].cuda()
target = target[:50].cuda()
else:
length = len(clips)
if int(length) - int(s) < 50:
for clip in range(s, length):
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(), video.float().cuda()),
1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0, s:length].cuda()
else:
for clip in range(s, length, (int(length) - int(s)) // 50):
if (clip - s) / ((int(length) - int(s)) // 50) >= 50:
break
video = clips[clip][0]
caption = clips[clip][1]
input_video = torch.cat(
(input_video.float().cuda(), video.float().cuda()),
1)
input_caption = torch.cat(
(input_caption, caption.long().cuda()), 0)
target = label[0, s:length:(length - s) // 50].cuda()
target = target[:50]
'''
video = clips[-1][0]
caption = clips[-1][1]
input_video = torch.cat((input_video.float().cuda(), video.float().cuda()),1)
input_caption = torch.cat((input_caption, caption.long().cuda()),0)
target = label[0,-1].cuda()
'''
input_video = input_video.view(1, -1, 3, crop_size, crop_size)
#input_caption = input_caption.view(1,-1,input_caption.size()[-1],1)
input_caption = input_caption.view(1, -1, 64, 1)
#print("target=",target.size())
input_video_var = torch.autograd.Variable(input_video)
input_caption_var = torch.autograd.Variable(input_caption)
target_var = torch.autograd.Variable(target)
# compute output
wExtraLoss = 1 if args.prune == '' else 0.1
output = model(input_video_var, input_caption_var)
#print("outputsize=",output.size())
#print("target_var=",target_var.size())
#print("target=",target.size())
loss_main = criterion(output.squeeze(), target_var.squeeze())
loss = loss_main
losses.update(loss_main.item(), )
auc = AUC(output.squeeze(), target_var.squeeze())
#print("auc=",auc)
AUCs.update(auc)
#print("after AUCs update")
# compute gradient and do SGD step
st = time.time()
loss.backward()
#print("{0:*^50}".format("after backward\t"+str(time.time()-st)))
if args.clip_gradient is not None:
total_norm = clip_grad_norm_(model.parameters(),
args.clip_gradient)
st = time.time()
optimizer.step()
#print("{0:*^50}".format("after step\t"+str(time.time()-st)))
st = time.time()
optimizer.zero_grad()
#print("{0:*^50}".format("after zero_grad\t"+str(time.time()-st)))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
'''
if i % args.print_freq == 0:
output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.7f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Loss_h {losses_extra.val:.4f} ({losses_extra.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, losses_extra=losses_extra, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
print(output)
log.write(output + '\n')
log.flush()
'''
txtoutput = ('Epoch: [{0}][{1}/{2}], lr: {lr:.7f}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
"output/target\t{output}/\t{target}\t"
'auc {auc.val:.4f} ({auc.avg:.3f})\t'.format(
epoch,
idx,
len(train_loader),
batch_time=batch_time,
output=output,
target=target,
data_time=data_time,
loss=losses,
auc=AUCs,
lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO
if idx % args.print_freq == 0:
print(txtoutput)
log.write(txtoutput + '\n')
log.flush()
print("**" * 50)
with open(os.path.join(args.root_model, args.store_name, "train.txt"),
"a+") as txt:
txt.write(txtoutput + "\t" + str(URL) + "\t" + str(id) + "\n")
#break
#exit()
tf_writer.add_scalar('loss/train', losses.avg, epoch)
#tf_writer.add_scalar('acc/train_top1', top1.avg, epoch)
#tf_writer.add_scalar('acc/train_top5', top5.avg, epoch)
tf_writer.add_scalar('auc/train', AUCs.avg, epoch)
tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
