def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None): batch_time = AverageMeter() losses = AverageMeter() aps = AverageMeter() pre_list = [] target_list = [] # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(val_loader): target = target.cuda().float() # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss pre_list.append(output.data.cpu().numpy()) target_list.append(target.cpu().numpy()) losses.update(loss.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Test: [{0}/{1}]\t' 'Time {batch_time.avg:.3f}\t' 'Loss {loss.avg:.4f}'.format(i, len(val_loader), batch_time=batch_time, loss=losses)) print(output) if log is not None: log.write(output + '\n') log.flush() ap_value = map_sklearn(np.vstack(target_list), np.vstack(pre_list)) output = ('Testing Results\t mAP: {}'.format(np.around(ap_value, 3))) print(output) if log is not None: log.write(output + '\n') log.flush() if tf_writer is not None: tf_writer.add_scalar('loss/test', losses.avg, epoch) tf_writer.add_scalar('acc/test_map', ap_value, epoch) return ap_value
def build_database(model_rgb, model_flow, dataset, threshold): assert isinstance(threshold, float) model_rgb.eval() model_flow.eval() end = time.time() data_time = AverageMeter() database = [[] for _ in range(num_class)] for i, (input, label, _, _, _) in enumerate(dataset): with torch.no_grad(): input_var = torch.from_numpy(input).cuda() target_var = torch.Tensor([label]).type(torch.cuda.LongTensor) assert isinstance(alpha, float) and (alpha < 1) and (alpha > 0), alpha logit_rgb, weight_rgb = model_rgb(input_var[:, :400]) logit_flow, weight_flow = model_flow(input_var[:, 400:]) if isinstance(label, int): score, score_rgb, weight_rgb, score_flow, weight_flow \ = get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, label, alpha) # [length] for sind, s in enumerate(score): if s > threshold: database[label].append({ 'feat_rgb': input[sind, :400], 'feat_flow': input[sind, 400:], 'feat_cat': input[sind, :], 'score': s, 'raw_score_rgb': score_rgb[sind], 'raw_score_flow': score_flow[sind], 'attention_rgb': weight_rgb[sind], 'attention_flow': weight_flow[sind] }) elif isinstance(label, list): for lab in label: assert isinstance(lab, int) score, score_rgb, weight_rgb, score_flow, weight_flow \ = get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, lab, alpha) if s > threshold: database[lab].append({ 'feat_rgb': input[sind, :400], 'feat_flow': input[sind, 400:], 'feat_cat': input[sind, :], 'score': s, 'raw_score_rgb': score_rgb[sind], 'raw_score_flow': score_flow[sind], 'attention_rgb': weight_rgb[sind], 'attention_flow': weight_flow[sind] }) print( "Database has generated. It has {} classes, and num of element in each class is: {}" .format(len(database), [len(dc) for dc in database])) return database
def make_predictions(model, testloader, seq_length, encode_feature_len, params): model.eval() batch_time = AverageMeter() data_time = AverageMeter() end_time = time.time() iteration = len(testloader) AA_Attention_Records = np.zeros([len(testloader.dataset), seq_length]) Predict_Array = np.zeros(len(testloader.dataset)) aa_feature_len = encode_feature_len + seq_length # for batch_idx, data in enumerate(testloader): aa_feature, index = data aa_feature = aa_feature.float() aa_feature = aa_feature.cuda() batch_size = aa_feature.size(0) h_0 = np.random.uniform(-0.5, high=0.5, size=(batch_size, aa_feature_len)) c_0 = np.random.uniform(-0.5, high=0.5, size=(batch_size, aa_feature_len)) h_0 = torch.from_numpy(h_0).float() c_0 = torch.from_numpy(c_0).float() h_0 = h_0.cuda() c_0 = c_0.cuda() data_time.update(time.time() - end_time, batch_size) with torch.no_grad(): pred_batch, attention_batch = model(aa_feature, (h_0, c_0), use_attention=True) index = index.detach().cpu().numpy() attention_batch = attention_batch.detach().cpu().numpy() AA_Attention_Records[index] = attention_batch[:, 0, :] pred = torch.softmax(pred_batch, dim=1) pred = pred.detach().cpu().numpy() Predict_Array[index] = pred[:, 1] batch_time.update(time.time() - end_time, batch_size) end_time = time.time() print('Iter: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format( batch_idx + 1, iteration, batch_time=batch_time, data_time=data_time, )) return Predict_Array, AA_Attention_Records
def validate(val_loader, model, criterion, epoch, log, tf_writer): losses = AverageMeter() top1 = AverageMeter() model.eval() with torch.no_grad(): for input, target in val_loader: input, target = input.cuda(), target.cuda() output = model(input) loss = criterion(output, target) # accuracy and loss prec1, = accuracy(output.data, target, topk=(1, )) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) output = ('Validate: Prec@1 {top1.avg:.2f} Loss {loss.avg:.3f}'.format( top1=top1, loss=losses)) print(output) log.write(output + '\n') log.flush() tf_writer.add_scalar('loss/val', losses.avg, epoch) tf_writer.add_scalar('acc/val_top1', top1.avg, epoch) return top1.avg
def train(train_loader, model, criterion, kl_loss, logsoftmax, softmax, optimizer, epoch, log, tf_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() loss_kl = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() total = 0 shift = 0 for i, moda in enumerate(args.modality): tmp = total if moda == 'RGB': total += 3 elif moda == 'Flow': total += 10 elif moda == 'RGBDiff': total += 18 if i == 0: shift = total if i == args.rank and i > 0: start_ind = tmp - shift end_ind = total - shift elif i == args.rank and i == 0: start_ind = 0 end_ind = total if args.rank == 0: inds = [] for x in range(args.num_segments): inds.extend(list(range(x * total + start_ind, x * total + end_ind))) send_inds = [] for x in range(args.num_segments): send_inds.extend( list(range(x * total + end_ind, x * total + total))) else: inds = [] for x in range(args.num_segments): inds.extend( list( range(x * (total - shift) + start_ind, x * (total - shift) + end_ind))) if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode5r model.train() if args.rank == 0: iter_through = train_loader else: iter_through = range( int(len([x for x in open(args.train_list[0])]) / args.batch_size)) end = time.time() for i, data in enumerate(iter_through): # measure data loading time data_time.update(time.time() - end) if args.rank == 0: input, target = data target = target.cuda(args.gpus[-1]) input = input.cuda(args.gpus[0]) if args.world_size > 1: torch.distributed.broadcast(input[:, send_inds].contiguous(), 0) torch.distributed.broadcast(target, 0) else: input = torch.zeros( (args.batch_size, (total - shift) * args.num_segments, 224, 224)).cuda(args.gpus[0]) target = torch.zeros((args.batch_size, ), dtype=torch.int64).cuda(args.gpus[-1]) torch.distributed.broadcast(input, 0) torch.distributed.broadcast(target, 0) input_var = torch.autograd.Variable(input[:, inds].contiguous()) target_var = torch.autograd.Variable(target) # compute output output = model(input_var).cuda(args.gpus[-1]) loss1 = criterion(output, target_var) if args.world_size > 1: reduce_output = output.clone().detach() distr.all_reduce(reduce_output) reduce_output = (reduce_output - output.detach()) / (args.world_size - 1) loss2 = kl_loss(logsoftmax(output), softmax(reduce_output.detach())) else: loss2 = torch.tensor(0.) loss = loss1 + loss2 # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss1.item(), input.size(0)) loss_kl.update(loss2.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # compute gradient and do SGD step loss.backward() if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) optimizer.step() optimizer.zero_grad() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss1.val:.4f} ({loss1.avg:.4f})\t' 'LossKL {loss2.val:.4f} ({loss2.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(iter_through), batch_time=batch_time, data_time=data_time, loss1=losses, loss2=loss_kl, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush() tf_writer.add_scalar('loss/train', losses.avg, epoch) tf_writer.add_scalar('loss/mutual', loss_kl.avg, epoch) tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) tf_writer.add_scalar('acc/train_top5', top5.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) target = target.cuda() input_var = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) # compute output output = model(input_var) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # compute gradient and do SGD step loss.backward() if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) optimizer.step() optimizer.zero_grad() # measure elapsed time batch_time.update(time.time() - end) end = time.time() # if i % 100 == 0: if i % args.print_freq == 0: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush() tf_writer.add_scalar('loss/train', losses.avg, epoch) tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) tf_writer.add_scalar('acc/train_top5', top5.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def validate(val_loader, model, decoder, criterion, epoch, log=None, tf_writer=None, index2wordDict=None): batch_time = AverageMeter() data_time = AverageMeter() #losses = AverageMeter() BLEUs = AverageMeter() # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): with open(os.path.join(args.root_model, args.store_name, "val.txt"), "a+") as txt: txt.write("epoch\t" + str(epoch) + "\n") for i, data in enumerate(val_loader): loss = 0 data_time.update(time.time() - end) #if isinstance(data, bool): # continue #if not torch.any(data): # continue if not data: continue try: [URL, id, label, clips] = data except Exception as e: print(e) print(data) with open("errr.txt", "a+") as txt: txt.write(str(data)) input_video = torch.tensor([], dtype=torch.float).cuda() input_caption = torch.tensor([], dtype=torch.long).cuda() #cap_nums = [] for clip in range(0, len(clips), max(len(clips) // 50, 1)): if list(input_video.size()) != [0]: if input_video.size()[1] > 50: break video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat((input_video, video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) #cap_nums.append(clip[2]) #for recording filename in result input_video = input_video.view(1, -1, 3, crop_size, crop_size) input_caption = input_caption.view(1, -1, input_caption.size()[-1], 1) #print(input_video.size()) #print(input_caption.size()) target = label.cuda() input_video_var = torch.autograd.Variable(input_video) input_caption_var = torch.autograd.Variable(input_caption) target_var = torch.autograd.Variable(target) # compute output wExtraLoss = 1 if args.prune == '' else 0.1 encoder_output = model(input_video_var, input_caption_var) #size=(1, frames, 2048) t_input = torch.zeros((1, 64, 2048)).cuda() for idx_e, e in enumerate(encoder_output[0]): t_input[0, idx_e] = e #print("encoder_output size", encoder_output.size()) decoder_outputs = [] #decoder_input = torch.tensor([[0]]).cuda() #SOS #decoder_hidden = decoder.initHidden() decoder_output = decoder(t_input) decoder_outputs = decoder_output.argmax(-1) #decoder_outputs = decoder_output[0,:len(target[0])].argmax(-1) #print("output size=",output.size()) #print("target)var=",target_var.size()) #print(output) #print(target_var) decoder_sentence = [ index2wordDict[index.item()] for index in decoder_outputs ] print(decoder_sentence) #print(target.size()) target_sentence = [ index2wordDict[index.item()] for index in target[0] ] print(target_sentence) bleu = sentence_bleu( target_sentence, decoder_sentence ) #default weight is [0.25,0.25,0.25,0.25] = bleu-4 #print("bleu =",bleu) BLEUs.update(bleu) #auc = AUC(output.squeeze(), target_var.squeeze()) #print("auc=",auc) #AUCs.update(auc) #losses.update(loss.item(), ) # measure elapsed time batch_time.update(time.time() - end) end = time.time() txtoutput = ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'BLEU {top5.val:.3f} ({top5.avg:.3f}\t' "output/target {output}/{target}").format( i, len(val_loader), batch_time=batch_time, output=decoder_sentence, target=target_sentence, top5=BLEUs) if i % args.print_freq == 0: print(txtoutput) if log is not None: log.write(txtoutput + '\n') log.flush() with open( os.path.join(args.root_model, args.store_name, "val.txt"), "a+") as txt: txt.write(txtoutput + "\n") #break txtoutput = ('Testing Results: BLEUs {auc.avg:.3f} '.format(auc=BLEUs)) with open(os.path.join(args.root_model, args.store_name, "val.txt"), "a+") as txt: txt.write(txtoutput + "\n") print(txtoutput) if log is not None: log.write(txtoutput + '\n') log.flush() if tf_writer is not None: #tf_writer.add_scalar('loss/test', losses.avg, epoch) #tf_writer.add_scalar('acc/test_top1', top1.avg, epoch) #tf_writer.add_scalar('acc/test_top5', top5.avg, epoch) tf_writer.add_scalar('bleu/test', BLEUs.avg, epoch) return BLEUs.avg
def validate(val_loader, model, criterion, iter, logger=None): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (r, f, path, label, video_index, seg_index, verify) in enumerate(val_loader): with torch.no_grad(): r = r.cuda() f = f.cuda() target_var = label.cuda(async=True).type(torch.cuda.LongTensor) output, _ = model(r, f) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec5 = accuracy(output, target_var, topk=(1, 5)) losses.update(loss.item(), r.shape[0]) top1.update(prec1.item(), r.shape[0]) top5.update(prec5.item(), r.shape[0]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print(('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.6f} ({loss.avg:.6f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5))) # if i == 99: break # Randomly Test 100 batch data (which is in the training data, so this is not a really evaluation). print(( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses))) return top1.avg, top5.avg
# feature_length = 800 path_list = [] file_list = [] label_list = [] vid = [] annotation_list = [] # annotation_dict = build_annotation_dict(annotation_root=annotation_root, index_file=index_file) ant = [] feature = [] last_ind = 0 last_path = "" last_label = 0 verbose = False testtime = AverageMeter() #### build network # fe = I3DFeatureExtractor() # fe = load_fe_from_i3d(fe, i3d_model_checkpoint) # fe = torch.nn.DataParallel(fe, device_ids=[i for i in range(torch.cuda.device_count())]).cuda() # fe.eval() #### # clf = Classifier(feature_length, num_class, isbn=True) # fe = I3DFeatureExtractor() # I3DClassifier = torch.nn.Sequential( # fe, # clf # ) # ckpt = torch.load(i3d_model_checkpoint)
def validate(val_loader, model, criterion, epoch): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() torch.set_grad_enabled(False) # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): for i, (input, target, name) in enumerate(val_loader): # discard final batch if i == len(val_loader) - 1: break target = target.cuda(async=True) # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if epoch > int(args.epochs * 0.7): batch_out = torch.argmax(output, dim=1) batch_reslut = (batch_out != target).tolist() for i in range(len(batch_reslut)): if batch_reslut[i] == 1: print(name[i]) if i % args.print_freq == 0: output = ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(output) output = ( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses)) print(output) return top1.avg
def validate(val_loader, model, criterion, iter=None, logger=None): batch_time = AverageMeter() losses = AverageMeter() accu = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (r, f, path, label, video_index, seg_index, verify) in enumerate(val_loader): with torch.no_grad(): r = r.cuda() f = f.cuda() target_var = label.cuda(async=True).type(torch.cuda.LongTensor) output = model([r, f]) loss = criterion(output, target_var) # measure accuracy and record loss prec1, = accuracy(output, target_var, topk=(1,)) losses.update(loss.item(), r.shape[0]) accu.update(prec1.item(), r.shape[0]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print(('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.6f} ({loss.avg:.6f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=accu))) print(('Testing Results: Prec@1 {top1.avg:.3f} Loss {loss.avg:.5f}' .format(top1=accu, loss=losses))) return accu.avg
def validate(model, epoch): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() ''' Make sure that the validation works so I can confirm that there is an improvement/loss in accuracy when running this model. Run model ''' val_loader = get_val_loader(model) criterion = torch.nn.CrossEntropyLoss().cuda() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(val_loader): target = target.cuda() # compute output output = model.net(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % 100 == 0: output = ( 'Test: [{0}/{1}]\t' #'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(output) output = ( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses)) print(output) return top1.avg
def v_train(train_loader, val_loader, model, v_model, vnet, criterion, valcriterion, optimizer, v_optimizer, optimizer_vnet, epoch, log, tf_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() val_loader_iter = iter(val_loader) if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) v_model.load_state_dict(model.state_dict()) target = target.cuda() input_var = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) # compute output output = v_model(input_var) # loss = criterion(output, target_var) cost = criterion(output, target_var) cost_v = torch.reshape(cost, (-1, 1)) v_lambda = vnet(cost_v.data) l_f_v = torch.sum(cost_v * v_lambda) / len(cost_v) v_model.zero_grad() grads = torch.autograd.grad(l_f_v, (v_model.module.params()), create_graph=True) # to be modified v_lr = args.lr * ((0.1**int(epoch >= 80)) * (0.1**int(epoch >= 100))) v_model.module.update_params(lr_inner=v_lr, source_params=grads) del grads # phase 2. pixel weights step try: inputs_val, targets_val = next(val_loader_iter) except StopIteration: val_loader_iter = iter(val_loader) inputs_val, targets_val = next(val_loader_iter) inputs_val, targets_val = inputs_val.cuda(), targets_val.cuda() y_g_hat = v_model(inputs_val) l_g_meta = valcriterion(y_g_hat, targets_val) # val loss optimizer_vnet.zero_grad() l_g_meta.backward() optimizer_vnet.step() # phase 1. network weight step (w) output = model(input_var) cost = criterion(output, target) cost_v = torch.reshape(cost, (-1, 1)) with torch.no_grad(): v_new = vnet(cost_v) loss = torch.sum(cost_v * v_new) / len(cost_v) optimizer.zero_grad() loss.backward() optimizer.step() # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # compute gradient and do SGD step # loss.backward() if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) # optimizer.step() # optimizer.zero_grad() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush() tf_writer.add_scalar('loss/train', losses.avg, epoch) tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) tf_writer.add_scalar('acc/train_top5', top5.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) if i == 20: os.system("gpustat") target = target.cuda() input_var = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) # compute output output = model(input_var) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # compute gradient and do SGD step loss.backward() no_grad_cnt = 0 if i % args.iter_size == 0: # scale down gradients when iter size is functioning if args.iter_size != 1: for g in optimizer.param_groups: for p in g['params']: if isinstance(p.grad, torch.Tensor): p.grad /= args.iter_size else: no_grad_cnt = no_grad_cnt + 1 if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) else: total_norm = 0 optimizer.step() optimizer.zero_grad() #if i == 0: # print("{}\nWARNING: There are {} params without gradient!!!!!\n{}".format("*"*50, no_grad_cnt, "*"*50)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush() tf_writer.add_scalar('loss/train', losses.avg, epoch) tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) tf_writer.add_scalar('acc/train_top5', top5.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def test(val_loader, model, criterion, epoch, log=None, tf_writer=None): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() nAUDC = 1 pMiss = 1 pred_dict = {} # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): if args.naudc: naudc_res = [] for i, (input, target, name) in enumerate(val_loader): batch_size = target.numel() target = target.cuda() # compute output output = model(input) # if args.dense_sample: # output = output.reshape(batch_size, 10, -1).mean(1) # loss = criterion(output, target) loss = criterion(output.squeeze(-1), target) # measure accuracy and record loss if args.loss_type == "bce" or args.loss_type == "wbce": [prec1, prec5], num = accuracy_bce(output.data, target, topk=(1, 5)) losses.update(loss.item(), output.size(0)) if prec1 != -1: top1.update(prec1.item(), num) top5.update(prec5.item(), num) elif args.loss_type == "nll": [prec1, prec5] = accuracy_nll(output.data, target, topk=(1, 5)) losses.update(loss.item(), output.size(0)) top1.update(prec1.item(), output.size(0)) top5.update(prec5.item(), output.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() rst = torch.softmax(output, dim=1).data.cpu().numpy().copy() tgt = target.cpu().numpy().copy() for idk in range(rst.shape[0]): pred_dict[str(name[idk])] = { "pred": rst[idk].tolist(), "gt": tgt[idk].tolist() } if args.naudc: rst = output.data.cpu().numpy().copy() this_rst_list = [] this_rst_list.append(rst) ensembled_predict = sum(this_rst_list) / len(this_rst_list) for p, g in zip(ensembled_predict, target.cpu().numpy()): naudc_res.append([p[None, ...], g]) if i % args.print_freq == 0: output = ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(output) logging.info(output) if log is not None: log.write(output + '\n') log.flush() json_str = json.dumps(pred_dict, indent=4) with open(os.path.join("./pred.json"), 'w') as save_json: save_json.write(json_str) if args.naudc: video_pred_topall = [ np.argsort(np.mean(x[0], axis=0).reshape(-1))[::-1][:args.topk] for x in naudc_res ] video_prob_topall = [ np.sort(np.mean(x[0], axis=0).reshape(-1))[::-1][:args.topk] for x in naudc_res ] with open(args.val_list) as f: vid_names = list(json.load(f)["database"].keys()) event_dict = get_eventdict(args) res_dict, file_dict, eve_dict = getoutput(vid_names, video_pred_topall, video_prob_topall, event_dict, args) json_str = json.dumps(res_dict, indent=4) with open(os.path.join(args.out_path, "output-mod.json"), 'w') as save_json: save_json.write(json_str) json_str = json.dumps(file_dict, indent=4) with open(os.path.join(args.out_path, "file-index.json"), 'w') as save_json: save_json.write(json_str) json_str = json.dumps(eve_dict, indent=4) with open(os.path.join(args.out_path, "activity-index.json"), 'w') as save_json: save_json.write(json_str) call_path = os.path.join(args.call_path, "ActEV_Scorer.py") s = os.path.join(args.out_path, "output_mod.json") r = os.path.join(args.out_path, "kitware_eo_s2-test_99.json") a = os.path.join(args.out_path, "activity-index.json") f = os.path.join(args.out_path, "file-index.json") o = args.out_path print("Start Calculatomg Pmiss and nAUDC") call = "python {} ActEV_SDL_V2 -s {} -r {} -a {} -f {} -o {} -v -n 36".format( call_path, s, r, a, f, o) os.system(call) csvFile = open(os.path.join(args.out_path, "scores_aggregated.csv"), "r") reader = csv.reader(csvFile) for item in reader: if "[email protected]" in item[0]: nAUDC = float(item[0].split("|")[-1]) if "[email protected]" in item[0]: pMiss = float(item[0].split("|")[-1]) output = ( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f} nAUDC {nAUDC:.5f} pMiss {pMiss:.5f}' .format(top1=top1, top5=top5, loss=losses, nAUDC=nAUDC, pMiss=pMiss)) print(output) logging.info(output) if log is not None: log.write(output + '\n') log.flush() if tf_writer is not None: tf_writer.add_scalar('loss/test', losses.avg, epoch) tf_writer.add_scalar('acc/test_top1', top1.avg, epoch) tf_writer.add_scalar('acc/test_top5', top5.avg, epoch) tf_writer.add_scalar('nAUDC/test', nAUDC, epoch) tf_writer.add_scalar('pmiss/test', pMiss, epoch) return losses.avg
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() AUCs = AverageMeter() # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): with open(os.path.join(args.root_model, args.store_name, "val.txt"), "a+") as txt: txt.write("epoch\t" + str(epoch) + "\n") for idx, data in enumerate(val_loader): totalOutput = torch.tensor([], dtype=torch.float).cuda() totalTarget = torch.tensor([], dtype=torch.float).cuda() data_time.update(time.time() - end) # if isinstance(data, bool): # continue #if not torch.any(data): # continue if not data: continue try: [URL, id, sift, label, clips] = data except Exception as e: print(e) print(data) with open("errr.txt", "a+") as txt: txt.write(str(data)) #cap_nums = [] for i, s in enumerate(sift): input_video = torch.tensor([], dtype=torch.float).cuda() input_caption = torch.tensor([], dtype=torch.long).cuda() #print("i={},s={}".format(i,s)) if i + 1 < len(sift): if int(sift[i + 1]) - int(s) < 50: #print( int(sift[i+1])-int(s)) for clip in range(s, sift[i + 1]): #print(clip) video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:sift[i + 1]].cuda() else: for clip in range(s, sift[i + 1], (int(sift[i + 1]) - int(s)) // 50): #print(clip) if (clip - s) / ( (int(sift[i + 1]) - int(s)) // 50) >= 50: break video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:sift[i + 1]:(sift[i + 1] - s) // 50].cuda() target = target[:50] else: length = len(clips) if int(length) - int(s) < 50: for clip in range(s, length): video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:length].cuda() else: for clip in range(s, length, (int(length) - int(s)) // 50): if (clip - s) / ( (int(length) - int(s)) // 50) >= 50: break video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:length:(length - s) // 50].cuda() target = target[:50] #print("input video.size()",input_video.size()) #print("input_caption.size()",input_caption.size()) input_video = input_video.view(1, -1, 3, crop_size, crop_size) #input_caption = input_caption.view(1,-1,input_caption.size()[-1],1) input_caption = input_caption.view(1, -1, 64, 1) input_video_var = torch.autograd.Variable(input_video) input_caption_var = torch.autograd.Variable(input_caption) target_var = torch.autograd.Variable(target) #print("input_video_var.size()",input_video_var.size()) #print("target size",target.size()) # compute output wExtraLoss = 1 if args.prune == '' else 0.1 output = model(input_video_var, input_caption_var) #print("output size=",output.size()) #print("target)var=",target_var.size()) #print(output) #print(target_var) #print("totalOutput size=",totalOutput.size()) #print("output size=",output.size()) #print("squeeze size=",output.squeeze().size()) #print(totalOutput) #print("/*"*50) #print(output) totalOutput = torch.cat( (totalOutput.float().cuda(), output.float().cuda()), dim=1) #print("totalTarget size()",totalTarget.size()) #print("target_Var.size()",target_var.size()) #print("target size",target.size()) #print(target_var) if len(target_var.size()) != 0: totalTarget = torch.cat((totalTarget.float().cuda(), target_var.float().cuda()), dim=0) loss = criterion(output.squeeze(), target.squeeze()) #print("loss = ",loss) #auc = AUC(output.squeeze(), target_var.squeeze()) #print("auc=",auc) #AUCs.update(auc) losses.update(loss.item(), ) # measure elapsed time batch_time.update(time.time() - end) end = time.time() auc = AUC(totalOutput.squeeze(), totalTarget.squeeze()) AUCs.update(auc) txtoutput = ( 'Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' "totalOutput\totalTarget\t{totalOutput}\t{totalTarget}" 'AUC {top5.val:.3f} ({top5.avg:.3f})'.format( idx, len(val_loader), batch_time=batch_time, loss=losses, totalOutput=totalOutput, target=totalTarget, top5=AUCs)) if idx % args.print_freq == 0: #output = ('Test: [{0}/{1}]\t' # 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' # 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' # "output/target {output}/{target}" # 'AUC {aucs.val:.3f} ({aucs.avg:.3f})'.format( # i, len(val_loader), batch_time=batch_time, loss=losses,output=output,target=target, # aucs=AUCs)) print(txtoutput) if log is not None: log.write(txtoutput + '\n') log.flush() with open( os.path.join(args.root_model, args.store_name, "val.txt"), "a+") as txt: txt.write(txtoutput + "\t" + str(URL) + "\t" + str(id) + "\n") #break txtoutput = ( 'Testing Results: auc {auc.avg:.3f} Loss {loss.avg:.5f}'.format( auc=AUCs, loss=losses)) with open(os.path.join(args.root_model, args.store_name, "val.txt"), "a+") as txt: txt.write(txtoutput + "\t" + str(URL) + "\t" + str(id) + "\n") print(txtoutput) if log is not None: log.write(txtoutput + '\n') log.flush() if tf_writer is not None: tf_writer.add_scalar('loss/test', losses.avg, epoch) #tf_writer.add_scalar('acc/test_top1', top1.avg, epoch) #tf_writer.add_scalar('acc/test_top5', top5.avg, epoch) tf_writer.add_scalar('auc/test', AUCs.avg, epoch) return AUCs.avg
def train_AETAllfast(train_dataloader, valid_dataloader, model, optimizer, ema_optimizer, mixmatch_criterion, epoch, use_cuda, aet_model, use_aet, aet_criterion, aet_optimizer, iteration, Temporature, Alpha, lambda_aet, epoch_logger, batch_logger, run_type, writer, num_classes, params, ema_model): model.train() ema_model.eval() #do not train,only use for validation if use_aet: aet_model1 = aet_model[0] aet_model2 = aet_model[1] aet_model3 = aet_model[2] aet_model4 = aet_model[3] aet_model5 = aet_model[4] aet_model1.train() aet_model2.train() aet_model3.train() aet_model4.train() aet_model5.train() aet_optimizer1 = aet_optimizer[0] aet_optimizer2 = aet_optimizer[1] aet_optimizer3 = aet_optimizer[2] aet_optimizer4 = aet_optimizer[3] aet_optimizer5 = aet_optimizer[4] batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() AET_loss1 = AverageMeter() AET_loss2 = AverageMeter() AET_loss3 = AverageMeter() AET_loss4 = AverageMeter() AET_loss5 = AverageMeter() Closs = AverageMeter() # loss_x in mixmatch Enloss = AverageMeter() # loss_u in mixmatch Entropy_Loss = AverageMeter() WS = AverageMeter() accuracies = AverageMeter() top5 = AverageMeter() WAET1 = AverageMeter() WAET2 = AverageMeter() WAET3 = AverageMeter() WAET4 = AverageMeter() WAET5 = AverageMeter() KL_Loss = AverageMeter() TeacherKL = AverageMeter() end_time = time.time() #labeled_train_iter = iter(train_dataloader) #unlabeled_train_iter = iter(valid_dataloader) prefetcher_label = data_prefetcher(train_dataloader) prefetcher_unlabel = data_prefetcher(valid_dataloader) labelled_reload = len(train_dataloader) unlabelled_reload = len(valid_dataloader) print("one iter for labelled data %d, one iter for unlabeled data %d" % (labelled_reload, unlabelled_reload)) for batch_idx in range(iteration): try: read_data = prefetcher_label.next() if (batch_idx % labelled_reload == 0 and batch_idx != 0) or read_data is None: print("reloading labeled dataloader again!!") prefetcher_label = data_prefetcher(train_dataloader) ( img1, img1_t ), img2t, img3t, img4t, img5t, img6t, img7t, aff_parat, transform_matrix1t, oper_paramst, target = read_data except: #problems happen here, we need to reload the dataloader by our self print( "reloading labeled dataloader again!!" ) ##very important to check if the reloading worked in fast mode prefetcher_label = data_prefetcher(train_dataloader) ( img1, img1_t ), img2t, img3t, img4t, img5t, img6t, img7t, aff_parat, transform_matrix1t, oper_paramst, target = prefetcher_label.next( ) #img1 = read_data[0] # original images #target = read_data[1] # target label try: read_data1 = prefetcher_unlabel.next() if (batch_idx % unlabelled_reload == 0 and batch_idx != 0) or read_data1 is None: print("reloading unlabeled dataloader again!!") prefetcher_unlabel = data_prefetcher(valid_dataloader) ( img1_u, img1_u2 ), img2, img3, img4, img5, img6, img7, aff_para, transform_matrix1, oper_params, _ = read_data1 except: print( "reloading unlabeled dataloader again!!" ) ##very important to check if the reloading worked in fast mode prefetcher_unlabel = data_prefetcher(valid_dataloader) ( img1_u, img1_u2 ), img2, img3, img4, img5, img6, img7, aff_para, transform_matrix1, oper_params, _ = prefetcher_unlabel.next( ) if batch_idx == 0 and epoch % 10 == 0: if writer != None: tmp1 = vutils.make_grid(img1_u, normalize=True, scale_each=True) writer.add_image('Image1_u', tmp1, epoch) tmp1 = vutils.make_grid(img1_u2, normalize=True, scale_each=True) writer.add_image('Image1_u2', tmp1, epoch) tmp1 = vutils.make_grid(img2, normalize=True, scale_each=True) writer.add_image('Image_Projective', tmp1, epoch) tmp1 = vutils.make_grid(img3, normalize=True, scale_each=True) writer.add_image('Image_Affine', tmp1, epoch) tmp1 = vutils.make_grid(img4, normalize=True, scale_each=True) writer.add_image('Image_Similarity', tmp1, epoch) tmp1 = vutils.make_grid(img5, normalize=True, scale_each=True) writer.add_image('Image_Euclidean', tmp1, epoch) tmp1 = vutils.make_grid(img6, normalize=True, scale_each=True) writer.add_image('Image_CCBS', tmp1, epoch) tmp1 = vutils.make_grid(img7, normalize=True, scale_each=True) writer.add_image('Image_Patch', tmp1, epoch) data_time.update(time.time() - end_time) batch_size = img1.size(0) targets_x = torch.zeros(batch_size, num_classes).scatter_(1, target.view(-1, 1), 1) if use_cuda: img1, targets_x = img1.cuda(), targets_x.cuda(non_blocking=True) img1_t = img1_t.cuda() target = target.cuda() img1_u = img1_u.cuda() img1_u2 = img1_u2.cuda() img2 = img2.cuda() img3 = img3.cuda() img4 = img4.cuda() img5 = img5.cuda() img6 = img6.cuda() img7 = img7.cuda() # img2t = img2t.cuda() # img3t = img3t.cuda() # img4t = img4t.cuda() # img5t = img5t.cuda() #img6t = img6t.cuda() #img7t = img7t.cuda() if use_aet: aff_para = aff_para.cuda() transform_matrix1 = transform_matrix1.cuda() oper_params = oper_params.cuda() with torch.no_grad(): # compute guessed labels of unlabel samples outputs_u = model(img1_u) outputs_u2 = model(img1_u2) #outputs_u3 = model(img2) #outputs_u4 = model(img3) #outputs_u5= model(img4) #outputs_u6 = model(img5) #outputs_u7 = model(img6) #outputs_u8 = model(img7) if params['mix_mode'] == 0: p = (torch.softmax(outputs_u, dim=1) + torch.softmax(outputs_u2, dim=1) ) / 2 #+ torch.softmax(outputs_u3, dim=1) # + torch.softmax(outputs_u4, dim=1)+ torch.softmax(outputs_u5, dim=1)+ torch.softmax(outputs_u6, dim=1) #+ torch.softmax(outputs_u7, dim=1)+ torch.softmax(outputs_u8, dim=1)) / 4 # mean the softmax maatrix else: outputs_u8 = model(img7) p = (torch.softmax(outputs_u, dim=1) + torch.softmax( outputs_u2, dim=1) + torch.softmax(outputs_u8, dim=1)) / 3 pt = p**(1 / Temporature) targets_u = pt / pt.sum(dim=1, keepdim=True) targets_u = targets_u.detach( ) #also serve as p(y|x) with fixed weight(no propagating) #now add KL divergence part # outputs_x = model(img1) # outputs_x2 = model(img1_t) # p_x=(torch.softmax(outputs_x, dim=1) + torch.softmax(outputs_x2, dim=1))/2 # ptx = p_x ** (1 / Temporature) # targets_xpred = ptx / ptx.sum(dim=1, keepdim=True) # targets_xpred = targets_xpred.detach() # also serve as p(y|x) with fixed weight(no propagating) mixup_count = 3 if params['mix_mode'] == 0: loss1, loss_ent, Lx, Lu, w = Calculate_CLoss( model, mixmatch_criterion, [img1, img1_u, img1_u2], [targets_x, targets_u, targets_u], Alpha, batch_size, 1, epoch + batch_idx / iteration, params, mixup_count, writer) else: loss1, loss_ent, Lx, Lu, w = Calculate_CLoss( model, mixmatch_criterion, [img1, img1_u, img1_u2, img7], [targets_x, targets_u, targets_u, targets_u], Alpha, batch_size, 1, epoch + batch_idx / iteration, params, mixup_count, writer) #if batch_idx==0 and epoch==0: # #write the result of mix up to show Closs.update(Lx.item(), 1 * batch_size) Enloss.update(Lu.item(), 1 * batch_size) WS.update(w, 1 * batch_size) loss = loss1 #here we do not add any parameter if use_aet: use_key = ['Attention'] feature_map1, atten1 = model(img1_u, use_key) if params['KL_Lambda'] != 0: #calculate KL mode use_key1 = use_key + ['classifier'] feature_map2, atten2 = model(img2, use_key1) pred_img2 = feature_map2[1] feature_map2 = feature_map2[0] else: feature_map2, atten2 = model(img2, use_key) pred_transform = aet_model1(feature_map1, feature_map2) transform_matrix1 = transform_matrix1.view(-1, 8) aet_loss = aet_criterion(pred_transform, transform_matrix1) loss += aet_loss * min( lambda_aet * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda']) WAET1.update( min( lambda_aet * np.clip((epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda']), img1.size(0)) if params['KL_Lambda'] != 0: #calculate KL mode use_key1 = use_key + ['classifier'] feature_map2, atten2 = model(img3, use_key1) pred_img3 = feature_map2[1] feature_map2 = feature_map2[0] else: feature_map2, atten2 = model(img3, use_key) pred_transform2 = aet_model2(feature_map1, feature_map2) aet_loss2 = aet_criterion(pred_transform2, aff_para) loss += aet_loss2 * min( params['lambda1'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda1']) WAET2.update( min( params['lambda1'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda1']), img1.size(0)) if params['KL_Lambda'] != 0: #calculate KL mode use_key1 = use_key + ['classifier'] feature_map2, atten2 = model(img4, use_key1) pred_img4 = feature_map2[1] feature_map2 = feature_map2[0] else: feature_map2, atten2 = model(img4, use_key) pred_transform3 = aet_model3(feature_map1, feature_map2) aet_loss3 = aet_criterion(pred_transform3, aff_para[:, :5]) loss += aet_loss3 * min( params['lambda2'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda2']) WAET3.update( min( params['lambda2'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda2']), img1.size(0)) if params['KL_Lambda'] != 0: #calculate KL mode use_key1 = use_key + ['classifier'] feature_map2, atten2 = model(img5, use_key1) pred_img5 = feature_map2[1] feature_map2 = feature_map2[0] else: feature_map2, atten2 = model(img5, use_key) pred_transform4 = aet_model4(feature_map1, feature_map2) aet_loss4 = aet_criterion(pred_transform4, aff_para[:, :4]) loss += aet_loss4 * min( params['lambda3'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda3']) WAET4.update( min( params['lambda3'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda3']), img1.size(0)) if params['KL_Lambda'] != 0: #calculate KL mode use_key1 = use_key + ['classifier'] feature_map2, atten2 = model(img6, use_key1) pred_img6 = feature_map2[1] feature_map2 = feature_map2[0] else: feature_map2, atten2 = model(img6, use_key) pred_transform5 = aet_model5(feature_map1, feature_map2) aet_loss5 = aet_criterion(pred_transform5, oper_params) loss += aet_loss5 * min( params['lambda4'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda4']) WAET5.update( min( params['lambda4'] * np.clip( (epoch * iteration + batch_idx) / (1024 * iteration), 0.0, 1.0), params['max_lambda4']), img1.size(0)) # Add KL divergence now for all the other augmentations if params['KL_Lambda'] != 0: #we do not have so much gpu memory, save everywhere, though it makes code massy KLloss = 0 count_KL = 0 if params['max_lambda'] != 0: #designed for ablation study KLloss += Calculate_KLloss_Result(pred_img2, targets_u) count_KL += 1 if params['max_lambda1'] != 0: KLloss += Calculate_KLloss_Result(pred_img3, targets_u) count_KL += 1 if params['max_lambda2'] != 0: KLloss += Calculate_KLloss_Result(pred_img4, targets_u) count_KL += 1 if params['max_lambda3'] != 0: KLloss += Calculate_KLloss_Result(pred_img5, targets_u) count_KL += 1 if params['max_lambda4'] != 0: KLloss += Calculate_KLloss_Result(pred_img6, targets_u) count_KL += 1 if params['mix_mode'] != 0: pred_img7 = model( img7 ) #again used for the final KL loss in agreement with the Teacher model KLloss += Calculate_KLloss_Result(pred_img7, targets_u) count_KL += 1 if count_KL != 0: KLloss = KLloss / count_KL KL_Loss.update(KLloss.item(), batch_size) if params['KL_Lambda'] != 0: loss += KLloss * params['KL_Lambda'] losses.update(loss.item(), img1.size(0)) if use_aet: AET_loss1.update(aet_loss.item(), img1.size(0)) AET_loss2.update(aet_loss2.item(), img1.size(0)) AET_loss3.update(aet_loss3.item(), img1.size(0)) AET_loss4.update(aet_loss4.item(), img1.size(0)) AET_loss5.update(aet_loss5.item(), img1.size(0)) aet_optimizer1.zero_grad() aet_optimizer2.zero_grad() aet_optimizer3.zero_grad() aet_optimizer4.zero_grad() aet_optimizer5.zero_grad() optimizer.zero_grad() loss.backward() optimizer.step() if params['use_ema']: ema_optimizer.step() if use_aet: aet_optimizer1.step() aet_optimizer2.step() aet_optimizer3.step() aet_optimizer4.step() aet_optimizer5.step() batch_time.update(time.time() - end_time) end_time = time.time() with torch.no_grad(): #save gpu space classify_output = model(img1) prec1, prec5 = Calculate_top_accuracy(classify_output.data, target.data, topk=(1, 5)) accuracies.update(prec1.item(), img1.size(0)) top5.update(prec5.item(), img1.size(0)) if use_aet: batch_logger.log({ 'epoch': epoch, 'batch': batch_idx + 1, 'iter': (epoch) * iteration + (batch_idx + 1), 'loss': losses.val, 'AET1': AET_loss1.val, 'AET2': AET_loss2.val, 'AET3': AET_loss3.val, 'AET4': AET_loss4.val, 'AET5': AET_loss5.val, 'Closs': Closs.val, 'Eloss': Enloss.val, 'KL': KL_Loss.val, 'wd': WS.val, 'waet': WAET1.val, 'top1': accuracies.val, 'top5': top5.val, 'lr1': optimizer.param_groups[0]['lr'], 'lr2': aet_optimizer1.param_groups[0]['lr'] }) print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'AET1 {AET1.val:.4f} ({AET1.avg:.4f})\t' 'AET2 {AET2.val:.4f} ({AET2.avg:.4f})\t' 'AET3 {AET3.val:.4f} ({AET3.avg:.4f})\t' 'AET4 {AET4.val:.4f} ({AET4.avg:.4f})\t' 'AET5 {AET5.val:.4f} ({AET5.avg:.4f})\t' 'Closs {Closs.val:.4f} ({Closs.avg:.4f})\t' 'Enloss {Enloss.val:.4f} ({Enloss.avg:.4f})\t' 'KL {KL.val:.4f} ({KL.avg:.4f})\t' 'Top1 {acc.val:.3f} ({acc.avg:.3f})\t' 'Top5 {acc1.val:.3f} ({acc1.avg:.3f})\t'.format( epoch, batch_idx + 1, iteration, batch_time=batch_time, data_time=data_time, AET1=AET_loss1, AET2=AET_loss2, AET3=AET_loss3, AET4=AET_loss4, AET5=AET_loss5, Closs=Closs, Enloss=Enloss, KL=KL_Loss, loss=losses, acc=accuracies, acc1=top5)) if use_aet: epoch_logger.log({ 'epoch': epoch, 'loss': losses.avg, 'AET1': AET_loss1.avg, 'AET2': AET_loss2.avg, 'AET3': AET_loss3.avg, 'AET4': AET_loss4.avg, 'AET5': AET_loss5.avg, 'Closs': Closs.avg, 'Eloss': Enloss.avg, 'KL': KL_Loss.avg, 'wd': WS.avg, 'waet': WAET1.avg, 'top1': accuracies.avg, 'top5': top5.avg, 'lr1': optimizer.param_groups[0]['lr'], 'lr2': aet_optimizer1.param_groups[0]['lr'] }) if writer != None: writer.add_scalars('Data/Loss', {'train_loss': losses.avg}, epoch) writer.add_scalars('Data/AET_Loss1', {'train_loss': AET_loss1.avg}, epoch) writer.add_scalars('Data/AET_Loss2', {'train_loss': AET_loss2.avg}, epoch) writer.add_scalars('Data/AET_Loss3', {'train_loss': AET_loss3.avg}, epoch) writer.add_scalars('Data/AET_Loss4', {'train_loss': AET_loss4.avg}, epoch) writer.add_scalars('Data/AET_Loss5', {'train_loss': AET_loss5.avg}, epoch) writer.add_scalars('Data/CLoss', {'train_loss': Closs.avg}, epoch) writer.add_scalars('Data/ELoss', {'train_loss': Enloss.avg}, epoch) #writer.add_scalars('Data/Entropy', {'train_loss': Entropy_Loss.avg}, epoch) writer.add_scalars('Data/KLloss', {'train_loss': KL_Loss.avg}, epoch) #writer.add_scalars('Data/KL_Teacher_loss', {'train_loss': TeacherKL.avg}, epoch) writer.add_scalars('Accuracy/top1', {'train_accu': accuracies.avg}, epoch) writer.add_scalars('Accuracy/top5', {'train_accu': top5.avg}, epoch) writer.add_scalar('Weight/Lambda1', WAET1.avg, epoch) writer.add_scalar('Weight/Lambda2', WAET2.avg, epoch) writer.add_scalar('Weight/Lambda3', WAET3.avg, epoch) writer.add_scalar('Weight/Lambda4', WAET4.avg, epoch) writer.add_scalar('Weight/Lambda5', WAET5.avg, epoch) writer.add_scalar('Weight/Beta', WS.avg, epoch) writer.add_scalar('LR/LR1', optimizer.param_groups[0]['lr'], epoch) writer.add_scalar('LR/LR2', aet_optimizer1.param_groups[0]['lr'], epoch) if epoch % 10 == 0: for name, param in model.named_parameters(): writer.add_histogram('CLF' + name, param.clone().cpu().data.numpy(), epoch) for name, param in aet_model1.named_parameters(): writer.add_histogram('AET1' + name, param.clone().cpu().data.numpy(), epoch) for name, param in aet_model2.named_parameters(): writer.add_histogram('AET2' + name, param.clone().cpu().data.numpy(), epoch) for name, param in aet_model3.named_parameters(): writer.add_histogram('AET3' + name, param.clone().cpu().data.numpy(), epoch) for name, param in aet_model4.named_parameters(): writer.add_histogram('AET4' + name, param.clone().cpu().data.numpy(), epoch) for name, param in aet_model5.named_parameters(): writer.add_histogram('AET5' + name, param.clone().cpu().data.numpy(), epoch) if params['use_ema']: ema_optimizer.step(bn=True) return losses.avg, Closs.avg, Enloss.avg
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer, args, rank): torch.cuda.empty_cache() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode model.train() end = time.time() model.zero_grad() loss_tmp = [] acc_tmp = [] for i, (input, target) in enumerate(train_loader): adjust_learning_rate(optimizer, epoch, args.lr_type, args.lr_steps, args, (epoch-1) + float(i) / len(train_loader)) i += 1 # if (i+1) % args.batch_multiplier == 0: # optimizer.step() # optimizer.zero_grad() # measure data loading time data_time.update(time.time() - end) if args.gpu is not None: input = input.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) input_var = input target_var = target # compute output output = model(input_var) loss = criterion(output, target_var) / args.batch_multiplier # divide batch_multiplier as grad accumulation # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item()*args.batch_multiplier, input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) loss_tmp.append(loss.item()*args.batch_multiplier) acc_tmp.append(prec1.item()) # compute gradient and do SGD step if i % args.batch_multiplier != 0: if args.multiprocessing_distributed: with model.no_sync(): loss.backward() else: loss.backward() else: loss.backward() if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) optimizer.step() # optimizer.zero_grad() if rank == 0: # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % (args.print_freq*args.batch_multiplier*5) == 0: tf_writer.add_scalar('loss/step', np.mean(loss_tmp), ((epoch-1)*len(train_loader)+i)/args.batch_multiplier) tf_writer.add_scalar('acc/step', np.mean(acc_tmp), ((epoch-1)*len(train_loader)+i)/args.batch_multiplier) loss_tmp = [] acc_tmp = [] if i % (args.print_freq * args.batch_multiplier) == 0: output = ('Epoch: [{0:3d}][{1:4d}/{2:4d}], lr: {lr:.5f}\t' 'Time {time:.1f}\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, int(i/args.batch_multiplier), int(len(train_loader)/args.batch_multiplier), time=(time.time()-global_time)/60., data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush() if i % (args.print_freq*args.batch_multiplier*10) == 0: for tag, value in model.named_parameters(): tag = tag.replace('.', '/') tf_writer.add_histogram('weights/'+tag, value.detach(), (epoch*len(train_loader)+i)/args.batch_multiplier) tf_writer.add_histogram('grads/'+tag, value.grad.detach().abs().mean(), (epoch*len(train_loader)+i)/args.batch_multiplier) if i % args.batch_multiplier == 0: optimizer.zero_grad() if rank == 0: tf_writer.add_scalar('loss/train', losses.avg, epoch-1) tf_writer.add_scalar('acc/train_top1', top1.avg, epoch-1)
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() if args.no_partialbn: try: model.module.partialBN(False) except: model.partialBN(False) else: try: model.module.partialBN(True) except: model.partialBN(True) model.train() end = time.time() for idx, (input, target) in enumerate(train_loader): data_time.update(time.time() - end) input, target = input.cuda(), target.cuda() output = model(input) loss = criterion(output, target) # accuracy and loss prec1, = accuracy(output.data, target, topk=(1, )) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) # gradient and optimizer loss.backward() if (idx + 1) % args.update_weight == 0: optimizer.step() optimizer.zero_grad() # time batch_time.update(time.time() - end) end = time.time() if (idx + 1) % args.print_freq == 0: output = ('Train: epoch-{0} ({1}/{2})\t' 'batch_time {batch_time.avg:.2f}\t\t' 'data_time {data_time.avg:.2f}\t\t' 'loss {loss.avg:.3f}\t' 'prec@1 {top1.avg:.2f}\t'.format(epoch, idx + 1, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1)) batch_time.reset() data_time.reset() losses.reset() top1.reset() print(output) log.write(output + '\n') log.flush() tf_writer.add_scalar('loss/train', losses.avg, epoch) tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(train_loader, model, criterion, optimizer, epoch, logger=None, scheduler=None): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) model.train() end = time.time() for i, (input, target) in enumerate(train_loader): data_time.update(time.time() - end) target = target.cuda() input_var = input.cuda() target_var = target output = model(input_var) loss = criterion(output, target_var) prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) optimizer.zero_grad() loss.backward() if args.clip_gradient is not None: clip_grad_norm_(model.parameters(), args.clip_gradient) optimizer.step() scheduler.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: logger.info(('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr']))) # TODO return losses.avg, top1.avg, top5.avg
def train(train_loader, model, criterion, optimizer, epoch): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to train mode model.train() end = time.time() for i, (r, f, path, label, video_index, seg_index, verify) in enumerate(train_loader): r = r.cuda() f = f.cuda() output, _ = model(r, f) target_var = label.cuda(async=True).type(torch.cuda.LongTensor) loss = criterion(output, target_var) # measure accuracy and record loss prec1, prec5 = accuracy(output, target_var, topk=(1, 5)) losses.update(loss.item(), r.shape[0]) top1.update(prec1.item(), r.shape[0]) top5.update(prec5.item(), r.shape[0]) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() if args.clip_gradient is not None: total_norm = torch.nn.utils.clip_grad_norm(model.parameters(), args.clip_gradient) optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print(('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'])))
def validate(val_loader, model, criterion, logger=None): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() model.eval() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(val_loader): target = target.cuda() output = model(input) loss = criterion(output, target) prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) loss = reduce_tensor(loss) prec1 = reduce_tensor(prec1) prec5 = reduce_tensor(prec5) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: logger.info( ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5))) logger.info(( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses))) return top1.avg, top5.avg, losses.avg
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None, scheduler=None): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): each_class_output_buffer = torch.Tensor() each_class_target_buffer = torch.Tensor() for i, (input, target, path) in enumerate(val_loader): target = target.cuda() # compute output output = model(input) loss = criterion(output, target) # tmp = torch.nn.functional.softmax(output[0]).cpu().detach().tolist() # for idx in range(3): # print("predict label: {}, conf: {:.2f}".format(idx2label[idx], float(tmp[idx]))) # measure accuracy and record loss # with open("/home/huyihui/workspace/tsm-huawei/predict_result3.txt", "a+", encoding="utf-8") as f: # f.write(f"{path[0]},{tmp[0]},{tmp[1]}\n") prec1, prec5 = accuracy(output.data, target, topk=(1, 2)) each_class_output_buffer = torch.cat( (each_class_output_buffer, output.detach().cpu().float()), 0) each_class_target_buffer = torch.cat( (each_class_target_buffer, target.detach().cpu().float()), 0) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(output) if log is not None: log.write(output + '\n') log.flush() if scheduler: scheduler.step(loss.avg) output = ( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses)) # acc = calculate_accuracy_each_class(each_class_output_buffer, each_class_target_buffer, 2) print(output) # print("-------------------\n", acc) if log is not None: log.write(output + '\n') log.flush() if tf_writer is not None: tf_writer.add_scalar('loss/test', losses.avg, epoch) tf_writer.add_scalar('acc/test_top1', top1.avg, epoch) tf_writer.add_scalar('acc/test_top5', top5.avg, epoch) return top1.avg
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() end = time.time() all_preds = [] all_target = [] with torch.no_grad(): for i, (input, target) in enumerate(val_loader): target = target.cuda() # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) all_target.append(target.cpu().numpy()) all_preds.append( output.data.topk(5, 1, True, True)[1].cpu().numpy()) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(output) if log is not None: log.write(output + '\n') log.flush() output = ( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses)) print(output) if log is not None: log.write(output + '\n') log.flush() #import ipdb; ipdb.set_trace() columns = [ 'idx', 'target', 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5' ] lines = [] with open('log/val_results_epoch{}.txt'.format(str(epoch)), 'w') as f: for b in range(len(all_target)): for i in range(len(all_target[b])): preds_formatted = [ s.rjust(6) for s in all_preds[b][i].astype(str).tolist() ] lines.append( str(all_target[b][i]).rjust(6) + '\t' + '\t'.join(preds_formatted)) header = '\t'.join([s.rjust(6) for s in columns]) f.write(header + '\n') for i in range(len(lines)): f.write(str(i).rjust(6) + '\t' + lines[i] + '\n') if tf_writer is not None: tf_writer.add_scalar('loss/test', losses.avg, epoch) tf_writer.add_scalar('acc/test_top1', top1.avg, epoch) tf_writer.add_scalar('acc/test_top5', top5.avg, epoch) return top1.avg
def train(train_loader, model, decoder, criterion, optimizer, epoch, log, tf_writer, index2wordDict): #global trainDataloader, valDataloader #print(len(train_loader)) #print(trainDataloader._getMode()) #print(valDataloader._getMode()) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() BLEUs = AverageMeter() teacher_forcing_ratio = 0.5 bptt = 64 #losses_extra = AverageMeter() #top1 = AverageMeter() #top5 = AverageMeter() #top1_extra = AverageMeter() #top5_extra = AverageMeter() if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode model.train() #print("308") end = time.time() #print("in train") with open(os.path.join(args.root_model, args.store_name, "train.txt"), "a+") as txt: txt.write("epoch\t" + str(epoch) + "\n") for i, data in enumerate(train_loader): loss = 0 #print("data fetch finish ",i) #print(data) #if isinstance(data, bool): #img DNE # continue if not data: continue data_time.update(time.time() - end) #print(data[0]) #print(data[1]) [URL, id, label, clips] = data input_video = torch.tensor([], dtype=torch.float).cuda() input_caption = torch.tensor([], dtype=torch.long).cuda() #cap_nums = [] for clip in range(0, len(clips), max(len(clips) // 50, 1)): #print(type(input_video.size())) #print(input_video.size()) if list(input_video.size()) != [0]: if input_video.size()[1] > 50: break video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat((input_video, video.float().cuda()), 1) input_caption = torch.cat((input_caption, caption.long().cuda()), 0) #cap_nums.append(clip[2]) #print(input_video.size()) #print(input_caption.size()) input_video = input_video.view(1, -1, 3, crop_size, crop_size) input_caption = input_caption.view(1, -1, input_caption.size()[-1], 1) #print(input_video.size()) #print(input_caption.size()) target = label.cuda() input_video_var = torch.autograd.Variable(input_video) input_caption_var = torch.autograd.Variable(input_caption) target_var = torch.autograd.Variable(target) # compute output wExtraLoss = 1 if args.prune == '' else 0.1 #print("input video size=",input_video.size()) encoder_output = model(input_video_var, input_caption_var) #size=(1, frames, 2048) print("encoder_output size=", encoder_output.size()) t_input = torch.zeros((1, 64, 2048)).cuda() for idx_e, e in enumerate(encoder_output[0]): t_input[0, idx_e] = e print("transformer input size(),", t_input.size()) decoder_outputs = [] src_mask = decoder.generate_square_subsequent_mask( target.size(1)).cuda() #print("target size=,",target.size()) #print("target= ", target) decoder_output = decoder(t_input, tar=target, mask=src_mask) #print("tar = ",tar) #print("tar size = ",tar.size()) decoder_outputs = decoder_output.argmax(-1) #print("decoder_outputs=",decoder_outputs) #for idx_t, tar in enumerate(target[0]): # decoder_output_word = t_input[0,idx_t].argmax().cpu() # decoder_outputs.append(decoder_output_word) # print(tar) # print(decoder_output_word) #loss += criterion(t_input[idx_t], tar) #print(decoder_output.transpose(0,1).size()) #print(target.size()) #print(criterion) loss = criterion( decoder_output.transpose(0, 1).squeeze(), target.squeeze()) #print("decoder_output_word=",decoder_output_word) #ttar = torch.zeros(17469, dtype=torch.long).unsqueeze(0).cuda() #ttar[0,tar.data] = torch.tensor(1, dtype=torch.long).cuda() #print("ttar size={}, decoder_output size={}".format(ttar.size(), decoder_output.size())) #print(decoder_output) #print(args.loss_type) #print(criterion) print("{0:*^50}".format("loss " + str(loss))) #exit() #print("target)var=",target_var.size()) #print(output) #print(target_var) #print("label size=",label.size()) #print(output.size()) #print(target_var.size()) #loss_main = criterion(output.squeeze(), target_var.squeeze()) #loss_main = 0 #lloss = torch.sub(output, target_var).squeeze() #for l in lloss: # loss_main += l**2 #extra_loss = criterion(extra, target_var)*wExtraLoss #loss = loss_main + extra_loss #loss = loss_main #print("loss=",loss) losses.update(loss.item(), ) #print(decoder_outputs) #print(decoder_outputs[0].item()) decoder_sentence = [ index2wordDict[index.item()] for index in decoder_outputs ] print(decoder_sentence) #print(target.size()) target_sentence = [index2wordDict[index.item()] for index in target[0]] print(target_sentence) bleu = sentence_bleu([ target_sentence ], decoder_sentence) #default weight is [0.25,0.25,0.25,0.25] = bleu-4 print("bleu =", bleu) BLEUs.update(bleu) # compute gradient and do SGD step st = time.time() loss.backward() #print("{0:*^50}".format("after backward\t"+str(time.time()-st))) if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) st = time.time() optimizer.step() #print("{0:*^50}".format("after step\t"+str(time.time()-st))) st = time.time() optimizer.zero_grad() #print("{0:*^50}".format("after zero_grad\t"+str(time.time()-st))) # measure elapsed time batch_time.update(time.time() - end) end = time.time() txtoutput = ('Epoch: [{0}][{1}/{2}], lr: {lr:.7f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' "output/target{output}/{target}" 'bleu {bleu.val:.4f} ({bleu.avg:.3f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, output=decoder_sentence, target=target_sentence, data_time=data_time, loss=losses, bleu=BLEUs, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO if i % args.print_freq == 0: print(txtoutput) log.write(txtoutput + '\n') log.flush() with open(os.path.join(args.root_model, args.store_name, "train.txt"), "a+") as txt: txt.write(txtoutput + "\n") print("**" * 50) #break #exit() tf_writer.add_scalar('loss/train', losses.avg, epoch) #tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) #tf_writer.add_scalar('acc/train_top5', top5.avg, epoch) tf_writer.add_scalar('auc/train', BLEUs.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)
def train(labeled_trainloader, unlabeled_trainloader, model, criterion, optimizer, epoch, log): batch_time = AverageMeter() data_time = AverageMeter() total_losses = AverageMeter() supervised_losses = AverageMeter() contrastive_losses = AverageMeter() group_contrastive_losses = AverageMeter() pl_losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() model = model.cuda() if args.no_partialbn: model.module.partialBN(False) else: model.module.partialBN(True) # switch to train mode model.train() if epoch >= args.sup_thresh or (args.use_finetuning and epoch >= args.finetune_start_epoch): data_loader = zip(labeled_trainloader, unlabeled_trainloader) else: data_loader = labeled_trainloader end = time.time() for i, data in enumerate(data_loader): # measure data loading time data_time.update(time.time() - end) #reseting losses contrastive_loss = torch.tensor(0.0).cuda() pl_loss = torch.tensor(0.0).cuda() loss = torch.tensor(0.0).cuda() group_contrastive_loss = torch.tensor(0.0).cuda() if epoch >= args.sup_thresh or (args.use_finetuning and epoch >= args.finetune_start_epoch): (labeled_data, unlabeled_data) = data images_fast, images_slow = unlabeled_data images_slow = images_slow.cuda() images_fast = images_fast.cuda() images_slow = torch.autograd.Variable(images_slow) images_fast = torch.autograd.Variable(images_fast) # contrastive_loss output_fast = model(images_fast) if not args.use_finetuning or epoch < args.finetune_start_epoch: output_slow = model(images_slow, unlabeled=True) output_fast_detach = output_fast.detach() if epoch >= args.sup_thresh and epoch < args.finetune_start_epoch: contrastive_loss = simclr_loss( torch.softmax(output_fast_detach, dim=1), torch.softmax(output_slow, dim=1)) if args.use_group_contrastive: grp_unlabeled_8seg = get_group(output_fast_detach) grp_unlabeled_4seg = get_group(output_slow) group_contrastive_loss = compute_group_contrastive_loss( grp_unlabeled_8seg, grp_unlabeled_4seg) elif args.use_finetuning and epoch >= args.finetune_start_epoch: pseudo_label = torch.softmax(output_fast_detach, dim=-1) max_probs, targets_pl = torch.max(pseudo_label, dim=-1) mask = max_probs.ge(args.threshold).float() targets_pl = torch.autograd.Variable(targets_pl) pl_loss = (F.cross_entropy( output_fast, targets_pl, reduction='none') * mask).mean() else: labeled_data = data input, target = labeled_data target = target.cuda() input = input.cuda() input = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) output = model(input) loss = criterion(output, target_var) total_loss = loss + args.gamma * contrastive_loss + group_contrastive_loss + args.gamma_finetune * pl_loss # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) if epoch >= args.sup_thresh: total_losses.update(total_loss.item(), input.size(0) + args.mu * input.size(0)) else: total_losses.update(total_loss.item(), input.size(0)) supervised_losses.update(loss.item(), input.size(0)) contrastive_losses.update(contrastive_loss.item(), input.size(0) + args.mu * input.size(0)) group_contrastive_losses.update( group_contrastive_loss.item(), input.size(0) + args.mu * input.size(0)) pl_losses.update(pl_loss.item(), input.size(0) + args.mu * input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # compute gradient and do SGD step total_loss.backward() if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) optimizer.step() optimizer.zero_grad() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ( 'Epoch: [{0}][{1}], lr: {lr:.5f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'TotalLoss {total_loss.val:.4f} ({total_loss.avg:.4f})\t' 'Supervised Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Contrastive_Loss {contrastive_loss.val:.4f} ({contrastive_loss.avg:.4f})\t' 'Group_contrastive_Loss {group_contrastive_loss.val:.4f} ({group_contrastive_loss.avg:.4f})\t' 'Pseudo_Loss {pl_loss.val:.4f} ({pl_loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, batch_time=batch_time, data_time=data_time, total_loss=total_losses, loss=supervised_losses, contrastive_loss=contrastive_losses, group_contrastive_loss=group_contrastive_losses, pl_loss=pl_losses, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush()
def build_database(model_rgb, model_flow, dataset, threshold): assert isinstance(threshold, float) model_rgb.eval() model_flow.eval() end = time.time() data_time = AverageMeter() database = [[] for _ in range(num_class)] for ind, (gen, label, _, _, length) in dataset: # gen(), label, rgb_record.path, video_index, self.segment_num logit_rgb = [] logit_flow = [] weight_rgb = [] weight_flow = [] feat_rgb = [] feat_flow = [] for seg in gen: data_time.update(time.time() - end) r, f = seg with torch.no_grad(): r = r.cuda() f = f.cuda() lr, wr, fr = model_rgb(r) lf, wf, ff = model_flow(f) logit_rgb.append(lr) logit_flow.append(lf) weight_rgb.append(wr) weight_flow.append(wf) feat_rgb.append(fr) feat_flow.append(f) logit_rgb = torch.cat(logit_rgb) logit_flow = torch.cat(logit_flow) weight_rgb = torch.cat(weight_rgb) weight_flow = torch.cat(weight_flow) feat_rgb = torch.cat(feat_rgb) feat_flow = torch.cat(feat_flow) if isinstance(label, int): score, score_rgb, weight_rgb, score_flow, weight_flow \ = get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, label, alpha) # [length] for sind, s in enumerate(score): if s > threshold: database[label].append({'feat_rgb': feat_rgb[sind], 'feat_flow': feat_flow[sind], 'feat_cat': torch.cat([feat_rgb[sind], feat_flow[sind]], 1), 'score': s, 'raw_score_rgb': score_rgb[sind], 'raw_score_flow': score_flow[sind], 'attention_rgb': weight_rgb[sind], 'attention_flow': weight_flow[sind]}) elif isinstance(label, list): for lab in label: score, score_rgb, weight_rgb, score_flow, weight_flow \ = get_score(logit_rgb, weight_rgb, logit_flow, weight_flow, lab, alpha) if s > threshold: database[lab].append({'feat_rgb': feat_rgb[sind], 'feat_flow': feat_flow[sind], 'feat_cat': torch.cat([feat_rgb[sind], feat_flow[sind]], 1), 'score': s, 'raw_score_rgb': score_rgb[sind], 'raw_score_flow': score_flow[sind], 'attention_rgb': weight_rgb[sind], 'attention_flow': weight_flow[sind]}) print("Database has generated. It has {} classes, and num of element in each class is: {}".format(len(database), [len(dc) for dc in database])) return database
def get_average_meters(number): return [AverageMeter() for _ in range(number)]
def validate(val_loader, model, criterion, epoch, log=None, tf_writer=None): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(val_loader): target = target.cuda() # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1.item(), input.size(0)) top5.update(prec5.item(), input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: output = ('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(output) if log is not None: log.write(output + '\n') log.flush() output = ( 'Testing Results: Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Loss {loss.avg:.5f}' .format(top1=top1, top5=top5, loss=losses)) print(output) if log is not None: log.write(output + '\n') log.flush() if tf_writer is not None: tf_writer.add_scalar('loss/test', losses.avg, epoch) tf_writer.add_scalar('acc/test_top1', top1.avg, epoch) tf_writer.add_scalar('acc/test_top5', top5.avg, epoch) return top1.avg
def train(train_loader, model, criterion, optimizer, epoch, log, tf_writer): #global trainDataloader, valDataloader #print(len(train_loader)) #print(trainDataloader._getMode()) #print(valDataloader._getMode()) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() AUCs = AverageMeter() #losses_extra = AverageMeter() #top1 = AverageMeter() #top5 = AverageMeter() #top1_extra = AverageMeter() #top5_extra = AverageMeter() #if args.no_partialbn: # model.module.partialBN(False) #else: # model.module.partialBN(True) # switch to train mode model.train() #print("308") end = time.time() #print("in train") with open(os.path.join(args.root_model, args.store_name, "train.txt"), "a+") as txt: txt.write("epoch\t" + str(epoch) + "\n") for idx, data in enumerate(train_loader): #print("data fetch finish ",i) #print(data) #if isinstance(data, bool): #img DNE # continue if not data: continue data_time.update(time.time() - end) #print(data[0]) #print(data[1]) [URL, id, sift, label, clips] = data #cap_nums = [] #print("sift=",sift) for i, s in enumerate(sift): input_video = torch.tensor([], dtype=torch.float).cuda() input_caption = torch.tensor([], dtype=torch.long).cuda() #print("i={},s={},lensift={},idx={},epoch={}".format(i, s, len(sift), idx, epoch)) if i + 1 < len(sift): #print(sift[i+1]) #print(s) #print(int(sift[i+1])-int(s)) if int(sift[i + 1]) - int(s) < 50: for clip in range(s, sift[i + 1]): video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) #print(input_video.size()) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) #print(input_caption.size()) #print(sift[i+1]) #print(s) #print(label[0,s: sift[i+1]]) target = label[0, s:sift[i + 1]].cuda() else: for clip in range(s, sift[i + 1], (int(sift[i + 1]) - int(s)) // 50): #print("clip={},s={},sift[i+1]={}".format(clip, s, sift[i+1])) if (clip - s) / ( (int(sift[i + 1]) - int(s)) // 50) >= 50: break video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:sift[i + 1]:( (int(sift[i + 1]) - int(s)) // 50)].cuda() target = target[:50].cuda() else: length = len(clips) if int(length) - int(s) < 50: for clip in range(s, length): video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:length].cuda() else: for clip in range(s, length, (int(length) - int(s)) // 50): if (clip - s) / ((int(length) - int(s)) // 50) >= 50: break video = clips[clip][0] caption = clips[clip][1] input_video = torch.cat( (input_video.float().cuda(), video.float().cuda()), 1) input_caption = torch.cat( (input_caption, caption.long().cuda()), 0) target = label[0, s:length:(length - s) // 50].cuda() target = target[:50] ''' video = clips[-1][0] caption = clips[-1][1] input_video = torch.cat((input_video.float().cuda(), video.float().cuda()),1) input_caption = torch.cat((input_caption, caption.long().cuda()),0) target = label[0,-1].cuda() ''' input_video = input_video.view(1, -1, 3, crop_size, crop_size) #input_caption = input_caption.view(1,-1,input_caption.size()[-1],1) input_caption = input_caption.view(1, -1, 64, 1) #print("target=",target.size()) input_video_var = torch.autograd.Variable(input_video) input_caption_var = torch.autograd.Variable(input_caption) target_var = torch.autograd.Variable(target) # compute output wExtraLoss = 1 if args.prune == '' else 0.1 output = model(input_video_var, input_caption_var) #print("outputsize=",output.size()) #print("target_var=",target_var.size()) #print("target=",target.size()) loss_main = criterion(output.squeeze(), target_var.squeeze()) loss = loss_main losses.update(loss_main.item(), ) auc = AUC(output.squeeze(), target_var.squeeze()) #print("auc=",auc) AUCs.update(auc) #print("after AUCs update") # compute gradient and do SGD step st = time.time() loss.backward() #print("{0:*^50}".format("after backward\t"+str(time.time()-st))) if args.clip_gradient is not None: total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient) st = time.time() optimizer.step() #print("{0:*^50}".format("after step\t"+str(time.time()-st))) st = time.time() optimizer.zero_grad() #print("{0:*^50}".format("after zero_grad\t"+str(time.time()-st))) # measure elapsed time batch_time.update(time.time() - end) end = time.time() ''' if i % args.print_freq == 0: output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.7f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Loss_h {losses_extra.val:.4f} ({losses_extra.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' .format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, losses_extra=losses_extra, top1=top1, top5=top5, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO print(output) log.write(output + '\n') log.flush() ''' txtoutput = ('Epoch: [{0}][{1}/{2}], lr: {lr:.7f}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' "output/target\t{output}/\t{target}\t" 'auc {auc.val:.4f} ({auc.avg:.3f})\t'.format( epoch, idx, len(train_loader), batch_time=batch_time, output=output, target=target, data_time=data_time, loss=losses, auc=AUCs, lr=optimizer.param_groups[-1]['lr'] * 0.1)) # TODO if idx % args.print_freq == 0: print(txtoutput) log.write(txtoutput + '\n') log.flush() print("**" * 50) with open(os.path.join(args.root_model, args.store_name, "train.txt"), "a+") as txt: txt.write(txtoutput + "\t" + str(URL) + "\t" + str(id) + "\n") #break #exit() tf_writer.add_scalar('loss/train', losses.avg, epoch) #tf_writer.add_scalar('acc/train_top1', top1.avg, epoch) #tf_writer.add_scalar('acc/train_top5', top5.avg, epoch) tf_writer.add_scalar('auc/train', AUCs.avg, epoch) tf_writer.add_scalar('lr', optimizer.param_groups[-1]['lr'], epoch)