def test(model, loader_test, data_length, device, criterion, batch_size, print_logger, step, use_top5=False, verbose=False): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() t1 = time.time() with torch.no_grad(): # switch to evaluate mode model.eval() end = time.time() for i, data in enumerate(loader_test): inputs = data[0]["data"].to(device) targets = data[0]["label"].squeeze().long().to(device) # for i, (inputs, targets) in enumerate(loader_test, 1): # inputs = inputs.to(device) # targets = targets.to(device) # compute output output = model(inputs) loss = criterion(output, targets) #measure accuracy and record loss prec1, prec5 = accuracy(output, targets, topk=(1, 5)) losses.update(loss.item(), batch_size) top1.update(prec1[0], batch_size) top5.update(prec5[0], batch_size) # measure elapsed time batch_time.update(time.time() - end) end = time.time() # plot progress # measure elapsed time t2 = time.time() print_logger.info('Test Step [{0}]: ' 'Loss {loss.avg:.4f} ' 'Prec@1(1,5) {top1.avg:.2f}, {top5.avg:.2f} ' 'Time {time}'.format(step, loss=losses, top1=top1, top5=top5, time=t2 - t1)) loader_test.reset() return top1.avg
def test(args, loader_test, model_s): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() cross_entropy = nn.CrossEntropyLoss() # switch to eval mode model_s.eval() with torch.no_grad(): for i, (inputs, targets) in enumerate(loader_test, 1): inputs = inputs.to(device) targets = targets.to(device) logits = model_s(inputs).to(device) loss = cross_entropy(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) print_logger.info('Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}' .format(top1=top1, top5=top5)) return top1.avg, top5.avg
def test(args, loader_test, model, criterion, writer_test, epoch=0): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model.eval() # num_iterations = len(loader_test) print("=> Evaluating...") logging.info('=> Evaluating...') with torch.no_grad(): # for i, (inputs, targets) in enumerate(loader_test, 1): for i, data in enumerate(loader_test): inputs = torch.cat([data[j]["data"] for j in range(num_gpu)], dim=0) targets = torch.cat([data[j]["label"] for j in range(num_gpu)], dim=0).squeeze().long() inputs = inputs.to(args.gpus[0]) targets = targets.to(args.gpus[0]) logits = model(inputs) loss = criterion(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1.item(), inputs.size(0)) top5.update(prec5.item(), inputs.size(0)) #print(f'* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}') print(f'* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}') logging.info('Top1: %e Top5: %e ', top1.avg, top5.avg) if not args.test_only: writer_test.add_scalar('test_top1', top1.avg, epoch) return top1.avg, top5.avg
def train(model, optimizer, trainLoader, args, epoch): model.train() losses = utils.AverageMeter() accurary = utils.AverageMeter() print_freq = len(trainLoader.dataset) // args.train_batch_size // 10 start_time = time.time() for batch, (inputs, targets) in enumerate(trainLoader): inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad() output = model(inputs) loss = loss_func(output, targets) loss.backward() losses.update(loss.item(), inputs.size(0)) optimizer.step() prec1 = utils.accuracy(output, targets) accurary.update(prec1[0], inputs.size(0)) if batch % print_freq == 0 and batch != 0: current_time = time.time() cost_time = current_time - start_time logger.info('Epoch[{}] ({}/{}):\t' 'Loss {:.4f}\t' 'Accurary {:.2f}%\t\t' 'Time {:.2f}s'.format(epoch, batch * args.train_batch_size, len(trainLoader.dataset), float(losses.avg), float(accurary.avg), cost_time)) start_time = current_time
def test(args, loader_test, model, criterion, writer_test, epoch=0): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model.eval() num_iterations = len(loader_test) print("=> Evaluating...") with torch.no_grad(): for i, (inputs, targets) in enumerate(loader_test, 1): inputs = inputs.to(args.gpus[0]) targets = targets.to(args.gpus[0]) logits = model(inputs) loss = criterion(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) print(f'* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}') if not args.test_only: writer_test.add_scalar('test_top1', top1.avg, epoch) return top1.avg, top5.avg
def test(args, loader_test, model, criterion, writer_test, epoch): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model.eval() num_iterations = len(loader_test) with torch.no_grad(): for i, (inputs, targets) in enumerate(loader_test, 1): num_iters = num_iterations * epoch + i inputs = inputs.to(device) targets = targets.to(device) logits = model(inputs) loss = criterion(logits, targets) writer_test.add_scalar('Test_loss (fine-tuned)', loss.item(), num_iters) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) writer_test.add_scalar('Prec@1', top1.avg, num_iters) writer_test.add_scalar('Prec@5', top5.avg, num_iters) print_logger.info(f'* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}') ''' if not args.test_only: writer_test.add_scalar('test_top1', top1.avg, epoch) ''' return top1.avg, top5.avg
def train(args, loader_train, model, criterion, optimizer, writer_train, epoch): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model.train() num_iterations = len(loader_train) for i, (inputs, targets) in enumerate(loader_train, 1): num_iters = num_iterations * epoch + i inputs = inputs.to(args.gpus[0]) targets = targets.to(args.gpus[0]) logits = model(inputs) loss = criterion(logits, targets) writer_train.add_scalar('Train_loss (fine-tuned)', loss.item(), num_iters) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) writer_train.add_scalar('Prec@1', top1.avg, num_iters) writer_train.add_scalar('Prec@5', top5.avg, num_iters) optimizer.zero_grad() loss.backward() optimizer.step()
def train(args, loader_train, model, criterion, optimizer, writer_train, epoch): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model.train() num_iterations = len(loader_train) for i, (inputs, targets) in enumerate(loader_train, 1): inputs = inputs.to(args.gpus[0]) targets = targets.to(args.gpus[0]) logits = model(inputs) loss = criterion(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) optimizer.zero_grad() loss.backward() optimizer.step()
def test(model, testLoader, topk=(1,)): model.eval() losses = utils.AverageMeter() accuracy = utils.AverageMeter() top5_accuracy = utils.AverageMeter() start_time = time.time() #testLoader = get_data_set('test') #i = 0 with torch.no_grad(): for batch_idx, batch_data in enumerate(testLoader): #i+=1 #if i > 5: #break inputs = batch_data[0]['data'].to(device) targets = batch_data[0]['label'].squeeze().long().to(device) targets = targets.cuda(non_blocking=True) outputs = model(inputs) loss = loss_func(outputs, targets) losses.update(loss.item(), inputs.size(0)) predicted = utils.accuracy(outputs, targets, topk=topk) accuracy.update(predicted[0], inputs.size(0)) top5_accuracy.update(predicted[1], inputs.size(0)) current_time = time.time() logger.info( 'Test Loss {:.4f}\tTop1 {:.2f}%\tTop5 {:.2f}%\tTime {:.2f}s\n' .format(float(losses.avg), float(accuracy.avg), float(top5_accuracy.avg), (current_time - start_time)) ) return top5_accuracy.avg, accuracy.avg
def test(model, testLoader): global best_acc model.eval() losses = utils.AverageMeter() accurary = utils.AverageMeter() start_time = time.time() with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testLoader): inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) loss = loss_func(outputs, targets) losses.update(loss.item(), inputs.size(0)) predicted = utils.accuracy(outputs, targets) accurary.update(predicted[0], inputs.size(0)) current_time = time.time() logger.info( 'Test Loss {:.4f}\tAccurary {:.2f}%\t\tTime {:.2f}s\n'.format( float(losses.avg), float(accurary.avg), (current_time - start_time))) return accurary.avg
def test(args, loader_test, model_s): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() cross_entropy = nn.CrossEntropyLoss() # switch to eval mode model_s.eval() with torch.no_grad(): for i, (inputs, targets) in enumerate(loader_test, 1): inputs = inputs.to(args.gpus[0]) targets = targets.to(args.gpus[0]) logits = model_s(inputs).to(args.gpus[0]) loss = cross_entropy(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) print('* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format( top1=top1, top5=top5)) mask = [] for name, weight in model_s.named_parameters(): if 'mask' in name: mask.append(weight.item()) print("* Pruned {} / {}".format(sum(m == 0 for m in mask), len(mask))) return top1.avg, top5.avg
def test(model, testLoader, topk=(1, )): model.eval() losses = utils.AverageMeter() accuracy = utils.AverageMeter() top5_accuracy = utils.AverageMeter() start_time = time.time() with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testLoader): inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) loss = loss_func(outputs, targets) losses.update(loss.item(), inputs.size(0)) predicted = utils.accuracy(outputs, targets, topk=topk) accuracy.update(predicted[0], inputs.size(0)) if len(topk) == 2: top5_accuracy.update(predicted[1], inputs.size(0)) current_time = time.time() if len(topk) == 1: logger.info( 'Test Loss {:.4f}\tAccuracy {:.2f}%\t\tTime {:.2f}s\n'.format( float(losses.avg), float(accuracy.avg), (current_time - start_time))) else: logger.info( 'Test Loss {:.4f}\tTop1 {:.2f}%\tTop5 {:.2f}%\tTime {:.2f}s\n'. format(float(losses.avg), float(accuracy.avg), float(top5_accuracy.avg), (current_time - start_time))) if len(topk) == 1: return accuracy.avg else: return top5_accuracy.avg
def test(model, testLoader, topk=(1, )): model.eval() losses = utils.AverageMeter('Loss', ':.4e') top1_accuracy = utils.AverageMeter('Acc@1', ':6.2f') top5_accuracy = utils.AverageMeter('Acc@5', ':6.2f') start_time = time.time() with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testLoader): inputs, targets = inputs.to(device), targets.to(device) # compute output outputs = model(inputs) loss = loss_func(outputs, targets) # measure accuracy and record loss losses.update(loss.item(), inputs.size(0)) pred = utils.accuracy(outputs, targets, topk=topk) top1_accuracy.update(pred[0], inputs.size(0)) top5_accuracy.update(pred[1], inputs.size(0)) # measure elapsed time current_time = time.time() print( f'Test Loss: {float(losses.avg):.6f}\t Top1: {float(top1_accuracy.avg):.6f}%\t' f'Top5: {float(top5_accuracy.avg):.6f}%\t Time: {float(current_time - start_time):.2f}s' ) return float(top1_accuracy.avg), float(top5_accuracy.avg)
def validate(self, epoch, val_loader, model, criterion): batch_time = utils.AverageMeter('Time', ':6.3f') losses = utils.AverageMeter('Loss', ':.4e') top1 = utils.AverageMeter('Acc@1', ':6.2f') top5 = utils.AverageMeter('Acc@5', ':6.2f') # switch to evaluation mode model.eval() with torch.no_grad(): end = time.time() for i, (images, target) in enumerate(val_loader): images = images.cuda() target = target.cuda() # compute output logits = model(images) loss = criterion(logits, target) # measure accuracy and record loss pred1, pred5 = utils.accuracy(logits, target, topk=(1, 5)) n = images.size(0) losses.update(loss.item(), n) top1.update(pred1[0], n) top5.update(pred5[0], n) # measure elapsed time batch_time.update(time.time() - end) end = time.time() self.logger.info( ' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'.format( top1=top1, top5=top5)) return losses.avg, top1.avg, top5.avg
def train_class(epoch, train_loader, model, criterion, optimizer): batch_time = utils.AverageMeter('Time', ':6.3f') data_time = utils.AverageMeter('Data', ':6.3f') losses = utils.AverageMeter('Loss', ':.4e') top1 = utils.AverageMeter('Acc@1', ':6.2f') top5 = utils.AverageMeter('Acc@5', ':6.2f') model.train() end = time.time() num_iter = len(train_loader) print_freq = num_iter // 10 i = 0 for batch_idx, (images, targets) in enumerate(train_loader): if args.debug: if i > 5: break i += 1 images = images.to(device) targets = targets.to(device) data_time.update(time.time() - end) # compute output logits, mask = model(images, targets) loss = criterion(logits, targets) for m in mask: loss += float(args.sparse_lambda) * torch.sum(m, 0).norm(2) # measure accuracy and record loss prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) n = images.size(0) losses.update(loss.item(), n) # accumulated loss top1.update(prec1.item(), n) top5.update(prec5.item(), n) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % print_freq == 0 and batch_idx != 0: logger.info( 'Epoch[{0}]({1}/{2}): ' 'Loss {loss.avg:.4f} ' 'Prec@1(1,5) {top1.avg:.2f}, {top5.avg:.2f}'.format( epoch, batch_idx, num_iter, loss=losses, top1=top1, top5=top5)) return losses.avg, top1.avg, top5.avg
def train(self, epoch, train_loader, model, criterion, optimizer, scheduler): batch_time = utils.AverageMeter('Time', ':6.3f') data_time = utils.AverageMeter('Data', ':6.3f') losses = utils.AverageMeter('Loss', ':.4e') top1 = utils.AverageMeter('Acc@1', ':6.2f') top5 = utils.AverageMeter('Acc@5', ':6.2f') model.train() end = time.time() for param_group in optimizer.param_groups: cur_lr = param_group['lr'] self.logger.info('learning_rate: ' + str(cur_lr)) num_iter = len(train_loader) for i, (images, target) in enumerate(train_loader): data_time.update(time.time() - end) images = images.cuda() target = target.cuda() # compute outputy logits = model(images) loss = criterion(logits, target) # measure accuracy and record loss prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5)) n = images.size(0) losses.update(loss.item(), n) #accumulated loss top1.update(prec1.item(), n) top5.update(prec5.item(), n) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % self.args.print_freq == 0: self.logger.info( 'Epoch[{0}]({1}/{2}): ' 'Loss {loss.avg:.4f} ' 'Prec@1(1,5) {top1.avg:.2f}, {top5.avg:.2f}'.format( epoch, i, num_iter, loss=losses, top1=top1, top5=top5)) scheduler.step() return losses.avg, top1.avg, top5.avg
def train(model, optimizer, trainLoader, args, epoch, topk=(1,)): model.train() losses = utils.AverageMeter() accuracy = utils.AverageMeter() top5_accuracy = utils.AverageMeter() print_freq = trainLoader._size // args.train_batch_size // 10 start_time = time.time() #trainLoader = get_data_set('train') #i = 0 for batch, batch_data in enumerate(trainLoader): #i+=1 #if i>5: #break inputs = batch_data[0]['data'].to(device) targets = batch_data[0]['label'].squeeze().long().to(device) train_loader_len = int(math.ceil(trainLoader._size / args.train_batch_size)) adjust_learning_rate(optimizer, epoch, batch, train_loader_len, args) output = model(inputs) loss = loss_func(output, targets) optimizer.zero_grad() loss.backward() losses.update(loss.item(), inputs.size(0)) optimizer.step() prec1 = utils.accuracy(output, targets, topk=topk) accuracy.update(prec1[0], inputs.size(0)) top5_accuracy.update(prec1[1], inputs.size(0)) if batch % print_freq == 0 and batch != 0: current_time = time.time() cost_time = current_time - start_time logger.info( 'Epoch[{}] ({}/{}):\t' 'Loss {:.4f}\t' 'Top1 {:.2f}%\t' 'Top5 {:.2f}%\t' 'Time {:.2f}s'.format( epoch, batch * args.train_batch_size, trainLoader._size, float(losses.avg), float(accuracy.avg), float(top5_accuracy.avg), cost_time ) ) start_time = current_time
def test(args, loader_test, model_s): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() cross_entropy = nn.CrossEntropyLoss() # switch to eval mode model_s.eval() with torch.no_grad(): #for i, (inputs, targets) in enumerate(loader_test, 1): for i, data in enumerate(loader_test): #if i > 20: # break inputs = torch.cat([data[j]["data"] for j in range(num_gpu)], dim=0) targets = torch.cat([data[j]["label"] for j in range(num_gpu)], dim=0).squeeze().long() targets = targets.cuda(non_blocking=True) inputs = inputs.cuda() #inputs = inputs.to(args.gpus[0]) #targets = targets.to(args.gpus[0]) logits = model_s(inputs).to(args.gpus[0]) loss = cross_entropy(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) print('* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format( top1=top1, top5=top5)) logging.info('Top1: %e Top5: %e ', top1.avg, top5.avg) mask = [] for name, weight in model_s.named_parameters(): if 'mask' in name: for i in range(len(weight)): mask.append(weight[i].item()) # num_pruned = sum(m == 0 for m in mask) print("* Pruned {} / {}".format(sum(m == 0 for m in mask), len(mask))) logging.info('Pruned: %e Total: %e ', sum(m == 0 for m in mask), len(mask)) return top1.avg, top5.avg
def train_class(model, optimizer, trainLoader, epoch, topk=(1, )): model.train() losses = utils.AverageMeter(':.4e') accurary = utils.AverageMeter(':6.3f') top5_accuracy = utils.AverageMeter(':6.3f') print_freq = len(trainLoader.dataset) // args.train_batch_size // 10 start_time = time.time() for batch, (inputs, targets) in enumerate(trainLoader): inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad() output, mask = model(inputs, targets) loss = loss_func(output, targets) for m in mask: loss += float(args.sparse_lambda) * torch.sum(m, 0).norm(2) loss.backward() losses.update(loss.item(), inputs.size(0)) optimizer.step() prec1 = utils.accuracy(output, targets, topk=topk) accurary.update(prec1[0], inputs.size(0)) if len(topk) == 2: top5_accuracy.update(prec1[1], inputs.size(0)) if batch % print_freq == 0 and batch != 0: current_time = time.time() cost_time = current_time - start_time if len(topk) == 1: logger.info('Epoch[{}] ({}/{}):\t' 'Loss {:.4f}\t' 'Accuracy {:.2f}%\t\t' 'Time {:.2f}s'.format( epoch, batch * args.train_batch_size, len(trainLoader.dataset), float(losses.avg), float(accurary.avg), cost_time)) else: logger.info('Epoch[{}] ({}/{}):\t' 'Loss {:.4f}\t' 'Top1 {:.2f}%\t' 'Top5 {:.2f}%\t' 'Time {:.2f}s'.format( epoch, batch * args.train_batch_size, len(trainLoader.dataset), float(losses.avg), float(accurary.avg), float(top5_accuracy.avg), cost_time)) start_time = current_time
def train(args, loader_train, model, criterion, optimizer, writer_train, epoch, model_kd = None): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model.train() # num_iterations = len(loader_train) # for i, (inputs, targets) in enumerate(loader_train, 1): for i, data in enumerate(loader_train): inputs = torch.cat([data[j]["data"] for j in range(num_gpu)], dim=0) targets = torch.cat([data[j]["label"] for j in range(num_gpu)], dim=0).squeeze().long() inputs = inputs.to(args.gpus[0]) targets = targets.to(args.gpus[0]) logits = model(inputs) loss = criterion(logits, targets) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) optimizer.zero_grad() loss.backward(retain_graph=True) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) if i % 500 == 0: print(f'* Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}') logging.info('Top1: %e Top5: %e ', top1.avg, top5.avg) #kd_flag = 0 if kd_flag: inputs = inputs.to(args.gpus[1]) features_kd = model_kd(inputs) alpha = 0.99 Temperature = 30 logits = logits.to(args.gpus[1]) KD_loss = nn.KLDivLoss()(F.log_softmax(logits / Temperature, dim=1), F.softmax(features_kd / Temperature, dim=1)) * ( alpha * Temperature * Temperature) #+ F.cross_entropy(logits, targets) * (1 - alpha) KD_loss.backward() # inputs = inputs.to(args.gpus[0]) optimizer.step()
def validate(val_loader, model, criterion, args): batch_time = utils.AverageMeter('Time', ':6.3f') losses = utils.AverageMeter('Loss', ':.4e') top1 = utils.AverageMeter('Acc@1', ':6.2f') top5 = utils.AverageMeter('Acc@5', ':6.2f') num_iter = len(val_loader) model.eval() with torch.no_grad(): end = time.time() i = 0 for batch_idx, (images, targets) in enumerate(val_loader): if args.debug: if i > 5: break i += 1 images = images.cuda() targets = targets.cuda() # compute output logits = model(images) loss = criterion(logits, targets) # measure accuracy and record loss pred1, pred5 = utils.accuracy(logits, targets, topk=(1, 5)) n = images.size(0) losses.update(loss.item(), n) top1.update(pred1[0], n) top5.update(pred5[0], n) # measure elapsed time batch_time.update(time.time() - end) end = time.time() logger.info(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}' .format(top1=top1, top5=top5)) return losses.avg, top1.avg, top5.avg
def test(model, testLoader): model.eval() losses = utils.AverageMeter('Loss', ':.4e') accuracy = utils.AverageMeter('Acc@1', ':6.2f') start_time = time.time() with torch.no_grad(): for batch_idx, (inputs, targets) in enumerate(testLoader): inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) loss = loss_func(outputs, targets) losses.update(loss.item(), inputs.size(0)) pred = utils.accuracy(outputs, targets) accuracy.update(pred[0], inputs.size(0)) current_time = time.time() print( f'Test Loss: {float(losses.avg):.4f}\t Acc: {float(accuracy.avg):.2f}%\t\t Time: {(current_time - start_time):.2f}s' ) return accuracy.avg
def test(args, loader_test, model_s, epoch): losses = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() cross_entropy = nn.CrossEntropyLoss() # switch to eval mode model_s.eval() num_iterations = len(loader_test) with torch.no_grad(): for i, (inputs, targets) in enumerate(loader_test, 1): num_iters = num_iterations * epoch + i inputs = inputs.to(device) targets = targets.to(device) logits = model_s(inputs).to(device) loss = cross_entropy(logits, targets) writer_test.add_scalar('Test_loss', loss.item(), num_iters) prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) losses.update(loss.item(), inputs.size(0)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) writer_test.add_scalar('Prec@1', top1.avg, num_iters) writer_test.add_scalar('Prec@5', top5.avg, num_iters) print_logger.info( 'Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}\n' '===============================================\n'.format(top1=top1, top5=top5)) return top1.avg, top5.avg
def test(model, topk=(1, )): model.eval() losses = utils.AverageMeter() accuracy = utils.AverageMeter() top5_accuracy = utils.AverageMeter() start_time = time.time() with torch.no_grad(): for batch_idx, batch_data in enumerate(testLoader): if len(topk) == 2: inputs = batch_data[0]['data'].to(device) targets = batch_data[0]['label'].squeeze().long().to(device) else: inputs = batch_data[0] targets = batch_data[1] inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) loss = loss_func(outputs, targets) losses.update(loss.item(), inputs.size(0)) predicted = utils.accuracy(outputs, targets, topk=topk) accuracy.update(predicted[0], inputs.size(0)) if len(topk) == 2: top5_accuracy.update(predicted[1], inputs.size(0)) current_time = time.time() if len(topk) == 1: print( 'Test Loss {:.4f}\tAccuracy {:.2f}%\t\tTime {:.2f}s\n'.format( float(losses.avg), float(accuracy.avg), (current_time - start_time))) else: print( 'Test Loss {:.4f}\tTop1 {:.2f}%\tTop5 {:.2f}%\tTime {:.2f}s\n'. format(float(losses.avg), float(accuracy.avg), float(top5_accuracy.avg), (current_time - start_time)))
def bn_update(model, loader, cumulative=False): """ BatchNorm buffers update (if any). Performs 1 epochs to estimate buffers average using train dataset. :param model: model being update :param loader: train dataset loader for buffers average estimation. :param cumulative: cumulative moving average or exponential moving average :return: approcimate train accuracy (util.AverageMeter) """ if not check_bn(model): return print("approcimate process:") train_acc = utils.AverageMeter() model.train() model.apply(reset_bn) if cumulative: momenta = {} model.apply(lambda module: _get_momenta(module, momenta)) for module in momenta.keys(): module.momentum = None approcimate_num = int(len(loader) * args.approcimate_rate) with torch.no_grad(): # freeze all the parameters # with tqdm(total=approcimate_num) as pbar: for i, (inputs, targets) in enumerate(loader): inputs, targets = inputs.to(device), targets.to(device) outputs = model(inputs) prec1 = utils.accuracy(outputs, targets) # pbar.set_description(f"train accuracy:{prec1}") train_acc.update(prec1[0], inputs.size(0)) # pbar.update(1) if i >= approcimate_num: break return train_acc
def test(args, loader_test, model, epoch=0): top1 = utils.AverageMeter() top5 = utils.AverageMeter() top1_t = utils.AverageMeter() top5_t = utils.AverageMeter() top1_s1 = utils.AverageMeter() top1_s2 = utils.AverageMeter() top1_s3 = utils.AverageMeter() top1_s4 = utils.AverageMeter() top5_s1 = utils.AverageMeter() top5_s2 = utils.AverageMeter() top5_s3 = utils.AverageMeter() top5_s4 = utils.AverageMeter() model.eval() num_iterations = len(loader_test) with torch.no_grad(): print_logger.info("=> Evaluating...") for i, (inputs, targets) in enumerate(loader_test, 1): inputs = inputs.cuda() targets = targets.cuda() # compute output logits_s, logits_t = model(inputs) best_prec_s_1 = 0. for j in range(args.num_stu): prec1, prec5 = utils.accuracy(logits_s[j], targets, topk=(1, 5)) eval('top1_s%d' % (j + 1)).update(prec1[0], inputs.size(0)) eval('top5_s%d' % (j + 1)).update(prec5[0], inputs.size(0)) if prec1 > best_prec_s_1: best_prec_s_1 = prec1 writer_test.add_scalar('test_stu_%d_top1' % (j + 1), prec1[0], num_iterations * epoch + i) prec1, prec5 = utils.accuracy(logits_t, targets, topk=(1, 5)) writer_test.add_scalar('test_tea_top1', prec1[0], num_iterations * epoch + i) top1_t.update(prec1[0], inputs.size(0)) top5_t.update(prec5[0], inputs.size(0)) for j in range(args.num_stu): if eval('top1_s%d' % (j + 1)).avg > top1.avg: top1.avg = eval('top1_s%d' % (j + 1)).avg top5.avg = eval('top5_s%d' % (j + 1)).avg #best_branch = j+ 1 print_logger.info('Epoch[{0}]({1}/{2}): ' 'Prec@1(1,5) {top1.avg:.2f}, {top5.avg:.2f}'.format( epoch, i, num_iterations, top1=top1, top5=top5)) for i in range(args.num_stu): print_logger.info('top1_s%d: %.2f' % (i + 1, eval('top1_s%d' % (i + 1)).avg)) print_logger.info('top1_t: %.2f' % (top1_t.avg)) if not args.test_only: writer_test.add_scalar('test_top1', top1.avg, epoch) return top1.avg, top5.avg
def train(args, loader_train, model, criterion, optimizer, epoch): losses = utils.AverageMeter() top1_t = utils.AverageMeter() top5_t = utils.AverageMeter() top1_s = utils.AverageMeter() top5_s = utils.AverageMeter() model.train() # update learning rate for param_group in optimizer.param_groups: writer_train.add_scalar('learning_rate', param_group['lr'], epoch) num_iterations = len(loader_train) for i, (inputs, targets) in enumerate(loader_train, 1): inputs = inputs.cuda() targets = targets.cuda() # compute output logits_s, logits_t = model(inputs) loss = criterion(logits_t, targets) best_prec_s_1 = torch.tensor(0.).cuda() best_prec_s_5 = torch.tensor(0.).cuda() best_branch = 1 for j in range(args.num_stu): loss += criterion(logits_s[j], targets) loss += args.t * args.t * utils.KL(logits_t / args.t, logits_s[j] / args.t) prec1, prec5 = utils.accuracy(logits_s[j], targets, topk=(1, 5)) writer_train.add_scalar('train_stu_%d_top1' % (j + 1), prec1.item(), num_iterations * epoch + i) if prec1 > best_prec_s_1: best_prec_s_1 = prec1 best_prec_s_5 = prec5 best_branch = j + 1 prec1 = best_prec_s_1 prec5 = best_prec_s_5 prec1_t, prec5_t = utils.accuracy(logits_t, targets, topk=(1, 5)) top1_t.update(prec1_t.item(), inputs.size(0)) top5_t.update(prec5_t.item(), inputs.size(0)) losses.update(loss.item(), inputs.size(0)) writer_train.add_scalar('train_top1', prec1.item(), num_iterations * epoch + i) writer_train.add_scalar('train_loss', loss.item(), num_iterations * epoch + i) top1_s.update(prec1.item(), inputs.size(0)) top5_s.update(prec5.item(), inputs.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time if i % args.print_freq == 0: print_logger.info( 'Epoch[{0}]({1}/{2}): ' 'Loss {loss.avg:.4f} ' 'TeacherPrec@1(1,5) {top1_t.avg:.2f}, {top5_t.avg:.2f} ' 'StuPrec@1(1,5) {top1_s.avg:.2f}, {top5_s.avg:.2f} ' 'Best branch: {best_branch: d}'.format( epoch, i, num_iterations, loss=losses, top1_t=top1_t, top5_t=top5_t, top1_s=top1_s, top5_s=top5_s, best_branch=best_branch)) return losses.avg, top1_s.avg
def train(args, loader_train, models, optimizers, epoch, writer_train): #losses_d = utils.AverageMeter() #losses_data = utils.AverageMeter() #losses_g = utils.AverageMeter() #losses_sparse = utils.AverageMeter() #losses_kl = utils.AverageMeter() top1 = utils.AverageMeter() top5 = utils.AverageMeter() model_t = models[0] model_s = models[1] model_d = models[2] #model_kd = models[3] bce_logits = nn.BCEWithLogitsLoss() optimizer_d = optimizers[0] optimizer_s = optimizers[1] optimizer_m = optimizers[2] # switch to train mode model_d.train() model_s.train() num_iterations = int(loader_train._size / batch_sizes) #num_iterations = len(loader_train) print(num_iterations) real_label = 1 fake_label = 0 exact_list = ["layer3"] num_pruned = -1 t0 = time.time() ''' prec1 = [60] #prec1 = 0 error_d = 0 error_sparse = 0 error_g = 0 error_data = 0 KD_loss = 0 alpha_d = args.miu * ( 0.9 - epoch / args.num_epochs * 0.9 ) sparse_lambda = args.sparse_lambda mask_step = args.mask_step lr_decay_step = args.lr_decay_step ''' #for i, (inputs, targets) in enumerate(loader_train, 1): for i, data in enumerate(loader_train): global iteration iteration = i tt0 = time.time() if i % 60 == 1: t0 = time.time() if i % 400 == 1: num_mask = [] for name, weight in model_s.named_parameters(): if 'mask' in name: for ii in range(len(weight)): num_mask.append(weight[ii].item()) num_pruned = sum(m == 0 for m in num_mask) if num_pruned > 1100: iteration = 1 #num_iters = num_iterations * epoch + i if i > 100 and top1.val < 30: iteration = 1 #iteration = 2 gl.set_value('iteration', iteration) inputs = torch.cat([data[j]["data"] for j in range(num_gpu)], dim=0) targets = torch.cat([data[j]["label"] for j in range(num_gpu)], dim=0).squeeze().long() targets = targets.cuda(non_blocking=True) inputs = inputs.cuda() #inputs = inputs.to(args.gpus[0]) #targets = targets.to(args.gpus[0]) features_t = model_t(inputs) features_s = model_s(inputs) #features_kd = model_kd(inputs) ############################ # (1) Update # D network ########################### #''' for p in model_d.parameters(): p.requires_grad = True optimizer_d.zero_grad() output_t = model_d(features_t.to(args.gpus[0]).detach()) labels_real = torch.full_like(output_t, real_label, device=args.gpus[0]) error_real = bce_logits(output_t, labels_real) output_s = model_d(features_s.to(args.gpus[0]).detach()) labels_fake = torch.full_like(output_t, fake_label, device=args.gpus[0]) error_fake = bce_logits(output_s, labels_fake) error_d = 0.1 * error_real + 0.1 * error_fake labels = torch.full_like(output_s, real_label, device=args.gpus[0]) #error_d += bce_logits(output_s, labels) error_d.backward() #losses_d.update(error_d.item(), inputs.size(0)) #writer_train.add_scalar( #'discriminator_loss', error_d.item(), num_iters) optimizer_d.step() #if i % args.print_freq == 0:#i >= 0:# if i < 0: print('=> D_Epoch[{0}]({1}/{2}):\t' 'Loss_d {loss_d.val:.4f} ({loss_d.avg:.4f})\t'.format( epoch, i, num_iterations, loss_d=losses_d)) #''' ############################ # (2) Update student network ########################### #''' for p in model_d.parameters(): p.requires_grad = False optimizer_s.zero_grad() optimizer_m.zero_grad() alpha = 0.9 - epoch / args.num_epochs * 0.9 Temperature = 10 KD_loss = 10 * nn.KLDivLoss()( F.log_softmax(features_s / Temperature, dim=1), F.softmax(features_t / Temperature, dim=1)) * ( alpha * Temperature * Temperature) + F.cross_entropy( features_s, targets) * (1 - alpha) KD_loss.backward(retain_graph=True) #losses_kl.update(KD_loss.item(), inputs.size(0)) # data_loss alpha = 0.9 - epoch / args.num_epochs * 0.9 #one_hot = torch.zeros(targets.shape[0], 1000).cuda() #one_hot = one_hot.scatter_(1, targets.reshape(targets.shape[0],1), 1).cuda() error_data = args.miu * ( alpha * F.mse_loss(features_t, features_s.to(args.gpus[0])) ) # + (1 - alpha) * F.mse_loss(one_hot, features_s.to(args.gpus[0]))) #losses_data.update(error_data.item(), inputs.size(0)) error_data.backward(retain_graph=True) # fool discriminator #tt3 = time.time() output_s = model_d(features_s.to(args.gpus[0])) labels = torch.full_like(output_s, real_label, device=args.gpus[0]) error_g = 0.1 * bce_logits(output_s, labels) #losses_g.update(error_g.item(), inputs.size(0)) #writer_train.add_scalar( #'generator_loss', error_g.item(), num_iters) error_g.backward(retain_graph=True) optimizer_s.step() #''' # train mask error_sparse = 0 decay = (epoch % args.lr_decay_step == 0 and i == 1) if i % (args.mask_step) == 0: mask = [] for name, param in model_s.named_parameters(): if 'mask' in name: mask.append(param.view(-1)) mask = torch.cat(mask) error_sparse = 0.00001 * args.sparse_lambda * F.l1_loss( mask, torch.zeros(mask.size()).to(args.gpus[0]), reduction='sum') error_sparse.backward() optimizer_m.step(decay) #losses_sparse.update(error_sparse.item(), inputs.size(0)) #writer_train.add_scalar( #'sparse_loss', error_sparse.item(), num_iters) prec1, prec5 = utils.accuracy(features_s.to(args.gpus[0]), targets.to(args.gpus[0]), topk=(1, 5)) top1.update(prec1[0], inputs.size(0)) top5.update(prec5[0], inputs.size(0)) if i % 60 == 0: t1 = time.time() print('=> G_Epoch[{0}]({1}/{2}):\n' 'Loss_s {loss_sparse:.4f} \t' 'Loss_data {loss_data:.4f}\t' 'Loss_d {loss_d:.4f} \n' 'Loss_g {loss_g:.4f} \t' 'Loss_kl {loss_kl:.4f} \n' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})\n' 'time {time:.4f}\t' 'pruned {np}'.format(epoch, i, num_iterations, loss_sparse=error_sparse, loss_data=error_data, loss_d=error_d, loss_g=error_g, loss_kl=KD_loss, top1=top1, top5=top5, time=t1 - t0, np=num_pruned)) logging.info( 'TRAIN epoch: %03d step : %03d Top1: %e Top5: %e error_g: %e error_data: %e error_d: %e Duration: %f Pruned: %d', epoch, i, top1.avg, top5.avg, error_g, error_data, error_d, t1 - t0, num_pruned)
def calculationFitness(honey, args): global best_honey global best_honey_state if args.arch == 'vgg': model = import_module(f'model.{args.arch}').BeeVGG(honeysource=honey, num_classes=1000).to(device) load_vgg_honey_model(model, args.random_rule) elif args.arch == 'resnet': model = import_module(f'model.{args.arch}').resnet(args.cfg,honey=honey).to(device) load_resnet_honey_model(model, args.random_rule) elif args.arch == 'googlenet': pass elif args.arch == 'densenet': pass #start_time = time.time() if len(args.gpus) != 1: model = nn.DataParallel(model, device_ids=args.gpus) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) #test(model, testLoader) model.train() #trainLoader = get_data_set('train') #i = 0 for epoch in range(args.calfitness_epoch): #print(epoch) for batch, batch_data in enumerate(trainLoader): #i += 1 #print(i) #if i > 5: #break #if i < 10: # continue #i = 0 inputs = batch_data[0]['data'].to(device) targets = batch_data[0]['label'].squeeze().long().to(device) train_loader_len = int(math.ceil(trainLoader._size / args.train_batch_size)) adjust_learning_rate(optimizer, epoch, batch, train_loader_len, args) #print('epoch{}\tlr{}'.format(epoch,lr)) optimizer.zero_grad() output = model(inputs) loss = loss_func(output, targets) loss.backward() optimizer.step() trainLoader.reset() #test(model, loader.testLoader) fit_accurary = utils.AverageMeter() model.eval() #testLoader = get_data_set('test') #i = 0 with torch.no_grad(): for batch_idx, batch_data in enumerate(testLoader): #print(i) #i += 1 #if i > 5: #reak #if i < 10: #continue #i = 0 inputs = batch_data[0]['data'].to(device) targets = batch_data[0]['label'].squeeze().long().to(device) outputs = model(inputs) predicted = utils.accuracy(outputs, targets,topk=(1,5)) fit_accurary.update(predicted[1], inputs.size(0)) testLoader.reset() #current_time = time.time() ''' logger.info( 'Honey Source fintness {:.2f}%\t\tTime {:.2f}s\n' .format(float(accurary.avg), (current_time - start_time)) ) ''' if fit_accurary.avg == 0: fit_accurary.avg = 0.01 if fit_accurary.avg > best_honey.fitness: best_honey_state = copy.deepcopy(model.module.state_dict() if len(args.gpus) > 1 else model.state_dict()) best_honey.code = copy.deepcopy(honey) best_honey.fitness = fit_accurary.avg return fit_accurary.avg
def train(epoch, train_loader, model, criterion, optimizer): batch_time = utils.AverageMeter('Time', ':6.3f') data_time = utils.AverageMeter('Data', ':6.3f') losses = utils.AverageMeter('Loss', ':.4e') top1 = utils.AverageMeter('Acc@1', ':6.2f') top5 = utils.AverageMeter('Acc@5', ':6.2f') model.train() end = time.time() if args.use_dali: num_iter = train_loader._size // args.train_batch_size else: num_iter = len(train_loader) print_freq = num_iter // 10 i = 0 if args.use_dali: for batch_idx, batch_data in enumerate(train_loader): if args.debug: if i > 5: break i += 1 images = batch_data[0]['data'].cuda() targets = batch_data[0]['label'].squeeze().long().cuda() data_time.update(time.time() - end) adjust_learning_rate(optimizer, epoch, batch_idx, num_iter) # compute output logits = model(images) loss = loss_func(logits, targets) # measure accuracy and record loss prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) n = images.size(0) losses.update(loss.item(), n) # accumulated loss top1.update(prec1.item(), n) top5.update(prec5.item(), n) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % print_freq == 0 and batch_idx != 0: logger.info( 'Epoch[{0}]({1}/{2}): ' 'Loss {loss.avg:.4f} ' 'Prec@1(1,5) {top1.avg:.2f}, {top5.avg:.2f}'.format( epoch, batch_idx, num_iter, loss=losses, top1=top1, top5=top5)) else: for batch_idx, (images, targets) in enumerate(train_loader): if args.debug: if i > 5: break i += 1 images = images.cuda() targets = targets.cuda() data_time.update(time.time() - end) adjust_learning_rate(optimizer, epoch, batch_idx, num_iter) # compute output logits = model(images) loss = loss_func(logits, targets) # measure accuracy and record loss prec1, prec5 = utils.accuracy(logits, targets, topk=(1, 5)) n = images.size(0) losses.update(loss.item(), n) # accumulated loss top1.update(prec1.item(), n) top5.update(prec5.item(), n) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % print_freq == 0 and batch_idx != 0: logger.info( 'Epoch[{0}]({1}/{2}): ' 'Loss {loss.avg:.4f} ' 'Prec@1(1,5) {top1.avg:.2f}, {top5.avg:.2f}'.format( epoch, batch_idx, num_iter, loss=losses, top1=top1, top5=top5)) return losses.avg, top1.avg, top5.avg