def main(): if args.resume: if not os.path.isfile('./checkpoint/{}.pkl'.format(args.model)): raise ValueError('no models saved....!!!!') print 'resume from checkpoint....' net = torch.load('./checkpoint/{}.pkl'.format(args.model)) else: if args.model == 'vgg16': net = vgg.VGG(args.model) elif args.model == 'vgg19': net = vgg.VGG(args.model) elif args.model == 'resnet18': net = resnet.ResNet18() elif args.model == 'resnet34': net = resnet.ResNet34() elif args.model == 'resnet50': net = resnet.ResNet50() elif args.model == 'resnet101': net = resnet.ResNet101() elif args.model == 'resnet152': net = resnet.ResNet152() elif args.model == 'densenet121': net = densenet.DenseNet121() elif args.model == 'densenet161': net = densenet.DenseNet161() elif args.model == 'densenet169': net = densenet.DenseNet169() elif args.model == 'densenet201': net = densenet.DenseNet201() else: raise ValueError('model not implemented...!!') net.cuda(args.gpu) net = nn.DataParallel(net, device_ids = range(torch.cuda.device_count())) criterion = nn.CrossEntropyLoss().cuda(args.gpu) optim = torch.optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=1e-4) lr_scheduler = torch.optim.lr_scheduler.StepLR(optim, step_size=100, gamma=0.1) for e in xrange(args.epoch): train(e, net, criterion, optim, lr_scheduler) test(e, net)
def get_emotion(current_face, gray): """ inputs: current_face: (xmin, ymin, w, h) gray: grayscale frame outputs: emotion: from -1 to 1, 1 being most positive, -1 being most negative """ cut_size = 44 transform_test = transforms.Compose([ transforms.TenCrop(cut_size), transforms.Lambda(lambda crops: torch.stack([transforms.ToTensor()(crop) for crop in crops])), ]) # crop face from grayscale frame xmin = current_face[0] xmax = current_face[0] + current_face[2] ymin = current_face[1] ymax = current_face[1] + current_face[3] face = gray[ymin:ymax,xmin:xmax] # resize and transform face = (resize(face, (48,48), mode='symmetric')*255).astype('uint8') img = face[:, :, np.newaxis] img = np.concatenate((img, img, img), axis=2) img = Image.fromarray(img) inputs = transform_test(img) class_names = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral'] # set device, load model device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') net = vgg.VGG('VGG19') checkpoint = torch.load('PrivateTest_model.t7') net.load_state_dict(checkpoint['net']) net.to(device) net.eval() ncrops, c, h, w = np.shape(inputs) inputs = inputs.view(-1, c, h, w) inputs = inputs.to(device) with torch.no_grad(): inputs = Variable(inputs) outputs = net(inputs) outputs_avg = outputs.view(ncrops, -1).mean(0) # avg over crops weights = np.array([-0.4,-0.1,-0.1,0.8,-0.4,0.2]) score = F.softmax(outputs_avg, dim=0) emotion_score = np.sum(score.cpu().detach().numpy()[:6]**0.5*weights) return emotion_score
def __init__(self): self.MODEL = vgg.VGG('VGG19') self.USE_CUDA = torch.cuda.is_available() self.NET_SAVE_PATH = "./source/trained_net/" self.NUM_EPOCHS = 50 self.BATCH_SIZE = 8 self.TOP_NUM = 4 self.NUM_WORKERS = 4 self.NUM_CLASSES = 374 self.LEARNING_RATE = 0.001 self.PRINT_BATCH = True self.NUM_PRINT_BATCH = 5
def main(): global args args = parser.parse_args() use_cuda = torch.cuda.is_available() best_acc = 0 # best test accuracy start_epoch = 0 # start from epoch 0 or last checkpoint epoch # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) data_dir = '/mnt/lustre/gaowei' trainset = torchvision.datasets.CIFAR10(root=data_dir, train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) testset = torchvision.datasets.CIFAR10(root=data_dir, train=False, download=True, transform=transform_test) test_loader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # Model if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isdir( 'checkpoint'), 'Error: no checkpoint directory found!' checkpoint = torch.load('./checkpoint/ckpt.t7') model = vgg.VGG('VGG16') model.load_state_dict(checkpoint['state_dict']) best_acc = checkpoint['acc'] start_epoch = checkpoint['epoch'] else: print('==> Building model..') model = vgg.VGG('VGG16') if use_cuda: model.cuda() model = torch.nn.DataParallel(model, device_ids=range( torch.cuda.device_count())) cudnn.benchmark = True criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.evaluate: prec1 = validate(val_loader, model, criterion) return best_prec1 = 0 for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer=optimizer, epoch=epoch, lr=args.lr, schedule=args.schedule, gamma=args.gamma) train(train_loader, model, criterion, optimizer, epoch) prec1 = validate(test_loader, model, criterion) if prec1 > best_prec1: if not os.path.isdir(args.save_path): os.mkdir(args.save_path) is_best = prec1 > best_prec1 save_path = os.path.join(args.save_path, 'checkpoint{0}'.format(epoch)) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, args.save_path) best_prec1 = prec1
pred_labels[j] = np.reshape(outputs.data.cpu().numpy(), (10)) diff = org_labels - pred_labels.argmax(1) diff[diff != 0] = 1 total_error = 100 * np.sum(diff) / N return total_error, np.argwhere(diff != 0)[:, 1] # ====================================================================================================================== if __name__ == '__main__': data_dir_ = "data/attacked/cifar/%s/p%d" % (args.model, args.pixels) if args.model == "vgg16": net = modelvgg.VGG('VGG16') elif args.model == "resnet18": net = modelresnet.ResNet18() # load model net.load_state_dict( torch.load('checkpoints/%s/%s_%d.pth' % (args.model, args.model, args.epoch), map_location=lambda storage, loc: storage)) if use_cuda: net.cuda() else: net.cpu() cudnn.benchmark = True net.eval()
for key in args_dict: print('{}: {}'.format(key, args_dict[key])) print('\n') idx_to_class = {} for key in testset.spk2label: idx_to_class[str(testset.spk2label[key].squeeze().item())] = key print(idx_to_class, '\n') ckpt = torch.load(args.cp_path, map_location=lambda storage, loc: storage) if args.model == 'cnn': model = base_cnn.CNN(n_classes=args.nclasses) elif args.model == 'vgg': model = vgg.VGG('VGG11', n_classes=args.nclasses) elif args.model == 'resnet': model = resnet.ResNet12(n_classes=args.nclasses) elif args.model == 'densenet': model = densenet.DenseNet121(n_classes=args.nclasses) elif args.model == 'tdnn': model = TDNN.TDNN(n_classes=args.nclasses) try: print(model.load_state_dict(ckpt['model_state'], strict=True)) print('\n') except RuntimeError as err: print("Runtime Error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0]) raise
help='latent layer dimension (default: 512)') parser.add_argument('--n-hidden', type=int, default=1, metavar='N', help='maximum number of frames per utterance (default: 1)') parser.add_argument('--emb-size', type=int, default=256, metavar='N', help='Embedding dimension (default: 256)') args = parser.parse_args() if args.model == 'vgg': model = vgg.VGG('VGG19', nh=args.n_hidden, n_h=args.hidden_size, emb_size=args.emb_size) elif args.model == 'resnet': model = resnet.ResNet50(nh=args.n_hidden, n_h=args.hidden_size, emb_size=args.emb_size) elif args.model == 'densenet': model = densenet.DenseNet121(nh=args.n_hidden, n_h=args.hidden_size, emb_size=args.emb_size) batch = torch.rand(3, 3, 224, 224) emb, out = model.forward(batch) print(emb.size(), out.size())
) else False transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) validset = datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) labels_list = [x[1] for x in validset] if args.model == 'vgg': model = vgg.VGG('VGG16', sm_type=args.softmax) elif args.model == 'resnet': model = resnet.ResNet18(sm_type=args.softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(sm_type=args.softmax) cp_list = glob.glob(args.cp_path + '*.pt') idxs_enroll, idxs_test, labels = create_trials_labels(labels_list) print('\n{} trials created out of which {} are target trials'.format( len(idxs_enroll), np.sum(labels))) best_model, best_eer = None, float('inf') for cp in cp_list:
transform_test = transforms.Compose([ transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) validset = datasets.ImageFolder(args.data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=args.batch_size, shuffle=False, num_workers=args.n_workers) labels_list = [x[1] for x in validset] if args.model == 'vgg': model = vgg.VGG('VGG19') elif args.model == 'resnet': model = resnet.ResNet50() elif args.model == 'DenseNet121': model = densenet.DenseNet121() ckpt = torch.load(args.cp_path, map_location=lambda storage, loc: storage) try: model.load_state_dict(ckpt['model_state'], strict=True) except RuntimeError as err: print("Runtime Error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0]) raise if args.cuda:
ckpt = torch.load(args.cp_path, map_location=lambda storage, loc: storage) try: dropout_prob, n_hidden, hidden_size, softmax = ckpt[ 'dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt[ 'sm_type'] except KeyError as err: print("Key Error: {0}".format(err)) print('\nProbably old cp has no info regarding classifiers arch!\n') n_hidden, hidden_size, softmax = get_classifier_config_from_cp(ckpt) dropout_prob = args.dropout_prob if args.model == 'vgg': model = vgg.VGG('VGG16', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet': model = resnet.ResNet18(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax) try: model.load_state_dict(ckpt['model_state'], strict=True)
def run(train_batch_size, epochs, lr, weight_decay, config, exp_id, log_dir, trained_model_file, model_name, disable_gpu=False): # 日志工具 def logging(s, log_path, print_=True, log_=True): if print_: print(s) if log_: with open(log_path, 'a+') as f_log: f_log.write(s + '\n') def get_logger(log_path, **kwargs): return functools.partial(logging, log_path=log_path, **kwargs) logging = get_logger('./logger/log.txt') # 加载数据集 if config['test_ratio']: train_loader, val_loader, test_loader = get_data_loaders(config, train_batch_size, exp_id) else: train_loader, val_loader = get_data_loaders(config, train_batch_size, exp_id) device = torch.device("cuda" if not disable_gpu and torch.cuda.is_available() else "cpu") if model_name == 'CNNIQA': model = cnn.CNNIQAnet() if model_name == 'lenet5': model = lenet5.LeNet5() if model_name == 'resnet18': model = resnet.ResNet18() if model_name == 'resnet34': model = resnet.ResNet34() if model_name == 'vgg19': model = vgg.VGG('VGG19') writer = SummaryWriter(log_dir=log_dir) model = model.to(device) # 将模型加载到指定设备上 # summary(model, input_size=(32, 32)) # must remove the number of N # print("model:", model) # logging("model: {}".format(model)) # if multi_gpu and torch.cuda.device_count() > 1: # model = nn.DataParallel(model) optimizer = Adam(model.parameters(), lr=lr, weight_decay=weight_decay) global best_criterion best_criterion = -1 # SROCC >= -1 # 训练器,调库 trainer = create_supervised_trainer(model, optimizer, loss_fn, device=device) # 校验器,调库 evaluator = create_supervised_evaluator(model, metrics={'IQA_performance': IQAPerformance()}, device=device) # 函数修饰器,以下函数都包含在trainer中,因此是一边训练一边验证、测试 # training/validation/testing = 0.6/0.2/0.2,每一个epoch训练完都进行validation和testing @trainer.on(Events.ITERATION_COMPLETED) def log_training_loss(engine): writer.add_scalar("training/loss", engine.state.output, engine.state.iteration) @trainer.on(Events.EPOCH_COMPLETED) def log_validation_results(engine): evaluator.run(val_loader) metrics = evaluator.state.metrics SROCC, KROCC, PLCC, RMSE, MAE, OR = metrics['IQA_performance'] # print("Validation Results - Epoch: {} SROCC: {:.4f} KROCC: {:.4f} PLCC: {:.4f} RMSE: {:.4f} MAE: {:.4f} OR: {:.2f}%" # .format(engine.state.epoch, SROCC, KROCC, PLCC, RMSE, MAE, 100 * OR)) logging("Validation Results - Epoch: {} SROCC: {:.4f} KROCC: {:.4f} PLCC: {:.4f} RMSE: {:.4f} MAE: {:.4f} OR: {:.2f}%" .format(engine.state.epoch, SROCC, KROCC, PLCC, RMSE, MAE, 100 * OR)) writer.add_scalar("validation/SROCC", SROCC, engine.state.epoch) writer.add_scalar("validation/KROCC", KROCC, engine.state.epoch) writer.add_scalar("validation/PLCC", PLCC, engine.state.epoch) writer.add_scalar("validation/RMSE", RMSE, engine.state.epoch) writer.add_scalar("validation/MAE", MAE, engine.state.epoch) writer.add_scalar("validation/OR", OR, engine.state.epoch) global best_criterion global best_epoch if SROCC > best_criterion: best_criterion = SROCC best_epoch = engine.state.epoch # 保存最佳模型,以SROCC指标为准 # _use_new_zipfile_serialization = False适用于pytorch1.6以前的版本 torch.save(model.state_dict(), trained_model_file, _use_new_zipfile_serialization=False) @trainer.on(Events.EPOCH_COMPLETED) def log_testing_results(engine): if config["test_ratio"] > 0 and config['test_during_training']: evaluator.run(test_loader) metrics = evaluator.state.metrics SROCC, KROCC, PLCC, RMSE, MAE, OR = metrics['IQA_performance'] # print("Testing Results - Epoch: {} SROCC: {:.4f} KROCC: {:.4f} PLCC: {:.4f} RMSE: {:.4f} MAE: {:.4f} OR: {:.2f}%" # .format(engine.state.epoch, SROCC, KROCC, PLCC, RMSE, MAE, 100 * OR)) logging("Testing Results - Epoch: {} SROCC: {:.4f} KROCC: {:.4f} PLCC: {:.4f} RMSE: {:.4f} MAE: {:.4f} OR: {:.2f}%" .format(engine.state.epoch, SROCC, KROCC, PLCC, RMSE, MAE, 100 * OR)) writer.add_scalar("testing/SROCC", SROCC, engine.state.epoch) writer.add_scalar("testing/KROCC", KROCC, engine.state.epoch) writer.add_scalar("testing/PLCC", PLCC, engine.state.epoch) writer.add_scalar("testing/RMSE", RMSE, engine.state.epoch) writer.add_scalar("testing/MAE", MAE, engine.state.epoch) writer.add_scalar("testing/OR", OR, engine.state.epoch) @trainer.on(Events.COMPLETED) def final_testing_results(engine): if config["test_ratio"]: model.load_state_dict(torch.load(trained_model_file)) evaluator.run(test_loader) metrics = evaluator.state.metrics SROCC, KROCC, PLCC, RMSE, MAE, OR = metrics['IQA_performance'] global best_epoch # best test results 是 validation的SROCC最高的一次 # print("Final Test Results - Epoch: {} SROCC: {:.4f} KROCC: {:.4f} PLCC: {:.4f} RMSE: {:.4f} MAE: {:.4f} OR: {:.2f}%" # .format(best_epoch, SROCC, KROCC, PLCC, RMSE, MAE, 100 * OR)) logging("Final Test Results - Epoch: {} SROCC: {:.4f} KROCC: {:.4f} PLCC: {:.4f} RMSE: {:.4f} MAE: {:.4f} OR: {:.2f}%" .format(best_epoch, SROCC, KROCC, PLCC, RMSE, MAE, 100 * OR)) np.save(save_result_file, (SROCC, KROCC, PLCC, RMSE, MAE, OR)) # kick everything off # 执行训练 trainer.run(train_loader, max_epochs=epochs) writer.close()
print(torch.cuda.get_device_name(torch.cuda.current_device())) # DATA device = 'cuda' if torch.cuda.is_available() else 'cpu' best_acc = 0 # best test accuracy start_epoch = 0 # start from epoch 0 or last checkpoint epoch trainloader, testloader, valloader = load_cifar(args.trainval_perc) ################################################### # MAKE AN INSTANCE OF A NETWORK AND (POSSIBLY) LOAD THE MODEL arch = [64, 64, 128, 128, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512] print('==> Building model..') net = vgg.VGG(arch) net = net.to(device) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # print(device)
metavar='S', help='latent layer dimension (default: 512)') parser.add_argument('--n-hidden', type=int, default=1, metavar='N', help='maximum number of frames per utterance (default: 1)') parser.add_argument('--softmax', choices=['softmax', 'am_softmax'], default='softmax', help='Softmax type') args = parser.parse_args() if args.model == 'vgg': model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size, sm_type=args.softmax) elif args.model == 'resnet': model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size, sm_type=args.softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size, sm_type=args.softmax) cp_list = glob.glob(args.cp_path + '*.pt') assert len(cp_list) > 0, 'No cp found in the given path!' for cp in cp_list:
def main(): # Load the parameters from json file args = parser.parse_args() json_path = os.path.join(args.model_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = utils.Params(json_path) # use GPU if available params.cuda = torch.cuda.is_available() # Set the random seed for reproducible experiments torch.manual_seed(138) if params.cuda: torch.cuda.manual_seed(138) # Set the logger utils.set_logger(os.path.join(args.model_dir, 'train.log')) # Create the input data pipeline logging.info("Loading the datasets...") # fetch dataloaders if args.dataset == 'cifar10': dataloaders = data_loaders.cifar10_dataloader(params.batch_size) else: dataloaders = data_loaders.mnist_dataloader(params.batch_size) train_dl = dataloaders['train'] val_dl = dataloaders['test'] # !!!!!!!!!!!!! logging.info("- done.") # Define model if args.model == 'lenet5': model = nets.LeNet5(params).cuda() if params.cuda else nets.LeNet5( params) elif args.model[:3] == 'vgg': model = vgg.VGG(args.model, params).cuda() if params.cuda else nvgg.VGG( args.model, params) elif args.model == 'mlp': model = nets.MLP(params).cuda() if params.cuda else nets.MLP(params) else: model = nets.SoftmaxReg( params).cuda() if params.cuda else nets.SoftmaxReg(params) # Define optimizer if params.optim == 'sgd': optimizer = optim.SGD(model.parameters(), lr=params.lr, momentum=params.momentum, weight_decay=(params.wd if params.dict.get('wd') is not None else 0.0)) else: optimizer = optim.Adam(model.parameters(), lr=params.lr, weight_decay=(params.wd if params.dict.get('wd') is not None else 0.0)) if params.dict.get('lr_adjust') is not None: scheduler = lr_scheduler.StepLR(optimizer, step_size=params.lr_adjust, gamma=0.1) else: scheduler = None # fetch loss function and metrics loss_fn = nn.NLLLoss() metrics = nets.metrics # Train the model logging.info("Starting training for {} epoch(s)".format(params.num_epochs)) train_and_evaluate(model, train_dl, val_dl, optimizer, scheduler, loss_fn, metrics, params, args.model_dir, args.restore_file)
def train(lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, hdf_path, valid_data_path, valid_hdf_path, checkpoint_path, softmax, n_classes, pretrained, max_gnorm, lr_factor): cp_name = get_cp_name(checkpoint_path) if hdf_path != 'none': transform_train = transforms.Compose([ transforms.ToPILImage(), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(30), transforms.RandomPerspective(p=0.2), transforms.ColorJitter(brightness=2), transforms.RandomGrayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) trainset = Loader(hdf_path, transform_train) else: transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(30), transforms.RandomPerspective(p=0.2), transforms.ColorJitter(brightness=2), transforms.RandomGrayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) trainset = datasets.ImageFolder(data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) if valid_hdf_path != 'none': transform_test = transforms.Compose([ transforms.ToPILImage(), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) validset = Loader(args.valid_hdf_path, transform_test) else: transform_test = transforms.Compose([ transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, pin_memory=True) if model == 'vgg': model_ = vgg.VGG('VGG19', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size) elif model == 'resnet': model_ = resnet.ResNet50(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size) elif model == 'densenet': model_ = densenet.DenseNet121(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size) if pretrained: print('\nLoading pretrained encoder from torchvision\n') if model == 'vgg': model_pretrained = torchvision.models.vgg19(pretrained=True) elif model == 'resnet': model_pretrained = torchvision.models.resnet50(pretrained=True) elif model == 'densenet': model_pretrained = torchvision.models.densenet121(pretrained=True) print( model_.load_state_dict(model_pretrained.state_dict(), strict=False)) print('\n') if cuda: device = get_freer_gpu() model_ = model_.cuda(device) torch.backends.cudnn.benchmark = True optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, patience=int(patience), lr_factor=lr_factor, label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Embedding size: {}'.format(emb_size)) print('Hidden layer size: {}'.format(hidden_size)) print('Number of hidden layers: {}'.format(n_hidden)) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print('Pretrained: {}'.format(pretrained)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print(' ') return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99
default=True, type=str2bool, help='Use visdom for loss visualization') args = parser.parse_args() if not os.path.exists(args.save_folder): os.mkdir(args.save_folder) # Load the data Cifar10 = Input.Cifar10() trainset = Cifar10.trainset trainloader = Cifar10.trainloader classes = Cifar10.classes classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # net = C.Net() # net = L.LeNet() net = V.VGG(vgg_name='VGG16') if args.cuda: # GPU selection net = torch.nn.DataParallel(net) net = net.cuda() else: net = torch.nn.DataParallel(net) Train(para=trainloader, net=net)
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test = transforms.ToTensor() #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) if model == 'vgg': model = vgg.VGG('VGG16') elif model == 'resnet': model = resnet.ResNet18() elif model == 'densenet': model = densenet.densenet_cifar() if cuda: model = model.cuda() optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, margin=margin, lambda_=lambda_, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, swap=swap, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') return 0.99
test_loader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers) cp_list = glob.glob(args.cp_path + '*.pt') best_model, best_acc = None, -float('inf') for cp in cp_list: ckpt = torch.load(cp, map_location=lambda storage, loc: storage) softmax = get_sm_from_cp(ckpt) if args.model == 'vgg': model = vgg.VGG('VGG16', sm_type=softmax) elif args.model == 'resnet': model = resnet.ResNet18(sm_type=softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(sm_type=softmax) try: model.load_state_dict(ckpt['model_state'], strict=True) except RuntimeError as err: print("Runtime Error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0]) raise if args.cuda: device = get_freer_gpu()
def train(lr, l2, momentum, smoothing, patience, model, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path, softmax): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers) if model == 'vgg': model_ = vgg.VGG('VGG16', nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) elif model == 'resnet': model_ = resnet.ResNet18(nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) elif model == 'densenet': model_ = densenet.densenet_cifar(nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) if args.cuda: device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, patience=int(patience), label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Hidden layer size size: {}'.format(int(hidden_size))) print('Number of hidden layers: {}'.format(int(n_hidden))) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print('Best Error Rate in file ' + cp_name + ' was: {}'.format(cost[2])) print(' ') return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99
transform_train = transforms.Compose([transforms.Resize(256), transforms.RandomCrop(224, padding=4), transforms.RandomHorizontalFlip(), transforms.RandomRotation(30), transforms.ColorJitter(brightness=2), transforms.RandomGrayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]) trainset = datasets.ImageFolder(args.data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=args.n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) if args.valid_hdf_path: transform_test = transforms.Compose([transforms.ToPILImage(), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]) validset = Loader(args.valid_hdf_path, transform_train) else: transform_test = transforms.Compose([transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=args.valid_batch_size, shuffle=True, num_workers=args.n_workers, pin_memory=True) if args.model == 'vgg': model = vgg.VGG('VGG19', sm_type=args.softmax, n_classes=args.nclasses) elif args.model == 'resnet': model = resnet.ResNet50(sm_type=args.softmax, n_classes=args.nclasses) elif args.model == 'densenet': model = densenet.DenseNet121(sm_type=args.softmax, n_classes=args.nclasses) if args.pretrained: print('\nLoading pretrained model\n') if args.model == 'vgg': model_pretrained = torchvision.models.vgg19(pretrained=True) elif args.model == 'resnet': model_pretrained = torchvision.models.resnet50(pretrained=True) elif args.model == 'densenet': model_pretrained = torchvision.models.densenet121(pretrained=True) print(model.load_state_dict(model_pretrained.state_dict(), strict=False))
label_mode=mode2) training_loader = DataLoader(training_dataset, BATCH_SIZE, shuffle=True, pin_memory=True) testing_dataset = CIFAR10(root, train=False, transform=img_transforms) testing_loader = DataLoader(testing_dataset, BATCH_SIZE, shuffle=False, pin_memory=True) loaders = {'train': training_loader, 'test': testing_loader} resnet18 = resnet.ResNet18() vgg16 = vgg.VGG('VGG16') alex = alexnet.alexnet() inception = inceptions.GoogLeNet() exec('model={}'.format(model_name)) if use_gpu: model = model.cuda() criterion = nn.CrossEntropyLoss() # optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay) # exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.95) optimizer = optim.SGD(model.parameters(), lr=lr) exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=1.)
def train(lr, l2, momentum, slack, patience, model, epochs, batch_size, valid_batch_size, train_mode, n_workers, cuda, data_path, valid_data_path, checkpoint_path): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) if model == 'vgg': model_ = vgg.VGG('VGG16') elif model == 'resnet': model_ = resnet.ResNet50() elif model == 'densenet': model_ = densenet.densenet_cifar() if cuda: device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, slack=slack, train_mode=train_mode, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Train mode: {}'.format(train_mode)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Slack: {}'.format(slack)) print('Patience: {}'.format(patience)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Train mode: {}'.format(train_mode)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Slack: {}'.format(slack)) print('Patience: {}'.format(patience)) print(' ') return 0.99
else: rproj_size = -1 except KeyError as err: print("Key Error: {0}".format(err)) print('\nProbably old cp has no info regarding classifiers arch!\n') n_hidden, hidden_size, softmax, n_classes = get_classifier_config_from_cp( ckpt) dropout_prob = args.dropout_prob emb_size = 350 rproj_size = -1 if args.model == 'vgg': model = vgg.VGG('VGG19', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size, r_proj_size=rproj_size) elif args.model == 'resnet': model = resnet.ResNet50(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size, r_proj_size=rproj_size) elif args.model == 'densenet': model = densenet.DenseNet121(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob,
worker_init_fn=set_np_randomseed) #validset = Loader(args.valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=args.valid_batch_size, shuffle=False, num_workers=args.n_workers) if args.model == 'vgg': model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax) elif args.model == 'resnet': model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax) if args.verbose > 0: print(model)
import torch import torchvision import torchvision.transforms as transforms import torch.optim as optim import torch.utils.data from models import vgg, resnet, densenet # Training settings parser = argparse.ArgumentParser(description='Test new architectures') parser.add_argument('--model', choices=['vgg', 'resnet', 'densenet'], default='resnet') parser.add_argument('--hidden-size', type=int, default=512, metavar='S', help='latent layer dimension (default: 512)') parser.add_argument('--n-hidden', type=int, default=1, metavar='N', help='maximum number of frames per utterance (default: 1)') args = parser.parse_args() if args.model == 'vgg': model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size) elif args.model == 'resnet': model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size) batch = torch.rand(3, 3, 84, 84) emb = model.forward(batch) print(emb.size()) out = model.out_proj(emb) print(out.size())
def main(): global args, best_prec1 args = parser.parse_args() if args.tensorboard: configure("runs/%s" % (args.name)) print("Launch...") print(sys.argv) # Global configuration of the datasets USE_COLOR = not args.monkey kwargs = {'num_workers': 4, 'pin_memory': True} ############################## # Database loading ############################## # Data loading code # TODO: For now only support 224x224 input size of 32x32. Need more work to support more resolutions if args.database == 'cifar-10': if not USE_COLOR: raise "CIFAR-10 does not handle training with gray images" # Data augumentation normalize = transforms.Normalize( mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) transform_test = transforms.Compose([transforms.ToTensor(), normalize]) # CIFAR-10 data_CIFAR10 = datasets.CIFAR10(args.tempdir, train=True, download=True, transform=transform_train) if args.split_data > 0: sampler = torch.utils.data.sampler.WeightedRandomSampler( weights=[1] * 10000, num_samples=args.split_data) shuffle = False else: sampler = None shuffle = True train_loader = torch.utils.data.DataLoader(data_CIFAR10, batch_size=args.batch_size, shuffle=shuffle, sampler=sampler, **kwargs) val_loader = torch.utils.data.DataLoader(datasets.CIFAR10( args.tempdir, train=False, transform=transform_test), batch_size=args.batch_size, shuffle=True, **kwargs) NUM_CLASS = 10 INPUT_SIZE = 32 elif args.database == 'cifar-100': if not USE_COLOR: raise "CIFAR-100 does not handle training with gray images" # Data augumentation # From: https://github.com/meliketoy/wide-resnet.pytorch/blob/master/config.py normalize = transforms.Normalize(mean=[0.5071, 0.4867, 0.4408], std=[0.2675, 0.2565, 0.2761]) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) transform_test = transforms.Compose([transforms.ToTensor(), normalize]) data_CIFAR100 = datasets.CIFAR100(args.tempdir, train=True, download=True, transform=transform_train) if args.split_data > 0: sampler = torch.utils.data.sampler.WeightedRandomSampler( weights=[1] * 10000, num_samples=args.split_data) shuffle = False else: sampler = None shuffle = True # CIFAR-100 train_loader = torch.utils.data.DataLoader(data_CIFAR100, batch_size=args.batch_size, shuffle=shuffle, sampler=sampler, **kwargs) val_loader = torch.utils.data.DataLoader(datasets.CIFAR100( args.tempdir, train=False, transform=transform_test), batch_size=args.batch_size, shuffle=True, **kwargs) NUM_CLASS = 100 INPUT_SIZE = 32 elif args.database == 'kth': if not USE_COLOR and args.gcn: raise "It is not possible to use grayimage and GCN" # Data auguementation if args.gcn: normalize = GCN() else: # TODO: Use the same normalization than CIFAR-10 # TODO: That might be suboptimal... if USE_COLOR: normalize = transforms.Normalize( mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) else: normalize = transforms.Normalize( mean=[x / 255.0 for x in [125.3]], std=[x / 255.0 for x in [63.0]]) add_transform = [] if not USE_COLOR: add_transform += [transforms.Grayscale(num_output_channels=1)] transform_train = transforms.Compose(add_transform + [ transforms.Resize((256, 256)), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose(add_transform + [ transforms.Resize((256, 256)), transforms.CenterCrop((224, 224)), transforms.ToTensor(), normalize ]) kth_train_dataset = datasets.ImageFolder(root=args.traindir, transform=transform_train) kth_test_dataset = datasets.ImageFolder(root=args.valdir, transform=transform_test) train_loader = torch.utils.data.DataLoader(kth_train_dataset, shuffle=True, batch_size=args.batch_size, **kwargs) val_loader = torch.utils.data.DataLoader(kth_test_dataset, shuffle=True, batch_size=args.batch_size, **kwargs) NUM_CLASS = args.num_clases INPUT_SIZE = 224 else: raise "Unknown database" if not USE_COLOR and not args.model == "dawn": raise "Only DAWN support training with gray images" ################################## # Create model ################################## # TODO: This condition only work if we have only two image size big_input = INPUT_SIZE != 32 # Only for scattering transform (as coefficient are precomptuted) scattering = None if args.model == 'densenet': no_init_conv = args.no_init_conv if (INPUT_SIZE > 128): # For these input size, init conv are necessary no_init_conv = False model = dn.DenseNet3(args.layers, NUM_CLASS, args.growth, reduction=args.reduce, bottleneck=args.bottleneck, dropRate=args.droprate, init_conv=not no_init_conv) elif args.model == 'vgg': model = vgg.VGG(NUM_CLASS, big_input=big_input) elif args.model == 'wcnn': model = wcnn.WCNN(NUM_CLASS, big_input=big_input, wavelet=args.wavelet, levels=args.levels) elif args.model == 'dawn': # Our model model = dawn.DAWN(NUM_CLASS, big_input=big_input, first_conv=args.first_conv, number_levels=args.levels, kernel_size=args.kernel_size, no_bootleneck=args.no_bootleneck, classifier=args.classifier, share_weights=args.share_weights, simple_lifting=args.simple_lifting, COLOR=USE_COLOR, regu_details=args.regu_details, regu_approx=args.regu_approx, haar_wavelet=args.haar_wavelet) elif args.model == 'scatter': from kymatio import Scattering2D from models.scatter.Scatter_WRN import Scattering2dCNN, ScatResNet if (INPUT_SIZE == 224): # KTH scattering = Scattering2D(J=args.scat, shape=(args.N, args.N), max_order=args.mode) scattering = scattering.cuda() model = ScatResNet(args.scat, INPUT_SIZE, NUM_CLASS, args.classifier, args.mode) else: # Precomputation scattering = Scattering2D(J=args.scat, shape=(args.N, args.N), max_order=args.mode) scattering = scattering.cuda() model = Scattering2dCNN(args.classifier, J=args.scat, N=args.N, num_classes=NUM_CLASS, blocks=args.blocks, mode=args.mode, use_avg_pool=args.use_avg_pool) elif args.model == 'resnet': if big_input: import torchvision model = torchvision.models.resnet18(pretrained=args.pretrained) model.fc = nn.Linear(512, NUM_CLASS) else: if args.use_normal: model = resnet.ResNetCIFARNormal( [args.size_normal, args.size_normal, args.size_normal], num_classes=NUM_CLASS) else: model = resnet.ResNetCIFAR([2, 2, 2, 2], num_classes=NUM_CLASS, levels=args.levels) else: raise "Unknown model" # get the number of model parameters print("Number of model parameters : {:,}".format( sum([p.data.nelement() for p in model.parameters()]))) print("Number of *trainable* model parameters: {:,}".format( sum(p.numel() for p in model.parameters() if p.requires_grad))) # for training on multiple GPUs. # Use CUDA_VISIBLE_DEVICES=0,1 to specify which GPUs to use if args.multigpu: model = torch.nn.DataParallel(model).cuda() else: model = model.cuda() # Print network summary if args.summary: # For display like Keras from torchsummary import summary summary(model, input_size=(3, INPUT_SIZE, INPUT_SIZE)) # CSV files statistics csv_logger = CSVStats() if args.monkey: # This is a special condition # Only to test and visualize the multi-resolution # output f = open('./data/baboon.png', 'rb') from PIL import Image img = Image.open(f) img.convert('RGB') img.show() # Do the transformation to_pil = transforms.ToPILImage() if USE_COLOR: unormalize = UnNormalize( mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) else: unormalize = UnNormalize(mean=[x / 255.0 for x in [125.3]], std=[x / 255.0 for x in [63.0]]) tensor_trans = transform_test(img) img_trans = to_pil(torch.clamp(unormalize(tensor_trans), 0.0, 1.0)) img_trans.show() img_trans.save("trans.png") # Make pytorch with the batch size tensor_trans = tensor_trans[None, :, :, :] # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) if args.monkey: dump_image(tensor_trans, args.start_epoch, model, unormalize) raise "stop here!" else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # define loss function (criterion) and pptimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, nesterov=True, weight_decay=args.weight_decay) if args.model == 'dawn': if args.warmup: model.initialization = True train_init(train_loader, model, criterion, optimizer) model.initialization = False # Switch for epoch in range(args.start_epoch, args.epochs): t0 = time.time() adjust_learning_rate(optimizer, epoch, args.drop) # TODO: Clean this code if args.model == 'dawn': args_model = 'dawn' #model, optimizer = amp.initialize(model, optimizer) # train for one epoch prec1_train, prec5_train, loss_train = train(train_loader, model, criterion, optimizer, epoch, args.model == args_model, scattering=scattering) # Optimize cache on cuda # This is quite important to avoid memory overflow # when training big models. The performance impact # seems minimal torch.cuda.empty_cache() # evaluate on validation set prec1_val, prec5_val, loss_val = validate(val_loader, model, criterion, epoch, args.model == args_model, scattering=scattering) if args.monkey: # In this case, we will output the Monkey image dump_image(tensor_trans, epoch, model, unormalize) # Print some statistics inside CSV csv_logger.add(prec1_train, prec1_val, prec5_train, prec5_val, loss_train, loss_val) # remember best prec@1 and save checkpoint is_best = prec1_val > best_prec1 best_prec1 = max(prec1_val, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, }, is_best) csv_logger.write() # Final print print( ' * Train[{:.3f} %, {:.3f} %, {:.3f} loss] Val [{:.3f} %, {:.3f}%, {:.3f} loss] Best: {:.3f} %' .format(prec1_train, prec5_train, loss_train, prec1_val, prec5_val, loss_val, best_prec1)) print('Time for', epoch, "/", args.epochs, time.time() - t0) print('Best accuracy: ', best_prec1)
# Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!' checkpoint = torch.load('./checkpoint/cnn_cifar100_' + args.method) model = checkpoint['model'] start_epoch = checkpoint['epoch'] train_losses = checkpoint['train_losses'] test_losses = checkpoint['test_losses'] train_errs = checkpoint['train_errs'] test_errs = checkpoint['test_errs'] else: print('==> Building model..') if args.net == 'vggnet': from models import vgg model = vgg.VGG('VGG16', num_classes=100) elif args.net == 'resnet': from models import resnet model = resnet.ResNet18(num_classes=100) elif args.net == 'wideresnet': from models import wideresnet model = wideresnet.WResNet_cifar10(num_classes=100, depth=16, multiplier=4) else: print('Network undefined!') if use_cuda: model.cuda() model = torch.nn.DataParallel(model, device_ids=range(torch.cuda.device_count())) cudnn.benchmark = True
pin_memory=True) if args.pretrained_path: print('\nLoading pretrained model from: {}\n'.format(args.pretrained_path)) ckpt = torch.load(args.pretrained_path, map_location=lambda storage, loc: storage) args.dropout_prob, args.n_hidden, args.hidden_size, args.emb_size = ckpt[ 'dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt[ 'emb_size'] print('\nUsing pretrained config for discriminator. Ignoring args.') if args.model == 'vgg': model = vgg.VGG('VGG19', nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax, n_classes=args.nclasses, emb_size=args.emb_size) elif args.model == 'resnet': model = resnet.ResNet50(nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax, n_classes=args.nclasses, emb_size=args.emb_size) elif args.model == 'densenet': model = densenet.DenseNet121(nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax,
) else False transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) validset = datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) labels_list = [x[1] for x in validset] if args.model == 'vgg': model = vgg.VGG('VGG16') elif args.model == 'resnet': model = resnet.ResNet18() elif args.model == 'densenet': model = densenet.densenet_cifar() ckpt = torch.load(args.cp_path, map_location=lambda storage, loc: storage) try: model.load_state_dict(ckpt['model_state'], strict=True) except RuntimeError as err: print("Runtime Error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0]) raise if args.cuda:
def train(lr, l2, momentum, smoothing, warmup, model, emb_size, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, hdf_path, valid_hdf_path, checkpoint_path, softmax, pretrained, pretrained_path, max_gnorm, stats, log_dir, eval_every, ablation): args_dict = locals() cp_name = get_cp_name(checkpoint_path) if pretrained_path != 'none': print('\nLoading pretrained model from: {}\n'.format( args.pretrained_path)) ckpt = torch.load(pretrained_path, map_location=lambda storage, loc: storage) dropout_prob, n_hidden, hidden_size, emb_size = ckpt[ 'dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt[ 'emb_size'] if 'r_proj_size' in ckpt: rproj_size = ckpt['r_proj_size'] else: rproj_size = -1 print('\nUsing pretrained config for discriminator. Ignoring args.') args_dict['dropout_prob'], args_dict['n_hidden'], args_dict[ 'hidden_size'], args_dict[ 'emb_size'] = dropout_prob, n_hidden, hidden_size, emb_size if log_dir != 'none': writer = SummaryWriter(log_dir=os.path.join(log_dir, cp_name), comment=model, purge_step=0) writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_eer': 0.0}) else: writer = None if stats == 'cars': mean, std = [0.4461, 0.4329, 0.4345], [0.2888, 0.2873, 0.2946] elif stats == 'cub': mean, std = [0.4782, 0.4925, 0.4418], [0.2330, 0.2296, 0.2647] elif stats == 'sop': mean, std = [0.5603, 0.5155, 0.4796], [0.2939, 0.2991, 0.3085] elif stats == 'imagenet': mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] if hdf_path != 'none': transform_train = transforms.Compose([ transforms.ToPILImage(), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(10), transforms.RandomPerspective(p=0.1), transforms.RandomGrayscale(p=0.1), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) trainset = Loader(hdf_path, transform_train) else: transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(10), transforms.RandomPerspective(p=0.1), transforms.RandomGrayscale(p=0.1), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) trainset = datasets.ImageFolder(data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) if valid_hdf_path != 'none': transform_test = transforms.Compose([ transforms.ToPILImage(), transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) validset = Loader(args.valid_hdf_path, transform_test) else: transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, pin_memory=True) nclasses = trainset.n_classes if isinstance(trainset, Loader) else len( trainset.classes) if model == 'vgg': model_ = vgg.VGG('VGG19', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=nclasses, emb_size=emb_size, r_proj_size=rproj_size) elif model == 'resnet': model_ = resnet.ResNet50(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=nclasses, emb_size=emb_size, r_proj_size=rproj_size) elif model == 'densenet': model_ = densenet.DenseNet121(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=nclasses, emb_size=emb_size, r_proj_size=rproj_size) if pretrained_path != 'none': if ckpt['sm_type'] == 'am_softmax': del (ckpt['model_state']['out_proj.w']) elif ckpt['sm_type'] == 'softmax': del (ckpt['model_state']['out_proj.w.weight']) del (ckpt['model_state']['out_proj.w.bias']) print(model_.load_state_dict(ckpt['model_state'], strict=False)) print('\n') if pretrained: print('\nLoading pretrained encoder from torchvision\n') if model == 'vgg': model_pretrained = torchvision.models.vgg19(pretrained=True) elif model == 'resnet': model_pretrained = torchvision.models.resnet50(pretrained=True) elif model == 'densenet': model_pretrained = torchvision.models.densenet121(pretrained=True) print( model_.load_state_dict(model_pretrained.state_dict(), strict=False)) print('\n') if cuda: device = get_freer_gpu() model_ = model_.cuda(device) torch.backends.cudnn.benchmark = True optimizer = TransformerOptimizer(optim.SGD(model_.parameters(), lr=lr, momentum=momentum, weight_decay=l2, nesterov=True), lr=lr, warmup_steps=warmup) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, ablation=ablation, cuda=cuda, logger=writer) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Embedding size: {}'.format(emb_size)) print('Hidden layer size: {}'.format(hidden_size)) print('Number of hidden layers: {}'.format(n_hidden)) print('Random projection size: {}'.format(rproj_size)) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Warmup iterations: {}'.format(warmup)) print('Softmax Mode is: {}'.format(softmax)) print('Pretrained: {}'.format(pretrained)) print('Pretrained path: {}'.format(pretrained_path)) print('Evaluate every {} iterations.'.format(eval_every)) print('Ablation Mode: {}'.format(ablation)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10, eval_every=eval_every) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print(' ') if log_dir != 'none': writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_eer': cost[0]}) return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99