def get_net(network: str, num_classes) -> torch.nn.Module: return VGG('VGG16', num_classes=num_classes) if network == 'VGG16' else \ ResNet34(num_classes=num_classes) if network == 'ResNet34' else \ PreActResNet18(num_classes=num_classes) if network == 'PreActResNet18' else \ GoogLeNet(num_classes=num_classes) if network == 'GoogLeNet' else \ densenet_cifar(num_classes=num_classes) if network == 'densenet_cifar' else \ ResNeXt29_2x64d(num_classes=num_classes) if network == 'ResNeXt29_2x64d' else \ MobileNet(num_classes=num_classes) if network == 'MobileNet' else \ MobileNetV2(num_classes=num_classes) if network == 'MobileNetV2' else \ DPN92(num_classes=num_classes) if network == 'DPN92' else \ ShuffleNetG2(num_classes=num_classes) if network == 'ShuffleNetG2' else \ SENet18(num_classes=num_classes) if network == 'SENet18' else \ ShuffleNetV2(1, num_classes=num_classes) if network == 'ShuffleNetV2' else \ EfficientNetB0( num_classes=num_classes) if network == 'EfficientNetB0' else None
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) validset = datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) labels_list = [x[1] for x in validset] if args.model == 'vgg': model = vgg.VGG('VGG16') elif args.model == 'resnet': model = resnet.ResNet18() elif args.model == 'densenet': model = densenet.densenet_cifar() ckpt = torch.load(args.cp_path, map_location=lambda storage, loc: storage) try: model.load_state_dict(ckpt['model_state'], strict=True) except RuntimeError as err: print("Runtime Error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0]) raise if args.cuda: device = get_freer_gpu() model = model.cuda(device) idxs_enroll, idxs_test, labels = create_trials_labels(labels_list)
import torch.utils.data from models import vgg, resnet, densenet # Training settings parser = argparse.ArgumentParser(description='Test new architectures') parser.add_argument('--model', choices=['vgg', 'resnet', 'densenet'], default='resnet') parser.add_argument('--hidden-size', type=int, default=512, metavar='S', help='latent layer dimension (default: 512)') parser.add_argument('--n-hidden', type=int, default=1, metavar='N', help='maximum number of frames per utterance (default: 1)') args = parser.parse_args() if args.model == 'vgg': model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size) elif args.model == 'resnet': model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size) batch = torch.rand(3, 3, 84, 84) emb = model.forward(batch) print(emb.size()) out = model.out_proj(emb) print(out.size()) emb = torch.cat([emb,emb],1) print(emb.size())
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) validset = datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform_test) labels_list = [x[1] for x in validset] if args.model == 'vgg': model = vgg.VGG('VGG16', sm_type=args.softmax) elif args.model == 'resnet': model = resnet.ResNet18(sm_type=args.softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(sm_type=args.softmax) cp_list = glob.glob(args.cp_path + '*.pt') idxs_enroll, idxs_test, labels = create_trials_labels(labels_list) print('\n{} trials created out of which {} are target trials'.format( len(idxs_enroll), np.sum(labels))) best_model, best_eer = None, float('inf') for cp in cp_list: ckpt = torch.load(cp, map_location=lambda storage, loc: storage) try: model.load_state_dict(ckpt['model_state'], strict=True) except:
dropout_prob = args.dropout_prob if args.model == 'vgg': model = vgg.VGG('VGG16', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet': model = resnet.ResNet18(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax) try: model.load_state_dict(ckpt['model_state'], strict=True) except RuntimeError as err: print("Runtime Error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0]) raise if args.cuda: device = get_freer_gpu() model = model.cuda(device) idxs_enroll, idxs_test, labels = create_trials_labels(labels_list)
def train(lr, l2, momentum, smoothing, patience, model, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path, softmax): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers) if model == 'vgg': model_ = vgg.VGG('VGG16', nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) elif model == 'resnet': model_ = resnet.ResNet18(nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) elif model == 'densenet': model_ = densenet.densenet_cifar(nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) if args.cuda: device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, patience=int(patience), label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Hidden layer size size: {}'.format(int(hidden_size))) print('Number of hidden layers: {}'.format(int(n_hidden))) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print('Best Error Rate in file ' + cp_name + ' was: {}'.format(cost[2])) print(' ') return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99
num_workers=args.n_workers) if args.model == 'vgg': model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax) elif args.model == 'resnet': model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size, dropout_prob=args.dropout_prob, sm_type=args.softmax) if args.verbose > 0: print(model) if args.cuda: device = get_freer_gpu() model = model.cuda(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.l2, momentum=args.momentum) trainer = TrainLoop(model,
default='softmax', help='Softmax type') args = parser.parse_args() if args.model == 'vgg': model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size, sm_type=args.softmax) elif args.model == 'resnet': model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size, sm_type=args.softmax) elif args.model == 'densenet': model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size, sm_type=args.softmax) cp_list = glob.glob(args.cp_path + '*.pt') assert len(cp_list) > 0, 'No cp found in the given path!' for cp in cp_list: ckpt = torch.load(cp, map_location=lambda storage, loc: storage) try: model.load_state_dict(ckpt['model_state'], strict=True) print('lol', cp.split('/')[-1]) except: pass
def train(lr, l2, momentum, slack, patience, model, epochs, batch_size, valid_batch_size, train_mode, n_workers, cuda, data_path, valid_data_path, checkpoint_path): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) if model == 'vgg': model_ = vgg.VGG('VGG16') elif model == 'resnet': model_ = resnet.ResNet50() elif model == 'densenet': model_ = densenet.densenet_cifar() if cuda: device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, slack=slack, train_mode=train_mode, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Train mode: {}'.format(train_mode)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Slack: {}'.format(slack)) print('Patience: {}'.format(patience)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Train mode: {}'.format(train_mode)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Slack: {}'.format(slack)) print('Patience: {}'.format(patience)) print(' ') return 0.99
alpha=args.step_size * normalization_factor) temp_hist = calculate_thickness(train_loader, model, PGD_attack, num_classes, args) hist_list_epochs[epoch_num] = temp_hist else: model_list = { 'resnet20': resnet20_cifar(), 'resnet32': resnet32_cifar(), 'resnet44': resnet44_cifar(), 'resnet56': resnet56_cifar(), 'resnet110': resnet110_cifar(), 'ResNet18': ResNet18(), 'ResNet50': ResNet50(), 'DenseNet': densenet_cifar(), 'VGG13': VGG('VGG13'), 'VGG19': VGG('VGG19') } model = model_list[args.arch] if args.noise_type == "None": Noisy_mixup = False elif args.noise_type == "Noisy": Noisy_mixup = True if args.arch=="ResNet18" and Noisy_mixup: model.linear = nn.Linear(in_features=512, out_features=num_classes+1, bias=True) model = model.cuda()
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test = transforms.ToTensor() #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) if model == 'vgg': model = vgg.VGG('VGG16') elif model == 'resnet': model = resnet.ResNet18() elif model == 'densenet': model = densenet.densenet_cifar() if cuda: model = model.cuda() optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, margin=margin, lambda_=lambda_, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, swap=swap, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') return 0.99
def __init__(self): super(Solver, self).__init__() global numberofclass #define the network if args.net_type == 'resnet': self.model = RN.ResNet(dataset=args.dataset, depth=args.depth, num_classes=numberofclass, bottleneck=args.bottleneck) elif args.net_type == 'pyramidnet': self.model = PYRM.PyramidNet(args.dataset, args.depth, args.alpha, numberofclass, args.bottleneck) elif args.net_type == 'wideresnet': self.model = WR.WideResNet(depth=args.depth, num_classes=numberofclass, widen_factor=args.width) elif args.net_type == 'vggnet': self.model = VGG.vgg16(num_classes=numberofclass) elif args.net_type == 'mobilenet': self.model = MN.mobile_half(num_classes=numberofclass) elif args.net_type == 'shufflenet': self.model = SN.ShuffleV2(num_classes=numberofclass) elif args.net_type == 'densenet': self.model = DN.densenet_cifar(num_classes=numberofclass) elif args.net_type == 'resnext-2': self.model = ResNeXt29_2x64d(num_classes=numberofclass) elif args.net_type == 'resnext-4': self.model = ResNeXt29_4x64d(num_classes=numberofclass) elif args.net_type == 'resnext-32': self.model = ResNeXt29_32x4d(num_classes=numberofclass) elif args.net_type == 'imagenetresnet18': self.model = multi_resnet18_kd(num_classes=numberofclass) elif args.net_type == 'imagenetresnet34': self.model = multi_resnet34_kd(num_classes=numberofclass) elif args.net_type == 'imagenetresnet50': self.model = multi_resnet50_kd(num_classes=numberofclass) elif args.net_type == 'imagenetresnet101': self.model = multi_resnet101_kd(num_classes=numberofclass) elif args.net_type == 'imagenetresnet152': self.model = multi_resnet152_kd(num_classes=numberofclass) else: raise Exception('unknown network architecture: {}'.format( args.net_type)) self.optimizer = torch.optim.SGD(self.model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) self.loss_lams = torch.zeros(numberofclass, numberofclass, dtype=torch.float32).cuda() self.loss_lams.requires_grad = False #define the loss function if args.method == 'ce': self.criterion = nn.CrossEntropyLoss() elif args.method == 'sce': if args.dataset == 'cifar10': self.criterion = SCELoss(alpha=0.1, beta=1.0, num_classes=numberofclass) else: self.criterion = SCELoss(alpha=6.0, beta=0.1, num_classes=numberofclass) elif args.method == 'ls': self.criterion = label_smooth(num_classes=numberofclass) elif args.method == 'gce': self.criterion = generalized_cross_entropy( num_classes=numberofclass) elif args.method == 'jo': self.criterion = joint_optimization(num_classes=numberofclass) elif args.method == 'bootsoft': self.criterion = boot_soft(num_classes=numberofclass) elif args.method == 'boothard': self.criterion = boot_hard(num_classes=numberofclass) elif args.method == 'forward': self.criterion = Forward(num_classes=numberofclass) elif args.method == 'backward': self.criterion = Backward(num_classes=numberofclass) elif args.method == 'disturb': self.criterion = DisturbLabel(num_classes=numberofclass) elif args.method == 'ols': self.criterion = nn.CrossEntropyLoss() self.criterion = self.criterion.cuda()
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path, softmax): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_train = transforms.Compose([ transforms.RandomCrop(84, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) #trainset = Loader(args.data_path) trainset = datasets.ImageFolder(args.data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=args.batch_size, shuffle=True, num_workers=args.n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) #validset = Loader(args.valid_data_path) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader( validset, batch_size=args.valid_batch_size, shuffle=True, num_workers=args.n_workers, pin_memory=True) if model == 'vgg': model_ = vgg.VGG('VGG16', sm_type=softmax) elif model == 'resnet': model_ = resnet.ResNet18(sm_type=softmax) elif model == 'densenet': model_ = densenet.densenet_cifar(sm_type=softmax) if cuda: torch.backends.cudnn.benchmark = True device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, margin=margin, lambda_=lambda_, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, swap=swap, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('lambda: {}'.format(lambda_)) print('Margin: {}'.format(margin)) print('Swap: {}'.format(swap)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') return 0.99