def get_net(network: str, num_classes) -> torch.nn.Module:
return VGG('VGG16', num_classes=num_classes) if network == 'VGG16' else \
ResNet34(num_classes=num_classes) if network == 'ResNet34' else \
PreActResNet18(num_classes=num_classes) if network == 'PreActResNet18' else \
GoogLeNet(num_classes=num_classes) if network == 'GoogLeNet' else \
densenet_cifar(num_classes=num_classes) if network == 'densenet_cifar' else \
ResNeXt29_2x64d(num_classes=num_classes) if network == 'ResNeXt29_2x64d' else \
MobileNet(num_classes=num_classes) if network == 'MobileNet' else \
MobileNetV2(num_classes=num_classes) if network == 'MobileNetV2' else \
DPN92(num_classes=num_classes) if network == 'DPN92' else \
ShuffleNetG2(num_classes=num_classes) if network == 'ShuffleNetG2' else \
SENet18(num_classes=num_classes) if network == 'SENet18' else \
ShuffleNetV2(1, num_classes=num_classes) if network == 'ShuffleNetV2' else \
EfficientNetB0(
num_classes=num_classes) if network == 'EfficientNetB0' else None
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
validset = datasets.CIFAR10(root=args.data_path,
train=False,
download=True,
transform=transform_test)
labels_list = [x[1] for x in validset]
if args.model == 'vgg':
model = vgg.VGG('VGG16')
elif args.model == 'resnet':
model = resnet.ResNet18()
elif args.model == 'densenet':
model = densenet.densenet_cifar()
ckpt = torch.load(args.cp_path, map_location=lambda storage, loc: storage)
try:
model.load_state_dict(ckpt['model_state'], strict=True)
except RuntimeError as err:
print("Runtime Error: {0}".format(err))
except:
print("Unexpected error:", sys.exc_info()[0])
raise
if args.cuda:
device = get_freer_gpu()
model = model.cuda(device)
idxs_enroll, idxs_test, labels = create_trials_labels(labels_list)
import torch.utils.data
from models import vgg, resnet, densenet
# Training settings
parser = argparse.ArgumentParser(description='Test new architectures')
parser.add_argument('--model', choices=['vgg', 'resnet', 'densenet'], default='resnet')
parser.add_argument('--hidden-size', type=int, default=512, metavar='S', help='latent layer dimension (default: 512)')
parser.add_argument('--n-hidden', type=int, default=1, metavar='N', help='maximum number of frames per utterance (default: 1)')
args = parser.parse_args()
if args.model == 'vgg':
model = vgg.VGG('VGG16', nh=args.n_hidden, n_h=args.hidden_size)
elif args.model == 'resnet':
model = resnet.ResNet18(nh=args.n_hidden, n_h=args.hidden_size)
elif args.model == 'densenet':
model = densenet.densenet_cifar(nh=args.n_hidden, n_h=args.hidden_size)
batch = torch.rand(3, 3, 84, 84)
emb = model.forward(batch)
print(emb.size())
out = model.out_proj(emb)
print(out.size())
emb = torch.cat([emb,emb],1)
print(emb.size())
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
validset = datasets.CIFAR10(root=args.data_path,
train=False,
download=True,
transform=transform_test)
labels_list = [x[1] for x in validset]
if args.model == 'vgg':
model = vgg.VGG('VGG16', sm_type=args.softmax)
elif args.model == 'resnet':
model = resnet.ResNet18(sm_type=args.softmax)
elif args.model == 'densenet':
model = densenet.densenet_cifar(sm_type=args.softmax)
cp_list = glob.glob(args.cp_path + '*.pt')
idxs_enroll, idxs_test, labels = create_trials_labels(labels_list)
print('\n{} trials created out of which {} are target trials'.format(
len(idxs_enroll), np.sum(labels)))
best_model, best_eer = None, float('inf')
for cp in cp_list:
ckpt = torch.load(cp, map_location=lambda storage, loc: storage)
try:
model.load_state_dict(ckpt['model_state'], strict=True)
except:
dropout_prob = args.dropout_prob
if args.model == 'vgg':
model = vgg.VGG('VGG16',
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax)
elif args.model == 'resnet':
model = resnet.ResNet18(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax)
elif args.model == 'densenet':
model = densenet.densenet_cifar(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax)
try:
model.load_state_dict(ckpt['model_state'], strict=True)
except RuntimeError as err:
print("Runtime Error: {0}".format(err))
except:
print("Unexpected error:", sys.exc_info()[0])
raise
if args.cuda:
device = get_freer_gpu()
model = model.cuda(device)
idxs_enroll, idxs_test, labels = create_trials_labels(labels_list)
def train(lr, l2, momentum, smoothing, patience, model, n_hidden, hidden_size,
dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda,
data_path, valid_data_path, checkpoint_path, softmax):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=True,
num_workers=n_workers)
if model == 'vgg':
model_ = vgg.VGG('VGG16',
nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
elif model == 'resnet':
model_ = resnet.ResNet18(nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
elif model == 'densenet':
model_ = densenet.densenet_cifar(nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
if args.cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
patience=int(patience),
label_smoothing=smoothing,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
print(' ')
print('Hyperparameters:')
print('Selected model: {}'.format(model))
print('Hidden layer size size: {}'.format(int(hidden_size)))
print('Number of hidden layers: {}'.format(int(n_hidden)))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Label smoothing: {}'.format(smoothing))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print(' ')
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print('Best Error Rate in file ' + cp_name +
' was: {}'.format(cost[2]))
print(' ')
return cost[0]
except:
print("Error:", sys.exc_info())
pass
print('Returning dummy cost due to failures while training.')
return 0.99
num_workers=args.n_workers)
if args.model == 'vgg':
model = vgg.VGG('VGG16',
nh=args.n_hidden,
n_h=args.hidden_size,
dropout_prob=args.dropout_prob,
sm_type=args.softmax)
elif args.model == 'resnet':
model = resnet.ResNet18(nh=args.n_hidden,
n_h=args.hidden_size,
dropout_prob=args.dropout_prob,
sm_type=args.softmax)
elif args.model == 'densenet':
model = densenet.densenet_cifar(nh=args.n_hidden,
n_h=args.hidden_size,
dropout_prob=args.dropout_prob,
sm_type=args.softmax)
if args.verbose > 0:
print(model)
if args.cuda:
device = get_freer_gpu()
model = model.cuda(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.l2,
momentum=args.momentum)
trainer = TrainLoop(model,
default='softmax',
help='Softmax type')
args = parser.parse_args()
if args.model == 'vgg':
model = vgg.VGG('VGG16',
nh=args.n_hidden,
n_h=args.hidden_size,
sm_type=args.softmax)
elif args.model == 'resnet':
model = resnet.ResNet18(nh=args.n_hidden,
n_h=args.hidden_size,
sm_type=args.softmax)
elif args.model == 'densenet':
model = densenet.densenet_cifar(nh=args.n_hidden,
n_h=args.hidden_size,
sm_type=args.softmax)
cp_list = glob.glob(args.cp_path + '*.pt')
assert len(cp_list) > 0, 'No cp found in the given path!'
for cp in cp_list:
ckpt = torch.load(cp, map_location=lambda storage, loc: storage)
try:
model.load_state_dict(ckpt['model_state'], strict=True)
print('lol', cp.split('/')[-1])
except:
pass
def train(lr, l2, momentum, slack, patience, model, epochs, batch_size,
valid_batch_size, train_mode, n_workers, cuda, data_path,
valid_data_path, checkpoint_path):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model_ = vgg.VGG('VGG16')
elif model == 'resnet':
model_ = resnet.ResNet50()
elif model == 'densenet':
model_ = densenet.densenet_cifar()
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
slack=slack,
train_mode=train_mode,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Train mode: {}'.format(train_mode))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Slack: {}'.format(slack))
print('Patience: {}'.format(patience))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Train mode: {}'.format(train_mode))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Slack: {}'.format(slack))
print('Patience: {}'.format(patience))
print(' ')
return 0.99
alpha=args.step_size * normalization_factor)
temp_hist = calculate_thickness(train_loader, model, PGD_attack, num_classes, args)
hist_list_epochs[epoch_num] = temp_hist
else:
model_list = {
'resnet20': resnet20_cifar(),
'resnet32': resnet32_cifar(),
'resnet44': resnet44_cifar(),
'resnet56': resnet56_cifar(),
'resnet110': resnet110_cifar(),
'ResNet18': ResNet18(),
'ResNet50': ResNet50(),
'DenseNet': densenet_cifar(),
'VGG13': VGG('VGG13'),
'VGG19': VGG('VGG19')
}
model = model_list[args.arch]
if args.noise_type == "None":
Noisy_mixup = False
elif args.noise_type == "Noisy":
Noisy_mixup = True
if args.arch=="ResNet18" and Noisy_mixup:
model.linear = nn.Linear(in_features=512, out_features=num_classes+1, bias=True)
model = model.cuda()
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs,
batch_size, valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.ToTensor()
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model = vgg.VGG('VGG16')
elif model == 'resnet':
model = resnet.ResNet18()
elif model == 'densenet':
model = densenet.densenet_cifar()
if cuda:
model = model.cuda()
optimizer = optim.SGD(model.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
swap=swap,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
def __init__(self):
super(Solver, self).__init__()
global numberofclass
#define the network
if args.net_type == 'resnet':
self.model = RN.ResNet(dataset=args.dataset,
depth=args.depth,
num_classes=numberofclass,
bottleneck=args.bottleneck)
elif args.net_type == 'pyramidnet':
self.model = PYRM.PyramidNet(args.dataset, args.depth, args.alpha,
numberofclass, args.bottleneck)
elif args.net_type == 'wideresnet':
self.model = WR.WideResNet(depth=args.depth,
num_classes=numberofclass,
widen_factor=args.width)
elif args.net_type == 'vggnet':
self.model = VGG.vgg16(num_classes=numberofclass)
elif args.net_type == 'mobilenet':
self.model = MN.mobile_half(num_classes=numberofclass)
elif args.net_type == 'shufflenet':
self.model = SN.ShuffleV2(num_classes=numberofclass)
elif args.net_type == 'densenet':
self.model = DN.densenet_cifar(num_classes=numberofclass)
elif args.net_type == 'resnext-2':
self.model = ResNeXt29_2x64d(num_classes=numberofclass)
elif args.net_type == 'resnext-4':
self.model = ResNeXt29_4x64d(num_classes=numberofclass)
elif args.net_type == 'resnext-32':
self.model = ResNeXt29_32x4d(num_classes=numberofclass)
elif args.net_type == 'imagenetresnet18':
self.model = multi_resnet18_kd(num_classes=numberofclass)
elif args.net_type == 'imagenetresnet34':
self.model = multi_resnet34_kd(num_classes=numberofclass)
elif args.net_type == 'imagenetresnet50':
self.model = multi_resnet50_kd(num_classes=numberofclass)
elif args.net_type == 'imagenetresnet101':
self.model = multi_resnet101_kd(num_classes=numberofclass)
elif args.net_type == 'imagenetresnet152':
self.model = multi_resnet152_kd(num_classes=numberofclass)
else:
raise Exception('unknown network architecture: {}'.format(
args.net_type))
self.optimizer = torch.optim.SGD(self.model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
self.loss_lams = torch.zeros(numberofclass,
numberofclass,
dtype=torch.float32).cuda()
self.loss_lams.requires_grad = False
#define the loss function
if args.method == 'ce':
self.criterion = nn.CrossEntropyLoss()
elif args.method == 'sce':
if args.dataset == 'cifar10':
self.criterion = SCELoss(alpha=0.1,
beta=1.0,
num_classes=numberofclass)
else:
self.criterion = SCELoss(alpha=6.0,
beta=0.1,
num_classes=numberofclass)
elif args.method == 'ls':
self.criterion = label_smooth(num_classes=numberofclass)
elif args.method == 'gce':
self.criterion = generalized_cross_entropy(
num_classes=numberofclass)
elif args.method == 'jo':
self.criterion = joint_optimization(num_classes=numberofclass)
elif args.method == 'bootsoft':
self.criterion = boot_soft(num_classes=numberofclass)
elif args.method == 'boothard':
self.criterion = boot_hard(num_classes=numberofclass)
elif args.method == 'forward':
self.criterion = Forward(num_classes=numberofclass)
elif args.method == 'backward':
self.criterion = Backward(num_classes=numberofclass)
elif args.method == 'disturb':
self.criterion = DisturbLabel(num_classes=numberofclass)
elif args.method == 'ols':
self.criterion = nn.CrossEntropyLoss()
self.criterion = self.criterion.cuda()
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs,
batch_size, valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path, softmax):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_train = transforms.Compose([
transforms.RandomCrop(84, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
#trainset = Loader(args.data_path)
trainset = datasets.ImageFolder(args.data_path, transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.n_workers,
worker_init_fn=set_np_randomseed,
pin_memory=True)
#validset = Loader(args.valid_data_path)
validset = datasets.ImageFolder(args.valid_data_path,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(
validset,
batch_size=args.valid_batch_size,
shuffle=True,
num_workers=args.n_workers,
pin_memory=True)
if model == 'vgg':
model_ = vgg.VGG('VGG16', sm_type=softmax)
elif model == 'resnet':
model_ = resnet.ResNet18(sm_type=softmax)
elif model == 'densenet':
model_ = densenet.densenet_cifar(sm_type=softmax)
if cuda:
torch.backends.cudnn.benchmark = True
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
swap=swap,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('lambda: {}'.format(lambda_))
print('Margin: {}'.format(margin))
print('Swap: {}'.format(swap))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
