def train(lr, l2, momentum, patience, latent_size, n_hidden, hidden_size, n_frames, model, ncoef, dropout_prob, epochs, batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, valid_n_cycles, cp_path, softmax): if cuda: device=get_freer_gpu() train_dataset=Loader_test(hdf5_name=train_hdf_file, max_nb_frames=int(n_frames)) train_loader=torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) valid_dataset = Loader(hdf5_name = valid_hdf_file, max_nb_frames = int(n_frames), n_cycles=valid_n_cycles) valid_loader=torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) if args.model == 'resnet_stats': model = model_.ResNet_stats(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet_mfcc': model = model_.ResNet_mfcc(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) if args.model == 'resnet_lstm': model = model_.ResNet_lstm(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet_small': model = model_.ResNet_small(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet_large': model = model_.ResNet_large(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) if cuda: model=model.cuda(device) else: device=None optimizer=optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2) trainer=TrainLoop(model, optimizer, train_loader, valid_loader, patience=int(patience), verbose=-1, device=device, cp_name=get_file_name(cp_path), save_cp=False, checkpoint_path=cp_path, pretrain=False, cuda=cuda) return trainer.train(n_epochs=epochs)
image_size = 28
img_transform = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])
img_transform_mnist = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])
source_dataset = datasets.MNIST(root=source_path, download=True, train=True, transform=img_transform_mnist)
source_loader = torch.utils.data.DataLoader(dataset=source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
train_list = os.path.join(target_path, 'mnist_m_train_labels.txt')
target_dataset = Loader(data_root=os.path.join(target_path, 'mnist_m_train'), data_list=train_list, transform=img_transform)
target_loader = torch.utils.data.DataLoader(dataset=target_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
model = models_.CNNModel()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9)
if args.cuda:
model = model.cuda()
torch.backends.cudnn.benchmark=True
trainer = TrainLoop(model, optimizer, source_loader, target_loader, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, target_name = args.target)
print('Cuda Mode: {}'.format(args.cuda))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('Source: {}'.format(args.source))
print('Target: {}'.format(args.target))
trainer.train(n_epochs=args.epochs, save_every=args.save_every)
weight_decay=args.l2,
nesterov=True),
lr=args.lr,
warmup_steps=args.warmup)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=args.max_gnorm,
margin=args.margin,
lambda_=args.lamb,
label_smoothing=args.smoothing,
warmup_its=args.warmup,
verbose=-1,
device=device,
cp_name=args.cp_name,
save_cp=True,
checkpoint_path=args.checkpoint_path,
swap=args.swap,
lr_red_epoch=args.lr_reduction_epoch,
lr_factor=args.lr_factor,
softmax=True,
pretrain=False,
mining=True,
cuda=args.cuda,
logger=writer)
print('\n')
print(model)
print('\n')
print('Device: {}'.format(device))
disc_list.append(disc)
if args.cuda:
generator = generator.cuda()
for disc in disc_list:
disc = disc.cuda()
torch.backends.cudnn.benchmark = True
optimizer = optim.Adam(generator.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
trainer = TrainLoop(generator,
fid_model,
disc_list,
optimizer,
train_loader,
nadir_slack=args.nadir_slack,
alpha=args.alpha,
train_mode=args.train_mode,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
job_id=args.job_id)
print('Cuda Mode is: {}'.format(args.cuda))
print('Train Mode is: {}'.format(args.train_mode))
print('Number of discriminators is: {}'.format(len(disc_list)))
trainer.train(n_epochs=args.epochs, save_every=args.save_every)
def train(lr, l2, momentum, smoothing, patience, model, n_hidden, hidden_size,
dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda,
data_path, valid_data_path, checkpoint_path, softmax):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=True,
num_workers=n_workers)
if model == 'vgg':
model_ = vgg.VGG('VGG16',
nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
elif model == 'resnet':
model_ = resnet.ResNet18(nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
elif model == 'densenet':
model_ = densenet.densenet_cifar(nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
if args.cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
patience=int(patience),
label_smoothing=smoothing,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
print(' ')
print('Hyperparameters:')
print('Selected model: {}'.format(model))
print('Hidden layer size size: {}'.format(int(hidden_size)))
print('Number of hidden layers: {}'.format(int(n_hidden)))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Label smoothing: {}'.format(smoothing))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print(' ')
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print('Best Error Rate in file ' + cp_name +
' was: {}'.format(cost[2]))
print(' ')
return cost[0]
except:
print("Error:", sys.exc_info())
pass
print('Returning dummy cost due to failures while training.')
return 0.99
def train(lr, l2, momentum, smoothing, warmup, model, emb_size, n_hidden,
hidden_size, dropout_prob, epochs, batch_size, valid_batch_size,
n_workers, cuda, data_path, valid_data_path, hdf_path,
valid_hdf_path, checkpoint_path, softmax, pretrained,
pretrained_path, max_gnorm, stats, log_dir, eval_every, ablation):
args_dict = locals()
cp_name = get_cp_name(checkpoint_path)
if pretrained_path != 'none':
print('\nLoading pretrained model from: {}\n'.format(
args.pretrained_path))
ckpt = torch.load(pretrained_path,
map_location=lambda storage, loc: storage)
dropout_prob, n_hidden, hidden_size, emb_size = ckpt[
'dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt[
'emb_size']
if 'r_proj_size' in ckpt:
rproj_size = ckpt['r_proj_size']
else:
rproj_size = -1
print('\nUsing pretrained config for discriminator. Ignoring args.')
args_dict['dropout_prob'], args_dict['n_hidden'], args_dict[
'hidden_size'], args_dict[
'emb_size'] = dropout_prob, n_hidden, hidden_size, emb_size
if log_dir != 'none':
writer = SummaryWriter(log_dir=os.path.join(log_dir, cp_name),
comment=model,
purge_step=0)
writer.add_hparams(hparam_dict=args_dict,
metric_dict={'best_eer': 0.0})
else:
writer = None
if stats == 'cars':
mean, std = [0.4461, 0.4329, 0.4345], [0.2888, 0.2873, 0.2946]
elif stats == 'cub':
mean, std = [0.4782, 0.4925, 0.4418], [0.2330, 0.2296, 0.2647]
elif stats == 'sop':
mean, std = [0.5603, 0.5155, 0.4796], [0.2939, 0.2991, 0.3085]
elif stats == 'imagenet':
mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
if hdf_path != 'none':
transform_train = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(10),
transforms.RandomPerspective(p=0.1),
transforms.RandomGrayscale(p=0.1),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
trainset = Loader(hdf_path, transform_train)
else:
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(10),
transforms.RandomPerspective(p=0.1),
transforms.RandomGrayscale(p=0.1),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
trainset = datasets.ImageFolder(data_path, transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
worker_init_fn=set_np_randomseed,
pin_memory=True)
if valid_hdf_path != 'none':
transform_test = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
validset = Loader(args.valid_hdf_path, transform_test)
else:
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
validset = datasets.ImageFolder(args.valid_data_path,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=True,
num_workers=n_workers,
pin_memory=True)
nclasses = trainset.n_classes if isinstance(trainset, Loader) else len(
trainset.classes)
if model == 'vgg':
model_ = vgg.VGG('VGG19',
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=nclasses,
emb_size=emb_size,
r_proj_size=rproj_size)
elif model == 'resnet':
model_ = resnet.ResNet50(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=nclasses,
emb_size=emb_size,
r_proj_size=rproj_size)
elif model == 'densenet':
model_ = densenet.DenseNet121(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=nclasses,
emb_size=emb_size,
r_proj_size=rproj_size)
if pretrained_path != 'none':
if ckpt['sm_type'] == 'am_softmax':
del (ckpt['model_state']['out_proj.w'])
elif ckpt['sm_type'] == 'softmax':
del (ckpt['model_state']['out_proj.w.weight'])
del (ckpt['model_state']['out_proj.w.bias'])
print(model_.load_state_dict(ckpt['model_state'], strict=False))
print('\n')
if pretrained:
print('\nLoading pretrained encoder from torchvision\n')
if model == 'vgg':
model_pretrained = torchvision.models.vgg19(pretrained=True)
elif model == 'resnet':
model_pretrained = torchvision.models.resnet50(pretrained=True)
elif model == 'densenet':
model_pretrained = torchvision.models.densenet121(pretrained=True)
print(
model_.load_state_dict(model_pretrained.state_dict(),
strict=False))
print('\n')
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
torch.backends.cudnn.benchmark = True
optimizer = TransformerOptimizer(optim.SGD(model_.parameters(),
lr=lr,
momentum=momentum,
weight_decay=l2,
nesterov=True),
lr=lr,
warmup_steps=warmup)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
max_gnorm=max_gnorm,
label_smoothing=smoothing,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
ablation=ablation,
cuda=cuda,
logger=writer)
for i in range(5):
print(' ')
print('Hyperparameters:')
print('Selected model: {}'.format(model))
print('Embedding size: {}'.format(emb_size))
print('Hidden layer size: {}'.format(hidden_size))
print('Number of hidden layers: {}'.format(n_hidden))
print('Random projection size: {}'.format(rproj_size))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Label smoothing: {}'.format(smoothing))
print('Warmup iterations: {}'.format(warmup))
print('Softmax Mode is: {}'.format(softmax))
print('Pretrained: {}'.format(pretrained))
print('Pretrained path: {}'.format(pretrained_path))
print('Evaluate every {} iterations.'.format(eval_every))
print('Ablation Mode: {}'.format(ablation))
print(' ')
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs,
save_every=epochs + 10,
eval_every=eval_every)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print(' ')
if log_dir != 'none':
writer.add_hparams(hparam_dict=args_dict,
metric_dict={'best_eer': cost[0]})
return cost[0]
except:
print("Error:", sys.exc_info())
pass
print('Returning dummy cost due to failures while training.')
return 0.99
if args.cuda:
device = get_freer_gpu()
model = model.to(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.l2,
momentum=args.momentum)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
margin=args.margin,
lambda_=args.lamb,
verbose=args.verbose,
save_cp=(not args.no_cp),
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
swap=args.swap,
cuda=args.cuda)
if args.verbose > 0:
print('Cuda Mode is: {}'.format(args.cuda))
print('Selected model: {}'.format(args.model))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('Momentum: {}'.format(args.momentum))
print('l2: {}'.format(args.l2))
print('lambda: {}'.format(args.lamb))
print('Margin: {}'.format(args.margin))
testset = datasets.CIFAR10(root=args.data_path,
train=False,
download=True,
transform=transform)
test_loader = torch.utils.data.DataLoader(testset,
batch_size=1000,
shuffle=False,
num_workers=args.workers)
save_testdata_statistics(fid_model,
test_loader,
downsample_=False,
cuda_mode=args.cuda)
optimizer = optim.Adam(generator.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
trainer = TrainLoop(generator,
fid_model,
disc,
optimizer,
train_loader,
its_disc=args.its_disc,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda)
print('Cuda Mode is: {}'.format(args.cuda))
trainer.train(n_epochs=args.epochs, save_every=args.save_every)
device = get_freer_gpu()
else:
device = None
if args.cuda:
model = model.cuda(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.l2)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
patience=args.patience,
verbose=-1,
device=device,
cp_name=args.cp_name,
save_cp=True,
checkpoint_path=args.checkpoint_path,
pretrain=False,
cuda=args.cuda)
best_eer = trainer.train(n_epochs=args.epochs, save_every=args.epochs + 10)
out_file = open(args.out_file, 'wb')
pickle.dump(best_eer[0], out_file)
out_file.close()
def train(lr, l2, momentum, max_gnorm, warmup, input_size, n_hidden,
hidden_size, dropout_prob, smoothing, n_cycles, epochs, batch_size,
valid_batch_size, n_workers, cuda, train_hdf_path, valid_hdf_path,
cp_path, logdir):
hp_dict = {
'lr': lr,
'l2': l2,
'momentum': momentum,
'max_gnorm': max_gnorm,
'warmup': warmup,
'input_size': input_size,
'n_hidden': n_hidden,
'hidden_size': hidden_size,
'dropout_prob': dropout_prob,
'smoothing': smoothing,
'n_cycles': n_cycles,
'epochs': epochs,
'batch_size': batch_size,
'valid_batch_size': valid_batch_size,
'n_workers': n_workers,
'cuda': cuda,
'train_hdf_path': train_hdf_path,
'valid_hdf_path': valid_hdf_path,
'cp_path': cp_path
}
cp_name = get_file_name(cp_path)
if args.logdir:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir=logdir + cp_name, purge_step=True)
writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': 0.5})
else:
writer = None
train_dataset = Loader(hdf5_clean=train_hdf_path + 'train_clean.hdf',
hdf5_attack=train_hdf_path + 'train_attack.hdf',
label_smoothing=smoothing,
n_cycles=n_cycles)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers)
valid_dataset = Loader(hdf5_clean=valid_hdf_path + 'valid_clean.hdf',
hdf5_attack=valid_hdf_path + 'valid_attack.hdf',
n_cycles=1)
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
model = model_.MLP(n_in=input_size,
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob)
if cuda:
device = get_freer_gpu()
model = model.cuda(device)
optimizer = TransformerOptimizer(optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
weight_decay=l2,
nesterov=True),
lr=lr,
warmup_steps=warmup)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=max_gnorm,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=cp_path,
cuda=cuda,
logger=writer)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best EER in file ' + cp_name + ' was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Hidden layer size size: {}'.format(int(hidden_size)))
print('Number of hidden layers: {}'.format(int(n_hidden)))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Warmup iterations: {}'.format(warmup))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Max. Grad. norm: {}'.format(max_gnorm))
print('Label smoothing: {}'.format(smoothing))
print(' ')
if args.logdir:
writer.add_hparams(hparam_dict=hp_dict,
metric_dict={'best_eer': cost})
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
cost = 0.99
if args.logdir:
writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': cost})
return cost
for i in range(args.ndiscriminators):
disc = model.Discriminator_toy(512, optim.Adam, args.lr,
(args.beta1, args.beta2)).train()
disc_list.append(disc)
optimizer = optim.Adam(generator.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
if args.hyper_mode:
trainer = TrainLoop(generator,
disc_list,
optimizer,
args.toy_dataset,
centers,
cov,
train_loader=train_loader,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
nadir_slack=args.nadir_slack)
else:
trainer = TrainLoop(generator,
disc_list,
optimizer,
args.toy_dataset,
centers,
cov,
train_loader=train_loader,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch)
ckpt = torch.load(args.pretrained_path, map_location = lambda storage, loc: storage)
model = model_.MLP(n_in=ckpt['input_size'], nh=ckpt['n_hidden'], n_h=ckpt['hidden_size'], dropout_prob=ckpt['dropout_prob'])
try:
model.load_state_dict(ckpt['model_state'], strict=True)
except RuntimeError as err:
print("Runtime Error: {0}".format(err))
model.load_state_dict(ckpt['model_state'], strict=False)
except:
print("Unexpected error:", sys.exc_info()[0])
raise
model = model.to(device)
optimizer = TransformerOptimizer(optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.l2, nesterov=True), lr=args.lr, warmup_steps=args.warmup)
trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, verbose=args.verbose, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, logger=writer)
if args.verbose > 0:
print('Cuda Mode: {}'.format(args.cuda))
print('Device: {}'.format(device))
print('Batch size: {}'.format(args.batch_size))
print('Validation batch size: {}'.format(args.valid_batch_size))
print('LR: {}'.format(args.lr))
print('Momentum: {}'.format(args.momentum))
print('l2: {}'.format(args.l2))
print('Max. grad norm: {}'.format(args.max_gnorm))
print('Warmup iterations: {}'.format(args.warmup))
print('Inputs dimensionality: {}'.format(args.input_size))
print('Number of hidden layers: {}'.format(args.n_hidden))
print('Size of hidden layers: {}'.format(args.hidden_size))
print('Label smoothing: {}'.format(args.smoothing))
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs,
batch_size, valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path, softmax):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_train = transforms.Compose([
transforms.RandomCrop(84, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
#trainset = Loader(args.data_path)
trainset = datasets.ImageFolder(args.data_path, transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.n_workers,
worker_init_fn=set_np_randomseed,
pin_memory=True)
#validset = Loader(args.valid_data_path)
validset = datasets.ImageFolder(args.valid_data_path,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(
validset,
batch_size=args.valid_batch_size,
shuffle=True,
num_workers=args.n_workers,
pin_memory=True)
if model == 'vgg':
model_ = vgg.VGG('VGG16', sm_type=softmax)
elif model == 'resnet':
model_ = resnet.ResNet18(sm_type=softmax)
elif model == 'densenet':
model_ = densenet.densenet_cifar(sm_type=softmax)
if cuda:
torch.backends.cudnn.benchmark = True
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
swap=swap,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('lambda: {}'.format(lambda_))
print('Margin: {}'.format(margin))
print('Swap: {}'.format(swap))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
parser.add_argument('--patience', type = int, default = 10, metavar = 'N', help = 'number of epochs without improvement to wait before stopping training (default: 30)')
parser.add_argument('--lr', type = float, default = 0.001, metavar = 'LR', help = 'learning rate (default: 0.001)')
parser.add_argument('--momentum', type = float, default = 0.9, metavar = 'mu', help = 'Momentum (default: 0.9)')
parser.add_argument('--l2', type = float, default = 0.001, metavar = 'lambda', help = 'L2 weight decay coefficient (default: 0.0001)')
parser.add_argument('--no-cuda', action = 'store_true', default = False, help = 'disables CUDA training')
parser.add_argument('--checkpoint-epoch', type = int, default = None, metavar = 'N', help = 'epoch to load for checkpointing. If None, training starts from scratch')
parser.add_argument('--checkpoint-path', type = str, default = None, metavar = 'Path', help = 'Path for checkpointing')
parser.add_argument('--seed', type = int, default = 12345, metavar = 'S', help = 'random seed (default: 12345)')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
model = models.model_eeg_rnn_shorterconvs()
if args.cuda:
model = model.cuda()
optimizer = torch.optim.SGD(model.parameters(), lr = args.lr, momentum = args.momentum, weight_decay = args.l2)
trainer = TrainLoop(model, optimizer, args.minibatch_size, checkpoint_path = args.checkpoint_path, checkpoint_epoch = args.checkpoint_epoch, cuda = args.cuda)
print('Cuda Mode is: {}'.format(args.cuda))
trainer.train(n_epochs = args.epochs, patience = args.patience)
if args.cuda:
device = get_freer_gpu()
model = model.cuda(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.l2,
momentum=args.momentum)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
patience=args.patience,
label_smoothing=args.smoothing,
verbose=args.verbose,
save_cp=(not args.no_cp),
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda)
if args.verbose > 0:
print('\nCuda Mode is: {}'.format(args.cuda))
print('Selected model: {}'.format(args.model))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('Momentum: {}'.format(args.momentum))
print('l2: {}'.format(args.l2))
print('Label smoothing: {}'.format(args.smoothing))
print('Patience: {}'.format(args.patience))
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs,
batch_size, valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.ToTensor()
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model = vgg.VGG('VGG16')
elif model == 'resnet':
model = resnet.ResNet18()
elif model == 'densenet':
model = densenet.densenet_cifar()
if cuda:
model = model.cuda()
optimizer = optim.SGD(model.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
swap=swap,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
comment=args.model,
purge_step=0 if args.checkpoint_epoch is None else
int(args.checkpoint_epoch * len(train_loader)))
args_dict = parse_args_for_log(args)
writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_eer': 0.0})
else:
writer = None
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=args.max_gnorm,
patience=args.patience,
lr_factor=args.lr_factor,
label_smoothing=args.smoothing,
verbose=args.verbose,
cp_name=args.cp_name,
save_cp=(not args.no_cp),
checkpoint_path=args.checkpoint_path,
ablation=args.ablation,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
logger=writer)
if args.verbose > 0:
print('\nCuda Mode is: {}'.format(args.cuda))
print('Selected model: {}'.format(args.model))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('Momentum: {}'.format(args.momentum))
print('l2: {}'.format(args.l2))
batch_size=args.batch_size,
num_workers=args.workers)
centers = toy_data.get_centers()
cov = toy_data.get_cov()
# hidden_size = 512
generator = model.Generator_toy(512).train()
disc = model.Discriminator_toy(512, optim.Adam, args.lr,
(args.beta1, args.beta2)).train()
optimizer = optim.Adam(generator.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
trainer = TrainLoop(generator,
disc,
optimizer,
args.toy_dataset,
centers,
cov,
train_loader=train_loader,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda)
print('Cuda Mode is: {}'.format(args.cuda))
trainer.train(n_epochs=args.epochs, save_every=args.save_every)
model = model.cuda(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.l2)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
margin=args.margin,
lambda_=args.lamb,
patience=args.patience,
verbose=-1,
device=device,
cp_name=args.cp_name,
save_cp=True,
checkpoint_path=args.checkpoint_path,
swap=args.swap,
softmax=True,
pretrain=False,
mining=True,
cuda=args.cuda)
best_eer = trainer.train(n_epochs=args.epochs, save_every=args.epochs + 10)
out_file = open(args.out_file, 'wb')
pickle.dump(best_eer, out_file)
out_file.close()
train_source_1 = args.data_path + 'train_' + args.source1 + '.hdf'
test_source_1 = args.data_path + 'test_' + args.source1 + '.hdf'
source_dataset = Loader_validation(hdf_path=train_source_1, transform=img_transform_train)
source_loader = torch.utils.data.DataLoader(dataset=source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
test_source_dataset = Loader_validation(hdf_path=test_source_1, transform=img_transform_test)
test_source_loader = torch.utils.data.DataLoader(dataset=test_source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
model = models.AlexNet(num_classes = 7, baseline = True)
state_dict = torch.load(args.model_path+'alexnet_caffe.pth.tar')
del state_dict["classifier.fc8.weight"]
del state_dict["classifier.fc8.bias"]
not_loaded = model.load_state_dict(state_dict, strict = False)
optimizer = optim.SGD(list(model.classifier.parameters()), lr=args.lr, momentum=args.momentum, weight_decay=args.l2, nesterov=True)
if args.cuda:
model = model.cuda()
torch.backends.cudnn.benchmark=True
trainer = TrainLoop(model, optimizer, source_loader, test_source_loader, args.nadir_slack, args.patience, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda)
err = trainer.train(n_epochs=args.epochs, save_every=args.save_every)
acc_runs.append(1-err)
print(acc_runs)
df = pandas.DataFrame(data={'Acc-{}'.format(args.source1): acc_runs, 'Seed': seeds[:args.n_runs]})
df.to_csv('./baseline_accuracy_runs_'+args.source1+'.csv', sep=',', index = False)
if args.optimizer == 'adam':
disc = models_zoo.Discriminator(optim.Adam, args.optimizer, args.lr, (args.beta1, args.beta2)).train()
elif args.optimizer == 'amsgrad':
disc = models_zoo.Discriminator(optim.Adam, args.optimizer, args.lr, (args.beta1, args.beta2), amsgrad = True).train()
elif args.optimizer == 'rmsprop':
disc = models_zoo.Discriminator(optim.RMSprop, args.optimizer, args.lr, (args.beta1, args.beta2)).train()
disc_list.append(disc)
if args.cuda:
generator = generator.cuda()
frames_generator = frames_generator.cuda()
for disc in disc_list:
disc = disc.cuda()
torch.backends.cudnn.benchmark=True
if args.optimizer == 'adam':
optimizer_g = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2))
elif args.optimizer == 'amsgrad':
optimizer_g = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), amsgrad = True)
elif args.optimizer == 'rmsprop':
optimizer_g = optim.RMSprop(generator.parameters(), lr=args.lr)
trainer = TrainLoop(generator, frames_generator, disc_list, optimizer_g, train_loader, nadir_slack=args.nadir_slack, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, hyper=not args.average_mode, cuda=args.cuda)
print('Cuda Mode is: {}'.format(args.cuda))
print('Selected model is: {}'.format(args.gen_arch))
trainer.train(n_epochs=args.epochs, save_every = args.save_every)
def train(lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden,
hidden_size, dropout_prob, epochs, batch_size, valid_batch_size,
n_workers, cuda, data_path, hdf_path, valid_data_path,
valid_hdf_path, checkpoint_path, softmax, n_classes, pretrained,
max_gnorm, lr_factor):
cp_name = get_cp_name(checkpoint_path)
if hdf_path != 'none':
transform_train = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(30),
transforms.RandomPerspective(p=0.2),
transforms.ColorJitter(brightness=2),
transforms.RandomGrayscale(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
trainset = Loader(hdf_path, transform_train)
else:
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(30),
transforms.RandomPerspective(p=0.2),
transforms.ColorJitter(brightness=2),
transforms.RandomGrayscale(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
trainset = datasets.ImageFolder(data_path, transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
worker_init_fn=set_np_randomseed,
pin_memory=True)
if valid_hdf_path != 'none':
transform_test = transforms.Compose([
transforms.ToPILImage(),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
validset = Loader(args.valid_hdf_path, transform_test)
else:
transform_test = transforms.Compose([
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
validset = datasets.ImageFolder(args.valid_data_path,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=True,
num_workers=n_workers,
pin_memory=True)
if model == 'vgg':
model_ = vgg.VGG('VGG19',
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
elif model == 'resnet':
model_ = resnet.ResNet50(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
elif model == 'densenet':
model_ = densenet.DenseNet121(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
if pretrained:
print('\nLoading pretrained encoder from torchvision\n')
if model == 'vgg':
model_pretrained = torchvision.models.vgg19(pretrained=True)
elif model == 'resnet':
model_pretrained = torchvision.models.resnet50(pretrained=True)
elif model == 'densenet':
model_pretrained = torchvision.models.densenet121(pretrained=True)
print(
model_.load_state_dict(model_pretrained.state_dict(),
strict=False))
print('\n')
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
torch.backends.cudnn.benchmark = True
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
max_gnorm=max_gnorm,
patience=int(patience),
lr_factor=lr_factor,
label_smoothing=smoothing,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
print(' ')
print('Hyperparameters:')
print('Selected model: {}'.format(model))
print('Embedding size: {}'.format(emb_size))
print('Hidden layer size: {}'.format(hidden_size))
print('Number of hidden layers: {}'.format(n_hidden))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Label smoothing: {}'.format(smoothing))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print('Pretrained: {}'.format(pretrained))
print(' ')
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print(' ')
return cost[0]
except:
print("Error:", sys.exc_info())
pass
print('Returning dummy cost due to failures while training.')
return 0.99
if args.n_classes > 2:
trainer = TrainLoop_mcc(model,
optimizer,
train_loader,
valid_loader,
patience=args.patience,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda)
else:
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
patience=args.patience,
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
logger=writer)
print('Cuda Mode: {}'.format(args.cuda))
print('Device: {}'.format(device))
print('Selected model: {}'.format(args.model))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('Momentum: {}'.format(args.momentum))
print('l2: {}'.format(args.l2))
trainer.train(n_epochs=args.epochs, save_every=args.save_every)
momentum=args.momentum,
weight_decay=args.l2,
nesterov=True),
lr=args.lr,
warmup_steps=args.warmup)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=args.max_gnorm,
margin=args.margin,
lambda_=args.lamb,
label_smoothing=args.smoothing,
warmup_its=args.warmup,
verbose=args.verbose,
device=device,
save_cp=(not args.no_cp),
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
swap=args.swap,
softmax=args.softmax,
pretrain=args.pretrain,
mining=args.mine_triplets,
cuda=args.cuda,
logger=writer)
if args.verbose > 0:
print(' ')
print('Cuda Mode: {}'.format(args.cuda))
print('Device: {}'.format(device))
print('Pretrain Mode: {}'.format(args.pretrain))
test_source_dataset = Loader_unif_sampling(hdf_path1=test_source_1, hdf_path2=test_source_2, hdf_path3=test_source_3, transform=img_transform_test)
test_source_loader = torch.utils.data.DataLoader(dataset=test_source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
target_dataset = Loader_validation(hdf_path=target_path, transform=img_transform_test)
target_loader = torch.utils.data.DataLoader(dataset=target_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
model = models.AlexNet(num_classes = 7, baseline = True)
state_dict = torch.load("../alexnet_caffe.pth.tar")
del state_dict["classifier.fc8.weight"]
del state_dict["classifier.fc8.bias"]
not_loaded = model.load_state_dict(state_dict, strict = False)
optimizer = optim.SGD(list(model.features.parameters())+list(model.classifier.parameters()), lr=args.lr, momentum=args.momentum, nesterov=True)
if args.cuda:
model = model.cuda()
torch.backends.cudnn.benchmark=True
trainer = TrainLoop(model, optimizer, source_loader, test_source_loader, target_loader, args.patience, args.l2, args.penalty_weight, args.penalty_anneal_epochs, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda)
err = trainer.train(n_epochs=args.epochs, save_every=args.save_every)
acc_runs.append(1-err)
print(acc_runs)
df = pandas.DataFrame(data={'Acc-{}'.format(args.target): acc_runs, 'Seed': seeds[:args.n_runs]})
df.to_csv('./irm_accuracy_runs_'+args.target+'.csv', sep=',', index = False)
optimizer = TransformerOptimizer(optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.l2,
nesterov=True),
lr=args.lr,
warmup_steps=args.warmup)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=args.max_gnorm,
label_smoothing=args.smoothing,
verbose=-1,
device=device,
cp_name=args.cp_name,
save_cp=True,
checkpoint_path=args.checkpoint_path,
pretrain=False,
cuda=args.cuda,
logger=writer)
print(' ')
print('CP name: {}'.format(args.cp_name))
print('Cuda Mode: {}'.format(args.cuda))
print('Device: {}'.format(device))
print('Selected model: {}'.format(args.model))
print('Number of discriminators: {}'.format(args.ndiscriminators))
print('Random projection size: {}'.format(args.rproj_size))
print('Softmax Mode is: {}'.format(args.softmax))
writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_acc': 0.0})
else:
writer = None
args_dict = None
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2),
weight_decay=args.l2)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=args.max_gnorm,
label_smoothing=args.smoothing,
verbose=args.verbose,
save_cp=(not args.no_cp),
checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
logger=writer)
if args.verbose > 0:
if args_dict is None:
args_dict = parse_args_for_log(args)
print('\n')
for key in args_dict:
print('{}: {}'.format(key, args_dict[key]))
print('\n')
trainer.train(n_epochs=args.epochs,
def train(lr, l2, momentum, slack, patience, model, epochs, batch_size,
valid_batch_size, train_mode, n_workers, cuda, data_path,
valid_data_path, checkpoint_path):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model_ = vgg.VGG('VGG16')
elif model == 'resnet':
model_ = resnet.ResNet50()
elif model == 'densenet':
model_ = densenet.densenet_cifar()
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
slack=slack,
train_mode=train_mode,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Train mode: {}'.format(train_mode))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Slack: {}'.format(slack))
print('Patience: {}'.format(patience))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Train mode: {}'.format(train_mode))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Slack: {}'.format(slack))
print('Patience: {}'.format(patience))
print(' ')
return 0.99
if key != 'domain_discriminator_list':
models_dict[key] = models_dict[key].cuda()
else:
for k, disc in enumerate(models_dict[key]):
models_dict[key][k] = disc.cuda()
torch.backends.cudnn.benchmark = True
trainer = TrainLoop(models_dict,
optimizer_task,
source_loader,
test_source_loader,
target_loader,
args.nadir_slack,
args.alpha,
args.patience,
args.factor,
args.smoothing,
args.warmup_its,
args.lr_threshold,
checkpoint_path=checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
ablation=args.ablation,
logging=args.logging,
train_mode=args.train_mode)
err, err_blind = trainer.train(n_epochs=args.epochs,
save_every=args.save_every)
acc_runs.append(1 - err)
acc_blind.append(err_blind)
df = pandas.DataFrame(data={
def train(lr, l2, max_gnorm, momentum, margin, lambda_, swap, latent_size, n_frames, model, ncoef, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, cp_path, softmax, delta, logdir):
if cuda:
device=get_freer_gpu()
if args.model == 'resnet_qrnn':
import cupy
cupy.cuda.Device(int(str(device).split(':')[-1])).use()
cp_name = get_file_name(cp_path)
if args.logdir:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir=logdir+cp_name, comment=args.model, purge_step=True)
else:
writer = None
train_dataset = Loader(hdf5_name = train_hdf_file, max_nb_frames = int(n_frames), delta = delta)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=n.workers, worker_init_fn=set_np_randomseed)
valid_dataset = Loader_valid(hdf5_name = valid_hdf_file, max_nb_frames = int(n_frames), delta = delta)
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed)
if model == 'resnet_mfcc':
model=model_.ResNet_mfcc(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_34':
model=model_.ResNet_34(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_lstm':
model=model_.ResNet_lstm(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_qrnn':
model=model_.ResNet_qrnn(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_stats':
model=model_.ResNet_stats(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif args.model == 'resnet_large':
model = model_.ResNet_large(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'resnet_small':
model = model_.ResNet_small(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'resnet_2d':
model = model_.ResNet_2d(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN':
model = model_.TDNN(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_att':
model = model_.TDNN_att(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_multihead':
model = model_.TDNN_multihead(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_lstm':
model = model_.TDNN_lstm(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_aspp':
model = model_.TDNN_aspp(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_mod':
model = model_.TDNN_mod(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_multipool':
model = model_.TDNN_multipool(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'transformer':
model = model_.transformer_enc(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
if cuda:
model=model.to(device)
else:
device=None
optimizer=optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2)
trainer=TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, margin=margin, lambda_=lambda_, verbose=-1, device=device, cp_name=cp_name, save_cp=True, checkpoint_path=cp_path, swap=swap, softmax=True, pretrain=False, mining=True, cuda=cuda, logger=writer)
return trainer.train(n_epochs=epochs)
