checkpoint_path=args.checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
logger=writer)
if args.verbose > 0:
print(' ')
print('Cuda Mode: {}'.format(args.cuda))
print('Device: {}'.format(device))
print('Softmax Mode: {}'.format(args.softmax))
print('Selected model: {}'.format(args.model))
print('Embeddings size: {}'.format(args.latent_size))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('momentum: {}'.format(args.momentum))
print('l2: {}'.format(args.l2))
print('Max. grad norm: {}'.format(args.max_gnorm))
print('Delta features: {}'.format(args.delta))
print('Max input length: {}'.format(args.n_frames))
print('Warmup iterations: {}'.format(args.warmup))
print('Label smoothing: {}'.format(args.smoothing))
print('Number of train speakers: {}'.format(train_dataset.n_speakers))
print('Number of train examples: {}'.format(len(train_dataset.utt_list)))
if args.valid_hdf_file is not None:
print('Number of valid speakers: {}'.format(valid_dataset.n_speakers))
print('Number of valid examples: {}'.format(len(
valid_dataset.utt_list)))
print(' ')
trainer.train(n_epochs=args.epochs, save_every=args.save_every)
def train(lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden,
hidden_size, dropout_prob, epochs, batch_size, valid_batch_size,
n_workers, cuda, data_path, hdf_path, valid_data_path,
valid_hdf_path, checkpoint_path, softmax, n_classes, pretrained,
max_gnorm, lr_factor):
cp_name = get_cp_name(checkpoint_path)
if hdf_path != 'none':
transform_train = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(30),
transforms.RandomPerspective(p=0.2),
transforms.ColorJitter(brightness=2),
transforms.RandomGrayscale(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
trainset = Loader(hdf_path, transform_train)
else:
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(30),
transforms.RandomPerspective(p=0.2),
transforms.ColorJitter(brightness=2),
transforms.RandomGrayscale(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
trainset = datasets.ImageFolder(data_path, transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
worker_init_fn=set_np_randomseed,
pin_memory=True)
if valid_hdf_path != 'none':
transform_test = transforms.Compose([
transforms.ToPILImage(),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
validset = Loader(args.valid_hdf_path, transform_test)
else:
transform_test = transforms.Compose([
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
validset = datasets.ImageFolder(args.valid_data_path,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=True,
num_workers=n_workers,
pin_memory=True)
if model == 'vgg':
model_ = vgg.VGG('VGG19',
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
elif model == 'resnet':
model_ = resnet.ResNet50(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
elif model == 'densenet':
model_ = densenet.DenseNet121(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
if pretrained:
print('\nLoading pretrained encoder from torchvision\n')
if model == 'vgg':
model_pretrained = torchvision.models.vgg19(pretrained=True)
elif model == 'resnet':
model_pretrained = torchvision.models.resnet50(pretrained=True)
elif model == 'densenet':
model_pretrained = torchvision.models.densenet121(pretrained=True)
print(
model_.load_state_dict(model_pretrained.state_dict(),
strict=False))
print('\n')
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
torch.backends.cudnn.benchmark = True
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
max_gnorm=max_gnorm,
patience=int(patience),
lr_factor=lr_factor,
label_smoothing=smoothing,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
print(' ')
print('Hyperparameters:')
print('Selected model: {}'.format(model))
print('Embedding size: {}'.format(emb_size))
print('Hidden layer size: {}'.format(hidden_size))
print('Number of hidden layers: {}'.format(n_hidden))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Label smoothing: {}'.format(smoothing))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print('Pretrained: {}'.format(pretrained))
print(' ')
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print(' ')
return cost[0]
except:
print("Error:", sys.exc_info())
pass
print('Returning dummy cost due to failures while training.')
return 0.99
test_source_loader,
target_loader,
args.nadir_slack,
args.alpha,
args.patience,
args.factor,
args.smoothing,
args.warmup_its,
args.lr_threshold,
checkpoint_path=checkpoint_path,
checkpoint_epoch=args.checkpoint_epoch,
cuda=args.cuda,
ablation=args.ablation,
logging=args.logging,
train_mode=args.train_mode)
_, results_acc, results_epochs = trainer.train(n_epochs=args.epochs,
save_every=args.save_every)
acc_runs.append(results_acc[-1])
acc_blind_runs.append(results_acc[-3])
df = pandas.DataFrame(data={
'Acc-{}'.format(args.target): acc_runs,
'Seed': seeds[:args.n_runs]
})
df.to_csv('./accuracy_runs_' + args.target + '.csv', sep=',', index=False)
df = pandas.DataFrame(data={
'Acc-{}'.format(args.target): acc_blind_runs,
'Seed': seeds[:args.n_runs]
})
df.to_csv('./accuracy_runs_' + args.target + '_blind.csv',
def train(lr, l2, momentum, margin, lambda_, patience, swap, latent_size,
n_frames, model, ncoef, epochs, batch_size, n_workers, cuda,
train_hdf_file, data_info_path, valid_hdf_file, n_cycles,
valid_n_cycles, cp_path, softmax, delta):
if cuda:
device = get_freer_gpu()
train_dataset = Loader_test(hdf5_name=train_hdf_file,
max_nb_frames=int(n_frames),
n_cycles=n_cycles,
delta=delta)
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
valid_dataset = Loader(hdf5_name=valid_hdf_file,
max_nb_frames=int(n_frames),
n_cycles=valid_n_cycles,
delta=delta)
valid_loader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
if model == 'resnet_mfcc':
model = model_.ResNet_mfcc(n_z=int(latent_size),
proj_size=len(train_dataset.speakers_list),
ncoef=ncoef,
sm_type=softmax,
delta=delta)
elif model == 'resnet_lstm':
model = model_.ResNet_lstm(n_z=int(latent_size),
proj_size=len(train_dataset.speakers_list),
ncoef=ncoef,
sm_type=softmax,
delta=delta)
elif model == 'resnet_stats':
model = model_.ResNet_stats(n_z=int(latent_size),
proj_size=len(train_dataset.speakers_list),
ncoef=ncoef,
sm_type=softmax,
delta=delta)
elif args.model == 'resnet_large':
model = model_.ResNet_large(n_z=int(latent_size),
proj_size=len(train_dataset.speakers_list),
ncoef=args.ncoef,
sm_type=softmax,
delta=delta)
elif args.model == 'resnet_small':
model = model_.ResNet_small(n_z=int(latent_size),
proj_size=len(train_dataset.speakers_list),
ncoef=args.ncoef,
sm_type=softmax,
delta=delta)
if cuda:
model = model.cuda(device)
else:
device = None
optimizer = optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
weight_decay=l2)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
device=device,
cp_name=get_file_name(cp_path),
save_cp=True,
checkpoint_path=cp_path,
swap=swap,
softmax=True,
pretrain=False,
mining=True,
cuda=cuda)
return trainer.train(n_epochs=epochs)
device = get_freer_gpu()
else:
device = None
if args.cuda:
model = model.cuda(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.l2)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
patience=args.patience,
verbose=-1,
device=device,
cp_name=args.cp_name,
save_cp=True,
checkpoint_path=args.checkpoint_path,
pretrain=False,
cuda=args.cuda)
best_eer = trainer.train(n_epochs=args.epochs, save_every=args.epochs + 10)
out_file = open(args.out_file, 'wb')
pickle.dump(best_eer[0], out_file)
out_file.close()
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs,
batch_size, valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.ToTensor()
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model = vgg.VGG('VGG16')
elif model == 'resnet':
model = resnet.ResNet18()
elif model == 'densenet':
model = densenet.densenet_cifar()
if cuda:
model = model.cuda()
optimizer = optim.SGD(model.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
swap=swap,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs,
batch_size, valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path, softmax):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]],
[x / 255 for x in [63.0, 62.1, 66.7]])
])
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model_ = vgg.VGG('VGG16', sm_type=softmax)
elif model == 'resnet':
model_ = resnet.ResNet18(sm_type=softmax)
elif model == 'densenet':
model_ = densenet.densenet_cifar(sm_type=softmax)
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
margin=margin,
lambda_=lambda_,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
swap=swap,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best cost in file ' + cp_name + 'was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('lambda: {}'.format(lambda_))
print('Margin: {}'.format(margin))
print('Swap: {}'.format(swap))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print(' ')
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
def train(lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden,
hidden_size, dropout_prob, epochs, batch_size, valid_batch_size,
n_workers, cuda, data_path, hdf_path, valid_data_path,
valid_hdf_path, checkpoint_path, softmax, n_classes, pretrained,
pretrained_path, max_gnorm, lr_factor, log_dir):
args_dict = locals()
cp_name = get_cp_name(checkpoint_path)
if log_dir != 'none':
writer = SummaryWriter(log_dir=log_dir, comment=model, purge_step=True)
writer.add_hparams(hparam_dict=args_dict,
metric_dict={'best_eer': 0.0})
else:
writer = None
if pretrained_path != 'none':
print('\nLoading pretrained model from: {}\n'.format(
args.pretrained_path))
ckpt = torch.load(pretrained_path,
map_location=lambda storage, loc: storage)
dropout_prob, n_hidden, hidden_size, emb_size = ckpt[
'dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt[
'emb_size']
print('\nUsing pretrained config for discriminator. Ignoring args.')
if hdf_path != 'none':
transform_train = transforms.Compose([
transforms.ToPILImage(),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(30),
transforms.RandomPerspective(p=0.2),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
trainset = Loader(hdf_path, transform_train)
else:
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(30),
transforms.RandomPerspective(p=0.2),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
trainset = datasets.ImageFolder(data_path, transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
worker_init_fn=set_np_randomseed,
pin_memory=True)
if valid_hdf_path != 'none':
transform_test = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
validset = Loader(args.valid_hdf_path, transform_test)
else:
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
validset = datasets.ImageFolder(args.valid_data_path,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=True,
num_workers=n_workers,
pin_memory=True)
if model == 'vgg':
model_ = vgg.VGG('VGG19',
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
elif model == 'resnet':
model_ = resnet.ResNet50(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
elif model == 'densenet':
model_ = densenet.DenseNet121(nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob,
sm_type=softmax,
n_classes=n_classes,
emb_size=emb_size)
if pretrained_path != 'none':
if ckpt['sm_type'] == 'am_softmax':
del (ckpt['model_state']['out_proj.w'])
elif ckpt['sm_type'] == 'softmax':
del (ckpt['model_state']['out_proj.w.weight'])
del (ckpt['model_state']['out_proj.w.bias'])
print(model_.load_state_dict(ckpt['model_state'], strict=False))
print('\n')
if pretrained:
print('\nLoading pretrained encoder from torchvision\n')
if model == 'vgg':
model_pretrained = torchvision.models.vgg19(pretrained=True)
elif model == 'resnet':
model_pretrained = torchvision.models.resnet50(pretrained=True)
elif model == 'densenet':
model_pretrained = torchvision.models.densenet121(pretrained=True)
print(
model_.load_state_dict(model_pretrained.state_dict(),
strict=False))
print('\n')
if cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
torch.backends.cudnn.benchmark = True
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
max_gnorm=max_gnorm,
patience=int(patience),
lr_factor=lr_factor,
label_smoothing=smoothing,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda,
logger=writer)
for i in range(5):
print(' ')
print('Hyperparameters:')
print('Selected model: {}'.format(model))
print('Embedding size: {}'.format(emb_size))
print('Hidden layer size: {}'.format(hidden_size))
print('Number of hidden layers: {}'.format(n_hidden))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Label smoothing: {}'.format(smoothing))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print('Pretrained: {}'.format(pretrained))
print(' ')
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print(' ')
return cost[0]
except:
print("Error:", sys.exc_info())
pass
print('Returning dummy cost due to failures while training.')
return 0.99
def train(lr, l2, momentum, max_gnorm, warmup, input_size, n_hidden,
hidden_size, dropout_prob, smoothing, n_cycles, epochs, batch_size,
valid_batch_size, n_workers, cuda, train_hdf_path, valid_hdf_path,
cp_path, logdir):
hp_dict = {
'lr': lr,
'l2': l2,
'momentum': momentum,
'max_gnorm': max_gnorm,
'warmup': warmup,
'input_size': input_size,
'n_hidden': n_hidden,
'hidden_size': hidden_size,
'dropout_prob': dropout_prob,
'smoothing': smoothing,
'n_cycles': n_cycles,
'epochs': epochs,
'batch_size': batch_size,
'valid_batch_size': valid_batch_size,
'n_workers': n_workers,
'cuda': cuda,
'train_hdf_path': train_hdf_path,
'valid_hdf_path': valid_hdf_path,
'cp_path': cp_path
}
cp_name = get_file_name(cp_path)
if args.logdir:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir=logdir + cp_name, purge_step=True)
writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': 0.5})
else:
writer = None
train_dataset = Loader(hdf5_clean=train_hdf_path + 'train_clean.hdf',
hdf5_attack=train_hdf_path + 'train_attack.hdf',
label_smoothing=smoothing,
n_cycles=n_cycles)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers)
valid_dataset = Loader(hdf5_clean=valid_hdf_path + 'valid_clean.hdf',
hdf5_attack=valid_hdf_path + 'valid_attack.hdf',
n_cycles=1)
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
model = model_.MLP(n_in=input_size,
nh=n_hidden,
n_h=hidden_size,
dropout_prob=dropout_prob)
if cuda:
device = get_freer_gpu()
model = model.cuda(device)
optimizer = TransformerOptimizer(optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
weight_decay=l2,
nesterov=True),
lr=lr,
warmup_steps=warmup)
trainer = TrainLoop(model,
optimizer,
train_loader,
valid_loader,
max_gnorm=max_gnorm,
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=cp_path,
cuda=cuda,
logger=writer)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best EER in file ' + cp_name + ' was: {}'.format(cost))
print(' ')
print('With hyperparameters:')
print('Hidden layer size size: {}'.format(int(hidden_size)))
print('Number of hidden layers: {}'.format(int(n_hidden)))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Warmup iterations: {}'.format(warmup))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Max. Grad. norm: {}'.format(max_gnorm))
print('Label smoothing: {}'.format(smoothing))
print(' ')
if args.logdir:
writer.add_hparams(hparam_dict=hp_dict,
metric_dict={'best_eer': cost})
return cost
except:
pass
print('Returning dummy cost due to failures while training.')
cost = 0.99
if args.logdir:
writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': cost})
return cost
parser.add_argument('--patience', type = int, default = 10, metavar = 'N', help = 'number of epochs without improvement to wait before stopping training (default: 30)')
parser.add_argument('--lr', type = float, default = 0.001, metavar = 'LR', help = 'learning rate (default: 0.001)')
parser.add_argument('--momentum', type = float, default = 0.9, metavar = 'mu', help = 'Momentum (default: 0.9)')
parser.add_argument('--l2', type = float, default = 0.001, metavar = 'lambda', help = 'L2 weight decay coefficient (default: 0.0001)')
parser.add_argument('--no-cuda', action = 'store_true', default = False, help = 'disables CUDA training')
parser.add_argument('--checkpoint-epoch', type = int, default = None, metavar = 'N', help = 'epoch to load for checkpointing. If None, training starts from scratch')
parser.add_argument('--checkpoint-path', type = str, default = None, metavar = 'Path', help = 'Path for checkpointing')
parser.add_argument('--seed', type = int, default = 12345, metavar = 'S', help = 'random seed (default: 12345)')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
model = models.model_eeg_rnn_shorterconvs()
if args.cuda:
model = model.cuda()
optimizer = torch.optim.SGD(model.parameters(), lr = args.lr, momentum = args.momentum, weight_decay = args.l2)
trainer = TrainLoop(model, optimizer, args.minibatch_size, checkpoint_path = args.checkpoint_path, checkpoint_epoch = args.checkpoint_epoch, cuda = args.cuda)
print('Cuda Mode is: {}'.format(args.cuda))
trainer.train(n_epochs = args.epochs, patience = args.patience)
def train(lr, l2, momentum, patience, swap, model, n_hidden, hidden_size,
dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda,
data_path, valid_data_path, checkpoint_path, softmax):
cp_name = get_cp_name(checkpoint_path)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
])
transform_test = transforms.ToTensor()
#trainset = Loader(data_path)
trainset = datasets.CIFAR10(root='./data',
train=True,
download=True,
transform=transform_train)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
shuffle=False,
num_workers=n_workers,
worker_init_fn=set_np_randomseed)
#validset = Loader(valid_data_path)
validset = datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
valid_loader = torch.utils.data.DataLoader(validset,
batch_size=valid_batch_size,
shuffle=False,
num_workers=n_workers)
if model == 'vgg':
model_ = vgg.VGG('VGG16',
nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
elif model == 'resnet':
model_ = resnet.ResNet18(nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
elif model == 'densenet':
model_ = densenet.densenet_cifar(nh=int(n_hidden),
n_h=int(hidden_size),
dropout_prob=dropout_prob,
sm_type=softmax)
if args.cuda:
device = get_freer_gpu()
model_ = model_.cuda(device)
optimizer = optim.SGD(model_.parameters(),
lr=lr,
weight_decay=l2,
momentum=momentum)
trainer = TrainLoop(model_,
optimizer,
train_loader,
valid_loader,
patience=int(patience),
verbose=-1,
cp_name=cp_name,
save_cp=True,
checkpoint_path=checkpoint_path,
cuda=cuda)
for i in range(5):
if i > 0:
print(' ')
print('Trial {}'.format(i + 1))
print(' ')
try:
cost = trainer.train(n_epochs=epochs, save_every=epochs + 10)
print(' ')
print('Best e2e EER in file ' + cp_name +
' was: {}'.format(cost[0]))
print('Best cos EER in file ' + cp_name +
' was: {}'.format(cost[1]))
print('Best Error Rate in file ' + cp_name +
' was: {}'.format(cost[2]))
print(' ')
print('With hyperparameters:')
print('Selected model: {}'.format(model))
print('Hidden layer size size: {}'.format(int(hidden_size)))
print('Number of hidden layers: {}'.format(int(n_hidden)))
print('Dropout rate: {}'.format(dropout_prob))
print('Batch size: {}'.format(batch_size))
print('LR: {}'.format(lr))
print('Momentum: {}'.format(momentum))
print('l2: {}'.format(l2))
print('Patience: {}'.format(patience))
print('Softmax Mode is: {}'.format(softmax))
print(' ')
return cost[0]
except:
pass
print('Returning dummy cost due to failures while training.')
return 0.99
def train(lr, l2, max_gnorm, momentum, margin, lambda_, swap, latent_size, n_frames, model, ncoef, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, cp_path, softmax, delta, logdir):
if cuda:
device=get_freer_gpu()
if args.model == 'resnet_qrnn':
import cupy
cupy.cuda.Device(int(str(device).split(':')[-1])).use()
cp_name = get_file_name(cp_path)
if args.logdir:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir=logdir+cp_name, comment=args.model, purge_step=True)
else:
writer = None
train_dataset = Loader(hdf5_name = train_hdf_file, max_nb_frames = int(n_frames), delta = delta)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=n.workers, worker_init_fn=set_np_randomseed)
valid_dataset = Loader_valid(hdf5_name = valid_hdf_file, max_nb_frames = int(n_frames), delta = delta)
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed)
if model == 'resnet_mfcc':
model=model_.ResNet_mfcc(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_34':
model=model_.ResNet_34(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_lstm':
model=model_.ResNet_lstm(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_qrnn':
model=model_.ResNet_qrnn(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif model == 'resnet_stats':
model=model_.ResNet_stats(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta)
elif args.model == 'resnet_large':
model = model_.ResNet_large(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'resnet_small':
model = model_.ResNet_small(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'resnet_2d':
model = model_.ResNet_2d(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN':
model = model_.TDNN(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_att':
model = model_.TDNN_att(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_multihead':
model = model_.TDNN_multihead(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_lstm':
model = model_.TDNN_lstm(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_aspp':
model = model_.TDNN_aspp(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_mod':
model = model_.TDNN_mod(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'TDNN_multipool':
model = model_.TDNN_multipool(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
elif args.model == 'transformer':
model = model_.transformer_enc(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta)
if cuda:
model=model.to(device)
else:
device=None
optimizer=optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2)
trainer=TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, margin=margin, lambda_=lambda_, verbose=-1, device=device, cp_name=cp_name, save_cp=True, checkpoint_path=cp_path, swap=swap, softmax=True, pretrain=False, mining=True, cuda=cuda, logger=writer)
return trainer.train(n_epochs=epochs)
args = parser.parse_args()
args.cuda = True if not args.no_cuda and torch.cuda.is_available() else False
print('Cuda Mode is: {}'.format(args.cuda))
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
dataset = IHDP(replications=args.reps)
in_size = len(dataset.binfeats) + len(dataset.contfeats)
optimim_params = {'lr':args.lr, 'weight_decay':args.l2}
trainer = TrainLoop(in_size, args.d, args.nh, args.h, args.n_pseudo_inputs, args.prior, torch.nn.functional.elu, args.opt, optimim_params, dataset, args.d, checkpoint_path=args.checkpoint_path, cuda=args.cuda)
trainer.train(n_epochs=args.epochs, n_reps=args.reps, batch_size=args.batch_size, save_every = args.save_every)
trainer.test()
if not args.no_plot:
log = trainer.history
for i, key in enumerate( log.keys() ):
plt.figure(i+1)
data = log[key]
plt.errorbar(np.arange(data.shape[1])*args.save_every, data.mean(0), yerr=sem(data, axis=0))
plt.title(key)
if args.save_plots:
plt.savefig(key+'.png')
#plt.show()