def train(lr, l2, momentum, patience, latent_size, n_hidden, hidden_size, n_frames, model, ncoef, dropout_prob, epochs, batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, valid_n_cycles, cp_path, softmax): if cuda: device=get_freer_gpu() train_dataset=Loader_test(hdf5_name=train_hdf_file, max_nb_frames=int(n_frames)) train_loader=torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) valid_dataset = Loader(hdf5_name = valid_hdf_file, max_nb_frames = int(n_frames), n_cycles=valid_n_cycles) valid_loader=torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) if args.model == 'resnet_stats': model = model_.ResNet_stats(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet_mfcc': model = model_.ResNet_mfcc(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) if args.model == 'resnet_lstm': model = model_.ResNet_lstm(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet_small': model = model_.ResNet_small(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) elif args.model == 'resnet_large': model = model_.ResNet_large(n_z=int(latent_size), nh=int(n_hidden), n_h=int(hidden_size), proj_size=len(train_dataset.speakers_list), ncoef=ncoef, dropout_prob=dropout_prob, sm_type=softmax) if cuda: model=model.cuda(device) else: device=None optimizer=optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2) trainer=TrainLoop(model, optimizer, train_loader, valid_loader, patience=int(patience), verbose=-1, device=device, cp_name=get_file_name(cp_path), save_cp=False, checkpoint_path=cp_path, pretrain=False, cuda=cuda) return trainer.train(n_epochs=epochs)
image_size = 28 img_transform = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) img_transform_mnist = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]) source_dataset = datasets.MNIST(root=source_path, download=True, train=True, transform=img_transform_mnist) source_loader = torch.utils.data.DataLoader(dataset=source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) train_list = os.path.join(target_path, 'mnist_m_train_labels.txt') target_dataset = Loader(data_root=os.path.join(target_path, 'mnist_m_train'), data_list=train_list, transform=img_transform) target_loader = torch.utils.data.DataLoader(dataset=target_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) model = models_.CNNModel() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9) if args.cuda: model = model.cuda() torch.backends.cudnn.benchmark=True trainer = TrainLoop(model, optimizer, source_loader, target_loader, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, target_name = args.target) print('Cuda Mode: {}'.format(args.cuda)) print('Batch size: {}'.format(args.batch_size)) print('LR: {}'.format(args.lr)) print('Source: {}'.format(args.source)) print('Target: {}'.format(args.target)) trainer.train(n_epochs=args.epochs, save_every=args.save_every)
weight_decay=args.l2, nesterov=True), lr=args.lr, warmup_steps=args.warmup) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, margin=args.margin, lambda_=args.lamb, label_smoothing=args.smoothing, warmup_its=args.warmup, verbose=-1, device=device, cp_name=args.cp_name, save_cp=True, checkpoint_path=args.checkpoint_path, swap=args.swap, lr_red_epoch=args.lr_reduction_epoch, lr_factor=args.lr_factor, softmax=True, pretrain=False, mining=True, cuda=args.cuda, logger=writer) print('\n') print(model) print('\n') print('Device: {}'.format(device))
disc_list.append(disc) if args.cuda: generator = generator.cuda() for disc in disc_list: disc = disc.cuda() torch.backends.cudnn.benchmark = True optimizer = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2)) trainer = TrainLoop(generator, fid_model, disc_list, optimizer, train_loader, nadir_slack=args.nadir_slack, alpha=args.alpha, train_mode=args.train_mode, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, job_id=args.job_id) print('Cuda Mode is: {}'.format(args.cuda)) print('Train Mode is: {}'.format(args.train_mode)) print('Number of discriminators is: {}'.format(len(disc_list))) trainer.train(n_epochs=args.epochs, save_every=args.save_every)
def train(lr, l2, momentum, smoothing, patience, model, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path, softmax): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers) if model == 'vgg': model_ = vgg.VGG('VGG16', nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) elif model == 'resnet': model_ = resnet.ResNet18(nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) elif model == 'densenet': model_ = densenet.densenet_cifar(nh=int(n_hidden), n_h=int(hidden_size), dropout_prob=dropout_prob, sm_type=softmax) if args.cuda: device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, patience=int(patience), label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Hidden layer size size: {}'.format(int(hidden_size))) print('Number of hidden layers: {}'.format(int(n_hidden))) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print('Best Error Rate in file ' + cp_name + ' was: {}'.format(cost[2])) print(' ') return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99
def train(lr, l2, momentum, smoothing, warmup, model, emb_size, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, hdf_path, valid_hdf_path, checkpoint_path, softmax, pretrained, pretrained_path, max_gnorm, stats, log_dir, eval_every, ablation): args_dict = locals() cp_name = get_cp_name(checkpoint_path) if pretrained_path != 'none': print('\nLoading pretrained model from: {}\n'.format( args.pretrained_path)) ckpt = torch.load(pretrained_path, map_location=lambda storage, loc: storage) dropout_prob, n_hidden, hidden_size, emb_size = ckpt[ 'dropout_prob'], ckpt['n_hidden'], ckpt['hidden_size'], ckpt[ 'emb_size'] if 'r_proj_size' in ckpt: rproj_size = ckpt['r_proj_size'] else: rproj_size = -1 print('\nUsing pretrained config for discriminator. Ignoring args.') args_dict['dropout_prob'], args_dict['n_hidden'], args_dict[ 'hidden_size'], args_dict[ 'emb_size'] = dropout_prob, n_hidden, hidden_size, emb_size if log_dir != 'none': writer = SummaryWriter(log_dir=os.path.join(log_dir, cp_name), comment=model, purge_step=0) writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_eer': 0.0}) else: writer = None if stats == 'cars': mean, std = [0.4461, 0.4329, 0.4345], [0.2888, 0.2873, 0.2946] elif stats == 'cub': mean, std = [0.4782, 0.4925, 0.4418], [0.2330, 0.2296, 0.2647] elif stats == 'sop': mean, std = [0.5603, 0.5155, 0.4796], [0.2939, 0.2991, 0.3085] elif stats == 'imagenet': mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] if hdf_path != 'none': transform_train = transforms.Compose([ transforms.ToPILImage(), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(10), transforms.RandomPerspective(p=0.1), transforms.RandomGrayscale(p=0.1), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) trainset = Loader(hdf_path, transform_train) else: transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(10), transforms.RandomPerspective(p=0.1), transforms.RandomGrayscale(p=0.1), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) trainset = datasets.ImageFolder(data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) if valid_hdf_path != 'none': transform_test = transforms.Compose([ transforms.ToPILImage(), transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) validset = Loader(args.valid_hdf_path, transform_test) else: transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=mean, std=std) ]) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, pin_memory=True) nclasses = trainset.n_classes if isinstance(trainset, Loader) else len( trainset.classes) if model == 'vgg': model_ = vgg.VGG('VGG19', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=nclasses, emb_size=emb_size, r_proj_size=rproj_size) elif model == 'resnet': model_ = resnet.ResNet50(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=nclasses, emb_size=emb_size, r_proj_size=rproj_size) elif model == 'densenet': model_ = densenet.DenseNet121(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=nclasses, emb_size=emb_size, r_proj_size=rproj_size) if pretrained_path != 'none': if ckpt['sm_type'] == 'am_softmax': del (ckpt['model_state']['out_proj.w']) elif ckpt['sm_type'] == 'softmax': del (ckpt['model_state']['out_proj.w.weight']) del (ckpt['model_state']['out_proj.w.bias']) print(model_.load_state_dict(ckpt['model_state'], strict=False)) print('\n') if pretrained: print('\nLoading pretrained encoder from torchvision\n') if model == 'vgg': model_pretrained = torchvision.models.vgg19(pretrained=True) elif model == 'resnet': model_pretrained = torchvision.models.resnet50(pretrained=True) elif model == 'densenet': model_pretrained = torchvision.models.densenet121(pretrained=True) print( model_.load_state_dict(model_pretrained.state_dict(), strict=False)) print('\n') if cuda: device = get_freer_gpu() model_ = model_.cuda(device) torch.backends.cudnn.benchmark = True optimizer = TransformerOptimizer(optim.SGD(model_.parameters(), lr=lr, momentum=momentum, weight_decay=l2, nesterov=True), lr=lr, warmup_steps=warmup) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, ablation=ablation, cuda=cuda, logger=writer) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Embedding size: {}'.format(emb_size)) print('Hidden layer size: {}'.format(hidden_size)) print('Number of hidden layers: {}'.format(n_hidden)) print('Random projection size: {}'.format(rproj_size)) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Warmup iterations: {}'.format(warmup)) print('Softmax Mode is: {}'.format(softmax)) print('Pretrained: {}'.format(pretrained)) print('Pretrained path: {}'.format(pretrained_path)) print('Evaluate every {} iterations.'.format(eval_every)) print('Ablation Mode: {}'.format(ablation)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10, eval_every=eval_every) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print(' ') if log_dir != 'none': writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_eer': cost[0]}) return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99
if args.cuda: device = get_freer_gpu() model = model.to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.l2, momentum=args.momentum) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, margin=args.margin, lambda_=args.lamb, verbose=args.verbose, save_cp=(not args.no_cp), checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, swap=args.swap, cuda=args.cuda) if args.verbose > 0: print('Cuda Mode is: {}'.format(args.cuda)) print('Selected model: {}'.format(args.model)) print('Batch size: {}'.format(args.batch_size)) print('LR: {}'.format(args.lr)) print('Momentum: {}'.format(args.momentum)) print('l2: {}'.format(args.l2)) print('lambda: {}'.format(args.lamb)) print('Margin: {}'.format(args.margin))
testset = datasets.CIFAR10(root=args.data_path, train=False, download=True, transform=transform) test_loader = torch.utils.data.DataLoader(testset, batch_size=1000, shuffle=False, num_workers=args.workers) save_testdata_statistics(fid_model, test_loader, downsample_=False, cuda_mode=args.cuda) optimizer = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2)) trainer = TrainLoop(generator, fid_model, disc, optimizer, train_loader, its_disc=args.its_disc, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda) print('Cuda Mode is: {}'.format(args.cuda)) trainer.train(n_epochs=args.epochs, save_every=args.save_every)
device = get_freer_gpu() else: device = None if args.cuda: model = model.cuda(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.l2) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, patience=args.patience, verbose=-1, device=device, cp_name=args.cp_name, save_cp=True, checkpoint_path=args.checkpoint_path, pretrain=False, cuda=args.cuda) best_eer = trainer.train(n_epochs=args.epochs, save_every=args.epochs + 10) out_file = open(args.out_file, 'wb') pickle.dump(best_eer[0], out_file) out_file.close()
def train(lr, l2, momentum, max_gnorm, warmup, input_size, n_hidden, hidden_size, dropout_prob, smoothing, n_cycles, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_path, valid_hdf_path, cp_path, logdir): hp_dict = { 'lr': lr, 'l2': l2, 'momentum': momentum, 'max_gnorm': max_gnorm, 'warmup': warmup, 'input_size': input_size, 'n_hidden': n_hidden, 'hidden_size': hidden_size, 'dropout_prob': dropout_prob, 'smoothing': smoothing, 'n_cycles': n_cycles, 'epochs': epochs, 'batch_size': batch_size, 'valid_batch_size': valid_batch_size, 'n_workers': n_workers, 'cuda': cuda, 'train_hdf_path': train_hdf_path, 'valid_hdf_path': valid_hdf_path, 'cp_path': cp_path } cp_name = get_file_name(cp_path) if args.logdir: from torch.utils.tensorboard import SummaryWriter writer = SummaryWriter(log_dir=logdir + cp_name, purge_step=True) writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': 0.5}) else: writer = None train_dataset = Loader(hdf5_clean=train_hdf_path + 'train_clean.hdf', hdf5_attack=train_hdf_path + 'train_attack.hdf', label_smoothing=smoothing, n_cycles=n_cycles) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=n_workers) valid_dataset = Loader(hdf5_clean=valid_hdf_path + 'valid_clean.hdf', hdf5_attack=valid_hdf_path + 'valid_attack.hdf', n_cycles=1) valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) model = model_.MLP(n_in=input_size, nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob) if cuda: device = get_freer_gpu() model = model.cuda(device) optimizer = TransformerOptimizer(optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2, nesterov=True), lr=lr, warmup_steps=warmup) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=cp_path, cuda=cuda, logger=writer) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best EER in file ' + cp_name + ' was: {}'.format(cost)) print(' ') print('With hyperparameters:') print('Hidden layer size size: {}'.format(int(hidden_size))) print('Number of hidden layers: {}'.format(int(n_hidden))) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Warmup iterations: {}'.format(warmup)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Max. Grad. norm: {}'.format(max_gnorm)) print('Label smoothing: {}'.format(smoothing)) print(' ') if args.logdir: writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': cost}) return cost except: pass print('Returning dummy cost due to failures while training.') cost = 0.99 if args.logdir: writer.add_hparams(hparam_dict=hp_dict, metric_dict={'best_eer': cost}) return cost
for i in range(args.ndiscriminators): disc = model.Discriminator_toy(512, optim.Adam, args.lr, (args.beta1, args.beta2)).train() disc_list.append(disc) optimizer = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2)) if args.hyper_mode: trainer = TrainLoop(generator, disc_list, optimizer, args.toy_dataset, centers, cov, train_loader=train_loader, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, nadir_slack=args.nadir_slack) else: trainer = TrainLoop(generator, disc_list, optimizer, args.toy_dataset, centers, cov, train_loader=train_loader, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch)
ckpt = torch.load(args.pretrained_path, map_location = lambda storage, loc: storage) model = model_.MLP(n_in=ckpt['input_size'], nh=ckpt['n_hidden'], n_h=ckpt['hidden_size'], dropout_prob=ckpt['dropout_prob']) try: model.load_state_dict(ckpt['model_state'], strict=True) except RuntimeError as err: print("Runtime Error: {0}".format(err)) model.load_state_dict(ckpt['model_state'], strict=False) except: print("Unexpected error:", sys.exc_info()[0]) raise model = model.to(device) optimizer = TransformerOptimizer(optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.l2, nesterov=True), lr=args.lr, warmup_steps=args.warmup) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, verbose=args.verbose, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, logger=writer) if args.verbose > 0: print('Cuda Mode: {}'.format(args.cuda)) print('Device: {}'.format(device)) print('Batch size: {}'.format(args.batch_size)) print('Validation batch size: {}'.format(args.valid_batch_size)) print('LR: {}'.format(args.lr)) print('Momentum: {}'.format(args.momentum)) print('l2: {}'.format(args.l2)) print('Max. grad norm: {}'.format(args.max_gnorm)) print('Warmup iterations: {}'.format(args.warmup)) print('Inputs dimensionality: {}'.format(args.input_size)) print('Number of hidden layers: {}'.format(args.n_hidden)) print('Size of hidden layers: {}'.format(args.hidden_size)) print('Label smoothing: {}'.format(args.smoothing))
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path, softmax): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_train = transforms.Compose([ transforms.RandomCrop(84, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) #trainset = Loader(args.data_path) trainset = datasets.ImageFolder(args.data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=args.batch_size, shuffle=True, num_workers=args.n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) #validset = Loader(args.valid_data_path) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader( validset, batch_size=args.valid_batch_size, shuffle=True, num_workers=args.n_workers, pin_memory=True) if model == 'vgg': model_ = vgg.VGG('VGG16', sm_type=softmax) elif model == 'resnet': model_ = resnet.ResNet18(sm_type=softmax) elif model == 'densenet': model_ = densenet.densenet_cifar(sm_type=softmax) if cuda: torch.backends.cudnn.benchmark = True device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, margin=margin, lambda_=lambda_, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, swap=swap, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('lambda: {}'.format(lambda_)) print('Margin: {}'.format(margin)) print('Swap: {}'.format(swap)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') return 0.99
parser.add_argument('--patience', type = int, default = 10, metavar = 'N', help = 'number of epochs without improvement to wait before stopping training (default: 30)') parser.add_argument('--lr', type = float, default = 0.001, metavar = 'LR', help = 'learning rate (default: 0.001)') parser.add_argument('--momentum', type = float, default = 0.9, metavar = 'mu', help = 'Momentum (default: 0.9)') parser.add_argument('--l2', type = float, default = 0.001, metavar = 'lambda', help = 'L2 weight decay coefficient (default: 0.0001)') parser.add_argument('--no-cuda', action = 'store_true', default = False, help = 'disables CUDA training') parser.add_argument('--checkpoint-epoch', type = int, default = None, metavar = 'N', help = 'epoch to load for checkpointing. If None, training starts from scratch') parser.add_argument('--checkpoint-path', type = str, default = None, metavar = 'Path', help = 'Path for checkpointing') parser.add_argument('--seed', type = int, default = 12345, metavar = 'S', help = 'random seed (default: 12345)') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) model = models.model_eeg_rnn_shorterconvs() if args.cuda: model = model.cuda() optimizer = torch.optim.SGD(model.parameters(), lr = args.lr, momentum = args.momentum, weight_decay = args.l2) trainer = TrainLoop(model, optimizer, args.minibatch_size, checkpoint_path = args.checkpoint_path, checkpoint_epoch = args.checkpoint_epoch, cuda = args.cuda) print('Cuda Mode is: {}'.format(args.cuda)) trainer.train(n_epochs = args.epochs, patience = args.patience)
if args.cuda: device = get_freer_gpu() model = model.cuda(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.l2, momentum=args.momentum) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, patience=args.patience, label_smoothing=args.smoothing, verbose=args.verbose, save_cp=(not args.no_cp), checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda) if args.verbose > 0: print('\nCuda Mode is: {}'.format(args.cuda)) print('Selected model: {}'.format(args.model)) print('Batch size: {}'.format(args.batch_size)) print('LR: {}'.format(args.lr)) print('Momentum: {}'.format(args.momentum)) print('l2: {}'.format(args.l2)) print('Label smoothing: {}'.format(args.smoothing)) print('Patience: {}'.format(args.patience))
def train(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test = transforms.ToTensor() #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) if model == 'vgg': model = vgg.VGG('VGG16') elif model == 'resnet': model = resnet.ResNet18() elif model == 'densenet': model = densenet.densenet_cifar() if cuda: model = model.cuda() optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, margin=margin, lambda_=lambda_, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, swap=swap, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') return 0.99
comment=args.model, purge_step=0 if args.checkpoint_epoch is None else int(args.checkpoint_epoch * len(train_loader))) args_dict = parse_args_for_log(args) writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_eer': 0.0}) else: writer = None trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, patience=args.patience, lr_factor=args.lr_factor, label_smoothing=args.smoothing, verbose=args.verbose, cp_name=args.cp_name, save_cp=(not args.no_cp), checkpoint_path=args.checkpoint_path, ablation=args.ablation, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, logger=writer) if args.verbose > 0: print('\nCuda Mode is: {}'.format(args.cuda)) print('Selected model: {}'.format(args.model)) print('Batch size: {}'.format(args.batch_size)) print('LR: {}'.format(args.lr)) print('Momentum: {}'.format(args.momentum)) print('l2: {}'.format(args.l2))
batch_size=args.batch_size, num_workers=args.workers) centers = toy_data.get_centers() cov = toy_data.get_cov() # hidden_size = 512 generator = model.Generator_toy(512).train() disc = model.Discriminator_toy(512, optim.Adam, args.lr, (args.beta1, args.beta2)).train() optimizer = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2)) trainer = TrainLoop(generator, disc, optimizer, args.toy_dataset, centers, cov, train_loader=train_loader, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda) print('Cuda Mode is: {}'.format(args.cuda)) trainer.train(n_epochs=args.epochs, save_every=args.save_every)
model = model.cuda(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.l2) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, margin=args.margin, lambda_=args.lamb, patience=args.patience, verbose=-1, device=device, cp_name=args.cp_name, save_cp=True, checkpoint_path=args.checkpoint_path, swap=args.swap, softmax=True, pretrain=False, mining=True, cuda=args.cuda) best_eer = trainer.train(n_epochs=args.epochs, save_every=args.epochs + 10) out_file = open(args.out_file, 'wb') pickle.dump(best_eer, out_file) out_file.close()
train_source_1 = args.data_path + 'train_' + args.source1 + '.hdf' test_source_1 = args.data_path + 'test_' + args.source1 + '.hdf' source_dataset = Loader_validation(hdf_path=train_source_1, transform=img_transform_train) source_loader = torch.utils.data.DataLoader(dataset=source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) test_source_dataset = Loader_validation(hdf_path=test_source_1, transform=img_transform_test) test_source_loader = torch.utils.data.DataLoader(dataset=test_source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) model = models.AlexNet(num_classes = 7, baseline = True) state_dict = torch.load(args.model_path+'alexnet_caffe.pth.tar') del state_dict["classifier.fc8.weight"] del state_dict["classifier.fc8.bias"] not_loaded = model.load_state_dict(state_dict, strict = False) optimizer = optim.SGD(list(model.classifier.parameters()), lr=args.lr, momentum=args.momentum, weight_decay=args.l2, nesterov=True) if args.cuda: model = model.cuda() torch.backends.cudnn.benchmark=True trainer = TrainLoop(model, optimizer, source_loader, test_source_loader, args.nadir_slack, args.patience, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda) err = trainer.train(n_epochs=args.epochs, save_every=args.save_every) acc_runs.append(1-err) print(acc_runs) df = pandas.DataFrame(data={'Acc-{}'.format(args.source1): acc_runs, 'Seed': seeds[:args.n_runs]}) df.to_csv('./baseline_accuracy_runs_'+args.source1+'.csv', sep=',', index = False)
if args.optimizer == 'adam': disc = models_zoo.Discriminator(optim.Adam, args.optimizer, args.lr, (args.beta1, args.beta2)).train() elif args.optimizer == 'amsgrad': disc = models_zoo.Discriminator(optim.Adam, args.optimizer, args.lr, (args.beta1, args.beta2), amsgrad = True).train() elif args.optimizer == 'rmsprop': disc = models_zoo.Discriminator(optim.RMSprop, args.optimizer, args.lr, (args.beta1, args.beta2)).train() disc_list.append(disc) if args.cuda: generator = generator.cuda() frames_generator = frames_generator.cuda() for disc in disc_list: disc = disc.cuda() torch.backends.cudnn.benchmark=True if args.optimizer == 'adam': optimizer_g = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2)) elif args.optimizer == 'amsgrad': optimizer_g = optim.Adam(generator.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), amsgrad = True) elif args.optimizer == 'rmsprop': optimizer_g = optim.RMSprop(generator.parameters(), lr=args.lr) trainer = TrainLoop(generator, frames_generator, disc_list, optimizer_g, train_loader, nadir_slack=args.nadir_slack, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, hyper=not args.average_mode, cuda=args.cuda) print('Cuda Mode is: {}'.format(args.cuda)) print('Selected model is: {}'.format(args.gen_arch)) trainer.train(n_epochs=args.epochs, save_every = args.save_every)
def train(lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, hdf_path, valid_data_path, valid_hdf_path, checkpoint_path, softmax, n_classes, pretrained, max_gnorm, lr_factor): cp_name = get_cp_name(checkpoint_path) if hdf_path != 'none': transform_train = transforms.Compose([ transforms.ToPILImage(), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(30), transforms.RandomPerspective(p=0.2), transforms.ColorJitter(brightness=2), transforms.RandomGrayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) trainset = Loader(hdf_path, transform_train) else: transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.RandomRotation(30), transforms.RandomPerspective(p=0.2), transforms.ColorJitter(brightness=2), transforms.RandomGrayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) trainset = datasets.ImageFolder(data_path, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed, pin_memory=True) if valid_hdf_path != 'none': transform_test = transforms.Compose([ transforms.ToPILImage(), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) validset = Loader(args.valid_hdf_path, transform_test) else: transform_test = transforms.Compose([ transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) validset = datasets.ImageFolder(args.valid_data_path, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, pin_memory=True) if model == 'vgg': model_ = vgg.VGG('VGG19', nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size) elif model == 'resnet': model_ = resnet.ResNet50(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size) elif model == 'densenet': model_ = densenet.DenseNet121(nh=n_hidden, n_h=hidden_size, dropout_prob=dropout_prob, sm_type=softmax, n_classes=n_classes, emb_size=emb_size) if pretrained: print('\nLoading pretrained encoder from torchvision\n') if model == 'vgg': model_pretrained = torchvision.models.vgg19(pretrained=True) elif model == 'resnet': model_pretrained = torchvision.models.resnet50(pretrained=True) elif model == 'densenet': model_pretrained = torchvision.models.densenet121(pretrained=True) print( model_.load_state_dict(model_pretrained.state_dict(), strict=False)) print('\n') if cuda: device = get_freer_gpu() model_ = model_.cuda(device) torch.backends.cudnn.benchmark = True optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, patience=int(patience), lr_factor=lr_factor, label_smoothing=smoothing, verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): print(' ') print('Hyperparameters:') print('Selected model: {}'.format(model)) print('Embedding size: {}'.format(emb_size)) print('Hidden layer size: {}'.format(hidden_size)) print('Number of hidden layers: {}'.format(n_hidden)) print('Dropout rate: {}'.format(dropout_prob)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Label smoothing: {}'.format(smoothing)) print('Patience: {}'.format(patience)) print('Softmax Mode is: {}'.format(softmax)) print('Pretrained: {}'.format(pretrained)) print(' ') if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best e2e EER in file ' + cp_name + ' was: {}'.format(cost[0])) print('Best cos EER in file ' + cp_name + ' was: {}'.format(cost[1])) print(' ') return cost[0] except: print("Error:", sys.exc_info()) pass print('Returning dummy cost due to failures while training.') return 0.99
if args.n_classes > 2: trainer = TrainLoop_mcc(model, optimizer, train_loader, valid_loader, patience=args.patience, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda) else: trainer = TrainLoop(model, optimizer, train_loader, valid_loader, patience=args.patience, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, logger=writer) print('Cuda Mode: {}'.format(args.cuda)) print('Device: {}'.format(device)) print('Selected model: {}'.format(args.model)) print('Batch size: {}'.format(args.batch_size)) print('LR: {}'.format(args.lr)) print('Momentum: {}'.format(args.momentum)) print('l2: {}'.format(args.l2)) trainer.train(n_epochs=args.epochs, save_every=args.save_every)
momentum=args.momentum, weight_decay=args.l2, nesterov=True), lr=args.lr, warmup_steps=args.warmup) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, margin=args.margin, lambda_=args.lamb, label_smoothing=args.smoothing, warmup_its=args.warmup, verbose=args.verbose, device=device, save_cp=(not args.no_cp), checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, swap=args.swap, softmax=args.softmax, pretrain=args.pretrain, mining=args.mine_triplets, cuda=args.cuda, logger=writer) if args.verbose > 0: print(' ') print('Cuda Mode: {}'.format(args.cuda)) print('Device: {}'.format(device)) print('Pretrain Mode: {}'.format(args.pretrain))
test_source_dataset = Loader_unif_sampling(hdf_path1=test_source_1, hdf_path2=test_source_2, hdf_path3=test_source_3, transform=img_transform_test) test_source_loader = torch.utils.data.DataLoader(dataset=test_source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) target_dataset = Loader_validation(hdf_path=target_path, transform=img_transform_test) target_loader = torch.utils.data.DataLoader(dataset=target_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) model = models.AlexNet(num_classes = 7, baseline = True) state_dict = torch.load("../alexnet_caffe.pth.tar") del state_dict["classifier.fc8.weight"] del state_dict["classifier.fc8.bias"] not_loaded = model.load_state_dict(state_dict, strict = False) optimizer = optim.SGD(list(model.features.parameters())+list(model.classifier.parameters()), lr=args.lr, momentum=args.momentum, nesterov=True) if args.cuda: model = model.cuda() torch.backends.cudnn.benchmark=True trainer = TrainLoop(model, optimizer, source_loader, test_source_loader, target_loader, args.patience, args.l2, args.penalty_weight, args.penalty_anneal_epochs, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda) err = trainer.train(n_epochs=args.epochs, save_every=args.save_every) acc_runs.append(1-err) print(acc_runs) df = pandas.DataFrame(data={'Acc-{}'.format(args.target): acc_runs, 'Seed': seeds[:args.n_runs]}) df.to_csv('./irm_accuracy_runs_'+args.target+'.csv', sep=',', index = False)
optimizer = TransformerOptimizer(optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.l2, nesterov=True), lr=args.lr, warmup_steps=args.warmup) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, label_smoothing=args.smoothing, verbose=-1, device=device, cp_name=args.cp_name, save_cp=True, checkpoint_path=args.checkpoint_path, pretrain=False, cuda=args.cuda, logger=writer) print(' ') print('CP name: {}'.format(args.cp_name)) print('Cuda Mode: {}'.format(args.cuda)) print('Device: {}'.format(device)) print('Selected model: {}'.format(args.model)) print('Number of discriminators: {}'.format(args.ndiscriminators)) print('Random projection size: {}'.format(args.rproj_size)) print('Softmax Mode is: {}'.format(args.softmax))
writer.add_hparams(hparam_dict=args_dict, metric_dict={'best_acc': 0.0}) else: writer = None args_dict = None optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.l2) trainer = TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=args.max_gnorm, label_smoothing=args.smoothing, verbose=args.verbose, save_cp=(not args.no_cp), checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, logger=writer) if args.verbose > 0: if args_dict is None: args_dict = parse_args_for_log(args) print('\n') for key in args_dict: print('{}: {}'.format(key, args_dict[key])) print('\n') trainer.train(n_epochs=args.epochs,
def train(lr, l2, momentum, slack, patience, model, epochs, batch_size, valid_batch_size, train_mode, n_workers, cuda, data_path, valid_data_path, checkpoint_path): cp_name = get_cp_name(checkpoint_path) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]) ]) #trainset = Loader(data_path) trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) train_loader = torch.utils.data.DataLoader( trainset, batch_size=batch_size, shuffle=False, num_workers=n_workers, worker_init_fn=set_np_randomseed) #validset = Loader(valid_data_path) validset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) valid_loader = torch.utils.data.DataLoader(validset, batch_size=valid_batch_size, shuffle=False, num_workers=n_workers) if model == 'vgg': model_ = vgg.VGG('VGG16') elif model == 'resnet': model_ = resnet.ResNet50() elif model == 'densenet': model_ = densenet.densenet_cifar() if cuda: device = get_freer_gpu() model_ = model_.cuda(device) optimizer = optim.SGD(model_.parameters(), lr=lr, weight_decay=l2, momentum=momentum) trainer = TrainLoop(model_, optimizer, train_loader, valid_loader, slack=slack, train_mode=train_mode, patience=int(patience), verbose=-1, cp_name=cp_name, save_cp=True, checkpoint_path=checkpoint_path, cuda=cuda) for i in range(5): if i > 0: print(' ') print('Trial {}'.format(i + 1)) print(' ') try: cost = trainer.train(n_epochs=epochs, save_every=epochs + 10) print(' ') print('Best cost in file ' + cp_name + 'was: {}'.format(cost)) print(' ') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Train mode: {}'.format(train_mode)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Slack: {}'.format(slack)) print('Patience: {}'.format(patience)) print(' ') return cost except: pass print('Returning dummy cost due to failures while training.') print('With hyperparameters:') print('Selected model: {}'.format(model)) print('Train mode: {}'.format(train_mode)) print('Batch size: {}'.format(batch_size)) print('LR: {}'.format(lr)) print('Momentum: {}'.format(momentum)) print('l2: {}'.format(l2)) print('Slack: {}'.format(slack)) print('Patience: {}'.format(patience)) print(' ') return 0.99
if key != 'domain_discriminator_list': models_dict[key] = models_dict[key].cuda() else: for k, disc in enumerate(models_dict[key]): models_dict[key][k] = disc.cuda() torch.backends.cudnn.benchmark = True trainer = TrainLoop(models_dict, optimizer_task, source_loader, test_source_loader, target_loader, args.nadir_slack, args.alpha, args.patience, args.factor, args.smoothing, args.warmup_its, args.lr_threshold, checkpoint_path=checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, ablation=args.ablation, logging=args.logging, train_mode=args.train_mode) err, err_blind = trainer.train(n_epochs=args.epochs, save_every=args.save_every) acc_runs.append(1 - err) acc_blind.append(err_blind) df = pandas.DataFrame(data={
def train(lr, l2, max_gnorm, momentum, margin, lambda_, swap, latent_size, n_frames, model, ncoef, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, cp_path, softmax, delta, logdir): if cuda: device=get_freer_gpu() if args.model == 'resnet_qrnn': import cupy cupy.cuda.Device(int(str(device).split(':')[-1])).use() cp_name = get_file_name(cp_path) if args.logdir: from torch.utils.tensorboard import SummaryWriter writer = SummaryWriter(log_dir=logdir+cp_name, comment=args.model, purge_step=True) else: writer = None train_dataset = Loader(hdf5_name = train_hdf_file, max_nb_frames = int(n_frames), delta = delta) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=n.workers, worker_init_fn=set_np_randomseed) valid_dataset = Loader_valid(hdf5_name = valid_hdf_file, max_nb_frames = int(n_frames), delta = delta) valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batch_size, shuffle=True, num_workers=n_workers, worker_init_fn=set_np_randomseed) if model == 'resnet_mfcc': model=model_.ResNet_mfcc(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta) elif model == 'resnet_34': model=model_.ResNet_34(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta) elif model == 'resnet_lstm': model=model_.ResNet_lstm(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta) elif model == 'resnet_qrnn': model=model_.ResNet_qrnn(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta) elif model == 'resnet_stats': model=model_.ResNet_stats(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=ncoef, sm_type=softmax, delta=delta) elif args.model == 'resnet_large': model = model_.ResNet_large(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'resnet_small': model = model_.ResNet_small(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'resnet_2d': model = model_.ResNet_2d(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN': model = model_.TDNN(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN_att': model = model_.TDNN_att(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN_multihead': model = model_.TDNN_multihead(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN_lstm': model = model_.TDNN_lstm(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN_aspp': model = model_.TDNN_aspp(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN_mod': model = model_.TDNN_mod(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'TDNN_multipool': model = model_.TDNN_multipool(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) elif args.model == 'transformer': model = model_.transformer_enc(n_z=int(latent_size), proj_size=train_dataset.n_speakers, ncoef=args.ncoef, sm_type=softmax, delta=delta) if cuda: model=model.to(device) else: device=None optimizer=optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2) trainer=TrainLoop(model, optimizer, train_loader, valid_loader, max_gnorm=max_gnorm, margin=margin, lambda_=lambda_, verbose=-1, device=device, cp_name=cp_name, save_cp=True, checkpoint_path=cp_path, swap=swap, softmax=True, pretrain=False, mining=True, cuda=cuda, logger=writer) return trainer.train(n_epochs=epochs)