def data(): labels = getLabels('train_answers.csv') [X_, y] = getDataset('train.csv', labels) X = X_[:, 4:] [Xtest_, ytest] = getDataset('test.csv', labels) Xtest = Xtest_[:, 4:] return [X, y, Xtest, ytest]
def run(dataset, net_type): # Hyper Parameter settings layer_type = cfg.layer_type activation_type = cfg.activation_type train_ens = cfg.train_ens valid_ens = cfg.valid_ens n_epochs = cfg.n_epochs lr_start = cfg.lr_start num_workers = cfg.num_workers valid_size = cfg.valid_size batch_size = cfg.batch_size beta_type = cfg.beta_type trainset, testset, inputs, outputs = data.getDataset(dataset) train_loader, valid_loader, test_loader = data.getDataloader( trainset, testset, valid_size, batch_size, num_workers) net = getModel(net_type, inputs, outputs, layer_type, activation_type).to(device) ckpt_dir = f'checkpoints/{dataset}/bayesian' ckpt_name = f'checkpoints/{dataset}/bayesian/model_{net_type}_{layer_type}.pt' if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir, exist_ok=True) criterion = metrics.ELBO(len(trainset)).to(device) optimizer = Adam(net.parameters(), lr=lr_start) lr_sched = lr_scheduler.ReduceLROnPlateau(optimizer, patience=6, verbose=True) valid_loss_max = np.Inf for epoch in range(n_epochs): # loop over the dataset multiple times cfg.curr_epoch_no = epoch train_loss, train_acc, train_kl = train_model(net, optimizer, criterion, train_loader, num_ens=train_ens, beta_type=beta_type) valid_loss, valid_acc = validate_model(net, criterion, valid_loader, num_ens=valid_ens) lr_sched.step(valid_loss) print( 'Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tValidation Loss: {:.4f} \tValidation Accuracy: {:.4f} \ttrain_kl_div: {:.4f}' .format(epoch, train_loss, train_acc, valid_loss, valid_acc, train_kl)) # save model if validation accuracy has increased if valid_loss <= valid_loss_max: print( 'Validation loss decreased ({:.6f} --> {:.6f}). Saving model ...' .format(valid_loss_max, valid_loss)) torch.save(net.state_dict(), ckpt_name) valid_loss_max = valid_loss
def run(dataset, net_type): # Hyper Parameter settings n_epochs = cfg.n_epochs lr = cfg.lr num_workers = cfg.num_workers valid_size = cfg.valid_size batch_size = cfg.batch_size trainset, testset, inputs, outputs = data.getDataset(dataset) train_loader, valid_loader, test_loader = data.getDataloader( trainset, testset, valid_size, batch_size, num_workers) net = getModel(net_type, inputs, outputs).to(device) ckpt_dir = f'checkpoints/{dataset}/frequentist' ckpt_name = f'checkpoints/{dataset}/frequentist/model_{net_type}.pt' if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir, exist_ok=True) criterion = nn.CrossEntropyLoss() optimizer = Adam(net.parameters(), lr=lr) lr_sched = lr_scheduler.ReduceLROnPlateau(optimizer, patience=6, verbose=True) valid_loss_min = np.Inf for epoch in range(1, n_epochs + 1): train_loss, train_acc = train_model(net, optimizer, criterion, train_loader) valid_loss, valid_acc = validate_model(net, criterion, valid_loader) lr_sched.step(valid_loss) train_loss = train_loss / len(train_loader.dataset) valid_loss = valid_loss / len(valid_loader.dataset) print( 'Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tValidation Loss: {:.4f} \tValidation Accuracy: {:.4f}' .format(epoch, train_loss, train_acc, valid_loss, valid_acc)) # save model if validation loss has decreased if valid_loss <= valid_loss_min: print( 'Validation loss decreased ({:.6f} --> {:.6f}). Saving model ...' .format(valid_loss_min, valid_loss)) torch.save(net.state_dict(), ckpt_name) valid_loss_min = valid_loss
def init_dataset(dataset): dataset_set, _, _, _ = data.getDataset(dataset) return dataset_set
def init_dataset(notmnist_dir): global mnist_set global notmnist_set mnist_set, _, _, _ = data.getDataset('MNIST') notmnist_set = torchvision.datasets.ImageFolder(root=notmnist_dir)
def test(): network.eval() test_loss = 0 correct = 0 with torch.no_grad(): for data, target in test_loader: output = network(data) test_loss += F.nll_loss(output, target, size_average=False).item() pred = output.data.max(1, keepdim=True)[1] correct += pred.eq(target.data.view_as(pred)).sum() test_loss /= len(test_loader.dataset) print('\nTest set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) if __name__ == "__main__": train_set, test_set, inputs, num_classes = data.getDataset() train_loader, valid_loader, test_loader = data.getDataloader( train_set, test_set, param.valid_size, param.batch_size_train, param.batch_size_test, param.num_workers) network = Net() optimizer = optim.SGD(network.parameters(), lr=param.learning_rate, momentum=param.momentum) test() for epoch in range(1, param.n_epochs + 1): train(epoch) test()
expr_dir = os.path.join(opt.checkpoints_dir, opt.name) util.mkdirs(expr_dir) opt.file_name = os.path.join(expr_dir, 'log.txt') with open(opt.file_name, 'wt') as log_file: log_file.write('------------ Options -------------\n') for k, v in sorted(args.items()): log_file.write('%s: %s\n' % (str(k), str(v))) log_file.write('-------------- End ----------------\n') Model = getModel(opt) model = Model() ''' model.initialize(opt) print("model [%s] was created" % (model.name())) ''' MRFDataset = getDataset(opt) opt.set_type = 'train' dataset_train = MRFDataset() dataset_train.initialize(opt) dataloader_train = torch.utils.data.DataLoader(dataset_train, batch_size=opt.batchSize, shuffle=True, num_workers=int(opt.nThreads)) dataloader_train.dataset.patchSize = opt.patchSize print("dataset_train [%s] was created" % (dataset_train.name())) # dataloader_train_val = torch.utils.data.DataLoader(dataset_train, # batch_size=1, shuffle=False, num_workers=1) opt.set_type = 'val'
# ############################################################################# import os, re, sys from lingdb import LingDB, Language from lingdb.exceptions import * from phonemes import vowels, consonants import data from data import selectors import phonemes # Lightweight "Database" objects representing each language dataset # (can be replaced with an actual DB later if the overhead is justified. # One dataset per semester that we have data (duh :P) datasets = { name: LingDB(data.getDataset(name)) for name in data.getDatasetNames() } def handleQuery(query, dataset): """Given a query dict, decide which type of query has been made, and return a dict mapping LingDB.MATCHES to a list of results corresponding to the languages matching that type of query, and NO_DATA to the languages that lacked sufficient data to contribute""" lingDB = datasets[dataset] trait = query["trait"] # Look up this query's function in the mapping table (from selectors.py) try:
def _get_splitmnist_datasets(num_tasks): datasets = [] for i in range(1, num_tasks + 1): name = 'SplitMNIST-{}.{}'.format(num_tasks, i) datasets.append(data.getDataset(name)) return datasets
def run(dataset, net_type, train=True): # Hyper Parameter settings train_ens = cfg.train_ens valid_ens = cfg.valid_ens test_ens = cfg.test_ens n_epochs = cfg.n_epochs lr_start = cfg.lr_start num_workers = cfg.num_workers valid_size = cfg.valid_size batch_size = cfg.batch_size trainset, testset, inputs, outputs = data.getDataset(dataset) train_loader, valid_loader, test_loader = data.getDataloader( trainset, testset, valid_size, batch_size, num_workers) net = getModel(net_type, inputs, outputs).to(device) ckpt_dir = f'checkpoints/{dataset}/bayesian' ckpt_name = f'checkpoints/{dataset}/bayesian/model_{net_type}.pt' if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir, exist_ok=True) criterion = metrics.ELBO(len(trainset)).to(device) if train: optimizer = Adam(net.parameters(), lr=lr_start) valid_loss_max = np.Inf for epoch in range(n_epochs): # loop over the dataset multiple times cfg.curr_epoch_no = epoch utils.adjust_learning_rate( optimizer, metrics.lr_linear(epoch, 0, n_epochs, lr_start)) train_loss, train_acc, train_kl = train_model(net, optimizer, criterion, train_loader, num_ens=train_ens) valid_loss, valid_acc = validate_model(net, criterion, valid_loader, num_ens=valid_ens) print( 'Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tValidation Loss: {:.4f} \tValidation Accuracy: {:.4f} \ttrain_kl_div: {:.4f}' .format(epoch, train_loss, train_acc, valid_loss, valid_acc, train_kl)) print( 'Training Loss: {:.4f} \tTraining Likelihood Loss: {:.4f} \tTraining Kl Loss: {:.4f}' .format(train_loss, train_loss - train_kl, train_kl)) # save model if validation accuracy has increased if valid_loss <= valid_loss_max: print( 'Validation loss decreased ({:.6f} --> {:.6f}). Saving model ...' .format(valid_loss_max, valid_loss)) torch.save(net.state_dict(), ckpt_name) valid_loss_max = valid_loss # test saved model best_model = getModel(net_type, inputs, outputs).to(device) best_model.load_state_dict(torch.load(ckpt_name)) test_loss, test_acc = test_model(best_model, criterion, test_loader, num_ens=test_ens) print('Test Loss: {:.4f} \tTest Accuracy: {:.4f} '.format( test_loss, test_acc)) print('Test uncertainities:') test_uncertainities(best_model, test_loader, num_ens=10)
def run(dataset, net_type, checkpoint='None', prune_criterion='EmptyCrit', pruning_limit=0.0, lower_limit=0.5, local_pruning=False): # Hyper Parameter settings layer_type = cfg.layer_type activation_type = cfg.activation_type priors = cfg.priors train_ens = cfg.train_ens valid_ens = cfg.valid_ens n_epochs = cfg.n_epochs lr_start = cfg.lr_start num_workers = cfg.num_workers valid_size = cfg.valid_size batch_size = cfg.batch_size beta_type = cfg.beta_type # LOAD STRUCTURED PRUNED MODEL if net_type == 'customconv6': import pickle with open('/nfs/homedirs/ayle/model_conv6_0.5.pickle', 'rb') as f: pre_pruned_model = pickle.load(f) else: pre_pruned_model = None trainset, testset, inputs, outputs = data.getDataset(dataset) train_loader, valid_loader, test_loader = data.getDataloader( trainset, testset, valid_size, batch_size, num_workers) net = getModel(net_type, inputs, outputs, priors, layer_type, activation_type, pre_pruned_model).to(device) # LOAD PRUNED UNSTRUCTURED MASK # import pickle # with open('/nfs/homedirs/ayle/mask.pickle', 'rb') as f: # mask = pickle.load(f) # # mask_keys = list(mask.keys()) # # count = 0 # for name, module in net.named_modules(): # if name.startswith('conv') or name.startswith('fc'): # module.mask = mask[mask_keys[count]] # count += 1 # print(module.mask.sum().float() / torch.numel(module.mask)) ckpt_dir = f'checkpoints/{dataset}/bayesian' ckpt_name = f'checkpoints/{dataset}/bayesian/model_{net_type}_{layer_type}_{activation_type}_{prune_criterion}_{pruning_limit}_after.pt' if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir, exist_ok=True) if checkpoint != 'None': net.load_state_dict(torch.load(checkpoint)) if layer_type == 'mgp': criterion = metrics.ELBO2(len(trainset)).to(device) else: criterion = metrics.ELBO(len(trainset)).to(device) optimizer = Adam(net.parameters(), lr=lr_start) lr_sched = lr_scheduler.ReduceLROnPlateau(optimizer, patience=6, verbose=True) valid_loss_max = np.Inf if prune_criterion == 'SNIPit': pruning_criterion = SNIPit(limit=pruning_limit, model=net, lower_limit=lower_limit) pruning_criterion.prune(pruning_limit, train_loader=train_loader, local=local_pruning) elif prune_criterion == 'SNR': pruning_criterion = SNR(limit=pruning_limit, model=net, lower_limit=lower_limit) pruning_criterion.prune(pruning_limit, train_loader=train_loader, local=local_pruning) elif prune_criterion == 'StructuredSNR': pruning_criterion = StructuredSNR(limit=pruning_limit, model=net, lower_limit=lower_limit) # pruning_criterion.prune(pruning_limit, train_loader=train_loader, local=local_pruning) init_num_params = sum([ np.prod(x.shape) for name, x in net.named_parameters() if "W_mu" in name ]) new_num_params = init_num_params for epoch in range(n_epochs): # loop over the dataset multiple times train_loss, train_acc, train_kl = train_model(net, optimizer, criterion, train_loader, num_ens=train_ens, beta_type=beta_type, epoch=epoch, num_epochs=n_epochs, layer_type=layer_type) valid_loss, valid_acc, _ = validate_model(net, criterion, valid_loader, num_ens=valid_ens, beta_type=beta_type, epoch=epoch, num_epochs=n_epochs, layer_type=layer_type) lr_sched.step(valid_loss) print( 'Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tValidation Loss: {:.4f} \tValidation Accuracy: {:.4f} \ttrain_kl_div: {:.4f}' .format(epoch, train_loss, train_acc, valid_loss, valid_acc, train_kl)) # save model if validation accuracy has increased if valid_loss <= valid_loss_max: print( 'Validation loss decreased ({:.6f} --> {:.6f}). Saving model ...' .format(valid_loss_max, valid_loss)) torch.save(net.state_dict(), ckpt_name) valid_loss_max = valid_loss # if epoch == 0 or epoch == 1: # if (epoch % 40 == 0) and (epoch > 1) and (epoch < 200) and (1 - new_num_params / init_num_params) < pruning_limit: # net.zero_grad() # optimizer.zero_grad() # # with torch.no_grad(): # pruning_criterion.prune(0.1, train_loader=train_loader, local=local_pruning) # # import pickle # with open('testt', 'wb') as f: # pickle.dump(net, f) # # with open('testt', 'rb') as f: # net = pickle.load(f).to(device) # # net.post_init_implementation() # criterion = metrics.ELBO(len(trainset)).to(device) # optimizer = Adam(net.parameters(), lr=lr_start) # lr_sched = lr_scheduler.ReduceLROnPlateau(optimizer, patience=6, verbose=True) # valid_loss_max = np.Inf # pruning_criterion = StructuredSNR(limit=pruning_limit, model=net, lower_limit=lower_limit) # # new_num_params = sum([np.prod(x.shape) for name, x in net.named_parameters() if "W_mu" in name]) # print('Overall sparsity', 1 - new_num_params / init_num_params) # import pickle # with open(ckpt_name, 'wb') as f: # pickle.dump(net, f) torch.save(net.state_dict(), ckpt_name)