def _new_trainer(): transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) train_dataset = serialize(MNIST, root='data/mnist', train=True, download=True, transform=transform) test_dataset = serialize(MNIST, root='data/mnist', train=False, download=True, transform=transform) multi_module = MultiModelSupervisedLearningModule( nn.CrossEntropyLoss, {'acc': pl._AccuracyWithLogits}) lightning = pl.Lightning(multi_module, cgo_trainer.Trainer(use_cgo=True, max_epochs=1, limit_train_batches=0.25, progress_bar_refresh_rate=0), train_dataloader=pl.DataLoader(train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100)) return lightning
def lightning(): return pl.Classification( train_dataloader=pl.DataLoader(train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100), max_epochs=1, limit_train_batches=0.1, # for faster training progress_bar_refresh_rate=progress_bar_refresh_rate)
def test_mnist(): _reset() transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) train_dataset = nni.trace(MNIST)(root='data/mnist', train=True, download=True, transform=transform) test_dataset = nni.trace(MNIST)(root='data/mnist', train=False, download=True, transform=transform) lightning = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100), max_epochs=2, limit_train_batches=0.25, # for faster training progress_bar_refresh_rate=progress_bar_refresh_rate) lightning._execute(MNISTModel) assert _get_final_result() > 0.7 _reset()
def _multi_trial_test(epochs, batch_size, port, benchmark): # initalize dataset. Note that 50k+10k is used. It's a little different from paper transf = [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip() ] normalize = [ transforms.ToTensor(), transforms.Normalize([0.49139968, 0.48215827, 0.44653124], [0.24703233, 0.24348505, 0.26158768]) ] train_dataset = serialize(CIFAR10, 'data', train=True, download=True, transform=transforms.Compose(transf + normalize)) test_dataset = serialize(CIFAR10, 'data', train=False, transform=transforms.Compose(normalize)) # specify training hyper-parameters training_module = NasBench101TrainingModule(max_epochs=epochs) # FIXME: need to fix a bug in serializer for this to work # lr_monitor = serialize(LearningRateMonitor, logging_interval='step') trainer = pl.Trainer(max_epochs=epochs, gpus=1) lightning = pl.Lightning( lightning_module=training_module, trainer=trainer, train_dataloader=pl.DataLoader(train_dataset, batch_size=batch_size, shuffle=True), val_dataloaders=pl.DataLoader(test_dataset, batch_size=batch_size), ) strategy = Random() model = NasBench101() exp = RetiariiExperiment(model, lightning, [], strategy) exp_config = RetiariiExeConfig('local') exp_config.trial_concurrency = 2 exp_config.max_trial_number = 20 exp_config.trial_gpu_number = 1 exp_config.training_service.use_active_gpu = False if benchmark: exp_config.benchmark = 'nasbench101' exp_config.execution_engine = 'benchmark' exp.run(exp_config, port)
def _test_searchspace_on_dataset(searchspace, dataset='cifar10', arch=None): _reset() # dataset supports cifar10 and imagenet model, mutators = extract_mutation_from_pt_module(searchspace) if arch is None: model = try_mutation_until_success(model, mutators, 10) arch = { mut.mutator.label: _unpack_if_only_one(mut.samples) for mut in model.history } print('Selected model:', arch) with fixed_arch(arch): model = model.python_class(**model.python_init_params) if dataset == 'cifar10': train_data = FakeData(size=200, image_size=(3, 32, 32), num_classes=10, transform=transforms.ToTensor()) valid_data = FakeData(size=200, image_size=(3, 32, 32), num_classes=10, transform=transforms.ToTensor()) elif dataset == 'imagenet': train_data = FakeData(size=200, image_size=(3, 224, 224), num_classes=1000, transform=transforms.ToTensor()) valid_data = FakeData(size=200, image_size=(3, 224, 224), num_classes=1000, transform=transforms.ToTensor()) train_dataloader = pl.DataLoader(train_data, batch_size=4, shuffle=True) valid_dataloader = pl.DataLoader(valid_data, batch_size=6) evaluator = pl.Classification( train_dataloader=train_dataloader, val_dataloaders=valid_dataloader, export_onnx=False, max_epochs=1, limit_train_batches=2, limit_val_batches=3, ) evaluator.fit(model) # cleanup to avoid affecting later test cases _reset()
def test_diabetes(): _reset() nni.trial._params = {'foo': 'bar', 'parameter_id': 0} nni.runtime.platform.test._last_metric = None train_dataset = DiabetesDataset(train=True) test_dataset = DiabetesDataset(train=False) lightning = pl.Regression(optimizer=torch.optim.SGD, train_dataloader=pl.DataLoader(train_dataset, batch_size=20), val_dataloaders=pl.DataLoader(test_dataset, batch_size=20), max_epochs=100, progress_bar_refresh_rate=progress_bar_refresh_rate) lightning._execute(FCNet(train_dataset.x.shape[1], 1)) assert _get_final_result() < 2e4 _reset()
def _multi_trial_test(epochs, batch_size, port): # initalize dataset. Note that 50k+10k is used. It's a little different from paper transf = [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip() ] normalize = [ transforms.ToTensor(), transforms.Normalize([x / 255 for x in [129.3, 124.1, 112.4]], [x / 255 for x in [68.2, 65.4, 70.4]]) ] train_dataset = serialize(CIFAR100, 'data', train=True, download=True, transform=transforms.Compose(transf + normalize)) test_dataset = serialize(CIFAR100, 'data', train=False, transform=transforms.Compose(normalize)) # specify training hyper-parameters training_module = NasBench201TrainingModule(max_epochs=epochs) # FIXME: need to fix a bug in serializer for this to work # lr_monitor = serialize(LearningRateMonitor, logging_interval='step') trainer = pl.Trainer(max_epochs=epochs, gpus=1) lightning = pl.Lightning( lightning_module=training_module, trainer=trainer, train_dataloader=pl.DataLoader(train_dataset, batch_size=batch_size, shuffle=True), val_dataloaders=pl.DataLoader(test_dataset, batch_size=batch_size), ) strategy = Random() model = NasBench201() exp = RetiariiExperiment(model, lightning, [], strategy) exp_config = RetiariiExeConfig('local') exp_config.trial_concurrency = 2 exp_config.max_trial_number = 20 exp_config.trial_gpu_number = 1 exp_config.training_service.use_active_gpu = False exp.run(exp_config, port)
def get_mnist_evaluator(): transform = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) train_dataset = nni.trace(MNIST)('data/mnist', download=True, train=True, transform=transform) train_loader = pl.DataLoader(train_dataset, 64) valid_dataset = nni.trace(MNIST)('data/mnist', download=True, train=False, transform=transform) valid_loader = pl.DataLoader(valid_dataset, 64) return pl.Classification( train_dataloader=train_loader, val_dataloaders=valid_loader, limit_train_batches=20, limit_val_batches=20, max_epochs=1 )
def test_multi_model_trainer_gpu(self): _reset() if not (torch.cuda.is_available() and torch.cuda.device_count() >= 2): pytest.skip('test requires GPU and torch+cuda') transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) train_dataset = serialize(MNIST, root='data/mnist', train=True, download=True, transform=transform) test_dataset = serialize(MNIST, root='data/mnist', train=False, download=True, transform=transform) multi_module = _MultiModelSupervisedLearningModule( nn.CrossEntropyLoss, {'acc': pl._AccuracyWithLogits}, n_models=2) lightning = pl.Lightning(multi_module, cgo_trainer.Trainer(use_cgo=True, max_epochs=1, limit_train_batches=0.25), train_dataloader=pl.DataLoader( train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100)) lightning._execute(_model_gpu) result = _get_final_result() assert len(result) == 2 for _ in result: assert _ > 0.8
def test_multiprocessing_dataloader(): # check whether multi-processing works # it's possible to have pickle errors dataset = nni.trace(MNIST)(root='data/mnist', train=False, download=True, transform=nni.trace(transforms.Compose)([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])) import nni.retiarii.evaluator.pytorch.lightning as pl dataloader = pl.DataLoader(dataset, batch_size=10, num_workers=2) x, y = next(iter(dataloader)) assert x.size() == torch.Size([10, 1, 28, 28]) assert y.size() == torch.Size([10])
(0.2023, 0.1994, 0.2010)), ]) train_dataset = serialize(CIFAR10, root='data/cifar10', train=True, download=True, transform=train_transform) test_dataset = serialize(CIFAR10, root='data/cifar10', train=False, download=True, transform=valid_transform) # trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=100), # val_dataloaders=pl.DataLoader(test_dataset, batch_size=100), # max_epochs=1, limit_train_batches=0.2) trainer = cgo.Classification(train_dataloader=pl.DataLoader( train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100), max_epochs=1, limit_train_batches=0.2) applied_mutators = [BlockMutator('mutable_0'), BlockMutator('mutable_1')] simple_strategy = TPEStrategy() exp = RetiariiExperiment(base_model, trainer, applied_mutators, simple_strategy) exp_config = RetiariiExeConfig('remote') exp_config.experiment_name = 'darts_search' exp_config.trial_concurrency = 3