def _new_trainer():
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
train_dataset = serialize(MNIST,
root='data/mnist',
train=True,
download=True,
transform=transform)
test_dataset = serialize(MNIST,
root='data/mnist',
train=False,
download=True,
transform=transform)
multi_module = MultiModelSupervisedLearningModule(
nn.CrossEntropyLoss, {'acc': pl._AccuracyWithLogits})
lightning = pl.Lightning(multi_module,
cgo_trainer.Trainer(use_cgo=True,
max_epochs=1,
limit_train_batches=0.25,
progress_bar_refresh_rate=0),
train_dataloader=pl.DataLoader(train_dataset,
batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=100))
return lightning
def lightning():
return pl.Classification(
train_dataloader=pl.DataLoader(train_dataset, batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=100),
max_epochs=1,
limit_train_batches=0.1, # for faster training
progress_bar_refresh_rate=progress_bar_refresh_rate)
def test_mnist():
_reset()
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])
train_dataset = nni.trace(MNIST)(root='data/mnist', train=True, download=True, transform=transform)
test_dataset = nni.trace(MNIST)(root='data/mnist', train=False, download=True, transform=transform)
lightning = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=100),
max_epochs=2, limit_train_batches=0.25, # for faster training
progress_bar_refresh_rate=progress_bar_refresh_rate)
lightning._execute(MNISTModel)
assert _get_final_result() > 0.7
_reset()
def _multi_trial_test(epochs, batch_size, port, benchmark):
# initalize dataset. Note that 50k+10k is used. It's a little different from paper
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [
transforms.ToTensor(),
transforms.Normalize([0.49139968, 0.48215827, 0.44653124],
[0.24703233, 0.24348505, 0.26158768])
]
train_dataset = serialize(CIFAR10,
'data',
train=True,
download=True,
transform=transforms.Compose(transf + normalize))
test_dataset = serialize(CIFAR10,
'data',
train=False,
transform=transforms.Compose(normalize))
# specify training hyper-parameters
training_module = NasBench101TrainingModule(max_epochs=epochs)
# FIXME: need to fix a bug in serializer for this to work
# lr_monitor = serialize(LearningRateMonitor, logging_interval='step')
trainer = pl.Trainer(max_epochs=epochs, gpus=1)
lightning = pl.Lightning(
lightning_module=training_module,
trainer=trainer,
train_dataloader=pl.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=batch_size),
)
strategy = Random()
model = NasBench101()
exp = RetiariiExperiment(model, lightning, [], strategy)
exp_config = RetiariiExeConfig('local')
exp_config.trial_concurrency = 2
exp_config.max_trial_number = 20
exp_config.trial_gpu_number = 1
exp_config.training_service.use_active_gpu = False
if benchmark:
exp_config.benchmark = 'nasbench101'
exp_config.execution_engine = 'benchmark'
exp.run(exp_config, port)
def _test_searchspace_on_dataset(searchspace, dataset='cifar10', arch=None):
_reset()
# dataset supports cifar10 and imagenet
model, mutators = extract_mutation_from_pt_module(searchspace)
if arch is None:
model = try_mutation_until_success(model, mutators, 10)
arch = {
mut.mutator.label: _unpack_if_only_one(mut.samples)
for mut in model.history
}
print('Selected model:', arch)
with fixed_arch(arch):
model = model.python_class(**model.python_init_params)
if dataset == 'cifar10':
train_data = FakeData(size=200,
image_size=(3, 32, 32),
num_classes=10,
transform=transforms.ToTensor())
valid_data = FakeData(size=200,
image_size=(3, 32, 32),
num_classes=10,
transform=transforms.ToTensor())
elif dataset == 'imagenet':
train_data = FakeData(size=200,
image_size=(3, 224, 224),
num_classes=1000,
transform=transforms.ToTensor())
valid_data = FakeData(size=200,
image_size=(3, 224, 224),
num_classes=1000,
transform=transforms.ToTensor())
train_dataloader = pl.DataLoader(train_data, batch_size=4, shuffle=True)
valid_dataloader = pl.DataLoader(valid_data, batch_size=6)
evaluator = pl.Classification(
train_dataloader=train_dataloader,
val_dataloaders=valid_dataloader,
export_onnx=False,
max_epochs=1,
limit_train_batches=2,
limit_val_batches=3,
)
evaluator.fit(model)
# cleanup to avoid affecting later test cases
_reset()
def test_diabetes():
_reset()
nni.trial._params = {'foo': 'bar', 'parameter_id': 0}
nni.runtime.platform.test._last_metric = None
train_dataset = DiabetesDataset(train=True)
test_dataset = DiabetesDataset(train=False)
lightning = pl.Regression(optimizer=torch.optim.SGD,
train_dataloader=pl.DataLoader(train_dataset, batch_size=20),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=20),
max_epochs=100,
progress_bar_refresh_rate=progress_bar_refresh_rate)
lightning._execute(FCNet(train_dataset.x.shape[1], 1))
assert _get_final_result() < 2e4
_reset()
def _multi_trial_test(epochs, batch_size, port):
# initalize dataset. Note that 50k+10k is used. It's a little different from paper
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [129.3, 124.1, 112.4]],
[x / 255 for x in [68.2, 65.4, 70.4]])
]
train_dataset = serialize(CIFAR100,
'data',
train=True,
download=True,
transform=transforms.Compose(transf + normalize))
test_dataset = serialize(CIFAR100,
'data',
train=False,
transform=transforms.Compose(normalize))
# specify training hyper-parameters
training_module = NasBench201TrainingModule(max_epochs=epochs)
# FIXME: need to fix a bug in serializer for this to work
# lr_monitor = serialize(LearningRateMonitor, logging_interval='step')
trainer = pl.Trainer(max_epochs=epochs, gpus=1)
lightning = pl.Lightning(
lightning_module=training_module,
trainer=trainer,
train_dataloader=pl.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=batch_size),
)
strategy = Random()
model = NasBench201()
exp = RetiariiExperiment(model, lightning, [], strategy)
exp_config = RetiariiExeConfig('local')
exp_config.trial_concurrency = 2
exp_config.max_trial_number = 20
exp_config.trial_gpu_number = 1
exp_config.training_service.use_active_gpu = False
exp.run(exp_config, port)
def get_mnist_evaluator():
transform = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
train_dataset = nni.trace(MNIST)('data/mnist', download=True, train=True, transform=transform)
train_loader = pl.DataLoader(train_dataset, 64)
valid_dataset = nni.trace(MNIST)('data/mnist', download=True, train=False, transform=transform)
valid_loader = pl.DataLoader(valid_dataset, 64)
return pl.Classification(
train_dataloader=train_loader, val_dataloaders=valid_loader,
limit_train_batches=20,
limit_val_batches=20,
max_epochs=1
)
def test_multi_model_trainer_gpu(self):
_reset()
if not (torch.cuda.is_available() and torch.cuda.device_count() >= 2):
pytest.skip('test requires GPU and torch+cuda')
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
train_dataset = serialize(MNIST,
root='data/mnist',
train=True,
download=True,
transform=transform)
test_dataset = serialize(MNIST,
root='data/mnist',
train=False,
download=True,
transform=transform)
multi_module = _MultiModelSupervisedLearningModule(
nn.CrossEntropyLoss, {'acc': pl._AccuracyWithLogits}, n_models=2)
lightning = pl.Lightning(multi_module,
cgo_trainer.Trainer(use_cgo=True,
max_epochs=1,
limit_train_batches=0.25),
train_dataloader=pl.DataLoader(
train_dataset, batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=100))
lightning._execute(_model_gpu)
result = _get_final_result()
assert len(result) == 2
for _ in result:
assert _ > 0.8
def test_multiprocessing_dataloader():
# check whether multi-processing works
# it's possible to have pickle errors
dataset = nni.trace(MNIST)(root='data/mnist',
train=False,
download=True,
transform=nni.trace(transforms.Compose)([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]))
import nni.retiarii.evaluator.pytorch.lightning as pl
dataloader = pl.DataLoader(dataset, batch_size=10, num_workers=2)
x, y = next(iter(dataloader))
assert x.size() == torch.Size([10, 1, 28, 28])
assert y.size() == torch.Size([10])
(0.2023, 0.1994, 0.2010)),
])
train_dataset = serialize(CIFAR10,
root='data/cifar10',
train=True,
download=True,
transform=train_transform)
test_dataset = serialize(CIFAR10,
root='data/cifar10',
train=False,
download=True,
transform=valid_transform)
# trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=100),
# val_dataloaders=pl.DataLoader(test_dataset, batch_size=100),
# max_epochs=1, limit_train_batches=0.2)
trainer = cgo.Classification(train_dataloader=pl.DataLoader(
train_dataset, batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=100),
max_epochs=1,
limit_train_batches=0.2)
applied_mutators = [BlockMutator('mutable_0'), BlockMutator('mutable_1')]
simple_strategy = TPEStrategy()
exp = RetiariiExperiment(base_model, trainer, applied_mutators,
simple_strategy)
exp_config = RetiariiExeConfig('remote')
exp_config.experiment_name = 'darts_search'
exp_config.trial_concurrency = 3
