def _new_trainer():
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
train_dataset = serialize(MNIST,
root='data/mnist',
train=True,
download=True,
transform=transform)
test_dataset = serialize(MNIST,
root='data/mnist',
train=False,
download=True,
transform=transform)
multi_module = MultiModelSupervisedLearningModule(
nn.CrossEntropyLoss, {'acc': pl._AccuracyWithLogits})
lightning = pl.Lightning(multi_module,
cgo_trainer.Trainer(use_cgo=True,
max_epochs=1,
limit_train_batches=0.25,
progress_bar_refresh_rate=0),
train_dataloader=pl.DataLoader(train_dataset,
batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=100))
return lightning
def _main(port):
base_model = ShuffleNetV2(32)
base_predictor = 'cortexA76cpu_tflite21'
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [
transforms.ToTensor(),
transforms.Normalize([0.49139968, 0.48215827, 0.44653124], [0.24703233, 0.24348505, 0.26158768])
]
train_dataset = serialize(CIFAR10, 'data', train=True, download=True, transform=transforms.Compose(transf + normalize))
test_dataset = serialize(CIFAR10, 'data', train=False, transform=transforms.Compose(normalize))
trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=64),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=64),
max_epochs=2, gpus=1)
simple_strategy = strategy.Random(model_filter=LatencyFilter(threshold=100, predictor=base_predictor))
exp = RetiariiExperiment(base_model, trainer, strategy=simple_strategy)
exp_config = RetiariiExeConfig('local')
exp_config.trial_concurrency = 2
exp_config.max_trial_number = 2
exp_config.trial_gpu_number = 1
exp_config.training_service.use_active_gpu = False
exp_config.execution_engine = 'base'
exp_config.dummy_input = [1, 3, 32, 32]
exp.run(exp_config, port)
print('Exported models:')
for model in exp.export_top_models(formatter='dict'):
print(model)
def _main(port):
base_model = ShuffleNetV2OneShot(32)
base_predictor = 'cortexA76cpu_tflite21'
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [
transforms.ToTensor(),
transforms.Normalize([0.49139968, 0.48215827, 0.44653124],
[0.24703233, 0.24348505, 0.26158768])
]
# FIXME
# CIFAR10 is used here temporarily.
# Actually we should load weight from supernet and evaluate on imagenet.
train_dataset = serialize(CIFAR10,
'data',
train=True,
download=True,
transform=transforms.Compose(transf + normalize))
test_dataset = serialize(CIFAR10,
'data',
train=False,
transform=transforms.Compose(normalize))
trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset,
batch_size=64),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=64),
max_epochs=2,
gpus=1)
simple_strategy = strategy.RegularizedEvolution(model_filter=LatencyFilter(
threshold=100, predictor=base_predictor),
sample_size=1,
population_size=2,
cycles=2)
exp = RetiariiExperiment(base_model, trainer, strategy=simple_strategy)
exp_config = RetiariiExeConfig('local')
exp_config.trial_concurrency = 2
# exp_config.max_trial_number = 2
exp_config.trial_gpu_number = 1
exp_config.training_service.use_active_gpu = False
exp_config.execution_engine = 'base'
exp_config.dummy_input = [1, 3, 32, 32]
exp.run(exp_config, port)
print('Exported models:')
for i, model in enumerate(exp.export_top_models(formatter='dict')):
print(model)
with open(f'architecture_final_{i}.json', 'w') as f:
json.dump(get_archchoice_by_model(model), f, indent=4)
def _multi_trial_test(epochs, batch_size, port, benchmark):
# initalize dataset. Note that 50k+10k is used. It's a little different from paper
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [
transforms.ToTensor(),
transforms.Normalize([0.49139968, 0.48215827, 0.44653124],
[0.24703233, 0.24348505, 0.26158768])
]
train_dataset = serialize(CIFAR10,
'data',
train=True,
download=True,
transform=transforms.Compose(transf + normalize))
test_dataset = serialize(CIFAR10,
'data',
train=False,
transform=transforms.Compose(normalize))
# specify training hyper-parameters
training_module = NasBench101TrainingModule(max_epochs=epochs)
# FIXME: need to fix a bug in serializer for this to work
# lr_monitor = serialize(LearningRateMonitor, logging_interval='step')
trainer = pl.Trainer(max_epochs=epochs, gpus=1)
lightning = pl.Lightning(
lightning_module=training_module,
trainer=trainer,
train_dataloader=pl.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=batch_size),
)
strategy = Random()
model = NasBench101()
exp = RetiariiExperiment(model, lightning, [], strategy)
exp_config = RetiariiExeConfig('local')
exp_config.trial_concurrency = 2
exp_config.max_trial_number = 20
exp_config.trial_gpu_number = 1
exp_config.training_service.use_active_gpu = False
if benchmark:
exp_config.benchmark = 'nasbench101'
exp_config.execution_engine = 'benchmark'
exp.run(exp_config, port)
def _multi_trial_test(epochs, batch_size, port):
# initalize dataset. Note that 50k+10k is used. It's a little different from paper
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [
transforms.ToTensor(),
transforms.Normalize([x / 255 for x in [129.3, 124.1, 112.4]],
[x / 255 for x in [68.2, 65.4, 70.4]])
]
train_dataset = serialize(CIFAR100,
'data',
train=True,
download=True,
transform=transforms.Compose(transf + normalize))
test_dataset = serialize(CIFAR100,
'data',
train=False,
transform=transforms.Compose(normalize))
# specify training hyper-parameters
training_module = NasBench201TrainingModule(max_epochs=epochs)
# FIXME: need to fix a bug in serializer for this to work
# lr_monitor = serialize(LearningRateMonitor, logging_interval='step')
trainer = pl.Trainer(max_epochs=epochs, gpus=1)
lightning = pl.Lightning(
lightning_module=training_module,
trainer=trainer,
train_dataloader=pl.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=batch_size),
)
strategy = Random()
model = NasBench201()
exp = RetiariiExperiment(model, lightning, [], strategy)
exp_config = RetiariiExeConfig('local')
exp_config.trial_concurrency = 2
exp_config.max_trial_number = 20
exp_config.trial_gpu_number = 1
exp_config.training_service.use_active_gpu = False
exp.run(exp_config, port)
def test_serialize():
module = serialize(Foo, 3)
assert json_loads(json_dumps(module)) == module
module = serialize(Foo, b=2, a=1)
assert json_loads(json_dumps(module)) == module
module = serialize(Foo, Foo(1), 5)
dumped_module = json_dumps(module)
assert len(
dumped_module
) > 200 # should not be too longer if the serialization is correct
module = serialize(Foo, serialize(Foo, 1), 5)
dumped_module = json_dumps(module)
assert len(
dumped_module
) < 200 # should not be too longer if the serialization is correct
assert json_loads(dumped_module) == module
def test_multi_model_trainer_gpu(self):
_reset()
if not (torch.cuda.is_available() and torch.cuda.device_count() >= 2):
pytest.skip('test requires GPU and torch+cuda')
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
train_dataset = serialize(MNIST,
root='data/mnist',
train=True,
download=True,
transform=transform)
test_dataset = serialize(MNIST,
root='data/mnist',
train=False,
download=True,
transform=transform)
multi_module = _MultiModelSupervisedLearningModule(
nn.CrossEntropyLoss, {'acc': pl._AccuracyWithLogits}, n_models=2)
lightning = pl.Lightning(multi_module,
cgo_trainer.Trainer(use_cgo=True,
max_epochs=1,
limit_train_batches=0.25),
train_dataloader=pl.DataLoader(
train_dataset, batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=100))
lightning._execute(_model_gpu)
result = _get_final_result()
assert len(result) == 2
for _ in result:
assert _ > 0.8
def test_mnist():
_reset()
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
train_dataset = serialize(MNIST,
root='data/mnist',
train=True,
download=True,
transform=transform)
test_dataset = serialize(MNIST,
root='data/mnist',
train=False,
download=True,
transform=transform)
lightning = pl.Classification(
train_dataloader=pl.DataLoader(train_dataset, batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset, batch_size=100),
max_epochs=2,
limit_train_batches=0.25, # for faster training
progress_bar_refresh_rate=progress_bar_refresh_rate)
lightning._execute(MNISTModel)
assert _get_final_result() > 0.7
_reset()
default='python')
parser.add_argument('--budget', default=2, type=int)
parser.add_argument('--port', default=8899, type=int)
args = parser.parse_args()
if args.net == 'simple':
base_model = SimpleNet(32)
else:
base_model = ComplexNet()
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
train_dataset = serialize(MNIST,
root='data/mnist',
train=True,
download=True,
transform=transform)
test_dataset = serialize(MNIST,
root='data/mnist',
train=False,
download=True,
transform=transform)
trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset,
batch_size=100),
val_dataloaders=pl.DataLoader(test_dataset,
batch_size=100),
max_epochs=2,
gpus=1,
limit_train_batches=0.1,
limit_val_batches=0.1)
