def _main(port): base_model = ShuffleNetV2(32) base_predictor = 'cortexA76cpu_tflite21' transf = [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip() ] normalize = [ transforms.ToTensor(), transforms.Normalize([0.49139968, 0.48215827, 0.44653124], [0.24703233, 0.24348505, 0.26158768]) ] train_dataset = serialize(CIFAR10, 'data', train=True, download=True, transform=transforms.Compose(transf + normalize)) test_dataset = serialize(CIFAR10, 'data', train=False, transform=transforms.Compose(normalize)) trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=64), val_dataloaders=pl.DataLoader(test_dataset, batch_size=64), max_epochs=2, gpus=1) simple_strategy = strategy.Random(model_filter=LatencyFilter(threshold=100, predictor=base_predictor)) exp = RetiariiExperiment(base_model, trainer, strategy=simple_strategy) exp_config = RetiariiExeConfig('local') exp_config.trial_concurrency = 2 exp_config.max_trial_number = 2 exp_config.trial_gpu_number = 1 exp_config.training_service.use_active_gpu = False exp_config.execution_engine = 'base' exp_config.dummy_input = [1, 3, 32, 32] exp.run(exp_config, port) print('Exported models:') for model in exp.export_top_models(formatter='dict'): print(model)
def _test_strategy(strategy_, support_value_choice=True): to_test = [ # (model, evaluator), support_or_net (_mnist_net('simple'), True), (_mnist_net('simple_value_choice'), support_value_choice), (_mnist_net('value_choice'), support_value_choice), (_mnist_net('repeat'), False), # no strategy supports repeat currently (_mnist_net('custom_op'), False), # this is definitely a NO (_multihead_attention_net(), support_value_choice), ] for (base_model, evaluator), support_or_not in to_test: if isinstance(strategy_, BaseStrategy): strategy = strategy_ else: strategy = strategy_(base_model, evaluator) print('Testing:', type(strategy).__name__, type(base_model).__name__, type(evaluator).__name__, support_or_not) experiment = RetiariiExperiment(base_model, evaluator, strategy=strategy) config = RetiariiExeConfig() config.execution_engine = 'oneshot' if support_or_not: experiment.run(config) assert isinstance(experiment.export_top_models()[0], dict) else: with pytest.raises(TypeError, match='not supported'): experiment.run(config)
def test_oneshot_experiment(): base_model = Net() evaluator = get_mnist_evaluator() search_strategy = strategy.RandomOneShot() exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy) exp_config = RetiariiExeConfig() exp_config.execution_engine = 'oneshot' exp.run(exp_config) assert isinstance(exp.export_top_models()[0], dict)
def _main(port): base_model = ShuffleNetV2OneShot(32) base_predictor = 'cortexA76cpu_tflite21' transf = [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip() ] normalize = [ transforms.ToTensor(), transforms.Normalize([0.49139968, 0.48215827, 0.44653124], [0.24703233, 0.24348505, 0.26158768]) ] # FIXME # CIFAR10 is used here temporarily. # Actually we should load weight from supernet and evaluate on imagenet. train_dataset = serialize(CIFAR10, 'data', train=True, download=True, transform=transforms.Compose(transf + normalize)) test_dataset = serialize(CIFAR10, 'data', train=False, transform=transforms.Compose(normalize)) trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=64), val_dataloaders=pl.DataLoader(test_dataset, batch_size=64), max_epochs=2, gpus=1) simple_strategy = strategy.RegularizedEvolution(model_filter=LatencyFilter( threshold=100, predictor=base_predictor), sample_size=1, population_size=2, cycles=2) exp = RetiariiExperiment(base_model, trainer, strategy=simple_strategy) exp_config = RetiariiExeConfig('local') exp_config.trial_concurrency = 2 # exp_config.max_trial_number = 2 exp_config.trial_gpu_number = 1 exp_config.training_service.use_active_gpu = False exp_config.execution_engine = 'base' exp_config.dummy_input = [1, 3, 32, 32] exp.run(exp_config, port) print('Exported models:') for i, model in enumerate(exp.export_top_models(formatter='dict')): print(model) with open(f'architecture_final_{i}.json', 'w') as f: json.dump(get_archchoice_by_model(model), f, indent=4)
def test_multitrial_experiment(pytestconfig): base_model = Net() evaluator = get_mnist_evaluator() search_strategy = strategy.Random() exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy) exp_config = RetiariiExeConfig('local') exp_config.trial_concurrency = 1 exp_config.max_trial_number = 1 exp_config._trial_command_params = nas_experiment_trial_params(pytestconfig.rootpath) exp.run(exp_config) ensure_success(exp) assert isinstance(exp.export_top_models()[0], dict) exp.stop()
def _test_experiment_in_separate_process(rootpath): try: base_model, evaluator = _mnist_net('simple', {'max_epochs': 1}) search_strategy = strategy.Random() exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy) exp_config = RetiariiExeConfig('local') exp_config.experiment_name = 'mnist_unittest' exp_config.trial_concurrency = 1 exp_config.max_trial_number = 1 exp_config._trial_command_params = nas_experiment_trial_params(rootpath) exp.run(exp_config) ensure_success(exp) assert isinstance(exp.export_top_models()[0], dict) finally: # https://stackoverflow.com/questions/34506638/how-to-register-atexit-function-in-pythons-multiprocessing-subprocess import atexit atexit._run_exitfuncs()
def test_multi_trial(model, pytestconfig): evaluator_kwargs = { 'max_epochs': 1 } base_model, evaluator = _mnist_net(model, evaluator_kwargs) search_strategy = strategy.Random() exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy) exp_config = RetiariiExeConfig('local') exp_config.experiment_name = 'mnist_unittest' exp_config.trial_concurrency = 1 exp_config.max_trial_number = 1 exp_config._trial_command_params = nas_experiment_trial_params(pytestconfig.rootpath) exp.run(exp_config) ensure_success(exp) assert isinstance(exp.export_top_models()[0], dict) exp.stop()
def _test_strategy(strategy_, support_value_choice=True, multi_gpu=False): evaluator_kwargs = {'max_epochs': 1} if multi_gpu: evaluator_kwargs.update(strategy='ddp', accelerator='gpu', devices=torch.cuda.device_count()) to_test = [ # (model, evaluator), support_or_net (_mnist_net('simple', evaluator_kwargs), True), (_mnist_net('simple_value_choice', evaluator_kwargs), support_value_choice), (_mnist_net('value_choice', evaluator_kwargs), support_value_choice), (_mnist_net('repeat', evaluator_kwargs), support_value_choice), # no strategy supports repeat currently (_mnist_net('custom_op', evaluator_kwargs), False), # this is definitely a NO (_multihead_attention_net(evaluator_kwargs), support_value_choice), ] for (base_model, evaluator), support_or_not in to_test: if isinstance(strategy_, BaseStrategy): strategy = strategy_ else: strategy = strategy_(base_model, evaluator) print('Testing:', type(strategy).__name__, type(base_model).__name__, type(evaluator).__name__, support_or_not) experiment = RetiariiExperiment(base_model, evaluator, strategy=strategy) config = RetiariiExeConfig() config.execution_engine = 'oneshot' if support_or_not: experiment.run(config) assert isinstance(experiment.export_top_models()[0], dict) else: with pytest.raises(TypeError, match='not supported'): experiment.run(config)
best_val_ppl ) # reports best validation ppl to nni as final result of one trial if __name__ == "__main__": train_iter = WikiText2(split='train') tokenizer = get_tokenizer('basic_english') vocab = build_vocab_from_iterator(map(tokenizer, train_iter), specials=['<unk>']) vocab.set_default_index(vocab['<unk>']) n_token = len(vocab) base_model = Transformer(n_token) evaluator = FunctionalEvaluator(fit) exp = RetiariiExperiment(base_model, evaluator, [], strategy.Random()) exp_config = RetiariiExeConfig('local') exp_config.experiment_name = 'transformer tuning' exp_config.trial_concurrency = 3 # please change configurations accordingly exp_config.max_trial_number = 25 exp_config.trial_gpu_number = 1 exp_config.training_service.use_active_gpu = False export_formatter = 'dict' exp.run(exp_config, 8081) print('Final model:') for model_code in exp.export_top_models(optimize_mode='minimize', formatter=export_formatter): print(model_code)
transform=transform) trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100), max_epochs=2, gpus=1, limit_train_batches=0.1, limit_val_batches=0.1) simple_strategy = strategy.Random() exp = RetiariiExperiment(base_model, trainer, [], simple_strategy) exp_config = RetiariiExeConfig('local') exp_config.experiment_name = 'mnist_search' exp_config.trial_concurrency = 2 exp_config.max_trial_number = args.budget exp_config.trial_gpu_number = 1 exp_config.training_service.use_active_gpu = True # Integration test GPU has a Xorg running export_formatter = 'dict' if args.exec == 'graph': exp_config.execution_engine = 'base' export_formatter = 'code' exp.run(exp_config, args.port) print('Final model:') for model_code in exp.export_top_models(formatter=export_formatter): print(model_code)
model = model_cls() # dump the model into an onnx if 'NNI_OUTPUT_DIR' in os.environ: dummy_input = torch.zeros(1, 3, 32, 32) torch.onnx.export(model, (dummy_input, ), Path(os.environ['NNI_OUTPUT_DIR']) / 'model.onnx') evaluate_model(model_cls) # %% # Relaunch the experiment, and a button is shown on Web portal. # # .. image:: ../../img/netron_entrance_webui.png # # Export Top Models # ----------------- # # Users can export top models after the exploration is done using ``export_top_models``. for model_dict in exp.export_top_models(formatter='dict'): print(model_dict) # %% # The output is ``json`` object which records the mutation actions of the top model. # If users want to output source code of the top model, # they can use :ref:`graph-based execution engine <graph-based-execution-engine>` for the experiment, # by simply adding the following two lines. exp_config.execution_engine = 'base' export_formatter = 'code'
transform=transform) test_dataset = serialize(MNIST, root='data/mnist', train=False, download=True, transform=transform) trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=100), val_dataloaders=pl.DataLoader(test_dataset, batch_size=100), max_epochs=2) # uncomment the following two lines to debug a generated model #debug_mutated_model(base_model, trainer, []) #exit(0) simple_strategy = strategy.Random() exp = RetiariiExperiment(base_model, trainer, [], simple_strategy) exp_config = RetiariiExeConfig('local') exp_config.experiment_name = 'mnist_search' exp_config.trial_concurrency = 2 exp_config.max_trial_number = 2 exp_config.training_service.use_active_gpu = False exp.run(exp_config, 8081 + random.randint(0, 100)) print('Final model:') for model_code in exp.export_top_models(): print(model_code)
def _main(): parser = argparse.ArgumentParser("SPOS Evolutional Search") parser.add_argument("--port", type=int, default=8084) parser.add_argument("--imagenet-dir", type=str, default="./data/imagenet") parser.add_argument("--checkpoint", type=str, default="./data/checkpoint-150000.pth.tar") parser.add_argument( "--spos-preprocessing", action="store_true", default=False, help="When true, image values will range from 0 to 255 and use BGR " "(as in original repo).") parser.add_argument("--seed", type=int, default=42) parser.add_argument("--workers", type=int, default=6) parser.add_argument("--train-batch-size", type=int, default=128) parser.add_argument("--train-iters", type=int, default=200) parser.add_argument("--test-batch-size", type=int, default=512) parser.add_argument("--log-frequency", type=int, default=10) parser.add_argument("--label-smoothing", type=float, default=0.1) parser.add_argument("--evolution-sample-size", type=int, default=10) parser.add_argument("--evolution-population-size", type=int, default=50) parser.add_argument("--evolution-cycles", type=int, default=10) parser.add_argument( "--latency-filter", type=str, default=None, help="Apply latency filter by calling the name of the applied hardware." ) parser.add_argument("--latency-threshold", type=float, default=100) args = parser.parse_args() # use a fixed set of image will improve the performance torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) np.random.seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True assert torch.cuda.is_available() base_model = ShuffleNetV2OneShot() criterion = CrossEntropyLabelSmooth(1000, args.label_smoothing) if args.latency_filter: latency_filter = LatencyFilter(threshold=args.latency_threshold, predictor=args.latency_filter) else: latency_filter = None evaluator = FunctionalEvaluator(evaluate_acc, criterion=criterion, args=args) evolution_strategy = strategy.RegularizedEvolution( model_filter=latency_filter, sample_size=args.evolution_sample_size, population_size=args.evolution_population_size, cycles=args.evolution_cycles) exp = RetiariiExperiment(base_model, evaluator, strategy=evolution_strategy) exp_config = RetiariiExeConfig('local') exp_config.trial_concurrency = 2 exp_config.trial_gpu_number = 1 exp_config.max_trial_number = args.evolution_cycles exp_config.training_service.use_active_gpu = False exp_config.execution_engine = 'base' exp_config.dummy_input = [1, 3, 224, 224] exp.run(exp_config, args.port) print('Exported models:') for i, model in enumerate(exp.export_top_models(formatter='dict')): print(model) with open(f'architecture_final_{i}.json', 'w') as f: json.dump(get_archchoice_by_model(model), f, indent=4)