Пример #1
0
def _main(port):
    base_model = ShuffleNetV2(32)
    base_predictor = 'cortexA76cpu_tflite21'
    transf = [
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip()
    ]
    normalize = [
        transforms.ToTensor(),
        transforms.Normalize([0.49139968, 0.48215827, 0.44653124], [0.24703233, 0.24348505, 0.26158768])
    ]
    train_dataset = serialize(CIFAR10, 'data', train=True, download=True, transform=transforms.Compose(transf + normalize))
    test_dataset = serialize(CIFAR10, 'data', train=False, transform=transforms.Compose(normalize))

    trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset, batch_size=64),
                                val_dataloaders=pl.DataLoader(test_dataset, batch_size=64),
                                max_epochs=2, gpus=1)

    simple_strategy = strategy.Random(model_filter=LatencyFilter(threshold=100, predictor=base_predictor))

    exp = RetiariiExperiment(base_model, trainer, strategy=simple_strategy)

    exp_config = RetiariiExeConfig('local')
    exp_config.trial_concurrency = 2
    exp_config.max_trial_number = 2
    exp_config.trial_gpu_number = 1
    exp_config.training_service.use_active_gpu = False
    exp_config.execution_engine = 'base'
    exp_config.dummy_input = [1, 3, 32, 32]

    exp.run(exp_config, port)

    print('Exported models:')
    for model in exp.export_top_models(formatter='dict'):
        print(model)
Пример #2
0
def _test_strategy(strategy_, support_value_choice=True):
    to_test = [
        # (model, evaluator), support_or_net
        (_mnist_net('simple'), True),
        (_mnist_net('simple_value_choice'), support_value_choice),
        (_mnist_net('value_choice'), support_value_choice),
        (_mnist_net('repeat'), False),  # no strategy supports repeat currently
        (_mnist_net('custom_op'), False),  # this is definitely a NO
        (_multihead_attention_net(), support_value_choice),
    ]

    for (base_model, evaluator), support_or_not in to_test:
        if isinstance(strategy_, BaseStrategy):
            strategy = strategy_
        else:
            strategy = strategy_(base_model, evaluator)
        print('Testing:',
              type(strategy).__name__,
              type(base_model).__name__,
              type(evaluator).__name__, support_or_not)
        experiment = RetiariiExperiment(base_model,
                                        evaluator,
                                        strategy=strategy)

        config = RetiariiExeConfig()
        config.execution_engine = 'oneshot'

        if support_or_not:
            experiment.run(config)
            assert isinstance(experiment.export_top_models()[0], dict)
        else:
            with pytest.raises(TypeError, match='not supported'):
                experiment.run(config)
Пример #3
0
def test_oneshot_experiment():
    base_model = Net()
    evaluator = get_mnist_evaluator()
    search_strategy = strategy.RandomOneShot()
    exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy)
    exp_config = RetiariiExeConfig()
    exp_config.execution_engine = 'oneshot'
    exp.run(exp_config)
    assert isinstance(exp.export_top_models()[0], dict)
Пример #4
0
def _main(port):
    base_model = ShuffleNetV2OneShot(32)
    base_predictor = 'cortexA76cpu_tflite21'
    transf = [
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip()
    ]
    normalize = [
        transforms.ToTensor(),
        transforms.Normalize([0.49139968, 0.48215827, 0.44653124],
                             [0.24703233, 0.24348505, 0.26158768])
    ]
    # FIXME
    # CIFAR10 is used here temporarily.
    # Actually we should load weight from supernet and evaluate on imagenet.
    train_dataset = serialize(CIFAR10,
                              'data',
                              train=True,
                              download=True,
                              transform=transforms.Compose(transf + normalize))
    test_dataset = serialize(CIFAR10,
                             'data',
                             train=False,
                             transform=transforms.Compose(normalize))

    trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset,
                                                               batch_size=64),
                                val_dataloaders=pl.DataLoader(test_dataset,
                                                              batch_size=64),
                                max_epochs=2,
                                gpus=1)

    simple_strategy = strategy.RegularizedEvolution(model_filter=LatencyFilter(
        threshold=100, predictor=base_predictor),
                                                    sample_size=1,
                                                    population_size=2,
                                                    cycles=2)
    exp = RetiariiExperiment(base_model, trainer, strategy=simple_strategy)

    exp_config = RetiariiExeConfig('local')
    exp_config.trial_concurrency = 2
    # exp_config.max_trial_number = 2
    exp_config.trial_gpu_number = 1
    exp_config.training_service.use_active_gpu = False
    exp_config.execution_engine = 'base'
    exp_config.dummy_input = [1, 3, 32, 32]

    exp.run(exp_config, port)

    print('Exported models:')
    for i, model in enumerate(exp.export_top_models(formatter='dict')):
        print(model)
        with open(f'architecture_final_{i}.json', 'w') as f:
            json.dump(get_archchoice_by_model(model), f, indent=4)
Пример #5
0
def test_multitrial_experiment(pytestconfig):
    base_model = Net()
    evaluator = get_mnist_evaluator()
    search_strategy = strategy.Random()
    exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy)
    exp_config = RetiariiExeConfig('local')
    exp_config.trial_concurrency = 1
    exp_config.max_trial_number = 1
    exp_config._trial_command_params = nas_experiment_trial_params(pytestconfig.rootpath)
    exp.run(exp_config)
    ensure_success(exp)
    assert isinstance(exp.export_top_models()[0], dict)
    exp.stop()
Пример #6
0
def _test_experiment_in_separate_process(rootpath):
    try:
        base_model, evaluator = _mnist_net('simple', {'max_epochs': 1})
        search_strategy = strategy.Random()
        exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy)
        exp_config = RetiariiExeConfig('local')
        exp_config.experiment_name = 'mnist_unittest'
        exp_config.trial_concurrency = 1
        exp_config.max_trial_number = 1
        exp_config._trial_command_params = nas_experiment_trial_params(rootpath)
        exp.run(exp_config)
        ensure_success(exp)
        assert isinstance(exp.export_top_models()[0], dict)
    finally:
        # https://stackoverflow.com/questions/34506638/how-to-register-atexit-function-in-pythons-multiprocessing-subprocess
        import atexit
        atexit._run_exitfuncs()
Пример #7
0
def test_multi_trial(model, pytestconfig):
    evaluator_kwargs = {
        'max_epochs': 1
    }

    base_model, evaluator = _mnist_net(model, evaluator_kwargs)

    search_strategy = strategy.Random()
    exp = RetiariiExperiment(base_model, evaluator, strategy=search_strategy)
    exp_config = RetiariiExeConfig('local')
    exp_config.experiment_name = 'mnist_unittest'
    exp_config.trial_concurrency = 1
    exp_config.max_trial_number = 1
    exp_config._trial_command_params = nas_experiment_trial_params(pytestconfig.rootpath)
    exp.run(exp_config)
    ensure_success(exp)
    assert isinstance(exp.export_top_models()[0], dict)
    exp.stop()
Пример #8
0
def _test_strategy(strategy_, support_value_choice=True, multi_gpu=False):
    evaluator_kwargs = {'max_epochs': 1}
    if multi_gpu:
        evaluator_kwargs.update(strategy='ddp',
                                accelerator='gpu',
                                devices=torch.cuda.device_count())

    to_test = [
        # (model, evaluator), support_or_net
        (_mnist_net('simple', evaluator_kwargs), True),
        (_mnist_net('simple_value_choice',
                    evaluator_kwargs), support_value_choice),
        (_mnist_net('value_choice', evaluator_kwargs), support_value_choice),
        (_mnist_net('repeat', evaluator_kwargs),
         support_value_choice),  # no strategy supports repeat currently
        (_mnist_net('custom_op',
                    evaluator_kwargs), False),  # this is definitely a NO
        (_multihead_attention_net(evaluator_kwargs), support_value_choice),
    ]

    for (base_model, evaluator), support_or_not in to_test:
        if isinstance(strategy_, BaseStrategy):
            strategy = strategy_
        else:
            strategy = strategy_(base_model, evaluator)
        print('Testing:',
              type(strategy).__name__,
              type(base_model).__name__,
              type(evaluator).__name__, support_or_not)
        experiment = RetiariiExperiment(base_model,
                                        evaluator,
                                        strategy=strategy)

        config = RetiariiExeConfig()
        config.execution_engine = 'oneshot'

        if support_or_not:
            experiment.run(config)
            assert isinstance(experiment.export_top_models()[0], dict)
        else:
            with pytest.raises(TypeError, match='not supported'):
                experiment.run(config)
Пример #9
0
        best_val_ppl
    )  # reports best validation ppl to nni as final result of one trial


if __name__ == "__main__":

    train_iter = WikiText2(split='train')
    tokenizer = get_tokenizer('basic_english')
    vocab = build_vocab_from_iterator(map(tokenizer, train_iter),
                                      specials=['<unk>'])
    vocab.set_default_index(vocab['<unk>'])

    n_token = len(vocab)
    base_model = Transformer(n_token)

    evaluator = FunctionalEvaluator(fit)
    exp = RetiariiExperiment(base_model, evaluator, [], strategy.Random())
    exp_config = RetiariiExeConfig('local')
    exp_config.experiment_name = 'transformer tuning'
    exp_config.trial_concurrency = 3  # please change configurations accordingly
    exp_config.max_trial_number = 25
    exp_config.trial_gpu_number = 1
    exp_config.training_service.use_active_gpu = False
    export_formatter = 'dict'

    exp.run(exp_config, 8081)
    print('Final model:')
    for model_code in exp.export_top_models(optimize_mode='minimize',
                                            formatter=export_formatter):
        print(model_code)
Пример #10
0
                             transform=transform)
    trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset,
                                                               batch_size=100),
                                val_dataloaders=pl.DataLoader(test_dataset,
                                                              batch_size=100),
                                max_epochs=2,
                                gpus=1,
                                limit_train_batches=0.1,
                                limit_val_batches=0.1)

    simple_strategy = strategy.Random()

    exp = RetiariiExperiment(base_model, trainer, [], simple_strategy)

    exp_config = RetiariiExeConfig('local')
    exp_config.experiment_name = 'mnist_search'
    exp_config.trial_concurrency = 2
    exp_config.max_trial_number = args.budget
    exp_config.trial_gpu_number = 1
    exp_config.training_service.use_active_gpu = True  # Integration test GPU has a Xorg running
    export_formatter = 'dict'

    if args.exec == 'graph':
        exp_config.execution_engine = 'base'
        export_formatter = 'code'

    exp.run(exp_config, args.port)
    print('Final model:')
    for model_code in exp.export_top_models(formatter=export_formatter):
        print(model_code)
Пример #11
0
    model = model_cls()
    # dump the model into an onnx
    if 'NNI_OUTPUT_DIR' in os.environ:
        dummy_input = torch.zeros(1, 3, 32, 32)
        torch.onnx.export(model, (dummy_input, ),
                          Path(os.environ['NNI_OUTPUT_DIR']) / 'model.onnx')
    evaluate_model(model_cls)


# %%
# Relaunch the experiment, and a button is shown on Web portal.
#
# .. image:: ../../img/netron_entrance_webui.png
#
# Export Top Models
# -----------------
#
# Users can export top models after the exploration is done using ``export_top_models``.

for model_dict in exp.export_top_models(formatter='dict'):
    print(model_dict)

# %%
# The output is ``json`` object which records the mutation actions of the top model.
# If users want to output source code of the top model,
# they can use :ref:`graph-based execution engine <graph-based-execution-engine>` for the experiment,
# by simply adding the following two lines.

exp_config.execution_engine = 'base'
export_formatter = 'code'
Пример #12
0
                              transform=transform)
    test_dataset = serialize(MNIST,
                             root='data/mnist',
                             train=False,
                             download=True,
                             transform=transform)
    trainer = pl.Classification(train_dataloader=pl.DataLoader(train_dataset,
                                                               batch_size=100),
                                val_dataloaders=pl.DataLoader(test_dataset,
                                                              batch_size=100),
                                max_epochs=2)

    # uncomment the following two lines to debug a generated model
    #debug_mutated_model(base_model, trainer, [])
    #exit(0)

    simple_strategy = strategy.Random()

    exp = RetiariiExperiment(base_model, trainer, [], simple_strategy)

    exp_config = RetiariiExeConfig('local')
    exp_config.experiment_name = 'mnist_search'
    exp_config.trial_concurrency = 2
    exp_config.max_trial_number = 2
    exp_config.training_service.use_active_gpu = False

    exp.run(exp_config, 8081 + random.randint(0, 100))
    print('Final model:')
    for model_code in exp.export_top_models():
        print(model_code)
Пример #13
0
def _main():
    parser = argparse.ArgumentParser("SPOS Evolutional Search")
    parser.add_argument("--port", type=int, default=8084)
    parser.add_argument("--imagenet-dir", type=str, default="./data/imagenet")
    parser.add_argument("--checkpoint",
                        type=str,
                        default="./data/checkpoint-150000.pth.tar")
    parser.add_argument(
        "--spos-preprocessing",
        action="store_true",
        default=False,
        help="When true, image values will range from 0 to 255 and use BGR "
        "(as in original repo).")
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--workers", type=int, default=6)
    parser.add_argument("--train-batch-size", type=int, default=128)
    parser.add_argument("--train-iters", type=int, default=200)
    parser.add_argument("--test-batch-size", type=int, default=512)
    parser.add_argument("--log-frequency", type=int, default=10)
    parser.add_argument("--label-smoothing", type=float, default=0.1)
    parser.add_argument("--evolution-sample-size", type=int, default=10)
    parser.add_argument("--evolution-population-size", type=int, default=50)
    parser.add_argument("--evolution-cycles", type=int, default=10)
    parser.add_argument(
        "--latency-filter",
        type=str,
        default=None,
        help="Apply latency filter by calling the name of the applied hardware."
    )
    parser.add_argument("--latency-threshold", type=float, default=100)

    args = parser.parse_args()

    # use a fixed set of image will improve the performance
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)
    np.random.seed(args.seed)
    random.seed(args.seed)
    torch.backends.cudnn.deterministic = True

    assert torch.cuda.is_available()

    base_model = ShuffleNetV2OneShot()
    criterion = CrossEntropyLabelSmooth(1000, args.label_smoothing)

    if args.latency_filter:
        latency_filter = LatencyFilter(threshold=args.latency_threshold,
                                       predictor=args.latency_filter)
    else:
        latency_filter = None

    evaluator = FunctionalEvaluator(evaluate_acc,
                                    criterion=criterion,
                                    args=args)
    evolution_strategy = strategy.RegularizedEvolution(
        model_filter=latency_filter,
        sample_size=args.evolution_sample_size,
        population_size=args.evolution_population_size,
        cycles=args.evolution_cycles)
    exp = RetiariiExperiment(base_model,
                             evaluator,
                             strategy=evolution_strategy)

    exp_config = RetiariiExeConfig('local')
    exp_config.trial_concurrency = 2
    exp_config.trial_gpu_number = 1
    exp_config.max_trial_number = args.evolution_cycles
    exp_config.training_service.use_active_gpu = False
    exp_config.execution_engine = 'base'
    exp_config.dummy_input = [1, 3, 224, 224]

    exp.run(exp_config, args.port)

    print('Exported models:')
    for i, model in enumerate(exp.export_top_models(formatter='dict')):
        print(model)
        with open(f'architecture_final_{i}.json', 'w') as f:
            json.dump(get_archchoice_by_model(model), f, indent=4)