Esempio n. 1
0
    def _test_scheduler_plugin(self, gamma, milestones, base_lr, epochs,
                               reset_lr, reset_scheduler, expected):
        class TestPlugin(StrategyPlugin):
            def __init__(self, expected_lrs):
                super().__init__()
                self.expected_lrs = expected_lrs

            def after_training_epoch(self, strategy, **kwargs):
                exp_id = strategy.training_exp_counter

                expected_lr = self.expected_lrs[exp_id][strategy.epoch]
                for group in strategy.optimizer.param_groups:
                    assert group['lr'] == expected_lr

        scenario = self.create_scenario()
        model = SimpleMLP(input_size=6, hidden_size=10)

        optim = SGD(model.parameters(), lr=base_lr)
        lrSchedulerPlugin = LRSchedulerPlugin(MultiStepLR(
            optim, milestones=milestones, gamma=gamma),
                                              reset_lr=reset_lr,
                                              reset_scheduler=reset_scheduler)

        cl_strategy = Naive(model,
                            optim,
                            CrossEntropyLoss(),
                            train_mb_size=32,
                            train_epochs=epochs,
                            eval_mb_size=100,
                            plugins=[lrSchedulerPlugin,
                                     TestPlugin(expected)])

        cl_strategy.train(scenario.train_stream[0])
        cl_strategy.train(scenario.train_stream[1])
Esempio n. 2
0
    def _test_scheduler_plugin(
        benchmark,
        model,
        optim,
        scheduler,
        epochs,
        reset_lr,
        reset_scheduler,
        expected,
        criterion=None,
        metric=None,
        eval_on_valid_stream=False,
    ):
        lr_scheduler_plugin = LRSchedulerPlugin(
            scheduler,
            reset_lr=reset_lr,
            reset_scheduler=reset_scheduler,
            metric=metric,
        )

        verifier_plugin = SchedulerPluginTestPlugin(expected)

        if criterion is None:
            criterion = CrossEntropyLoss()
        if eval_on_valid_stream:
            cl_strategy = Naive(
                model,
                optim,
                criterion,
                train_mb_size=32,
                train_epochs=epochs,
                eval_mb_size=100,
                plugins=[lr_scheduler_plugin, verifier_plugin],
                eval_every=1,
                evaluator=None,
            )

            cl_strategy.train(
                benchmark.train_stream[0],
                shuffle=False,
                eval_streams=[benchmark.valid_stream[0]],
            )
            cl_strategy.train(
                benchmark.train_stream[1],
                shuffle=False,
                eval_streams=[benchmark.valid_stream[1]],
            )
        else:
            cl_strategy = Naive(
                model,
                optim,
                criterion,
                train_mb_size=32,
                train_epochs=epochs,
                eval_mb_size=100,
                plugins=[lr_scheduler_plugin, verifier_plugin],
                evaluator=None,
            )

            cl_strategy.train(benchmark.train_stream[0], shuffle=False)
            cl_strategy.train(benchmark.train_stream[1], shuffle=False)
Esempio n. 3
0
    def test_scheduler_reduce_on_plateau_plugin(self):
        # Regression test for issue #858
        n_epochs = 20
        criterion = CrossEntropyLoss()

        def _prepare_rng_critical_parts(seed=1234):
            torch.random.manual_seed(seed)
            return (
                PluginTests.create_benchmark(seed=seed),
                _PlainMLP(input_size=6, hidden_size=10),
            )

        self._verify_rop_tests_reproducibility(
            _prepare_rng_critical_parts, n_epochs, criterion
        )

        # Everything is in order, now we can test the plugin support for the
        # ReduceLROnPlateau scheduler!

        for reset_lr, reset_scheduler in itertools.product(
            (True, False), (True, False)
        ):
            with self.subTest(
                reset_lr=reset_lr, reset_scheduler=reset_scheduler
            ):
                # First, obtain the reference (expected) lr timeline by running
                # a plain PyTorch training loop with ReduceLROnPlateau.
                benchmark, model = _prepare_rng_critical_parts()
                model.train()
                expected_lrs = []

                optimizer = SGD(model.parameters(), lr=0.001)
                scheduler = ReduceLROnPlateau(optimizer)
                for exp in benchmark.train_stream:
                    if reset_lr:
                        for group in optimizer.param_groups:
                            group["lr"] = 0.001

                    if reset_scheduler:
                        scheduler = ReduceLROnPlateau(optimizer)

                    expected_lrs.append([])
                    train_loss = Mean()
                    for epoch in range(n_epochs):
                        train_loss.reset()
                        for x, y, t in TaskBalancedDataLoader(
                            exp.dataset,
                            oversample_small_groups=True,
                            num_workers=0,
                            batch_size=32,
                            shuffle=False,
                            pin_memory=False,
                        ):
                            optimizer.zero_grad()
                            outputs = model(x)
                            loss = criterion(outputs, y)
                            train_loss.update(loss, weight=len(x))
                            loss.backward()
                            optimizer.step()
                        scheduler.step(train_loss.result())
                        for group in optimizer.param_groups:
                            expected_lrs[-1].append(group["lr"])
                            break

                # Now we have the correct timeline stored in expected_lrs.
                # Let's test the plugin!
                benchmark, model = _prepare_rng_critical_parts()
                optimizer = SGD(model.parameters(), lr=0.001)
                scheduler = ReduceLROnPlateau(optimizer)

                PluginTests._test_scheduler_plugin(
                    benchmark,
                    model,
                    optimizer,
                    scheduler,
                    n_epochs,
                    reset_lr,
                    reset_scheduler,
                    expected_lrs,
                    criterion=criterion,
                    metric="train_loss",
                )

        # Other tests
        benchmark, model = _prepare_rng_critical_parts()
        optimizer = SGD(model.parameters(), lr=0.001)
        scheduler = ReduceLROnPlateau(optimizer)
        scheduler2 = MultiStepLR(optimizer, [1, 2, 3])

        # The metric must be set
        with self.assertRaises(Exception):
            LRSchedulerPlugin(scheduler, metric=None)

        # Doesn't make sense to set the metric when using a non-metric
        # based scheduler (should warn)
        with self.assertWarns(Warning):
            LRSchedulerPlugin(scheduler2, metric="train_loss")

        # Must raise an error on unsupported metric
        with self.assertRaises(Exception):
            LRSchedulerPlugin(scheduler, metric="cuteness")
Esempio n. 4
0
def run_experiment(config):
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    torch.manual_seed(config.seed)
    torch.cuda.manual_seed(config.seed)
    np.random.seed(config.seed)
    random.seed(config.seed)
    torch.backends.cudnn.enabled = False
    torch.backends.cudnn.deterministic = True

    per_pixel_mean = get_dataset_per_pixel_mean(
        CIFAR100(
            expanduser("~") + "/.avalanche/data/cifar100/",
            train=True,
            download=True,
            transform=transforms.Compose([transforms.ToTensor()]),
        )
    )

    transforms_group = dict(
        eval=(
            transforms.Compose(
                [
                    transforms.ToTensor(),
                    lambda img_pattern: img_pattern - per_pixel_mean,
                ]
            ),
            None,
        ),
        train=(
            transforms.Compose(
                [
                    transforms.ToTensor(),
                    lambda img_pattern: img_pattern - per_pixel_mean,
                    icarl_cifar100_augment_data,
                ]
            ),
            None,
        ),
    )

    train_set = CIFAR100(
        expanduser("~") + "/.avalanche/data/cifar100/",
        train=True,
        download=True,
    )
    test_set = CIFAR100(
        expanduser("~") + "/.avalanche/data/cifar100/",
        train=False,
        download=True,
    )

    train_set = AvalancheDataset(
        train_set,
        transform_groups=transforms_group,
        initial_transform_group="train",
    )
    test_set = AvalancheDataset(
        test_set,
        transform_groups=transforms_group,
        initial_transform_group="eval",
    )

    scenario = nc_benchmark(
        train_dataset=train_set,
        test_dataset=test_set,
        n_experiences=config.nb_exp,
        task_labels=False,
        seed=config.seed,
        shuffle=False,
        fixed_class_order=config.fixed_class_order,
    )

    evaluator = EvaluationPlugin(
        EpochAccuracy(),
        ExperienceAccuracy(),
        StreamAccuracy(),
        loggers=[InteractiveLogger()],
    )

    model: IcarlNet = make_icarl_net(num_classes=100)
    model.apply(initialize_icarl_net)

    optim = SGD(
        model.parameters(),
        lr=config.lr_base,
        weight_decay=config.wght_decay,
        momentum=0.9,
    )
    sched = LRSchedulerPlugin(
        MultiStepLR(optim, config.lr_milestones, gamma=1.0 / config.lr_factor)
    )

    strategy = ICaRL(
        model.feature_extractor,
        model.classifier,
        optim,
        config.memory_size,
        buffer_transform=transforms.Compose([icarl_cifar100_augment_data]),
        fixed_memory=True,
        train_mb_size=config.batch_size,
        train_epochs=config.epochs,
        eval_mb_size=config.batch_size,
        plugins=[sched],
        device=device,
        evaluator=evaluator,
    )

    for i, exp in enumerate(scenario.train_stream):
        eval_exps = [e for e in scenario.test_stream][: i + 1]
        strategy.train(exp, num_workers=4)
        strategy.eval(eval_exps, num_workers=4)
Esempio n. 5
0
            model = model.cuda()
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

        optimizer = SGD(
            model.parameters(),
            lr=HYPER_PARAMETER[mode]["start_lr"],
            weight_decay=HYPER_PARAMETER[mode]["weight_decay"],
            momentum=HYPER_PARAMETER[mode]["momentum"],
        )
        scheduler = make_scheduler(
            optimizer,
            HYPER_PARAMETER[mode]["step_schedular_decay"],
            HYPER_PARAMETER[mode]["schedular_step"],
        )

        plugin_list = [LRSchedulerPlugin(scheduler)]
        cl_strategy = Naive(
            model,
            optimizer,
            CrossEntropyLoss(),
            train_mb_size=HYPER_PARAMETER[mode]["batch_size"],
            train_epochs=num_epoch,
            eval_mb_size=HYPER_PARAMETER[mode]["batch_size"],
            evaluator=eval_plugin,
            device=device,
            plugins=plugin_list,
        )

        # TRAINING LOOP
        print("Starting experiment...")
        results = []
Esempio n. 6
0
    def _test_scheduler_plugin(benchmark,
                               model,
                               optim,
                               scheduler,
                               epochs,
                               reset_lr,
                               reset_scheduler,
                               expected,
                               criterion=None,
                               metric=None,
                               eval_on_valid_stream=False,
                               granularity='epoch',
                               expected_granularity='iteration',
                               peval_mode='epoch',
                               first_epoch_only=False,
                               first_exp_only=False,
                               max_exps=None):
        lr_scheduler_plugin = LRSchedulerPlugin(
            scheduler,
            reset_lr=reset_lr,
            reset_scheduler=reset_scheduler,
            metric=metric,
            step_granularity=granularity,
            first_epoch_only=first_epoch_only,
            first_exp_only=first_exp_only)

        verifier_plugin = SchedulerPluginTestPlugin(
            expected, expected_granularity=expected_granularity)

        if criterion is None:
            criterion = CrossEntropyLoss()
        if eval_on_valid_stream:
            cl_strategy = Naive(
                model,
                optim,
                criterion,
                train_mb_size=32,
                train_epochs=epochs,
                eval_mb_size=100,
                plugins=[lr_scheduler_plugin, verifier_plugin],
                eval_every=1,
                peval_mode=peval_mode,
                evaluator=None,
            )

            for exp_id in range(len(benchmark.train_stream)):
                if max_exps is not None and exp_id >= max_exps:
                    break
                cl_strategy.train(
                    benchmark.train_stream[exp_id],
                    shuffle=False,
                    eval_streams=[benchmark.valid_stream[exp_id]],
                )
            # cl_strategy.train(
            #     benchmark.train_stream[1],
            #     shuffle=False,
            #     eval_streams=[benchmark.valid_stream[1]],
            # )
        else:
            cl_strategy = Naive(
                model,
                optim,
                criterion,
                train_mb_size=32,
                train_epochs=epochs,
                eval_mb_size=100,
                plugins=[lr_scheduler_plugin, verifier_plugin],
                peval_mode=peval_mode,
                evaluator=None,
            )

            for exp_id in range(len(benchmark.train_stream)):
                if max_exps is not None and exp_id >= max_exps:
                    break
                cl_strategy.train(benchmark.train_stream[exp_id],
                                  shuffle=False)