def random_validation_split_strategy( validation_size: Union[int, float], shuffle: bool, experience: Experience): """ The default splitting strategy used by :func:`benchmark_with_validation_stream`. This splitting strategy simply splits the experience in two experiences ( train and validation) of size `validation_size`. When taking inspiration for your custom splitting strategy, please consider that all parameters preceding `experience` are filled by :func:`benchmark_with_validation_stream` by using `partial` from the `functools` standard library. A custom splitting strategy must have only a single parameter: the experience. Consider wrapping your custom splitting strategy with `partial` if more parameters are needed. Also consider that the stream name of the experience can be obtained by using `experience.origin_stream.name`. :param validation_size: The number of instances to allocate to the validation experience. Can be an int value or a float between 0 and 1. :param shuffle: If True, instances will be shuffled before splitting. Otherwise, the first instances will be allocated to the training dataset by leaving the last ones to the validation dataset. :param experience: The experience to split. :return: A tuple containing 2 elements: the new training and validation datasets. """ exp_dataset = experience.dataset exp_indices = list(range(len(exp_dataset))) if shuffle: exp_indices = \ torch.as_tensor(exp_indices)[ torch.randperm(len(exp_indices)) ].tolist() if 0.0 <= validation_size <= 1.0: valid_n_instances = int(validation_size * len(exp_dataset)) else: valid_n_instances = int(validation_size) if valid_n_instances > len(exp_dataset): raise ValueError( f'Can\'t create the validation experience: nott enough ' f'instances. Required {valid_n_instances}, got only' f'{len(exp_dataset)}') train_n_instances = len(exp_dataset) - valid_n_instances result_train_dataset = AvalancheSubset( exp_dataset, indices=exp_indices[:train_n_instances]) result_valid_dataset = AvalancheSubset( exp_dataset, indices=exp_indices[train_n_instances:]) return result_train_dataset, result_valid_dataset
def create_sub_experience_list(experience): """Creates a list of sub-experiences from an experience. It returns a list of experiences, where each experience is a subset of the original experience. :param experience: single Experience. :return: list of Experience. """ # Shuffle the indices indices = torch.randperm(len(experience.dataset)) num_sub_exps = len(indices) mb_size = 1 sub_experience_list = [] for subexp_id in range(num_sub_exps): subexp_indices = indices[subexp_id * mb_size:(subexp_id + 1) * mb_size] sub_experience = copy.copy(experience) subexp_ds = AvalancheSubset(sub_experience.dataset, indices=subexp_indices) sub_experience.dataset = subexp_ds sub_experience_list.append(sub_experience) return sub_experience_list
def class_balanced_split_strategy(validation_size: Union[int, float], experience: Experience): """Class-balanced train/validation splits. This splitting strategy splits `experience` into two experiences (train and validation) of size `validation_size` using a class-balanced split. Sample of each class are chosen randomly. :param validation_size: The percentage of samples to allocate to the validation experience as a float between 0 and 1. :param experience: The experience to split. :return: A tuple containing 2 elements: the new training and validation datasets. """ if not isinstance(validation_size, float): raise ValueError("validation_size must be an integer") if not 0.0 <= validation_size <= 1.0: raise ValueError("validation_size must be a float in [0, 1].") exp_dataset = experience.dataset if validation_size > len(exp_dataset): raise ValueError(f"Can't create the validation experience: not enough " f"instances. Required {validation_size}, got only" f"{len(exp_dataset)}") exp_indices = list(range(len(exp_dataset))) exp_classes = experience.classes_in_this_experience # shuffle exp_indices exp_indices = torch.as_tensor(exp_indices)[torch.randperm( len(exp_indices))] # shuffle the targets as well exp_targets = torch.as_tensor(experience.dataset.targets)[exp_indices] train_exp_indices = [] valid_exp_indices = [] for cid in exp_classes: # split indices for each class separately. c_indices = exp_indices[exp_targets == cid] valid_n_instances = int(validation_size * len(c_indices)) valid_exp_indices.extend(c_indices[:valid_n_instances]) train_exp_indices.extend(c_indices[valid_n_instances:]) result_train_dataset = AvalancheSubset(exp_dataset, indices=train_exp_indices) result_valid_dataset = AvalancheSubset(exp_dataset, indices=valid_exp_indices) return result_train_dataset, result_valid_dataset
def fixed_size_experience_split_strategy( experience_size: int, shuffle: bool, drop_last: bool, experience: ClassificationExperience, ): """ The default splitting strategy used by :func:`data_incremental_benchmark`. This splitting strategy simply splits the experience in smaller experiences of size `experience_size`. When taking inspiration for your custom splitting strategy, please consider that all parameters preceding `experience` are filled by :func:`data_incremental_benchmark` by using `partial` from the `functools` standard library. A custom splitting strategy must have only a single parameter: the experience. Consider wrapping your custom splitting strategy with `partial` if more parameters are needed. Also consider that the stream name of the experience can be obtained by using `experience.origin_stream.name`. :param experience_size: The experience size (number of instances). :param shuffle: If True, instances will be shuffled before splitting. :param drop_last: If True, the last mini-experience will be dropped if not of size `experience_size` :param experience: The experience to split. :return: The list of datasets that will be used to create the mini-experiences. """ exp_dataset = experience.dataset exp_indices = list(range(len(exp_dataset))) result_datasets = [] if shuffle: exp_indices = torch.as_tensor(exp_indices)[torch.randperm( len(exp_indices))].tolist() init_idx = 0 while init_idx < len(exp_indices): final_idx = init_idx + experience_size # Exclusive if final_idx > len(exp_indices): if drop_last: break final_idx = len(exp_indices) result_datasets.append( AvalancheSubset(exp_dataset, indices=exp_indices[init_idx:final_idx])) init_idx = final_idx return result_datasets