def __init__(self, config):
super().__init__()
self.config = config
self.batch_size = get_batch_size()
self.train_loader = MultiDatasetLoader("train")
self.val_loader = MultiDatasetLoader("val")
self.test_loader = MultiDatasetLoader("test")
self.train_loader.load(self.config)
self.val_loader.load(self.config)
self.test_loader.load(self.config)
def setUp(self):
torch.manual_seed(1234)
np.random.seed(1234)
self.multi_dataset = MultiDatasetLoader()
self.multi_dataset._num_datasets = 3
self.multi_dataset.current_index = 0
numbers_dataset_a = NumbersDataset(4, "a")
numbers_dataset_b = NumbersDataset(40, "b")
numbers_dataset_c = NumbersDataset(4000, "c")
self.multi_dataset._datasets = [
numbers_dataset_a,
numbers_dataset_b,
numbers_dataset_c,
]
self.multi_dataset._loaders = [
self._get_dataloader(numbers_dataset_a),
self._get_dataloader(numbers_dataset_b),
self._get_dataloader(numbers_dataset_c),
]
self.multi_dataset.current_loader = self.multi_dataset.loaders[0]
self.multi_dataset.config = {
"training": {
"dataset_size_proportional_sampling": True,
"max_epochs": None
}
}
self.multi_dataset._per_dataset_lengths = [4, 40, 4000]
self.multi_dataset._total_length = sum(
self.multi_dataset._per_dataset_lengths)
def setUp(self):
torch.manual_seed(1234)
np.random.seed(1234)
self.multi_dataset = MultiDatasetLoader()
self.multi_dataset._num_datasets = 3
self.multi_dataset.current_index = 0
numbers_dataset_a = NumbersDataset(4, "a")
numbers_dataset_b = NumbersDataset(40, "b")
numbers_dataset_c = NumbersDataset(4000, "c")
self.multi_dataset.dataset_list = ["a", "b", "c"]
self.multi_dataset._loaders = {
"a": self._get_dataloader(numbers_dataset_a),
"b": self._get_dataloader(numbers_dataset_b),
"c": self._get_dataloader(numbers_dataset_c),
}
self.original_config = registry.get("config")
registry.register(
"config",
OmegaConf.create({
"training": {
"dataset_size_proportional_sampling": True,
"max_epochs": None,
}
}),
)
self.multi_dataset._per_dataset_lengths = [4, 40, 4000]
self.multi_dataset._total_length = 4044
self.multi_dataset._total_length = sum(
self.multi_dataset._per_dataset_lengths)
def load_datasets(self):
self.train_dataset = MultiDatasetLoader("train")
self.val_dataset = MultiDatasetLoader("val")
self.test_dataset = MultiDatasetLoader("test")
self.train_dataset.load(self.config)
self.val_dataset.load(self.config)
self.test_dataset.load(self.config)
# If number of datasets is one, this will return the first loader
self.train_loader = self.train_dataset
self.val_loader = self.val_dataset
self.test_loader = self.test_dataset
self.mapping = {
"train": self.train_dataset,
"val": self.val_dataset,
"test": self.test_dataset,
}
self.test_reporter = None
self.should_not_log = self.config.training.should_not_log