def test_transformation_concat_dataset_groups(self):
original_dataset = AvalancheDataset(MNIST('./data/mnist',
download=True),
transform_groups=dict(
eval=(None, None),
train=(ToTensor(), None)))
original_dataset2 = AvalancheDataset(MNIST('./data/mnist',
download=True),
transform_groups=dict(
train=(None, None),
eval=(ToTensor(), None)))
dataset = AvalancheConcatDataset([original_dataset, original_dataset2])
self.assertEqual(
len(original_dataset) + len(original_dataset2), len(dataset))
x, *_ = dataset[0]
x2, *_ = dataset[len(original_dataset)]
self.assertIsInstance(x, Tensor)
self.assertIsInstance(x2, Image)
dataset = dataset.eval()
x3, *_ = dataset[0]
x4, *_ = dataset[len(original_dataset)]
self.assertIsInstance(x3, Image)
self.assertIsInstance(x4, Tensor)
def __call__(self, strategy, **kwargs):
curr_task_id = strategy.experience.task_label
curr_data = strategy.experience.dataset
# how many experiences to divide the memory over
if self.adaptive_size: # Observed number of experiences
exp_cnt = (strategy.training_exp_counter + 1)
else: # Fixed number of experiences
exp_cnt = self.num_experiences
exp_mem_size = self.mem_size // exp_cnt
# Add current experience data to memory
if curr_task_id not in self.ext_mem:
self.ext_mem[curr_task_id] = AvalancheDataset(curr_data)
else: # Merge data with previous seen data
self.ext_mem[curr_task_id] = AvalancheConcatDataset(
[curr_data, self.ext_mem[curr_task_id]])
# Distribute remaining samples using counts
cutoff_per_exp = self.divide_remaining_samples(
exp_mem_size, strategy.training_exp_counter + 1)
# Use counts to remove samples from memory
self.cutoff_memory(cutoff_per_exp)
def test_dataload_batch_balancing(self):
scenario = get_fast_scenario()
model = SimpleMLP(input_size=6, hidden_size=10)
batch_size = 32
replayPlugin = ReplayPlugin(mem_size=20)
cl_strategy = Naive(model,
SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.001),
CrossEntropyLoss(),
train_mb_size=batch_size,
train_epochs=1,
eval_mb_size=100,
plugins=[replayPlugin])
for step in scenario.train_stream:
adapted_dataset = step.dataset
dataloader = MultiTaskJoinedBatchDataLoader(
adapted_dataset,
AvalancheConcatDataset(replayPlugin.ext_mem.values()),
oversample_small_tasks=True,
num_workers=0,
batch_size=batch_size,
shuffle=True)
for mini_batch in dataloader:
lengths = []
for task_id in mini_batch.keys():
lengths.append(len(mini_batch[task_id][1]))
if sum(lengths) == batch_size:
difference = max(lengths) - min(lengths)
self.assertLessEqual(difference, 1)
self.assertLessEqual(sum(lengths), batch_size)
cl_strategy.train(step)
def __init__(self, max_size: int):
"""Init.
:param max_size: max number of input samples in the replay memory.
"""
self.max_size = max_size
""" Maximum size of the buffer. """
self._buffer: AvalancheDataset = AvalancheConcatDataset([])
def train_dataset_adaptation(self, **kwargs):
"""Concatenates all the datastream."""
self.adapted_dataset = self._experiences[0].dataset
for exp in self._experiences[1:]:
cat_data = AvalancheConcatDataset(
[self.adapted_dataset, exp.dataset])
self.adapted_dataset = cat_data
self.adapted_dataset = self.adapted_dataset.train()
def test_transformation_concat_dataset(self):
original_dataset = MNIST('./data/mnist', download=True)
original_dataset2 = MNIST('./data/mnist', download=True)
dataset = AvalancheConcatDataset([original_dataset, original_dataset2])
self.assertEqual(
len(original_dataset) + len(original_dataset2), len(dataset))
def after_train_dataset_adaptation(self, strategy: 'BaseStrategy',
**kwargs):
if strategy.training_exp_counter != 0:
memory = AvalancheTensorDataset(
torch.cat(self.x_memory).cpu(),
list(itertools.chain.from_iterable(self.y_memory)),
transform=self.buffer_transform, target_transform=None)
strategy.adapted_dataset = \
AvalancheConcatDataset((strategy.adapted_dataset, memory))
def train_dataset_adaptation(self, **kwargs):
"""
Concatenates all the previous experiences.
"""
if self.dataset is None:
self.dataset = self.experience.dataset
else:
self.dataset = AvalancheConcatDataset(
[self.dataset, self.experience.dataset])
self.adapted_dataset = self.dataset
def after_train_dataset_adaptation(self, strategy: 'BaseStrategy',
**kwargs):
""" Before training we make sure to organize the memory following
GDumb approach and updating the dataset accordingly.
"""
# for each pattern, add it to the memory or not
dataset = strategy.experience.dataset
current_counter = self.counter[strategy.experience.task_label]
current_mem = self.ext_mem[strategy.experience.task_label]
for i, (pattern, target_value, _) in enumerate(dataset):
target = torch.tensor(target_value)
if len(pattern.size()) == 1:
pattern = pattern.unsqueeze(0)
if current_counter == {}:
# any positive (>0) number is ok
patterns_per_class = 1
else:
patterns_per_class = int(self.mem_size /
len(current_counter.keys()))
if target_value not in current_counter or \
current_counter[target_value] < patterns_per_class:
# add new pattern into memory
if sum(current_counter.values()) >= self.mem_size:
# full memory: replace item from most represented class
# with current pattern
to_remove = max(current_counter, key=current_counter.get)
for j in range(len(current_mem.tensors[1])):
if current_mem.tensors[1][j].item() == to_remove:
current_mem.tensors[0][j] = pattern
current_mem.tensors[1][j] = target
break
current_counter[to_remove] -= 1
else:
# memory not full: add new pattern
if current_mem is None:
current_mem = TensorDataset(pattern,
target.unsqueeze(0))
else:
current_mem = TensorDataset(
torch.cat([pattern, current_mem.tensors[0]],
dim=0),
torch.cat(
[target.unsqueeze(0), current_mem.tensors[1]],
dim=0))
current_counter[target_value] += 1
self.ext_mem[strategy.experience.task_label] = current_mem
strategy.adapted_dataset = AvalancheConcatDataset(
self.ext_mem.values())
def after_train_dataset_adaptation(self, strategy: "SupervisedTemplate",
**kwargs):
if strategy.clock.train_exp_counter != 0:
memory = AvalancheTensorDataset(
torch.cat(self.x_memory).cpu(),
list(itertools.chain.from_iterable(self.y_memory)),
transform=self.buffer_transform,
target_transform=None,
)
strategy.adapted_dataset = AvalancheConcatDataset(
(strategy.adapted_dataset, memory))
def test_basic(self):
scenario = get_fast_scenario()
ds = [el.dataset for el in scenario.train_stream]
data = AvalancheConcatDataset(ds)
dl = TaskBalancedDataLoader(data)
for el in dl:
pass
dl = GroupBalancedDataLoader(ds)
for el in dl:
pass
dl = ReplayDataLoader(data, data)
for el in dl:
pass
def before_training_exp(self, strategy, num_workers=0, shuffle=True,
**kwargs):
"""
Dataloader to build batches containing examples from both memories and
the training dataset
"""
if len(self.ext_mem) == 0:
return
strategy.dataloader = ReplayDataLoader(
strategy.adapted_dataset,
AvalancheConcatDataset(self.ext_mem.values()),
oversample_small_tasks=True,
num_workers=num_workers,
batch_size=strategy.train_mb_size,
shuffle=shuffle)
def test_basic(self):
benchmark = get_fast_benchmark()
ds = [el.dataset for el in benchmark.train_stream]
data = AvalancheConcatDataset(ds)
dl = TaskBalancedDataLoader(data)
for el in dl:
pass
dl = GroupBalancedDataLoader(ds)
for el in dl:
pass
dl = ReplayDataLoader(data, data)
for el in dl:
pass
def update_from_dataset(self, new_data: AvalancheDataset):
"""Update the buffer using the given dataset.
:param new_data:
:return:
"""
new_weights = torch.rand(len(new_data))
cat_weights = torch.cat([new_weights, self._buffer_weights])
cat_data = AvalancheConcatDataset([new_data, self.buffer])
sorted_weights, sorted_idxs = cat_weights.sort(descending=True)
buffer_idxs = sorted_idxs[:self.max_size]
self.buffer = AvalancheSubset(cat_data, buffer_idxs)
self._buffer_weights = sorted_weights[:self.max_size]
def after_training_exp(self, strategy, **kwargs):
""" After training we update the external memory with the patterns of
the current training batch/task, adding the patterns from the new
experience and removing those from past experiences to comply the limit
of the total number of patterns in memory """
curr_task_id = strategy.experience.task_label
curr_data = strategy.experience.dataset
# Additional set of the current batch to be concatenated
# to the external memory.
rm_add = None
# how many patterns to save for next iter
single_task_mem_size = min(self.mem_size, len(curr_data))
h = single_task_mem_size // (strategy.training_exp_counter + 1)
remaining_example = single_task_mem_size % (
strategy.training_exp_counter + 1)
# We recover it using the random_split method and getting rid of the
# second split.
rm_add, _ = random_split(curr_data, [h, len(curr_data) - h])
# replace patterns randomly in memory
ext_mem = self.ext_mem
if curr_task_id not in ext_mem:
ext_mem[curr_task_id] = rm_add
else:
rem_len = len(ext_mem[curr_task_id]) - len(rm_add)
_, saved_part = random_split(ext_mem[curr_task_id],
[len(rm_add), rem_len])
ext_mem[curr_task_id] = AvalancheConcatDataset(
[saved_part, rm_add])
# remove exceeding patterns, the amount of pattern kept is such that the
# sum is equal to mem_size and that the number of patterns between the
# tasks is balanced
for task_id in ext_mem.keys():
current_mem_size = h if remaining_example <= 0 else h + 1
remaining_example -= 1
if (current_mem_size < len(ext_mem[task_id])
and task_id != curr_task_id):
rem_len = len(ext_mem[task_id]) - current_mem_size
_, saved_part = random_split(ext_mem[task_id],
[rem_len, current_mem_size])
ext_mem[task_id] = saved_part
self.ext_mem = ext_mem
def before_training_exp(self,
strategy,
num_workers=0,
shuffle=True,
**kwargs):
"""
Random retrieval from a class-balanced memory.
Dataloader builds batches containing examples from both memories and
the training dataset.
"""
if len(self.replay_mem) == 0:
return
strategy.dataloader = ReplayDataLoader(
strategy.adapted_dataset,
AvalancheConcatDataset(self.replay_mem.values()),
oversample_small_tasks=True,
num_workers=num_workers,
batch_size=strategy.train_mb_size * 2,
shuffle=shuffle)
def before_training_exp(self,
strategy,
num_workers=0,
shuffle=True,
**kwargs):
"""
Random retrieval from a class-balanced memory.
Dataloader builds batches containing examples from both memories and
the training dataset.
This implementation requires the use of early stopping, otherwise the
entire memory will be iterated.
"""
if len(self.replay_mem) == 0:
return
self.it_cnt = 0
strategy.dataloader = ReplayDataLoader(
strategy.adapted_dataset,
AvalancheConcatDataset(self.replay_mem.values()),
oversample_small_tasks=False,
num_workers=num_workers,
batch_size=strategy.train_mb_size * 2,
force_data_batch_size=strategy.train_mb_size,
shuffle=shuffle)
def test_dataload_batch_balancing(self):
benchmark = get_fast_benchmark()
batch_size = 32
replayPlugin = ReplayPlugin(mem_size=20)
model = SimpleMLP(input_size=6, hidden_size=10)
cl_strategy = Naive(model,
SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.001),
CrossEntropyLoss(),
train_mb_size=batch_size,
train_epochs=1,
eval_mb_size=100,
plugins=[replayPlugin])
for step in benchmark.train_stream:
adapted_dataset = step.dataset
dataloader = ReplayDataLoader(adapted_dataset,
AvalancheConcatDataset(
replayPlugin.ext_mem.values()),
oversample_small_tasks=True,
num_workers=0,
batch_size=batch_size,
shuffle=True)
for mini_batch in dataloader:
mb_task_labels = mini_batch[-1]
lengths = []
for task_id in adapted_dataset.task_set:
len_task = (mb_task_labels == task_id).sum()
lengths.append(len_task)
if sum(lengths) == batch_size:
difference = max(lengths) - min(lengths)
self.assertLessEqual(difference, 1)
self.assertLessEqual(sum(lengths), batch_size)
cl_strategy.train(step)
def __call__(self, strategy, **kwargs):
new_data = strategy.experience.dataset
# Get sample idxs per class
cl_idxs = {}
for idx, target in enumerate(new_data.targets):
if target not in cl_idxs:
cl_idxs[target] = []
cl_idxs[target].append(idx)
# Make AvalancheSubset per class
cl_datasets = {}
for c, c_idxs in cl_idxs.items():
cl_datasets[c] = AvalancheSubset(new_data, indices=c_idxs)
# Update seen classes
self.seen_classes.update(cl_datasets.keys())
# how many experiences to divide the memory over
div_cnt = len(self.seen_classes) if self.adaptive_size \
else self.total_num_classes
class_mem_size = self.mem_size // div_cnt
# Add current classes data to memory
for c, c_mem in cl_datasets.items():
if c not in self.ext_mem:
self.ext_mem[c] = c_mem
else: # Merge data with previous seen data
self.ext_mem[c] = AvalancheConcatDataset(
[c_mem, self.ext_mem[c]])
# Distribute remaining samples using counts
cutoff_per_exp = self.divide_remaining_samples(class_mem_size, div_cnt)
# Use counts to remove samples from memory
self.cutoff_memory(cutoff_per_exp)
def dataset(self) -> AvalancheDataset:
return AvalancheConcatDataset(list(self.class_id2dataset.values()))
def buffer(self):
return AvalancheConcatDataset(
[g.buffer for g in self.buffer_groups.values()])
def update_from_dataset(self, strategy, new_data):
self.buffer = AvalancheConcatDataset([self.buffer, new_data])
self.resize(strategy, self.max_size)
def test_avalanche_concat_dataset_collate_fn_inheritance(self):
tensor_x = torch.rand(200, 3, 28, 28)
tensor_y = torch.randint(0, 100, (200, ))
tensor_z = torch.randint(0, 100, (200, ))
tensor_x2 = torch.rand(200, 3, 28, 28)
tensor_y2 = torch.randint(0, 100, (200, ))
tensor_z2 = torch.randint(0, 100, (200, ))
def my_collate_fn(patterns):
x_values = torch.stack([pat[0] for pat in patterns], 0)
y_values = torch.tensor([pat[1] for pat in patterns]) + 1
z_values = torch.tensor([-1 for _ in patterns])
t_values = torch.tensor([pat[3] for pat in patterns])
return x_values, y_values, z_values, t_values
def my_collate_fn2(patterns):
x_values = torch.stack([pat[0] for pat in patterns], 0)
y_values = torch.tensor([pat[1] for pat in patterns]) + 2
z_values = torch.tensor([-2 for _ in patterns])
t_values = torch.tensor([pat[3] for pat in patterns])
return x_values, y_values, z_values, t_values
dataset1 = TensorDataset(tensor_x, tensor_y, tensor_z)
dataset2 = AvalancheTensorDataset(tensor_x2,
tensor_y2,
tensor_z2,
collate_fn=my_collate_fn)
concat = AvalancheConcatDataset([dataset1, dataset2],
collate_fn=my_collate_fn2) # Ok
x, y, z, t = dataset2[0:5]
self.assertIsInstance(x, Tensor)
self.assertTrue(torch.equal(tensor_x2[0:5], x))
self.assertTrue(torch.equal(tensor_y2[0:5] + 1, y))
self.assertTrue(torch.equal(torch.full((5, ), -1, dtype=torch.long),
z))
self.assertTrue(torch.equal(torch.zeros(5, dtype=torch.long), t))
x2, y2, z2, t2 = concat[0:5]
self.assertIsInstance(x2, Tensor)
self.assertTrue(torch.equal(tensor_x[0:5], x2))
self.assertTrue(torch.equal(tensor_y[0:5] + 2, y2))
self.assertTrue(
torch.equal(torch.full((5, ), -2, dtype=torch.long), z2))
self.assertTrue(torch.equal(torch.zeros(5, dtype=torch.long), t2))
dataset1_classification = AvalancheTensorDataset(
tensor_x,
tensor_y,
tensor_z,
dataset_type=AvalancheDatasetType.CLASSIFICATION)
dataset2_segmentation = AvalancheDataset(
dataset2, dataset_type=AvalancheDatasetType.SEGMENTATION)
with self.assertRaises(ValueError):
bad_concat_types = dataset1_classification + dataset2_segmentation
with self.assertRaises(ValueError):
bad_concat_collate = AvalancheConcatDataset(
[dataset1, dataset2_segmentation], collate_fn=my_collate_fn)
ok_concat_classification = dataset1_classification + dataset2
self.assertEqual(AvalancheDatasetType.CLASSIFICATION,
ok_concat_classification.dataset_type)
ok_concat_classification2 = dataset2 + dataset1_classification
self.assertEqual(AvalancheDatasetType.CLASSIFICATION,
ok_concat_classification2.dataset_type)
def after_train_dataset_adaptation(self, strategy: BaseStrategy, **kwargs):
""" Before training we make sure to organize the memory following
GDumb approach and updating the dataset accordingly.
"""
# for each pattern, add it to the memory or not
dataset = strategy.experience.dataset
pbar = tqdm.tqdm(dataset,
desc="Exhausting dataset to create GDumb buffer")
for pattern, target, task_id in pbar:
target = torch.as_tensor(target)
target_value = target.item()
if len(pattern.size()) == 1:
pattern = pattern.unsqueeze(0)
current_counter = self.counter.setdefault(task_id,
defaultdict(int))
current_mem = self.ext_mem.setdefault(task_id, ([], []))
if current_counter == {}:
# any positive (>0) number is ok
patterns_per_class = 1
else:
patterns_per_class = int(self.mem_size /
len(current_counter.keys()))
if (target_value not in current_counter
or current_counter[target_value] < patterns_per_class):
# add new pattern into memory
if sum(current_counter.values()) >= self.mem_size:
# full memory: replace item from most represented class
# with current pattern
to_remove = max(current_counter, key=current_counter.get)
# dataset_size = len(current_mem)
# for j in range(dataset_size):
# if current_mem.tensors[1][j].item() == to_remove:
# current_mem.tensors[0][j] = pattern
# current_mem.tensors[1][j] = target
# break
dataset_size = len(current_mem[0])
for j in range(dataset_size):
if current_mem[1][j].item() == to_remove:
current_mem[0][j] = pattern
current_mem[1][j] = target
break
current_counter[to_remove] -= 1
else:
# memory not full: add new pattern
current_mem[0].append(pattern)
current_mem[1].append(target)
# Indicate that we've changed the number of stored instances of this
# class.
current_counter[target_value] += 1
task_datasets: Dict[Any, TensorDataset] = {}
for task_id, task_mem_tuple in self.ext_mem.items():
patterns, targets = task_mem_tuple
task_dataset = TensorDataset(torch.stack(patterns, dim=0),
torch.stack(targets, dim=0))
task_datasets[task_id] = task_dataset
logger.debug(
f"There are {len(task_dataset)} entries from task {task_id} in the new "
f"dataset.")
adapted_dataset = AvalancheConcatDataset(task_datasets.values())
strategy.adapted_dataset = adapted_dataset
def create_multi_dataset_generic_scenario(
train_dataset_list: Sequence[SupportedDataset],
test_dataset_list: Sequence[SupportedDataset],
task_labels: Sequence[int],
complete_test_set_only: bool = False,
train_transform=None, train_target_transform=None,
eval_transform=None, eval_target_transform=None,
dataset_type: AvalancheDatasetType = None) \
-> GenericCLScenario:
"""
Creates a generic scenario given a list of datasets and the respective task
labels. Each training dataset will be considered as a separate training
experience. Contents of the datasets will not be changed, including the
targets.
When loading the datasets from a set of fixed filelist, consider using
the :func:`create_generic_scenario_from_filelists` helper method instead.
In its base form, this function accepts a list of test datsets that must
contain the same amount of datasets of the training list.
Those pairs are then used to create the "past", "cumulative"
(a.k.a. growing) and "future" test sets. However, in certain Continual
Learning scenarios only the concept of "complete" test set makes sense. In
that case, the ``complete_test_set_only`` should be set to True (see the
parameter description for more info).
Beware that pattern transformations must already be included in the
datasets (when needed).
:param train_dataset_list: A list of training datasets.
:param test_dataset_list: A list of test datasets.
:param task_labels: A list of task labels. Must contain the same amount of
elements of the ``train_dataset_list`` parameter. For
Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
a list of zeros. For Multi Task scenario, this is usually a list of
ascending task labels (starting from 0).
:param complete_test_set_only: If True, only the complete test set will
be returned by the scenario. This means that the ``test_dataset_list``
parameter must be list with a single element (the complete test set).
Defaults to False, which means that ``train_dataset_list`` and
``test_dataset_list`` must contain the same amount of datasets.
:param train_transform: The transformation to apply to the training data,
e.g. a random crop, a normalization or a concatenation of different
transformations (see torchvision.transform documentation for a
comprehensive list of possible transformations). Defaults to None.
:param train_target_transform: The transformation to apply to training
patterns targets. Defaults to None.
:param eval_transform: The transformation to apply to the test data,
e.g. a random crop, a normalization or a concatenation of different
transformations (see torchvision.transform documentation for a
comprehensive list of possible transformations). Defaults to None.
:param eval_target_transform: The transformation to apply to test
patterns targets. Defaults to None.
:param dataset_type: The type of the dataset. Defaults to None, which
means that the type will be obtained from the input datasets. If input
datasets are not instances of :class:`AvalancheDataset`, the type
UNDEFINED will be used.
:returns: A :class:`GenericCLScenario` instance.
"""
# TODO: more generic handling of task labels
transform_groups = dict(train=(train_transform, train_target_transform),
eval=(eval_transform, eval_target_transform))
# GenericCLScenario accepts a single training+test sets couple along with
# the respective list of patterns indexes to include in each experience.
# This means that we have to concat the list of train and test sets
# and create, for each experience, a of indexes.
# Each dataset describes a different experience so the lists of indexes will
# just be ranges of ascending indexes.
train_structure = []
pattern_train_task_labels = []
concat_train_dataset = AvalancheConcatDataset(
train_dataset_list,
transform_groups=transform_groups,
initial_transform_group='train',
dataset_type=dataset_type)
next_idx = 0
for dataset_idx, train_dataset in enumerate(train_dataset_list):
end_idx = next_idx + len(train_dataset)
train_structure.append(range(next_idx, end_idx))
pattern_train_task_labels.append(
ConstantSequence(task_labels[dataset_idx], len(train_dataset)))
next_idx = end_idx
pattern_train_task_labels = LazyConcatIntTargets(pattern_train_task_labels)
test_structure = []
if complete_test_set_only:
# If complete_test_set_only is True, we can leave test_structure = []
# In this way, GenericCLScenario will consider the whole test set.
#
# We don't offer a way to reduce the test set here. However, consider
# that the user may reduce the test set by creating a subset and passing
# it to this function directly.
if len(test_dataset_list) != 1:
raise ValueError('Test must contain 1 element when'
'complete_test_set_only is True')
concat_test_dataset = as_avalanche_dataset(test_dataset_list[0],
dataset_type=dataset_type)
concat_test_dataset = concat_test_dataset.train().add_transforms(
train_transform, train_target_transform)
concat_test_dataset = concat_test_dataset.eval().add_transforms(
eval_transform, eval_target_transform)
pattern_test_task_labels = ConstantSequence(0,
len(concat_test_dataset))
else:
concat_test_dataset = AvalancheConcatDataset(
test_dataset_list,
transform_groups=transform_groups,
initial_transform_group='eval',
dataset_type=dataset_type)
test_structure = []
pattern_test_task_labels = []
next_idx = 0
for dataset_idx, test_dataset in enumerate(test_dataset_list):
end_idx = next_idx + len(test_dataset)
test_structure.append(range(next_idx, end_idx))
pattern_test_task_labels.append(
ConstantSequence(task_labels[dataset_idx], len(test_dataset)))
next_idx = end_idx
pattern_test_task_labels = LazyConcatIntTargets(
pattern_test_task_labels)
task_labels = [[x] for x in task_labels]
# GenericCLScenario constructor will also check that the same amount of
# train/test sets + task_labels have been defined.
return GenericCLScenario(concat_train_dataset,
concat_test_dataset,
concat_train_dataset,
concat_test_dataset,
train_structure,
test_structure,
task_labels,
pattern_train_task_labels,
pattern_test_task_labels,
complete_test_set_only=complete_test_set_only)