def test_rf_unit(self, device):
float_dtype = get_float(device)
parent_rf_size_x = parent_rf_size_y = 4
n_channels = 4
image_grid_size_x = image_grid_size_y = 16
dimensions = (image_grid_size_y, image_grid_size_x, n_channels)
parent_rf_dims = Size2D(parent_rf_size_y, parent_rf_size_x)
unit = ReceptiveFieldUnit(AllocatingCreator(device),
dimensions,
parent_rf_dims,
flatten_output_grid_dimensions=True)
input_image = torch.zeros(image_grid_size_y,
image_grid_size_x,
n_channels,
dtype=float_dtype,
device=device)
input_image[0, parent_rf_size_x, 0] = 1
unit.step(input_image)
node_output = unit.output_tensor
n_parent_rfs = (image_grid_size_x // parent_rf_size_x) * (
image_grid_size_y // parent_rf_size_y)
assert node_output.shape == torch.Size(
[n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels])
assert node_output[1, 0, 0, 0] == 1
# assert node_output.interpret_shape == [n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels]
back_projection = unit.inverse_projection(node_output)
# assert back_projection.interpret_shape == input_image.shape
assert back_projection.equal(input_image)
def test_test_cycles():
"""Set the testing position to the last element and make step. Then test weather the cycle starts in the
begining. """
device = 'cuda'
# init node
params = DatasetSeObjectsParams()
params.dataset_size = SeDatasetSize.SIZE_24
params.dataset_config = DatasetConfig.TEST_ONLY
params.random_order = False
params.save_gpu_memory = False
node = DatasetSeObjectsNode(params, seed=None)
node.allocate_memory_blocks(AllocatingCreator(device))
# make step
node.step()
# read outputs
first_image = node.outputs.image_output.tensor
first_label = node.outputs.task_to_agent_label_ground_truth.tensor
# hack the position and make another step
node._unit._pos = len(node._unit._test_images)
node.step()
# read another outputs
output_image = node.outputs.image_output.tensor
output_label = node.outputs.task_to_agent_label_ground_truth.tensor
# assert they are both the same
assert same(first_image, output_image)
assert same(first_label, output_label)
def test_node_accessor(generator, device):
"""Validate the accessor properties."""
# common params
params = DatasetMNISTParams()
params.one_hot_labels = False
seq = None
# different iterators might result in different behavior here
if type(generator) is List:
seq = DatasetSequenceMNISTNodeParams(seqs=generator)
elif generator == 0:
params.random_order = True
else:
params.random_order = False
node = DatasetSequenceMNISTNode(params,
seq_params=seq,
dataset=get_dataset(),
seed=123)
node.allocate_memory_blocks(AllocatingCreator(device=device))
node.step()
bitmap = MnistNodeAccessor.get_data(node)
label = MnistNodeAccessor.get_label_id(node)
assert type(label) is int
assert type(bitmap) is torch.Tensor
assert sequences_equal(bitmap.shape, [28, 28])
assert 0 <= label < 10
def test_to_one_hot_speed(self, capsys):
@measure_time(iterations=100, function_repetitions=100)
def measured_step():
mb.tensor.copy_(
torch.rand(input_shape, dtype=get_float(device),
device=device))
to_one_hot.step()
input_shape = (150, )
device = 'cuda'
vector = torch.zeros(input_shape,
dtype=get_float(device),
device=device)
mb = MemoryBlock()
mb.tensor = torch.tensor(vector,
device=device,
dtype=get_float(device))
to_one_hot = ToOneHotNode(mode=ToOneHotMode.RANDOM)
Connector.connect(mb, to_one_hot.inputs.input)
to_one_hot.allocate_memory_blocks(AllocatingCreator(device))
with capsys.disabled():
measured_step()
def test_train_test_position_persistence():
"""If we switch from train to test, do some testing and then back to train, we should continue where stopped"""
device = 'cuda'
params = get_common_params()
params.switch_train_resets_train_pos = False
node = DatasetSeObjectsNode(params, seed=None)
node.allocate_memory_blocks(AllocatingCreator(device))
node.switch_training(training_on=True)
assert node._unit._pos == -1
first_steps = 10
_, _, _, = collect_data(node, first_steps)
# we are at the expected position
last_train_pos = -1 + first_steps
assert node._unit._pos == last_train_pos
assert train_pos(node) == last_train_pos
# switch to training (redundant) and check position not changed
node.switch_training(training_on=True)
assert node._unit._pos == last_train_pos
assert train_pos(node) == last_train_pos
second_steps = 3
# check the training position continues increasing
_, _, _ = collect_data(node, second_steps)
last_train_pos += second_steps
assert node._unit._pos == last_train_pos
assert train_pos(node) == last_train_pos
# switch to testing and collect some data
# _pos should reset (start testing), but the training_pos in the tensor should stay the same
node.switch_training(training_on=False)
assert node._unit._pos == -1
assert train_pos(node) == last_train_pos
testing_steps = 7
test_images, test_labels, _ = collect_data(node, testing_steps)
assert node._unit._pos == -1 + testing_steps # pos in the training set changes
assert train_pos(
node) == last_train_pos # train pos in the tensor does not
# switch back to training and check expected positions
third_train_steps = 2
node.switch_training(training_on=True)
_, _, _ = collect_data(node, third_train_steps)
last_train_pos += third_train_steps
assert train_pos(node) == last_train_pos
assert node._unit._pos == last_train_pos
# one more testing
node.switch_training(training_on=False)
test_images_2, test_labels_2, _ = collect_data(node, testing_steps)
assert same(test_images[0], test_images_2[0])
assert same(test_labels[0], test_labels_2[0])
def test_expansion(input_shape, dim, desired_size, expected_output_shape):
input_data = torch.zeros(input_shape)
expected_output = torch.zeros(expected_output_shape)
expand_unit = Expand(AllocatingCreator(device='cpu'), input_shape, dim=dim, desired_size=desired_size)
expand_unit.step(input_data)
assert same(expected_output, expand_unit.output)
def test_seq_node():
seq = [1, 2, 3, 4, 5, 10, 2, 2]
iterator = SequenceGenerator.from_list(seq)
node = SequenceNode(seq=iterator)
node._prepare_unit(AllocatingCreator(device='cpu'))
for element in seq * 2:
node.step()
assert element == node.outputs.output.tensor.item()
def test_node_accessor_and_determinism(device):
node = RandomNumberNode(lower_bound=LOWER_BOUND,
upper_bound=UPPER_BOUND,
seed=SEED)
node.allocate_memory_blocks(AllocatingCreator(device))
# expected sequence
assert generate_and_validate_sequence(node)
# expected after re-allocating MBs
node.allocate_memory_blocks(AllocatingCreator(device))
assert generate_and_validate_sequence(node)
# sequence independent on the global seeds
node.allocate_memory_blocks(AllocatingCreator(device))
set_global_seeds(None)
assert generate_and_validate_sequence(node)
def test_creator_methods(create_func):
measuring_creator = MeasuringCreator()
allocating_creator = AllocatingCreator('cpu')
expected = create_func(allocating_creator)
surrogate = create_func(measuring_creator)
assert expected.shape == surrogate.shape
def test_node_accessor_and_determinism_seed(device):
# new node with a different seed, should produce different results
node = RandomNumberNode(lower_bound=LOWER_BOUND,
upper_bound=UPPER_BOUND,
seed=None)
node.allocate_memory_blocks(AllocatingCreator(device))
assert not generate_and_validate_sequence(node)
def test_mse_multiple_steps():
device = 'cpu'
creator = AllocatingCreator(device=device)
dtype = creator.float32
input_tensors1 = [
creator.tensor([0, 1, 2, 0, 1, 2], device=device, dtype=dtype),
creator.tensor([0, 3, 0, 0, 1, 0], device=device, dtype=dtype)
] # squared diff = 12
input_tensors2 = [
creator.tensor([1, 1, 0, 0, 1, 2], device=device, dtype=dtype),
creator.tensor([0, 3, 2, 1, 3, 0], device=device, dtype=dtype)
] # squared diff = 18
input_tensors3 = [
creator.tensor([1, 1, 2, 0, 1, 1], device=device, dtype=dtype),
creator.tensor([-2, 3, 1, 3, 1, 0], device=device, dtype=dtype)
] # squared diff = 24
expected_result = creator.tensor(
[3], dtype=dtype,
device=device) # MSE=3 for a concatenation of the input tensors
mse_unit = MseUnit(creator,
input_shape=input_tensors1[0].shape,
buffer_size=3)
mse_unit.step(input_tensors1[0], input_tensors1[1])
mse_unit.step(input_tensors2[0], input_tensors2[1])
mse_unit.step(input_tensors3[0], input_tensors3[1])
assert same(expected_result, mse_unit._mean_square_error_output)
def test_join_node_inverse_flatten():
device = 'cpu'
creator = AllocatingCreator(device)
dtype = creator.float32
# The result of the inverse projection should only be one tensor.
expected_results = [
creator.tensor([[1, 2, 3, 4], [5, 6, 7, 8]],
dtype=dtype,
device=device),
creator.tensor([9, 10], dtype=dtype, device=device)
]
input_memory_blocks = [MemoryBlock(), MemoryBlock()]
input_memory_blocks[0].tensor = creator.zeros((2, 4))
input_memory_blocks[1].tensor = creator.zeros((2, ))
output_tensor = creator.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
dtype=dtype,
device=device)
join_node = JoinNode(flatten=True)
Connector.connect(input_memory_blocks[0], join_node.inputs[0])
Connector.connect(input_memory_blocks[1], join_node.inputs[1])
output_inverse_packet = InversePassOutputPacket(output_tensor,
join_node.outputs.output)
join_node.allocate_memory_blocks(creator)
results = join_node.recursive_inverse_projection_from_output(
output_inverse_packet)
for expected, result in zip(expected_results, results):
assert same(expected, result.tensor)
def test_create_node(self, device):
unit = DatasetAlphabetUnit(
AllocatingCreator(device),
DatasetAlphabetParams(
symbols="abcd",
sequence_probs=DatasetAlphabetSequenceProbsModeParams(
seqs=['abcd'])))
assert [4, 7, 5] == list(unit.all_symbols.shape)
assert [7, 5] == list(unit.output_data.shape)
def test_join_inverse_dim_0():
creator = AllocatingCreator(device='cpu')
dtype = creator.float32
dim = 0
input_shapes = [(2, 4), (2, 4)]
expected_results = [
creator.tensor([[1, 2, 3, 4], [5, 6, 7, 8]],
dtype=dtype,
device=creator.device),
creator.tensor([[9, 10, 11, 12], [13, 14, 15, 16]],
dtype=dtype,
device=creator.device)
]
_test_join_inverse(dim, input_shapes, expected_results, creator, dtype,
creator.device)
def test_mode_sequence_probs(self, device):
unit = DatasetAlphabetUnit(
AllocatingCreator(device),
DatasetAlphabetParams(
symbols="abcd",
sequence_probs=DatasetAlphabetSequenceProbsModeParams(
seqs=['abc'])))
generator = self.label_generator(unit)
result = [next(generator) for _ in range(7)]
assert [0, 1, 2, 0, 1, 2, 0] == result
def create_tp_buffer(flock_size=2,
buffer_size=5,
n_cluster_centers=3,
n_frequent_seqs=3,
context_size=4,
n_providers=1,
device='cuda'):
return TPFlockBuffer(AllocatingCreator(device), flock_size, buffer_size,
n_cluster_centers, n_frequent_seqs, context_size,
n_providers)
def test_inverse_projection(expected_input_shape, dim, desired_size, output_shape):
output_data = torch.zeros(output_shape)
expected_input = torch.zeros(expected_input_shape)
expand_unit = Expand(AllocatingCreator(device='cpu'), expected_input_shape, dim=dim, desired_size=desired_size)
# expand_unit.step(input_data)
projection = expand_unit.inverse_projection(output_data)
assert same(expected_input, projection)
def test_join_inverse_dim_1():
creator = AllocatingCreator(device='cpu')
dtype = creator.float32
dim = 1
input_shapes = [(4, 2), (4, 2)]
expected_results = [
creator.tensor([[1, 2], [5, 6], [9, 10], [13, 14]],
dtype=dtype,
device=creator.device),
creator.tensor([[3, 4], [7, 8], [11, 12], [15, 16]],
dtype=dtype,
device=creator.device)
]
_test_join_inverse(dim, input_shapes, expected_results, creator, dtype,
creator.device)
def make_nnet(input_size: int = 16,
num_classes: int = 5,
seed: int = 123,
buffer_s: int = 16,
batch_s: int = 8,
mixed_mode: bool = True):
device = 'cuda'
# set topology params and configs
_params = NNetParams(NNetParams.default_params())
# _params.set_params(_nn_node_params) # params defined in this file
# ._params.set_params(kwargs) # params defined in the constructor
# small input sizes for testing
_params.input_shape = (3, input_size, input_size)
_params.output_size = num_classes
_params.seed = seed
_params.batch_size = batch_s
_params.buffer_size = buffer_s
_params.num_epochs = 3
_params.mixed_mode = mixed_mode
# observation storage params
_observation_types = {
'x': (_params.buffer_size, *_params.input_shape), # observations
'y': (_params.buffer_size, _params.output_size), # labels
}
# data storage
_storage = ObservationStorage(_params.buffer_size, _observation_types)
_storage.to('cpu' if _params.mixed_mode else device)
# network needs to have the global seeds to have set before creating (outside of the node in this case)
set_global_seeds(seed=_params.seed)
# neural network setup
_network = NNet(input_shape=_params.input_shape,
output_shape=_params.output_size).to(
'cuda' if _params.mixed_mode else device)
# neural net optimizer
_optimizer = optim.Adam(_network.parameters(), lr=_params.lr)
# NNet Node
_nnet_node = NNetNode(_network,
_optimizer,
_storage,
_params,
name='Neural Network Node')
creator = AllocatingCreator(device=device)
_nnet_node.allocate_memory_blocks(creator)
return _params, _nnet_node
def test_sequence_iteration(expected_sequence, seq_params):
node = DatasetSequenceMNISTNode(params=DatasetMNISTParams(one_hot_labels=False), seq_params=seq_params)
node.allocate_memory_blocks(AllocatingCreator(device='cpu'))
actual_sequence = []
for x in range(len(node._seq_params.seqs[0])):
node.step()
actual_sequence.append(node.outputs.label.tensor.clone())
actual_sequence = torch.cat(actual_sequence, dim=0)
assert same(expected_sequence, actual_sequence)
def test_load_dataset(self):
images_tensor = DatasetLoader._load_dataset(
os.path.join('tests', 'data', 'datasets', 'testing_dataset'))
device = 'cpu'
creator = AllocatingCreator(device=device)
expected_images_tensor = self._create_expected_images_tensor(creator)
assert same(expected_images_tensor[:2], images_tensor[:2])
# jpeg uses lossy compression
assert same(expected_images_tensor[2], images_tensor[2], eps=0.01)
def test_scatter_node(self, device, input, mask, output_shape, dimension, expected_result):
float_dtype = get_float(device)
input_mb = MemoryBlock()
input_mb.tensor = torch.tensor(input, device=device, dtype=float_dtype)
expected = torch.tensor(expected_result, device=device, dtype=float_dtype)
sn = ScatterNode(mapping=mask, output_shape=output_shape, dimension=dimension, device=device)
Connector.connect(input_mb, sn.inputs.input)
sn.allocate_memory_blocks(AllocatingCreator(device))
sn.step()
assert same(sn.outputs.output.tensor, expected)
def test_sample_learning_batch_combinations(self, method, flock_indices,
elements_written):
flock_size = 3
creator = AllocatingCreator('cpu')
f_size = len(flock_indices) if flock_indices is not None else 3
buffer = SPFlockBuffer(creator,
buffer_size=20,
n_cluster_centers=3,
flock_size=flock_size,
input_size=5)
if flock_indices is not None:
buffer.set_flock_indices(
creator.tensor(flock_indices, dtype=torch.int64))
buffer.total_data_written[:] = elements_written
buffer.clusters.stored_data[:, :elements_written] = creator.tensor(
[0, 1, 0])
buffer.inputs.stored_data[:, :elements_written, :] = creator.tensor(
[1.3, 0.2, 0.6, 0.4, 0.1])
# use some dummy value here to check that it is rewriting all the lines in res
dummy_value = -2.1
sampled_data = creator.full((f_size, elements_written, 5),
fill_value=dummy_value)
buffer.sample_learning_batch(elements_written, sampled_data, method)
assert (sampled_data == dummy_value).any().item() == 0
def get_subflock_creation_testing_flock(
flock_size=10,
subflock_size=6,
input_size=5,
buffer_size=7,
batch_size=5,
n_cluster_centers=4,
device='cpu',
sampling_method=SamplingMethod.LAST_N) -> Tuple[SPFlock, torch.Tensor]:
# Generate a flock with some data
float_dtype = get_float(device)
params = ExpertParams()
params.flock_size = flock_size
params.n_cluster_centers = n_cluster_centers
params.spatial.input_size = input_size
params.spatial.buffer_size = buffer_size
params.spatial.batch_size = batch_size
params.spatial.sampling_method = sampling_method
flock = SPFlock(params, AllocatingCreator(device))
flock.buffer.inputs.stored_data = torch.rand(
(flock_size, buffer_size, input_size),
dtype=float_dtype,
device=device)
flock.buffer.clusters.stored_data = torch.rand(
(flock_size, buffer_size, n_cluster_centers),
dtype=float_dtype,
device=device) # NOTE: This is not one-hot and thus not real data
flock.buffer.current_ptr = torch.rand(flock_size,
device=device).type(torch.int64)
flock.buffer.total_data_written = torch.rand(
flock_size, device=device).type(torch.int64)
# The bookeeping values which are copied across
flock.cluster_boosting_durations = torch.rand(
(flock_size, n_cluster_centers), device=device).type(torch.int64)
flock.prev_boosted_clusters = torch.clamp(
torch.round(torch.rand((flock_size, n_cluster_centers),
device=device)), 0, 1).type(torch.uint8)
flock.boosting_targets = torch.rand((flock_size, n_cluster_centers),
device=device).type(torch.int64)
# Indices which are to be subflocked
indices = torch.tensor(np.random.choice(flock_size,
subflock_size,
replace=False),
dtype=torch.int64,
device=device)
return flock, indices
def __init__(self, device: str):
super().__init__('Graph', 0, 0)
self.device = device
self.float_dtype = get_float(self.device)
self._measuring_creator = MeasuringCreator()
self._allocating_creator = AllocatingCreator(device)
self._seed = None
self._do_before_step = None
self._visible_nodes = set()
self._id_generator = IdGenerator()
def test_extra_buffers():
num_indices = 5
indices = torch.arange(num_indices, dtype=torch.int64)
proc = init_process(indices, subflocking=False)
creator = AllocatingCreator("cpu")
buff = DummyBuffer(creator, 10, 100, (1, 1))
proc._get_buffer(buff)
with pytest.raises(AssertionError):
proc._get_buffer(buff)
def test_rnd_node_accessor_return_type(device):
lower_bound = 50
upper_bound = 100
node = RandomNumberNode(lower_bound=lower_bound, upper_bound=upper_bound)
node.allocate_memory_blocks(AllocatingCreator(device=device))
node._step()
random_number = RandomNumberNodeAccessor.get_output_id(node)
assert type(random_number) is int
assert lower_bound <= random_number < upper_bound
def test_load_dataset_shippet(self):
images_tensor = DatasetLoader._load_dataset(os.path.join(
'tests', 'data', 'datasets', 'testing_dataset'),
limit_to_n=2)
device = 'cpu'
creator = AllocatingCreator(device=device)
expected_images_tensor = self._create_expected_images_tensor(creator)
assert len(images_tensor) == 2
assert same(expected_images_tensor[:2], images_tensor[:2])
def test_deterministic_samples_per_class_choice(device: str,
sequence_type: str,
class_filter):
"""Test random/ordered/sequential sampling of the dataset with and without class filter.
Everything should produce valid and repeatable results if seed is not None.
"""
params = DatasetMNISTParams()
generator_params = None
if sequence_type == 'ordered':
params.random_order = False
elif sequence_type == 'random':
params.random_order = True
else:
generator_params = DatasetSequenceMNISTNodeParams([[1, 2, 3], [5, 2]])
# parametrized, try filtered and non-filtered versions
params.class_filter = class_filter
params.examples_per_class = 10
n_samples = 15
creator = AllocatingCreator(device=device)
node = DatasetSequenceMNISTNode(params,
seq_params=generator_params,
dataset=get_dataset(),
seed=10)
node.allocate_memory_blocks(creator)
labels, bitmaps = collect_data(node, n_samples)
node = DatasetSequenceMNISTNode(params,
seq_params=generator_params,
dataset=get_dataset(),
seed=10)
node.allocate_memory_blocks(creator)
labels_b, bitmaps_b = collect_data(node, n_samples)
assert len(bitmaps_b) == len(bitmaps)
# go through the generated sequences and compare labels and bitmaps, should be identical
for bitmap_a, label_a, bitmap_b, label_b in zip(bitmaps, labels, bitmaps_b,
labels_b):
# the same sequences of labels
assert label_a == label_b
# bitmaps are identical (randomly sampled the same samples from the dataset)
assert same(bitmap_a, bitmap_b)
def get_subflock_integration_testing_flock(
params, subflock_size,
device) -> Tuple[SPFlock, torch.Tensor, np.array]:
flock = SPFlock(params, AllocatingCreator(device))
float_dtype = get_float(device)
flock.buffer.inputs.stored_data = torch.rand(flock.buffer.inputs.dims,
dtype=float_dtype,
device=device)
# NOTE: This is not one-hot and thus not real data
flock.buffer.clusters.stored_data = torch.rand(flock.buffer.clusters.dims,
dtype=float_dtype,
device=device)
flock.buffer.current_ptr = torch.randint(0,
params.spatial.buffer_size,
flock.buffer.current_ptr.size(),
dtype=torch.int64,
device=device)
flock.buffer.total_data_written = torch.randint(
0,
params.spatial.buffer_size,
flock.buffer.current_ptr.size(),
dtype=torch.int64,
device=device)
flock.cluster_boosting_durations = torch.randint(
0,
20,
flock.cluster_boosting_durations.size(),
dtype=torch.int64,
device=device)
flock.prev_boosted_clusters = \
torch.clamp(torch.randint(0, 2, flock.prev_boosted_clusters.size(), device=device), 0, 1).type(torch.uint8)
flock.boosting_targets = torch.randint(0,
params.flock_size,
flock.boosting_targets.size(),
dtype=torch.int64,
device=device)
flock.forward_clusters = torch.rand(flock.forward_clusters.size(),
dtype=float_dtype,
device=device)
# Create a subflock and populate it
indices_np = sorted(
np.random.choice(params.flock_size, subflock_size, replace=False))
indices = torch.tensor(list(map(int, indices_np)),
dtype=torch.int64,
device=device).unsqueeze(dim=1)
return flock, indices, indices_np
