def test_overlapping_rfs(self, device):
float_dtype = get_float(device)
x = 3
y = 4
input_image_1 = torch.arange(0, x * y, dtype=float_dtype, device=device).view(y, x)
input_image_2 = torch.rand_like(input_image_1, dtype=float_dtype, device=device)
grids = Grids(Size2D(y, x), parent_rf_dims=Size2D(2, 2), parent_rf_stride=Stride(1, 1),
flatten_output_grid_dimensions=True)
mapping = Mapping.from_default_input(grids, chunk_size=1, device=device)
expert_input = mapping.map(input_image_1)
assert expert_input.equal(torch.tensor(
[[0, 1, 3, 4],
[1, 2, 4, 5],
[3, 4, 6, 7],
[4, 5, 7, 8],
[6, 7, 9, 10],
[7, 8, 10, 11]],
dtype=float_dtype,
device=device
).view(6, 2, 2, 1))
back_projection_1 = mapping.inverse_map(expert_input)
assert input_image_1.equal(back_projection_1)
back_projection_2 = mapping.inverse_map(mapping.map(input_image_2))
assert input_image_2.equal(back_projection_2)
def test_occupancy(self, device):
float_dtype = get_float(device)
grids = Grids(Size2D(4, 3), parent_rf_dims=Size2D(2, 2), parent_rf_stride=Stride(1, 1),
flatten_output_grid_dimensions=True)
mapping = Mapping.from_default_input(grids, device, chunk_size=1)
assert mapping._occupancies.equal(torch.tensor(
[[1, 2, 1],
[2, 4, 2],
[2, 4, 2],
[1, 2, 1]], dtype=float_dtype, device=device).view(-1))
mapping = Mapping.from_default_input(grids, device, chunk_size=2)
assert mapping._occupancies.size() == torch.Size([4 * 3 * 2])
def test_rf_unit(self, device):
float_dtype = get_float(device)
parent_rf_size_x = parent_rf_size_y = 4
n_channels = 4
image_grid_size_x = image_grid_size_y = 16
dimensions = (image_grid_size_y, image_grid_size_x, n_channels)
parent_rf_dims = Size2D(parent_rf_size_y, parent_rf_size_x)
unit = ReceptiveFieldUnit(AllocatingCreator(device),
dimensions,
parent_rf_dims,
flatten_output_grid_dimensions=True)
input_image = torch.zeros(image_grid_size_y,
image_grid_size_x,
n_channels,
dtype=float_dtype,
device=device)
input_image[0, parent_rf_size_x, 0] = 1
unit.step(input_image)
node_output = unit.output_tensor
n_parent_rfs = (image_grid_size_x // parent_rf_size_x) * (
image_grid_size_y // parent_rf_size_y)
assert node_output.shape == torch.Size(
[n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels])
assert node_output[1, 0, 0, 0] == 1
# assert node_output.interpret_shape == [n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels]
back_projection = unit.inverse_projection(node_output)
# assert back_projection.interpret_shape == input_image.shape
assert back_projection.equal(input_image)
def test_expert_dimensions(self):
"""Tests multi-dimensional expert indexes."""
device = 'cpu'
parent_rf_size_x = parent_rf_size_y = 4
n_channels = 4
image_grid_size_x = image_grid_size_y = 16
input_dimensions = (image_grid_size_y, image_grid_size_x, n_channels)
parent_rf_dims = Size2D(parent_rf_size_x, parent_rf_size_y)
parent_grid_dimensions = (4, 4)
graph = Topology(device)
node = ReceptiveFieldNode(input_dimensions, parent_rf_dims)
graph.add_node(node)
memory_block = MemoryBlock()
memory_block.tensor = torch.zeros(image_grid_size_y,
image_grid_size_x,
n_channels,
device=device)
memory_block.tensor[0, parent_rf_size_x, 0] = 1
Connector.connect(memory_block, node.inputs.input)
graph.prepare()
graph.step()
node_output = node.outputs.output.tensor
assert node_output.shape == torch.Size(parent_grid_dimensions +
(parent_rf_size_y,
parent_rf_size_x, n_channels))
assert node_output[0, 1, 0, 0, 0] == 1
def __init__(self):
super().__init__(device='cuda')
self._se_config = SpaceEngineersConnectorConfig()
self.SX = self._se_config.render_width
self.SY = self._se_config.render_height
# setup the params
# SeToyArchDebugTopology.config_se_communication()
expert_params = self.get_expert_params()
# create the nodes
self._actions_descriptor = SpaceEngineersActionsDescriptor()
self._node_se_connector = SpaceEngineersConnectorNode(
self._actions_descriptor, self._se_config)
self._node_action_monitor = ActionMonitorNode(self._actions_descriptor)
self._blank_action = ConstantNode(
shape=self._actions_descriptor.ACTION_COUNT, constant=0)
self._blank_task_data = ConstantNode(
shape=self._se_config.agent_to_task_buffer_size, constant=0)
self._blank_task_control = ConstantNode(
shape=self._se_config.TASK_CONTROL_SIZE, constant=0)
# parent_rf_dims = (self.lrf_width, self.lrf_height, 3)
self._node_lrf = ReceptiveFieldNode((self.SY, self.SX, 3),
Size2D(self.SX // EOX,
self.SY // EOY))
self._node_flock = ExpertFlockNode(expert_params)
self.add_node(self._node_se_connector)
self.add_node(self._node_action_monitor)
self.add_node(self._blank_action)
self.add_node(self._blank_task_data)
self.add_node(self._blank_task_control)
self.add_node(self._node_flock)
self.add_node(self._node_lrf)
Connector.connect(self._blank_action.outputs.output,
self._node_action_monitor.inputs.action_in)
Connector.connect(self._node_action_monitor.outputs.action_out,
self._node_se_connector.inputs.agent_action)
Connector.connect(self._blank_task_data.outputs.output,
self._node_se_connector.inputs.agent_to_task_label)
Connector.connect(self._blank_task_control.outputs.output,
self._node_se_connector.inputs.task_control)
# SE -> Flock (no LRF)
# Connector.connect(self._node_se_connector.outputs.image_output,
# self._node_flock.inputs.sp.data_input)
# Image -> LRF -> Flock
Connector.connect(self._node_se_connector.outputs.image_output,
self._node_lrf.inputs[0])
Connector.connect(self._node_lrf.outputs[0],
self._node_flock.inputs.sp.data_input)
def _install_experts(self):
im_width = self.se_io.get_image_width()
im_height = self.se_io.get_image_height()
self.input_dims = torch.Size((im_height, im_width, 3))
self.parent_rf_dims = Size2D(im_height // self.EXPERTS_IN_X,
im_width // self.EXPERTS_IN_X)
lrf_node = ReceptiveFieldNode(input_dims=self.input_dims,
parent_rf_dims=self.parent_rf_dims)
self.add_node(lrf_node)
self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params)
self._mid_node = ExpertFlockNode(self.mid_expert_params)
self._conv_node = ConvExpertFlockNode(self.conv_flock_params)
self.add_node(self._top_level_flock_node)
self.add_node(self._mid_node)
self.add_node(self._conv_node)
def rescale(inputs, outputs):
if self.TRAINING:
outputs[0].copy_(
inputs[0] *
1000) # large constant to make the label more important
else:
outputs[0].copy_(inputs[0] * float('nan'))
self._rescale_node = LambdaNode(rescale, 1,
[(self.se_io.get_num_labels(), )])
self.add_node(self._rescale_node)
unsqueeze_node_0 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_0)
unsqueeze_node_1 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_1)
Connector.connect(self.se_io.outputs.image_output,
lrf_node.inputs.input)
Connector.connect(lrf_node.outputs.output,
self._conv_node.inputs.sp.data_input)
Connector.connect(self._conv_node.outputs.tp.projection_outputs,
unsqueeze_node_0.inputs.input)
Connector.connect(unsqueeze_node_0.outputs.output,
self._mid_node.inputs.sp.data_input)
Connector.connect(self._mid_node.outputs.tp.projection_outputs,
self._join_node.inputs[0])
Connector.connect(self.se_io.outputs.task_to_agent_label,
self._rescale_node.inputs[0])
Connector.connect(self._rescale_node.outputs[0],
self._join_node.inputs[1])
Connector.connect(self._join_node.outputs.output,
unsqueeze_node_1.inputs.input)
Connector.connect(unsqueeze_node_1.outputs.output,
self._top_level_flock_node.inputs.sp.data_input)
def _install_lrf(self, image_size: int, experts_on_x: int):
self._lrf_input_dims = torch.Size((image_size, image_size, 3))
self._parent_rf_dims = Size2D(image_size // experts_on_x,
image_size // experts_on_x)
self._node_lrf = ReceptiveFieldNode(
input_dims=self._lrf_input_dims,
parent_rf_dims=self._parent_rf_dims)
self.add_node(self._node_lrf)
def __init__(self,
child_grid_dims: Size2D,
parent_rf_dims: Size2D,
parent_rf_stride: Optional[Stride] = None,
flatten_output_grid_dimensions=False,
device: str = 'cpu'):
self._device = device
self._check_parameters(child_grid_dims, parent_rf_dims,
parent_rf_stride)
self._child_grid_dims = Size2D(*child_grid_dims)
self._parent_receptive_field_dims = Size2D(*parent_rf_dims)
if parent_rf_stride is None:
self._parent_receptive_field_stride_dims = Stride(*parent_rf_dims)
else:
self._parent_receptive_field_stride_dims = Stride(
*parent_rf_stride)
if flatten_output_grid_dimensions:
self._parent_grid_dims = (self.n_parent_rfs, )
else:
self._parent_grid_dims = (self.n_parent_rfs_y, self.n_parent_rfs_x)
def setup_class(cls, device: str = 'cpu'):
super().setup_class(device)
cls.float_dtype = get_float(cls._device)
cls.parent_rf_size_x = 2
cls.parent_rf_size_y = 3
cls.n_channels = 5
cls.image_grid_size_x = 4
cls.image_grid_size_y = 6
cls.dimensions = (cls.image_grid_size_y, cls.image_grid_size_x,
cls.n_channels)
cls.parent_rf_dims = Size2D(cls.parent_rf_size_y, cls.parent_rf_size_x)
cls.parent_rf_stride_dims = Stride(3, 1)
def test_image_expert_mapping(self, device):
float_dtype = get_float(device)
n_channels = 3
image_grid_size_x = 16
image_grid_size_y = 8
image_grid_dims = Size2D(image_grid_size_y, image_grid_size_x)
parent_rf_size_x = parent_rf_size_y = 4
parent_rf_dims = Size2D(parent_rf_size_y, parent_rf_size_x)
grids = Grids(image_grid_dims, parent_rf_dims, flatten_output_grid_dimensions=True)
mapping = Mapping.from_sensory_input(grids, device, n_channels)
input_image = torch.zeros(image_grid_size_y, image_grid_size_x, n_channels, dtype=float_dtype, device=device)
input_image[0, parent_rf_size_x, 0] = 1
input_image[1, parent_rf_size_x + 2, 0] = 2
input_image[parent_rf_size_y, parent_rf_size_x, 0] = 3
expert_input = mapping.map(input_image)
assert expert_input[1, 0, 0, 0] == 1
assert expert_input[1, 1, 2, 0] == 2
assert expert_input[image_grid_size_x // parent_rf_size_x + 1, 0, 0, 0] == 3
def test_expert_expert_mapping(self, device):
float_dtype = get_float(device)
child_output_size = 1000
number_of_child_rfs_x = number_of_child_rfs_y = 10
parent_rf_size_x = parent_rf_size_y = 2
grids = Grids(
Size2D(number_of_child_rfs_y, number_of_child_rfs_x),
parent_rf_dims=Size2D(parent_rf_size_y, parent_rf_size_x),
flatten_output_grid_dimensions=True)
mapping = Mapping.from_child_expert_output(grids, device, child_output_size)
children_output = torch.zeros(grids.child_grid_height, grids.child_grid_width, child_output_size,
dtype=float_dtype,
device=device)
children_output[0, 2, 0] = 1
children_output[number_of_child_rfs_y - parent_rf_size_y - 1, 0, 0] = 1
parents_input = mapping.map(children_output)
eyxc_view = parents_input.view(grids.n_parent_rfs, parent_rf_size_y, parent_rf_size_x, -1)
assert eyxc_view[1, 0, 0, 0] == 1
assert eyxc_view[grids.n_parent_rfs - 2 * grids.n_parent_rfs_x, parent_rf_size_y - 1, 0, 0] == 1
def test_rf_node(self, device):
float_dtype = get_float(device)
parent_rf_size_x = parent_rf_size_y = 4
n_channels = 4
image_grid_size_x = image_grid_size_y = 16
dimensions = (image_grid_size_y, image_grid_size_x, n_channels)
parent_rf_dims = Size2D(parent_rf_size_y, parent_rf_size_x)
graph = Topology(device)
node = ReceptiveFieldNode(dimensions,
parent_rf_dims,
flatten_output_grid_dimensions=True)
graph.add_node(node)
memory_block = MemoryBlock()
memory_block.tensor = torch.zeros(image_grid_size_y,
image_grid_size_x,
n_channels,
dtype=float_dtype,
device=device)
memory_block.tensor[0, parent_rf_size_x, 0] = 1
Connector.connect(memory_block, node.inputs.input)
graph.prepare()
graph.step()
node_output = node.outputs.output.tensor
n_parent_rfs = (image_grid_size_y // parent_rf_size_y) * (
image_grid_size_x // parent_rf_size_x)
assert node_output.shape == torch.Size(
[n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels])
assert node_output[1, 0, 0, 0] == 1
back_projection = node.recursive_inverse_projection_from_output(
InversePassOutputPacket(node_output, node.outputs.output))
# assert back_projection.interpret_shape == input_image.shape
assert same(back_projection[0].tensor, memory_block.tensor)
def canvas_shape(self, value: List[int]):
validate_list_of_size(value, 2)
self._params.canvas_shape = Size2D(*value)
def test_move_point(self, move_strategy, vector, point, expected):
params = DatasetSimplePointGravityParams(Size2D(3, 3), Point2D(1, 1), 1, move_strategy)
unit = DatasetSimplePointGravityUnit(AllocatingCreator(device='cpu'), params, random=None)
result = unit._move_point(np.array(vector), Point2D(*point))
assert Point2D(*expected) == result
class SeT0ConvSPTopology(SeT0TopologicalGraph):
_flock_nodes: List[SpatialPoolerFlockNode]
EXPERTS_IN_X = 8
BATCH = 10000
TRAINING = True
parent_params = ExpertParams()
parent_params.flock_size = 1
parent_params.n_cluster_centers = 200
parent_params.compute_reconstruction = True
parent_params.spatial.learning_period = 400
parent_params.spatial.learning_rate = 0.2
if not TRAINING:
parent_params.spatial.learning_rate *= 0
parent_params.spatial.buffer_size = 1000
parent_params.spatial.batch_size = 1000
parent_params.spatial.cluster_boost_threshold = 200
conv_flock_params = ExpertParams()
conv_flock_params.flock_size = EXPERTS_IN_X**2
conv_flock_params.n_cluster_centers = 200
conv_flock_params.spatial.learning_period = 1000
conv_flock_params.spatial.learning_rate = 0.2
if not TRAINING:
conv_flock_params.spatial.learning_rate *= 0
conv_flock_params.spatial.buffer_size = BATCH
conv_flock_params.spatial.batch_size = BATCH
conv_flock_params.spatial.cluster_boost_threshold = 200
input_dims = torch.Size((24, 24, 3))
parent_rf_dims = Size2D(24 // EXPERTS_IN_X, 24 // EXPERTS_IN_X)
# parent_rf_stride_dims = (16, 16)
def _install_experts(self):
lrf_node = ReceptiveFieldNode(input_dims=self.input_dims,
parent_rf_dims=self.parent_rf_dims)
self.add_node(lrf_node)
self._top_level_flock_node = SpatialPoolerFlockNode(self.parent_params,
name="Parent 1 SP")
self._conv_node = ConvSpatialPoolerFlockNode(self.conv_flock_params,
name="Conv SP flock")
self.add_node(self._top_level_flock_node)
self.add_node(self._conv_node)
scale = 1000
def rescale_up(inputs, outputs):
outputs[0].copy_(inputs[0] * scale)
def rescale_down(inputs, outputs):
outputs[0].copy_(inputs[0] / scale)
self._rescale_up_node = LambdaNode(rescale_up,
1, [(20, )],
name="upscale 1000")
self.add_node(self._rescale_up_node)
self._rescale_down_node = LambdaNode(
rescale_down,
1, [(1, self._top_level_expert_output_size() + 20)],
name="downscale 1000")
self.add_node(self._rescale_down_node)
unsqueeze_node = UnsqueezeNode(0)
self.add_node(unsqueeze_node)
Connector.connect(self.se_io.outputs.image_output,
lrf_node.inputs.input)
Connector.connect(lrf_node.outputs.output,
self._conv_node.inputs.sp.data_input)
Connector.connect(self._conv_node.outputs.sp.forward_clusters,
self._join_node.inputs[0])
Connector.connect(self.se_io.outputs.task_to_agent_label,
self._rescale_up_node.inputs[0])
Connector.connect(self._rescale_up_node.outputs[0],
self._join_node.inputs[1])
Connector.connect(self._join_node.outputs.output,
unsqueeze_node.inputs.input)
Connector.connect(unsqueeze_node.outputs.output,
self._top_level_flock_node.inputs.sp.data_input)
Connector.connect(
self._top_level_flock_node.outputs.sp.current_reconstructed_input,
self._rescale_down_node.inputs[0])
def _get_agent_output(self):
return self._rescale_down_node.outputs[0]
def _top_level_expert_output_size(self):
return self.conv_flock_params.n_cluster_centers * self.conv_flock_params.flock_size
def test_invalid_params(self):
with raises(FailedValidationException):
image_grid_dims = Size2D(16, 16)
parent_rf_dims = Size2D(3, 4)
Grids(image_grid_dims, parent_rf_dims)
class TestGrids:
def test_invalid_params(self):
with raises(FailedValidationException):
image_grid_dims = Size2D(16, 16)
parent_rf_dims = Size2D(3, 4)
Grids(image_grid_dims, parent_rf_dims)
@pytest.mark.parametrize(
'child_dims, parent_dims, stride, expected_result', [
(Size2D(4, 12), Size2D(2, 3), None, [(0, 0), (0, 1), (0, 2),
(0, 3), (1, 0), (1, 1),
(1, 2), (1, 3)]),
(Size2D(4, 12), Size2D(2, 3), Stride(1, 3), [(0, 0), (0, 1),
(0, 2), (0, 3),
(1, 0), (1, 1),
(1, 2), (1, 3),
(2, 0), (2, 1),
(2, 2), (2, 3)]),
(Size2D(8, 9), Size2D(2, 3), None, [(0, 0), (0, 1), (0, 2), (1, 0),
(1, 1), (1, 2), (2, 0), (2, 1),
(2, 2), (3, 0), (3, 1),
(3, 2)]),
(Size2D(4, 3), Size2D(2, 2), Stride(1, 1), [
(0, 0),
(0, 1),
(1, 0),
(1, 1),
(2, 0),
(2, 1),
]),
])
def test_gen_parent_receptive_fields(self, child_dims, parent_dims, stride,
expected_result):
grids = Grids(child_dims, parent_dims, stride)
result = list(grids.gen_parent_receptive_fields())
assert expected_result == result
@pytest.mark.parametrize(
'grids, expected_result, expected_shape',
[(Grids(child_grid_dims=Size2D(1, 5),
parent_rf_dims=Size2D(1, 3),
parent_rf_stride=Stride(1, 1)),
[[[[[nan, nan], [nan, nan], [0, 0]]], [[[nan, nan], [0, 0], [0, 1]]],
[[[0, 0], [0, 1], [0, 2]]], [[[0, 1], [0, 2], [nan, nan]]],
[[[0, 2], [nan, nan], [nan, nan]]]]], [1, 5, 1, 3, 2]),
(Grids(child_grid_dims=Size2D(1, 5),
parent_rf_dims=Size2D(1, 3),
parent_rf_stride=Stride(1, 2)),
[[[[[nan, nan], [nan, nan], [0, 0]]],
[[[nan, nan], [0, 0], [nan, nan]]], [[[0, 0], [nan, nan], [0, 1]]],
[[[nan, nan], [0, 1], [nan, nan]]],
[[[0, 1], [nan, nan], [nan, nan]]]]], [1, 5, 1, 3, 2]),
(Grids(child_grid_dims=Size2D(3, 1),
parent_rf_dims=Size2D(2, 1),
parent_rf_stride=Stride(1, 1)), [
[[[[nan, nan]], [[0, 0]]]],
[[[[0, 0]], [[1, 0]]]],
[[[[1, 0]], [[nan, nan]]]],
], [3, 1, 2, 1, 2]),
(Grids(child_grid_dims=Size2D(4, 4),
parent_rf_dims=Size2D(2, 2),
parent_rf_stride=Stride(1, 2)), [
[[[[nan, nan], [nan, nan]], [[nan, nan], [0, 0]]],
[[[nan, nan], [nan, nan]], [[0, 0], [nan, nan]]],
[[[nan, nan], [nan, nan]], [[nan, nan], [0, 1]]],
[[[nan, nan], [nan, nan]], [[0, 1], [nan, nan]]]],
[[[[nan, nan], [0, 0]], [[nan, nan], [1, 0]]],
[[[0, 0], [nan, nan]], [[1, 0], [nan, nan]]],
[[[nan, nan], [0, 1]], [[nan, nan], [1, 1]]],
[[[0, 1], [nan, nan]], [[1, 1], [nan, nan]]]],
[[[[nan, nan], [1, 0]], [[nan, nan], [2, 0]]],
[[[1, 0], [nan, nan]], [[2, 0], [nan, nan]]],
[[[nan, nan], [1, 1]], [[nan, nan], [2, 1]]],
[[[1, 1], [nan, nan]], [[2, 1], [nan, nan]]]],
[[[[nan, nan], [2, 0]], [[nan, nan], [nan, nan]]],
[[[2, 0], [nan, nan]], [[nan, nan], [nan, nan]]],
[[[nan, nan], [2, 1]], [[nan, nan], [nan, nan]]],
[[[2, 1], [nan, nan]], [[nan, nan], [nan, nan]]]],
], [4, 4, 2, 2, 2])])
def test_gen_positioned_parent_child_map(self, grids, expected_result,
expected_shape):
result = grids.gen_positioned_parent_child_map()
expected_tensor = torch.tensor(expected_result).long()
assert expected_shape == list(
expected_tensor.shape), "Check expected integrity"
assert expected_shape == list(result.shape), "Check result shape"
assert same(expected_tensor, result)
def __init__(self,
expert_width: int = 1,
input_square_side: int = 64,
n_cluster_centers: int = 8,
stride: int = None,
training_phase_steps: int = 200,
testing_phase_steps: int = 800,
seed: int = 0):
super().__init__(device='cuda')
if stride is None:
stride = expert_width
self.training_phase_steps = training_phase_steps
self.testing_phase_steps = testing_phase_steps
self.testing_phase = True
self.n_testing_phase = -1
self.training_step = -1
self._step_counter = 0
assert (input_square_side - (expert_width - stride)) % stride == 0, \
f'(input_square_side - (expert_width - stride)) ' \
f'({(input_square_side - (expert_width - stride))}) must be divisible' \
f' by stride ({stride})'
self.n_experts_width = (input_square_side -
(expert_width - stride)) // stride
self.input_square_side = input_square_side
self.one_expert_lrf_width = expert_width
self.sp_params = ExpertParams()
self.sp_params.n_cluster_centers = n_cluster_centers
self.sp_params.spatial.input_size = self.one_expert_lrf_width * self.one_expert_lrf_width * 3
self.sp_params.flock_size = int(self.n_experts_width**2)
self.sp_params.spatial.buffer_size = 510
self.sp_params.spatial.batch_size = 500
self.sp_params.compute_reconstruction = True
self.reconstructed_data = torch.zeros(
(self.input_square_side, self.input_square_side, 3),
dtype=self.float_dtype,
device=self.device)
self.image_difference = torch.zeros(
(self.input_square_side, self.input_square_side, 3),
dtype=self.float_dtype,
device=self.device)
se_nav_params = DatasetSENavigationParams(
SeDatasetSize.SIZE_64,
sampling_method=SamplingMethod.RANDOM_SAMPLING)
self._node_se_nav = DatasetSeNavigationNode(se_nav_params, seed=0)
parent_rf_dims = Size2D(self.one_expert_lrf_width,
self.one_expert_lrf_width)
self._node_lrf = ReceptiveFieldNode(
(input_square_side, input_square_side, 3), parent_rf_dims,
Stride(stride, stride))
# necessary to clone the params, because the GUI changes them during the simulation (restart needed)
self._node_spatial_pooler = SpatialPoolerFlockNode(
self.sp_params.clone(), seed=seed)
self._node_spatial_pooler_backup = SpatialPoolerFlockNode(
self.sp_params.clone(), seed=seed)
self.add_node(self._node_se_nav)
self.add_node(self._node_lrf)
self.add_node(self._node_spatial_pooler)
self.add_node(self._node_spatial_pooler_backup)
Connector.connect(self._node_se_nav.outputs.image_output,
self._node_lrf.inputs.input)
Connector.connect(self._node_lrf.outputs.output,
self._node_spatial_pooler.inputs.sp.data_input)
Connector.connect(
self._node_lrf.outputs.output,
self._node_spatial_pooler_backup.inputs.sp.data_input)
self.se_nav_node = self._node_se_nav
def test_reverse_map_concat(self, device):
# device = "cpu"
float_dtype = get_float(device)
def context(part: int, parent_id: int):
context_size = 5
start = 2 * parent_id + part
return torch.arange(start * context_size,
(start + 1) * context_size,
dtype=float_dtype,
device=device)
def nans(count: int):
return torch.full((count, ), FLOAT_NAN, device=device)
data_size = 2 * 5
grids = Grids(Size2D(3, 4),
parent_rf_dims=Size2D(2, 3),
parent_rf_stride=Stride(1, 1),
device=device)
assert 4 == len(list(grids.gen_parent_receptive_fields()))
mapping = ReverseMapping(grids, device, data_size)
data = torch.arange(4 * 2 * 5, device=device).view(2, 2, 2, 5).float()
result = mapping.reverse_map_concat(data)
assert (3, 4, 2, 3, 2, 5) == result.shape
# r = result.view(12, 6, 10)
# r = result.view(12, 6, 2, 5)
# r = r.transpose(1, 2).contiguous()
r = result.view(12, 6, 2, 5)
for part in [0, 1]:
child = 0
assert same(nans(25),
r[child, 0:5,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 0).equal(r[child, 5, part, :]), f'Part {part}'
child = 1
assert same(nans(20),
r[child, 0:4,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 0).equal(r[child, 4, part, :]), f'Part {part}'
assert context(part, 1).equal(r[child, 5, part, :]), f'Part {part}'
child = 2
assert same(nans(15),
r[child, 0:3,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 0).equal(r[child, 3, part, :]), f'Part {part}'
assert context(part, 1).equal(r[child, 4, part, :]), f'Part {part}'
assert same(nans(5),
r[child, 5,
part, :].contiguous().view(-1)), f'Part {part}'
child = 3
assert same(nans(15),
r[child, 0:3,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 1).equal(r[child, 3, part, :]), f'Part {part}'
assert same(nans(10),
r[child, 4:6,
part, :].contiguous().view(-1)), f'Part {part}'
child = 4
assert same(nans(10),
r[child, 0:2,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 0).equal(r[child, 2, part, :]), f'Part {part}'
assert same(nans(10),
r[child, 3:5,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 2).equal(r[child, 5, part, :]), f'Part {part}'
child = 5
assert same(nans(5),
r[child, 0,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 0).equal(r[child, 1, part, :]), f'Part {part}'
assert context(part, 1).equal(r[child, 2, part, :]), f'Part {part}'
assert same(nans(5),
r[child, 3,
part, :].contiguous().view(-1)), f'Part {part}'
assert context(part, 2).equal(r[child, 4, part, :]), f'Part {part}'
assert context(part, 3).equal(r[child, 5, part, :]), f'Part {part}'
def __init__(self,
expert_width: int = 1,
input_square_side: int = 28,
n_cluster_centers: int = 8,
stride: int = None,
is_convolutional: bool = False,
training_phase_steps: int = 200,
testing_phase_steps: int = 800,
seed: int = 0):
super().__init__(device='cuda')
if stride is None:
stride = expert_width
self.training_phase_steps = training_phase_steps
self.testing_phase_steps = testing_phase_steps
self.testing_phase = True
self.n_testing_phase = -1
self.training_step = -1
self._step_counter = 0
assert (input_square_side - (expert_width - stride)) % stride == 0, \
f'(input_square_side - (expert_width - stride)) ' \
f'({(input_square_side - (expert_width - stride))}) must be divisible' \
f' by stride ({stride})'
self.n_experts_width = (input_square_side -
(expert_width - stride)) // stride
self.input_square_side = input_square_side
self.one_expert_lrf_width = expert_width
self.sp_params = ExpertParams()
self.sp_params.n_cluster_centers = n_cluster_centers
self.sp_params.spatial.input_size = self.one_expert_lrf_width * self.one_expert_lrf_width
self.sp_params.flock_size = int(self.n_experts_width**2)
self.sp_params.spatial.buffer_size = 800
self.sp_params.spatial.batch_size = 500
self.sp_params.compute_reconstruction = True
self.mnist_params = DatasetMNISTParams(one_hot_labels=False,
examples_per_class=100)
self.mnist_test_params = DatasetMNISTParams(one_hot_labels=False,
examples_per_class=1000)
self.reconstructed_data = torch.zeros(
(self.input_square_side, self.input_square_side),
dtype=self.float_dtype,
device=self.device)
self.image_difference = torch.zeros(
(self.input_square_side, self.input_square_side),
dtype=self.float_dtype,
device=self.device)
# init nodes
random_noise_params = RandomNoiseParams(amplitude=0.0001)
self._node_mnist = DatasetMNISTNode(self.mnist_params, seed=0)
self._node_mnist_test = DatasetMNISTNode(self.mnist_test_params,
seed=1)
parent_rf_dims = Size2D(self.one_expert_lrf_width,
self.one_expert_lrf_width)
self._node_lrf = ReceptiveFieldNode(
(input_square_side, input_square_side, 1), parent_rf_dims,
Stride(stride, stride))
sp_class = ConvSpatialPoolerFlockNode if is_convolutional else SpatialPoolerFlockNode
# necessary to clone the params, because the GUI changes them during the simulation (restart needed)
self._node_spatial_pooler = sp_class(self.sp_params.clone(), seed=seed)
self._node_spatial_pooler_backup = sp_class(self.sp_params.clone(),
seed=seed)
self._noise_node = RandomNoiseNode(random_noise_params)
self.add_node(self._node_mnist)
self.add_node(self._node_mnist_test)
self.add_node(self._noise_node)
self.add_node(self._node_lrf)
self.add_node(self._node_spatial_pooler)
self.add_node(self._node_spatial_pooler_backup)
Connector.connect(self._node_mnist.outputs.data,
self._noise_node.inputs[0])
Connector.connect(self._noise_node.outputs[0],
self._node_lrf.inputs[0])
Connector.connect(self._node_lrf.outputs[0],
self._node_spatial_pooler.inputs.sp.data_input)
Connector.connect(
self._node_lrf.outputs[0],
self._node_spatial_pooler_backup.inputs.sp.data_input)
self.mnist_node = self._node_mnist
def __init__(self):
super().__init__('cuda')
# MNIST node producing two sequences.
# Receptive field looking at the input with window size = 14, 14, stride 7, 7.
# 3x3 experts looking at the outputs of the RF node.
# 1 expert combining the outputs.
mnist_seq_params: DatasetSequenceMNISTNodeParams = DatasetSequenceMNISTNodeParams(
[[0, 1, 2], [3, 1, 4]])
mnist_params = DatasetMNISTParams(class_filter=[0, 1, 2, 3, 4],
one_hot_labels=False,
examples_per_class=1)
n_channels_1 = 1
input_dims_1 = torch.Size((28, 28, n_channels_1))
parent_rf_dims_1 = Size2D(14, 14)
parent_rf_stride_dims_1 = Stride(7, 7)
self.expert_params = ExpertParams()
self.expert_params.flock_size = 9
self.expert_params.spatial.input_size = reduce(
lambda a, x: a * x, parent_rf_dims_1 + (n_channels_1, ))
self.expert_params.n_cluster_centers = 10
self.expert_params.spatial.buffer_size = 100
self.expert_params.spatial.batch_size = 50
self.expert_params.spatial.learning_period = 1
self.expert_params.spatial.cluster_boost_threshold = 100
self.expert_params.spatial.max_boost_time = 200
self.expert_params.temporal.seq_length = 3
self.expert_params.temporal.seq_lookahead = 1
self.expert_params.temporal.buffer_size = 100
self.expert_params.temporal.batch_size = 50
self.expert_params.temporal.learning_period = 50 + self.expert_params.temporal.seq_lookbehind
self.expert_params.temporal.incoming_context_size = 1
self.expert_params.temporal.max_encountered_seqs = 100
self.expert_params.temporal.n_frequent_seqs = 20
self.expert_params.temporal.forgetting_limit = 1000
self.parent_expert_params = self.expert_params.clone()
self.parent_expert_params.flock_size = 1
self.parent_expert_params.spatial.input_size = \
self.expert_params.flock_size * 2 * self.expert_params.n_cluster_centers
# Create the nodes.
mnist_node_1 = DatasetSequenceMNISTNode(params=mnist_params,
seq_params=mnist_seq_params)
mnist_node_2 = DatasetSequenceMNISTNode(params=mnist_params,
seq_params=mnist_seq_params)
receptive_field_node_1_1 = ReceptiveFieldNode(input_dims_1,
parent_rf_dims_1,
parent_rf_stride_dims_1)
receptive_field_node_1_2 = ReceptiveFieldNode(input_dims_1,
parent_rf_dims_1,
parent_rf_stride_dims_1)
expert_flock_node_1 = ExpertFlockNode(self.expert_params)
expert_flock_node_2 = ExpertFlockNode(self.expert_params)
join_node = JoinNode(dim=0, n_inputs=2)
expert_parent_node = ExpertFlockNode(self.parent_expert_params)
self.add_node(mnist_node_1)
self.add_node(receptive_field_node_1_1)
self.add_node(expert_flock_node_1)
self.add_node(mnist_node_2)
self.add_node(receptive_field_node_1_2)
self.add_node(expert_flock_node_2)
self.add_node(join_node)
self.add_node(expert_parent_node)
unsqueeze_node_0 = UnsqueezeNode(0)
self.add_node(unsqueeze_node_0)
Connector.connect(mnist_node_1.outputs.data,
receptive_field_node_1_1.inputs.input)
Connector.connect(receptive_field_node_1_1.outputs.output,
expert_flock_node_1.inputs.sp.data_input)
Connector.connect(expert_flock_node_1.outputs.tp.projection_outputs,
join_node.inputs[0])
Connector.connect(mnist_node_2.outputs.data,
receptive_field_node_1_2.inputs.input)
Connector.connect(receptive_field_node_1_2.outputs.output,
expert_flock_node_2.inputs.sp.data_input)
Connector.connect(expert_flock_node_2.outputs.tp.projection_outputs,
join_node.inputs[1])
Connector.connect(join_node.outputs.output,
unsqueeze_node_0.inputs.input)
Connector.connect(unsqueeze_node_0.outputs.output,
expert_parent_node.inputs.sp.data_input)
def create_rf_mapping(input_dims: Tuple[int, int, int], parent_rf_dims, parent_rf_stride_dims,
flatten_output_grid_dimensions, device):
bottom_grid_y, bottom_grid_x, chunk_size = input_dims
grids = Grids(Size2D(bottom_grid_y, bottom_grid_x), parent_rf_dims, parent_rf_stride=parent_rf_stride_dims,
flatten_output_grid_dimensions=flatten_output_grid_dimensions)
return ReverseMapping(grids, device, chunk_size)
def __init__(self):
super().__init__('cuda')
expert1_cluster_centers = 6
expert2_cluster_centers = 10 # 50
n_channels_1 = 1
rf1_input_size = (6, 8, n_channels_1)
rf1_rf_size = Size2D(2, 2)
rf2_input_size = (3, 4, expert1_cluster_centers)
rf2_rf_size = Size2D(2, 2)
rf2_stride = Stride(1, 1)
self.expert_params = ExpertParams()
self.expert_params.flock_size = 3 * 4
self.expert_params.n_cluster_centers = expert1_cluster_centers
self.expert_params.spatial.buffer_size = 100
self.expert_params.spatial.batch_size = 50
self.expert_params.spatial.learning_period = 1
self.expert_params.spatial.cluster_boost_threshold = 100
self.expert_params.spatial.max_boost_time = 200
self.expert_params.temporal.seq_length = 3
self.expert_params.temporal.seq_lookahead = 1
self.expert_params.temporal.buffer_size = 100 # 1000
self.expert_params.temporal.batch_size = 50 # 500
self.expert_params.temporal.learning_period = 50 + self.expert_params.temporal.seq_lookbehind
self.expert_params.temporal.max_encountered_seqs = 100 # 2000
self.expert_params.temporal.n_frequent_seqs = 20
self.expert_params.temporal.forgetting_limit = 1000 # 10000
self.expert_params_2 = self.expert_params.clone()
self.expert_params_2.flock_size = 2 * 3
self.expert_params_2.n_cluster_centers = expert2_cluster_centers
self.expert_params_2.temporal.incoming_context_size = 1
# self.expert_params_2.temporal.max_encountered_seqs = 2000
# self.expert_params_2.temporal.n_frequent_seqs = 500
# self.expert_params_2.temporal.forgetting_limit = 5000
# self.expert_params_2.temporal.context_without_rewards_size = 80
self.expert_params.temporal.n_providers = rf2_rf_size[0] * rf2_rf_size[
1] * 2
self.expert_params.temporal.incoming_context_size = self.expert_params_2.n_cluster_centers
# context_input_dims = (2, self.expert_params.temporal.context_without_rewards_size * 2)
context_input_dims = (*rf2_input_size[:2], *rf2_rf_size, 2, 3,
expert2_cluster_centers)
# Create the nodes.
receptive_field_1_node = ReceptiveFieldNode(rf1_input_size,
rf1_rf_size,
Stride(*rf1_rf_size))
receptive_field_2_node = ReceptiveFieldNode(rf2_input_size,
rf2_rf_size, rf2_stride)
receptive_field_reverse_node = ReceptiveFieldReverseNode(
context_input_dims, rf2_input_size, rf2_rf_size, rf2_stride)
expert_flock_1_node = ConvExpertFlockNode(self.expert_params)
expert_flock_2_node = ConvExpertFlockNode(self.expert_params_2)
dataset_simple_point_gravity_node = DatasetSimplePointGravityNode(
DatasetSimplePointGravityParams(
canvas_shape=Size2D(6, 8),
point_pos=Point2D(2, 3),
attractor_distance=2,
move_strategy=MoveStrategy.LIMITED_TO_LEFT_DOWN_QUADRANT))
self.add_node(dataset_simple_point_gravity_node)
self.add_node(receptive_field_1_node)
self.add_node(expert_flock_1_node)
self.add_node(receptive_field_2_node)
self.add_node(expert_flock_2_node)
self.add_node(receptive_field_reverse_node)
Connector.connect(dataset_simple_point_gravity_node.outputs.data,
receptive_field_1_node.inputs.input)
Connector.connect(receptive_field_1_node.outputs.output,
expert_flock_1_node.inputs.sp.data_input)
Connector.connect(expert_flock_1_node.outputs.tp.projection_outputs,
receptive_field_2_node.inputs.input)
Connector.connect(receptive_field_2_node.outputs.output,
expert_flock_2_node.inputs.sp.data_input)
Connector.connect(expert_flock_2_node.outputs.output_context,
receptive_field_reverse_node.inputs.input)
Connector.connect(receptive_field_reverse_node.outputs.output,
expert_flock_1_node.inputs.tp.context_input,
is_backward=True)
