def test_rf_unit(self, device): float_dtype = get_float(device) parent_rf_size_x = parent_rf_size_y = 4 n_channels = 4 image_grid_size_x = image_grid_size_y = 16 dimensions = (image_grid_size_y, image_grid_size_x, n_channels) parent_rf_dims = Size2D(parent_rf_size_y, parent_rf_size_x) unit = ReceptiveFieldUnit(AllocatingCreator(device), dimensions, parent_rf_dims, flatten_output_grid_dimensions=True) input_image = torch.zeros(image_grid_size_y, image_grid_size_x, n_channels, dtype=float_dtype, device=device) input_image[0, parent_rf_size_x, 0] = 1 unit.step(input_image) node_output = unit.output_tensor n_parent_rfs = (image_grid_size_x // parent_rf_size_x) * ( image_grid_size_y // parent_rf_size_y) assert node_output.shape == torch.Size( [n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels]) assert node_output[1, 0, 0, 0] == 1 # assert node_output.interpret_shape == [n_parent_rfs, parent_rf_size_y, parent_rf_size_x, n_channels] back_projection = unit.inverse_projection(node_output) # assert back_projection.interpret_shape == input_image.shape assert back_projection.equal(input_image)
def test_test_cycles(): """Set the testing position to the last element and make step. Then test weather the cycle starts in the begining. """ device = 'cuda' # init node params = DatasetSeObjectsParams() params.dataset_size = SeDatasetSize.SIZE_24 params.dataset_config = DatasetConfig.TEST_ONLY params.random_order = False params.save_gpu_memory = False node = DatasetSeObjectsNode(params, seed=None) node.allocate_memory_blocks(AllocatingCreator(device)) # make step node.step() # read outputs first_image = node.outputs.image_output.tensor first_label = node.outputs.task_to_agent_label_ground_truth.tensor # hack the position and make another step node._unit._pos = len(node._unit._test_images) node.step() # read another outputs output_image = node.outputs.image_output.tensor output_label = node.outputs.task_to_agent_label_ground_truth.tensor # assert they are both the same assert same(first_image, output_image) assert same(first_label, output_label)
def test_node_accessor(generator, device): """Validate the accessor properties.""" # common params params = DatasetMNISTParams() params.one_hot_labels = False seq = None # different iterators might result in different behavior here if type(generator) is List: seq = DatasetSequenceMNISTNodeParams(seqs=generator) elif generator == 0: params.random_order = True else: params.random_order = False node = DatasetSequenceMNISTNode(params, seq_params=seq, dataset=get_dataset(), seed=123) node.allocate_memory_blocks(AllocatingCreator(device=device)) node.step() bitmap = MnistNodeAccessor.get_data(node) label = MnistNodeAccessor.get_label_id(node) assert type(label) is int assert type(bitmap) is torch.Tensor assert sequences_equal(bitmap.shape, [28, 28]) assert 0 <= label < 10
def test_to_one_hot_speed(self, capsys): @measure_time(iterations=100, function_repetitions=100) def measured_step(): mb.tensor.copy_( torch.rand(input_shape, dtype=get_float(device), device=device)) to_one_hot.step() input_shape = (150, ) device = 'cuda' vector = torch.zeros(input_shape, dtype=get_float(device), device=device) mb = MemoryBlock() mb.tensor = torch.tensor(vector, device=device, dtype=get_float(device)) to_one_hot = ToOneHotNode(mode=ToOneHotMode.RANDOM) Connector.connect(mb, to_one_hot.inputs.input) to_one_hot.allocate_memory_blocks(AllocatingCreator(device)) with capsys.disabled(): measured_step()
def test_train_test_position_persistence(): """If we switch from train to test, do some testing and then back to train, we should continue where stopped""" device = 'cuda' params = get_common_params() params.switch_train_resets_train_pos = False node = DatasetSeObjectsNode(params, seed=None) node.allocate_memory_blocks(AllocatingCreator(device)) node.switch_training(training_on=True) assert node._unit._pos == -1 first_steps = 10 _, _, _, = collect_data(node, first_steps) # we are at the expected position last_train_pos = -1 + first_steps assert node._unit._pos == last_train_pos assert train_pos(node) == last_train_pos # switch to training (redundant) and check position not changed node.switch_training(training_on=True) assert node._unit._pos == last_train_pos assert train_pos(node) == last_train_pos second_steps = 3 # check the training position continues increasing _, _, _ = collect_data(node, second_steps) last_train_pos += second_steps assert node._unit._pos == last_train_pos assert train_pos(node) == last_train_pos # switch to testing and collect some data # _pos should reset (start testing), but the training_pos in the tensor should stay the same node.switch_training(training_on=False) assert node._unit._pos == -1 assert train_pos(node) == last_train_pos testing_steps = 7 test_images, test_labels, _ = collect_data(node, testing_steps) assert node._unit._pos == -1 + testing_steps # pos in the training set changes assert train_pos( node) == last_train_pos # train pos in the tensor does not # switch back to training and check expected positions third_train_steps = 2 node.switch_training(training_on=True) _, _, _ = collect_data(node, third_train_steps) last_train_pos += third_train_steps assert train_pos(node) == last_train_pos assert node._unit._pos == last_train_pos # one more testing node.switch_training(training_on=False) test_images_2, test_labels_2, _ = collect_data(node, testing_steps) assert same(test_images[0], test_images_2[0]) assert same(test_labels[0], test_labels_2[0])
def test_expansion(input_shape, dim, desired_size, expected_output_shape): input_data = torch.zeros(input_shape) expected_output = torch.zeros(expected_output_shape) expand_unit = Expand(AllocatingCreator(device='cpu'), input_shape, dim=dim, desired_size=desired_size) expand_unit.step(input_data) assert same(expected_output, expand_unit.output)
def test_seq_node(): seq = [1, 2, 3, 4, 5, 10, 2, 2] iterator = SequenceGenerator.from_list(seq) node = SequenceNode(seq=iterator) node._prepare_unit(AllocatingCreator(device='cpu')) for element in seq * 2: node.step() assert element == node.outputs.output.tensor.item()
def test_node_accessor_and_determinism(device): node = RandomNumberNode(lower_bound=LOWER_BOUND, upper_bound=UPPER_BOUND, seed=SEED) node.allocate_memory_blocks(AllocatingCreator(device)) # expected sequence assert generate_and_validate_sequence(node) # expected after re-allocating MBs node.allocate_memory_blocks(AllocatingCreator(device)) assert generate_and_validate_sequence(node) # sequence independent on the global seeds node.allocate_memory_blocks(AllocatingCreator(device)) set_global_seeds(None) assert generate_and_validate_sequence(node)
def test_creator_methods(create_func): measuring_creator = MeasuringCreator() allocating_creator = AllocatingCreator('cpu') expected = create_func(allocating_creator) surrogate = create_func(measuring_creator) assert expected.shape == surrogate.shape
def test_node_accessor_and_determinism_seed(device): # new node with a different seed, should produce different results node = RandomNumberNode(lower_bound=LOWER_BOUND, upper_bound=UPPER_BOUND, seed=None) node.allocate_memory_blocks(AllocatingCreator(device)) assert not generate_and_validate_sequence(node)
def test_mse_multiple_steps(): device = 'cpu' creator = AllocatingCreator(device=device) dtype = creator.float32 input_tensors1 = [ creator.tensor([0, 1, 2, 0, 1, 2], device=device, dtype=dtype), creator.tensor([0, 3, 0, 0, 1, 0], device=device, dtype=dtype) ] # squared diff = 12 input_tensors2 = [ creator.tensor([1, 1, 0, 0, 1, 2], device=device, dtype=dtype), creator.tensor([0, 3, 2, 1, 3, 0], device=device, dtype=dtype) ] # squared diff = 18 input_tensors3 = [ creator.tensor([1, 1, 2, 0, 1, 1], device=device, dtype=dtype), creator.tensor([-2, 3, 1, 3, 1, 0], device=device, dtype=dtype) ] # squared diff = 24 expected_result = creator.tensor( [3], dtype=dtype, device=device) # MSE=3 for a concatenation of the input tensors mse_unit = MseUnit(creator, input_shape=input_tensors1[0].shape, buffer_size=3) mse_unit.step(input_tensors1[0], input_tensors1[1]) mse_unit.step(input_tensors2[0], input_tensors2[1]) mse_unit.step(input_tensors3[0], input_tensors3[1]) assert same(expected_result, mse_unit._mean_square_error_output)
def test_join_node_inverse_flatten(): device = 'cpu' creator = AllocatingCreator(device) dtype = creator.float32 # The result of the inverse projection should only be one tensor. expected_results = [ creator.tensor([[1, 2, 3, 4], [5, 6, 7, 8]], dtype=dtype, device=device), creator.tensor([9, 10], dtype=dtype, device=device) ] input_memory_blocks = [MemoryBlock(), MemoryBlock()] input_memory_blocks[0].tensor = creator.zeros((2, 4)) input_memory_blocks[1].tensor = creator.zeros((2, )) output_tensor = creator.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=dtype, device=device) join_node = JoinNode(flatten=True) Connector.connect(input_memory_blocks[0], join_node.inputs[0]) Connector.connect(input_memory_blocks[1], join_node.inputs[1]) output_inverse_packet = InversePassOutputPacket(output_tensor, join_node.outputs.output) join_node.allocate_memory_blocks(creator) results = join_node.recursive_inverse_projection_from_output( output_inverse_packet) for expected, result in zip(expected_results, results): assert same(expected, result.tensor)
def test_create_node(self, device): unit = DatasetAlphabetUnit( AllocatingCreator(device), DatasetAlphabetParams( symbols="abcd", sequence_probs=DatasetAlphabetSequenceProbsModeParams( seqs=['abcd']))) assert [4, 7, 5] == list(unit.all_symbols.shape) assert [7, 5] == list(unit.output_data.shape)
def test_join_inverse_dim_0(): creator = AllocatingCreator(device='cpu') dtype = creator.float32 dim = 0 input_shapes = [(2, 4), (2, 4)] expected_results = [ creator.tensor([[1, 2, 3, 4], [5, 6, 7, 8]], dtype=dtype, device=creator.device), creator.tensor([[9, 10, 11, 12], [13, 14, 15, 16]], dtype=dtype, device=creator.device) ] _test_join_inverse(dim, input_shapes, expected_results, creator, dtype, creator.device)
def test_mode_sequence_probs(self, device): unit = DatasetAlphabetUnit( AllocatingCreator(device), DatasetAlphabetParams( symbols="abcd", sequence_probs=DatasetAlphabetSequenceProbsModeParams( seqs=['abc']))) generator = self.label_generator(unit) result = [next(generator) for _ in range(7)] assert [0, 1, 2, 0, 1, 2, 0] == result
def create_tp_buffer(flock_size=2, buffer_size=5, n_cluster_centers=3, n_frequent_seqs=3, context_size=4, n_providers=1, device='cuda'): return TPFlockBuffer(AllocatingCreator(device), flock_size, buffer_size, n_cluster_centers, n_frequent_seqs, context_size, n_providers)
def test_inverse_projection(expected_input_shape, dim, desired_size, output_shape): output_data = torch.zeros(output_shape) expected_input = torch.zeros(expected_input_shape) expand_unit = Expand(AllocatingCreator(device='cpu'), expected_input_shape, dim=dim, desired_size=desired_size) # expand_unit.step(input_data) projection = expand_unit.inverse_projection(output_data) assert same(expected_input, projection)
def test_join_inverse_dim_1(): creator = AllocatingCreator(device='cpu') dtype = creator.float32 dim = 1 input_shapes = [(4, 2), (4, 2)] expected_results = [ creator.tensor([[1, 2], [5, 6], [9, 10], [13, 14]], dtype=dtype, device=creator.device), creator.tensor([[3, 4], [7, 8], [11, 12], [15, 16]], dtype=dtype, device=creator.device) ] _test_join_inverse(dim, input_shapes, expected_results, creator, dtype, creator.device)
def make_nnet(input_size: int = 16, num_classes: int = 5, seed: int = 123, buffer_s: int = 16, batch_s: int = 8, mixed_mode: bool = True): device = 'cuda' # set topology params and configs _params = NNetParams(NNetParams.default_params()) # _params.set_params(_nn_node_params) # params defined in this file # ._params.set_params(kwargs) # params defined in the constructor # small input sizes for testing _params.input_shape = (3, input_size, input_size) _params.output_size = num_classes _params.seed = seed _params.batch_size = batch_s _params.buffer_size = buffer_s _params.num_epochs = 3 _params.mixed_mode = mixed_mode # observation storage params _observation_types = { 'x': (_params.buffer_size, *_params.input_shape), # observations 'y': (_params.buffer_size, _params.output_size), # labels } # data storage _storage = ObservationStorage(_params.buffer_size, _observation_types) _storage.to('cpu' if _params.mixed_mode else device) # network needs to have the global seeds to have set before creating (outside of the node in this case) set_global_seeds(seed=_params.seed) # neural network setup _network = NNet(input_shape=_params.input_shape, output_shape=_params.output_size).to( 'cuda' if _params.mixed_mode else device) # neural net optimizer _optimizer = optim.Adam(_network.parameters(), lr=_params.lr) # NNet Node _nnet_node = NNetNode(_network, _optimizer, _storage, _params, name='Neural Network Node') creator = AllocatingCreator(device=device) _nnet_node.allocate_memory_blocks(creator) return _params, _nnet_node
def test_sequence_iteration(expected_sequence, seq_params): node = DatasetSequenceMNISTNode(params=DatasetMNISTParams(one_hot_labels=False), seq_params=seq_params) node.allocate_memory_blocks(AllocatingCreator(device='cpu')) actual_sequence = [] for x in range(len(node._seq_params.seqs[0])): node.step() actual_sequence.append(node.outputs.label.tensor.clone()) actual_sequence = torch.cat(actual_sequence, dim=0) assert same(expected_sequence, actual_sequence)
def test_load_dataset(self): images_tensor = DatasetLoader._load_dataset( os.path.join('tests', 'data', 'datasets', 'testing_dataset')) device = 'cpu' creator = AllocatingCreator(device=device) expected_images_tensor = self._create_expected_images_tensor(creator) assert same(expected_images_tensor[:2], images_tensor[:2]) # jpeg uses lossy compression assert same(expected_images_tensor[2], images_tensor[2], eps=0.01)
def test_scatter_node(self, device, input, mask, output_shape, dimension, expected_result): float_dtype = get_float(device) input_mb = MemoryBlock() input_mb.tensor = torch.tensor(input, device=device, dtype=float_dtype) expected = torch.tensor(expected_result, device=device, dtype=float_dtype) sn = ScatterNode(mapping=mask, output_shape=output_shape, dimension=dimension, device=device) Connector.connect(input_mb, sn.inputs.input) sn.allocate_memory_blocks(AllocatingCreator(device)) sn.step() assert same(sn.outputs.output.tensor, expected)
def test_sample_learning_batch_combinations(self, method, flock_indices, elements_written): flock_size = 3 creator = AllocatingCreator('cpu') f_size = len(flock_indices) if flock_indices is not None else 3 buffer = SPFlockBuffer(creator, buffer_size=20, n_cluster_centers=3, flock_size=flock_size, input_size=5) if flock_indices is not None: buffer.set_flock_indices( creator.tensor(flock_indices, dtype=torch.int64)) buffer.total_data_written[:] = elements_written buffer.clusters.stored_data[:, :elements_written] = creator.tensor( [0, 1, 0]) buffer.inputs.stored_data[:, :elements_written, :] = creator.tensor( [1.3, 0.2, 0.6, 0.4, 0.1]) # use some dummy value here to check that it is rewriting all the lines in res dummy_value = -2.1 sampled_data = creator.full((f_size, elements_written, 5), fill_value=dummy_value) buffer.sample_learning_batch(elements_written, sampled_data, method) assert (sampled_data == dummy_value).any().item() == 0
def get_subflock_creation_testing_flock( flock_size=10, subflock_size=6, input_size=5, buffer_size=7, batch_size=5, n_cluster_centers=4, device='cpu', sampling_method=SamplingMethod.LAST_N) -> Tuple[SPFlock, torch.Tensor]: # Generate a flock with some data float_dtype = get_float(device) params = ExpertParams() params.flock_size = flock_size params.n_cluster_centers = n_cluster_centers params.spatial.input_size = input_size params.spatial.buffer_size = buffer_size params.spatial.batch_size = batch_size params.spatial.sampling_method = sampling_method flock = SPFlock(params, AllocatingCreator(device)) flock.buffer.inputs.stored_data = torch.rand( (flock_size, buffer_size, input_size), dtype=float_dtype, device=device) flock.buffer.clusters.stored_data = torch.rand( (flock_size, buffer_size, n_cluster_centers), dtype=float_dtype, device=device) # NOTE: This is not one-hot and thus not real data flock.buffer.current_ptr = torch.rand(flock_size, device=device).type(torch.int64) flock.buffer.total_data_written = torch.rand( flock_size, device=device).type(torch.int64) # The bookeeping values which are copied across flock.cluster_boosting_durations = torch.rand( (flock_size, n_cluster_centers), device=device).type(torch.int64) flock.prev_boosted_clusters = torch.clamp( torch.round(torch.rand((flock_size, n_cluster_centers), device=device)), 0, 1).type(torch.uint8) flock.boosting_targets = torch.rand((flock_size, n_cluster_centers), device=device).type(torch.int64) # Indices which are to be subflocked indices = torch.tensor(np.random.choice(flock_size, subflock_size, replace=False), dtype=torch.int64, device=device) return flock, indices
def __init__(self, device: str): super().__init__('Graph', 0, 0) self.device = device self.float_dtype = get_float(self.device) self._measuring_creator = MeasuringCreator() self._allocating_creator = AllocatingCreator(device) self._seed = None self._do_before_step = None self._visible_nodes = set() self._id_generator = IdGenerator()
def test_extra_buffers(): num_indices = 5 indices = torch.arange(num_indices, dtype=torch.int64) proc = init_process(indices, subflocking=False) creator = AllocatingCreator("cpu") buff = DummyBuffer(creator, 10, 100, (1, 1)) proc._get_buffer(buff) with pytest.raises(AssertionError): proc._get_buffer(buff)
def test_rnd_node_accessor_return_type(device): lower_bound = 50 upper_bound = 100 node = RandomNumberNode(lower_bound=lower_bound, upper_bound=upper_bound) node.allocate_memory_blocks(AllocatingCreator(device=device)) node._step() random_number = RandomNumberNodeAccessor.get_output_id(node) assert type(random_number) is int assert lower_bound <= random_number < upper_bound
def test_load_dataset_shippet(self): images_tensor = DatasetLoader._load_dataset(os.path.join( 'tests', 'data', 'datasets', 'testing_dataset'), limit_to_n=2) device = 'cpu' creator = AllocatingCreator(device=device) expected_images_tensor = self._create_expected_images_tensor(creator) assert len(images_tensor) == 2 assert same(expected_images_tensor[:2], images_tensor[:2])
def test_deterministic_samples_per_class_choice(device: str, sequence_type: str, class_filter): """Test random/ordered/sequential sampling of the dataset with and without class filter. Everything should produce valid and repeatable results if seed is not None. """ params = DatasetMNISTParams() generator_params = None if sequence_type == 'ordered': params.random_order = False elif sequence_type == 'random': params.random_order = True else: generator_params = DatasetSequenceMNISTNodeParams([[1, 2, 3], [5, 2]]) # parametrized, try filtered and non-filtered versions params.class_filter = class_filter params.examples_per_class = 10 n_samples = 15 creator = AllocatingCreator(device=device) node = DatasetSequenceMNISTNode(params, seq_params=generator_params, dataset=get_dataset(), seed=10) node.allocate_memory_blocks(creator) labels, bitmaps = collect_data(node, n_samples) node = DatasetSequenceMNISTNode(params, seq_params=generator_params, dataset=get_dataset(), seed=10) node.allocate_memory_blocks(creator) labels_b, bitmaps_b = collect_data(node, n_samples) assert len(bitmaps_b) == len(bitmaps) # go through the generated sequences and compare labels and bitmaps, should be identical for bitmap_a, label_a, bitmap_b, label_b in zip(bitmaps, labels, bitmaps_b, labels_b): # the same sequences of labels assert label_a == label_b # bitmaps are identical (randomly sampled the same samples from the dataset) assert same(bitmap_a, bitmap_b)
def get_subflock_integration_testing_flock( params, subflock_size, device) -> Tuple[SPFlock, torch.Tensor, np.array]: flock = SPFlock(params, AllocatingCreator(device)) float_dtype = get_float(device) flock.buffer.inputs.stored_data = torch.rand(flock.buffer.inputs.dims, dtype=float_dtype, device=device) # NOTE: This is not one-hot and thus not real data flock.buffer.clusters.stored_data = torch.rand(flock.buffer.clusters.dims, dtype=float_dtype, device=device) flock.buffer.current_ptr = torch.randint(0, params.spatial.buffer_size, flock.buffer.current_ptr.size(), dtype=torch.int64, device=device) flock.buffer.total_data_written = torch.randint( 0, params.spatial.buffer_size, flock.buffer.current_ptr.size(), dtype=torch.int64, device=device) flock.cluster_boosting_durations = torch.randint( 0, 20, flock.cluster_boosting_durations.size(), dtype=torch.int64, device=device) flock.prev_boosted_clusters = \ torch.clamp(torch.randint(0, 2, flock.prev_boosted_clusters.size(), device=device), 0, 1).type(torch.uint8) flock.boosting_targets = torch.randint(0, params.flock_size, flock.boosting_targets.size(), dtype=torch.int64, device=device) flock.forward_clusters = torch.rand(flock.forward_clusters.size(), dtype=float_dtype, device=device) # Create a subflock and populate it indices_np = sorted( np.random.choice(params.flock_size, subflock_size, replace=False)) indices = torch.tensor(list(map(int, indices_np)), dtype=torch.int64, device=device).unsqueeze(dim=1) return flock, indices, indices_np