def _exploration_cluster(self, exploration_random_numbers: torch.Tensor,
action_outputs: torch.Tensor,
action_rewards: torch.Tensor):
"""For all those exploring, they have a self.exploration_probability chance of a totally random cluster being picked to explore instead
of the sequence they were going to do in _exploration_sequence. Update rewards accordingly.
"""
cluster_exploration = (exploration_random_numbers <
self.cluster_exploration_prob *
self.exploration_probability).float()
random_indices = torch.randint(low=0,
high=self.n_cluster_centers,
size=(self._flock_size, ),
device=self.device)
# one-hot representation of the actions (=clusters) we want to explore right now
exploration_actions = id_to_one_hot(random_indices,
self.n_cluster_centers,
dtype=self._float_dtype)
exploration_rewards = exploration_actions * EXPLORATION_REWARD
# overwrite the actions for experts which are exploring by exploration actions
weighted_sum_(tensor_a=exploration_actions,
weight_a=cluster_exploration,
tensor_b=action_outputs,
weight_b=1.0 - cluster_exploration,
output=action_outputs)
# Do the same for the rewards
weighted_sum_(tensor_a=exploration_rewards,
weight_a=cluster_exploration,
tensor_b=action_rewards,
weight_b=1.0 - cluster_exploration,
output=action_rewards)
def test_to_one_hot_in_process(self, device):
flock_size = 2
n_cluster_centers = 3
float_dtype = get_float(device)
all_indices = torch.arange(flock_size,
dtype=torch.int64,
device=device)
process = SPProcessStub(all_indices,
do_subflocking=True,
n_cluster_centers=n_cluster_centers,
input_size=1,
device=device)
closest_cluster_centers = torch.tensor([[0, 1], [2, 2], [1, 2]],
dtype=torch.int64,
device=device)
result = id_to_one_hot(closest_cluster_centers,
process._n_cluster_centers,
dtype=float_dtype)
expected_result = torch.tensor(
[[[1, 0, 0], [0, 1, 0]], [[0, 0, 1], [0, 0, 1]],
[[0, 1, 0], [0, 0, 1]]],
dtype=float_dtype,
device=device)
assert same(expected_result, result)
def _exploration_sequence(self, exploring: torch.Tensor,
exploration_random_numbers: torch.Tensor,
sequence_likelihoods_active: torch.Tensor,
buffer: TPFlockBuffer):
"""Decide if exploring. If yes, overwrite the sequence_likelihoods by a random sequence which will get
converted to a corresponding cluster center later. Update the rewards with a corresponding exploration reward.
Also stores the exploration flag into the buffer."""
# Randomly decide which experts will be exploring
exploration_random_numbers.uniform_(0, 1.0)
exploring.copy_(
exploration_random_numbers < self.exploration_probability)
buffer.exploring.store(exploring)
random_indices = torch.randint(low=0,
high=self.n_frequent_seqs,
size=(self._flock_size, ),
device=self.device)
# one-hot representation of the sequences we want to explore right now
exploration_sequences = id_to_one_hot(random_indices,
self.n_frequent_seqs,
dtype=self._float_dtype)
exploration_sequences *= EXPLORATION_REWARD
# overwrite the actions for experts which are exploring by exploration actions
weighted_sum_(tensor_a=exploration_sequences,
weight_a=exploring,
tensor_b=sequence_likelihoods_active,
weight_b=1.0 - exploring,
output=sequence_likelihoods_active)
def get_landmark(self, y: float, x: float):
"""Finds a nearest landmark for a given yx position. Position expected from range [0, MAX_POSITION).
Args:
y: y position in range [0,MAX_POSITION)
x: x position in range [0,MAX_POSITION)
Returns:
ID of the nearest landmark from [0, horizontal_segments * vertical_segments] as a scalar Tensor
(or one-hot).
"""
if math.isnan(y) or math.isnan(x):
return self._handle_nan()
sy = self.segment_sizes[0].item()
sx = self.segment_sizes[1].item()
y_landmark = np.floor(y / sy)
x_landmark = np.floor(x / sx)
res = y_landmark * self.horizontal_segments + x_landmark
result = torch.tensor([res],
dtype=self._float_dtype,
device=self._device)
return result, id_to_one_hot(result.long(),
self.num_landmarks).squeeze(0)
def test_id_to_one_hot(indexes, number_of_elements, dtype, expected_output,
device):
float_type = get_float(device)
indexes_tensor = torch.tensor(indexes, dtype=torch.int64, device=device)
if dtype is None: # test default dtype
result = id_to_one_hot(indexes_tensor, number_of_elements)
ground_truth = torch.tensor(expected_output,
dtype=float_type,
device=device)
else:
result = id_to_one_hot(indexes_tensor, number_of_elements, dtype=dtype)
ground_truth = torch.tensor(expected_output,
dtype=dtype,
device=device)
assert same(ground_truth, result)
def step(self):
self.output_data.copy_(self.all_symbols[self._current])
self.output_label[0] = self._current
self.output_sequence_id[0] = self.seq.current_sequence_id
self.output_sequence_id_one_hot.copy_(
id_to_one_hot(self.output_sequence_id,
self.output_sequence_id_one_hot.shape[1]))
self._current = next(self.seq)
def _forward(self, input_clusters, input_context, input_rewards, sp_mask):
self._verify_context_and_rewards(input_context, input_rewards)
self._every_step_computations()
trained_forward_indices, untrained_forward_indices, common_mask = self._determine_forward_pass(
input_clusters, sp_mask)
# Create and run trained
if self.trained_forward_process is not None:
# Free memory of old process
del self.trained_forward_process
# TODO this causes blinking of process observers in UI. Solve it e.g. by caching the last value in observer.
# Do not assign to self to prevent observer to fetch invalid data
trained_process = self._trained_forward_process_factory.create(
self, input_clusters, self.input_context, self.input_rewards,
trained_forward_indices, self._device)
self._run_process(trained_process)
if trained_process is not None:
self.trained_forward_process = trained_process
untrained_process = self._untrained_forward_process_factory.create(
self, input_clusters, self.input_context, self.input_rewards,
untrained_forward_indices, self._device)
self._run_process(untrained_process)
# Find all TPs who are annoyed (i.e over the frustration threshold) and generate a random action for them
# Also: activate their exploration bits in the buffer, so as to learn that this action they have currently
# taken doesn't do anything
annoyed = self.frustration > self.frustration_threshold
annoyed_experts = annoyed.sum()
action_ids = torch.randint(high=self.n_cluster_centers,
size=(annoyed_experts.item(), ),
dtype=torch.int64,
device=self._device)
actions = id_to_one_hot(action_ids, vector_len=self.n_cluster_centers)
self.action_rewards[annoyed, :] = actions * EXPLORATION_REWARD
self.action_outputs[annoyed, :] = actions
if annoyed_experts > 0:
annoyed_indices = annoyed.nonzero().view(-1)
self.buffer.set_flock_indices(annoyed_indices)
self.buffer.exploring.store(
torch.ones((annoyed_indices.numel(), ),
dtype=self._float_dtype,
device=self._device))
self.buffer.unset_flock_indices()
return common_mask
def step(self):
self._step += 1
if self._step % self._next_generation != 0:
return
self._step = 0
if self._random_generation_intervals:
self._next_generation = self._random.randint(
low=1, high=self._generate_new_every_n + 1)
self._current_value[0] = self._random.randint(low=self._lower_bound,
high=self._upper_bound)
self._scalar_output[0] = self._current_value[0]
self._one_hot_output.copy_(
id_to_one_hot(self._current_value, self._upper_bound).squeeze())
def compute_nn_classifier_accuracy(inputs: torch.Tensor,
labels: torch.Tensor,
n_classes: int,
custom_input_length: int = None,
learn_rate: float = 0.1,
max_loss_difference: float = 0.005,
loss_window_length: int = 5,
max_epochs: int = 100,
use_hidden_layer: bool = False,
log_loss: bool = False) -> float:
"""Uses a simple (low-VC dimension) classifier to learn the labels.
Accuracy of this classifier is returned as a measure of quality of the representation.
Args:
inputs: matrix of data [n_samples, input_size] - float (long shaped as [n_samples] if inputs_are_ids is True)
labels: vector of labels [n_samples] - long
n_classes: how many classes there is
custom_input_length: by default, the inputs are expected to be [n_samples, input_size], if this parameter is set,
inputs are expected to be vector of [n_samples] scalars, which is converted into one-hot format of this length
learn_rate: 0.01, it uses SGD
max_loss_difference: if the max difference between the loss values is smaller than this, stop training
loss_window_length: from how long history of the loss values to determine when to stop training?
max_epochs: in case that the data are too difficult, the training does not converge, this stops it
use_hidden_layer: use a classifier with hidden layer (should be false)
log_loss: should the loss be printed in time?
Returns:
accuracy of the classifier ~ quality of the input representation
"""
if len(inputs.shape) != 2 and custom_input_length is None:
raise ValueError(
'input tensor is expected to have 2D shape [n_samples, data_length], view it appropriately'
)
if custom_input_length is not None:
if len(inputs.shape) != 1:
raise ValueError(
'in case inputs_are_ids, the shape should be 1D [n_samples]')
inputs = id_to_one_hot(inputs, vector_len=custom_input_length)
trainer = train_nn_classifier(inputs, labels, n_classes, learn_rate,
max_loss_difference, loss_window_length,
max_epochs, use_hidden_layer, log_loss)
return trainer.compute_accuracy(inputs, labels)
def step(self):
"""Returns nothing sets values of the tensors, which are marked as Node outputs (MemoryBlocks)."""
# the data are held on CPU because they can be quite big and they are sent to GPU just when needed
data, self.label_tensor = next(self._data_seq)
# TODO (Time-Optim) the iterator could be changed so that the copying to output tensors is avoided?
# copy the results to outputs
self.output_data.copy_(data.to(self._device))
self.label_tensor = self.label_tensor.to(self._device)
if self._seq is not None:
self.output_sequence_id[0] = self._seq.current_sequence_id
if self._params.one_hot_labels:
self.output_label.copy_(id_to_one_hot(self.label_tensor, 10))
else:
self.output_label.copy_(self.label_tensor)
def compute_closest_cluster_centers(self, cluster_centers: torch.Tensor, data: torch.Tensor) \
-> Tuple[torch.Tensor, torch.Tensor]:
"""Calculates the closest cluster for a batch of datapoints for each member of the flock.
Each member of the flock receives and clusters the datapoints and returns the closest cluster for each of them.
Args:
cluster_centers: [flock_size, n_cluster_centers, input_size]
data (torch.Tensor): [flock_size, batch_size, input_size] containing datapoints
for each member of the flock
Returns:
one_hot_cluster_centers (torch.Tensor): [flock_size, batch_size, n_cluster_centers] the one-hot clustering
of each of the datapoints given their respective spatial poolers.
datapoint_variances (torch.Tensor): [flock_size, n_cluster_centers] - variance of all points belonging
to each cluster center or -1 if it has zero points.
"""
# [flock_size, batch_size, n_cluster_centers]
distances = self._compute_squared_distances(cluster_centers, data)
# [flock_size, batch_size]
closest = self._closest_cluster_center(distances)
# Get one-hot representations of the assigned cluster centers
# [flock_size, batch_size, n_cluster_centers]
one_hot_cluster_centers = id_to_one_hot(closest,
self._n_cluster_centers,
dtype=self._float_dtype)
# Set squared distance between datapoint and irrelevant (not closest) clusters centers to zero.
# Then sum over batch.
# [flock_size, n_cluster_centers]
datapoint_variances = torch.sum(one_hot_cluster_centers * distances,
dim=1)
# identify cluster centers with zero points
# [flock_size, n_cluster_centers]
indices = torch.sum(one_hot_cluster_centers, dim=1) == 0
# set their variance to -1
datapoint_variances -= indices.type(self._float_dtype)
return one_hot_cluster_centers, datapoint_variances
def get_landmarks(
self,
yx_positions: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
"""Compute landmarks for no_positions stored in a 2D tensor.
Args:
yx_positions: tensor of size [num_positions, 2] containing list of YX positions in the square map
Returns:
Tensor of size [num_positions] (or [num_positions, num_landmarks]), which contains nearest landmark
for each position.
"""
yx_landmarks = torch.floor(yx_positions / self.segment_sizes)
result = (yx_landmarks[:, 0] *
self.horizontal_segments) + yx_landmarks[:, 1]
one_hot_result = id_to_one_hot(
result.view(-1).long(), self.num_landmarks)
return result, one_hot_result
def train_svm_classifier(inputs: torch.Tensor,
labels: torch.Tensor,
n_classes: int,
custom_input_length: int = None) -> "SvmModel":
model = SvmModel(num_classes=n_classes)
if len(inputs.shape) != 2 and custom_input_length is None:
raise ValueError(
'input tensor is expected to have 2D shape [n_samples, data_length], view it appropriately'
)
if custom_input_length is not None:
inputs = id_to_one_hot(inputs, custom_input_length)
if len(labels.shape) >= 2:
labels = one_hot_to_id(labels)
inputs = np.array(inputs.cpu())
labels = np.array(labels.cpu())
model.train(inputs, labels)
return model
def _check_and_standardize(
data: Union[torch.Tensor, List[int], List[torch.Tensor]],
classifier_input_size: int = None,
device: str = 'cpu',
verify_input_tensor: bool = False) -> torch.Tensor:
"""Converts input data or labels into the format used internally.
Args:
data: The data (input data or labels) to be standardized
classifier_input_size: Size of one-hot vectors; must be specified when input is integer ids
device: Device of tensors; must be specified when input is list
verify_input_tensor: True if data is input data (not labels)
Returns:
The data in a standard format
"""
if type(data) is list and type(data[0]) is int:
data = torch.tensor(data, device=device, dtype=torch.long)
if type(data) is list and type(data[0]) is torch.Tensor:
data = torch.stack(data)
if verify_input_tensor and data.dim(
) != 2 and classifier_input_size is None:
raise ValueError(
'input tensor is expected to have 2D shape [n_samples, data_length], view it appropriately'
)
if classifier_input_size is not None:
if data.dim() != 1:
raise ValueError(
'in case classifier_input_size is not None, the shape should be 1D [n_samples]'
)
data = id_to_one_hot(data, vector_len=classifier_input_size)
if torch.isnan(data).any():
logger.error(f'data is containing NaNs')
return data
def argmin(inputs: List[torch.Tensor], outputs: List[torch.Tensor]):
outputs[0].copy_(id_to_one_hot(inputs[0].argmin(), num_predictors))
