def _initialize_weights(self):
if self._should_load(TASK_HIDDEN_KEY):
self.task_hidden_weights = self.loaded_weights[TASK_HIDDEN_KEY].T
else:
self.task_hidden_weights = pnl.random_matrix(
self.num_tasks, self.hidden_layer_size, 2,
-1) * self.weight_init_scale
if self._should_load(INPUT_HIDDEN_KEY):
self.input_hidden_weights = self.loaded_weights[INPUT_HIDDEN_KEY].T
else:
self.input_hidden_weights = pnl.random_matrix(
self.num_features, self.hidden_layer_size, 2,
-1) * self.weight_init_scale
if self._should_load(HIDDEN_OUTPUT_KEY):
self.hidden_output_weights = self.loaded_weights[
HIDDEN_OUTPUT_KEY].T
else:
self.hidden_output_weights = pnl.random_matrix(
self.hidden_layer_size, self.num_features, 2,
-1) * self.weight_init_scale
if self._should_load(TASK_OUTPUT_KEY):
self.task_output_weights = self.loaded_weights[TASK_OUTPUT_KEY].T
else:
self.task_output_weights = pnl.random_matrix(
self.num_tasks, self.num_features, 2,
-1) * self.weight_init_scale
def _generate_process(self, weight_key, in_size, out_size, in_layer, out_layer, name, learning_rate=0.0):
"""
Generate a simple PNL process between two layers.
:param weight_key: The key to use to load weights for this network, if pre-loaded weights exist
:param in_size: The size of the input to this process
:param out_size: The size of the output from this process
:param in_layer: The layer (`pnl.TransferMechanism`) object that is the input to this process
:param out_layer: The layer (`pnl.TransferMechanism`) object that is the output to this process
:param name: A name to give this layer
:param learning_rate: A learning rate for this process. If none given, remains zero.
:return: The generated process specified by the input parameters
"""
if self._should_load(weight_key):
weights = self.loaded_weights[weight_key]
else:
weights = pnl.random_matrix(in_size, out_size, 2, -1) * self.weight_init_scale
learning = learning_rate != 0
process = pnl.Process(pathway=[in_layer, weights, out_layer],
name=name, learning=learning and pnl.LEARNING or None)
self.processes.append(process)
if learning:
learning_mechanism = process.pathway[1].learning_mechanism
learning_mechanism.learning_rate = learning_rate
self.learning_mechanisms_to_learning_rates[learning_mechanism] = learning_rate
return process
def _generate_indirect_layer(self):
# TODO: initial weights - does a hollow matrix make sense? Yes, but with smaller initial values
matrix_size = 2 * self.num_features + self.indirect_layer_size
matrix = pnl.random_matrix(matrix_size, matrix_size, 2,
-1) * self.weight_init_scale
np.fill_diagonal(matrix, 0)
# TODO: why is the parameter here called enable_learning rather than learning?
self.indirect_shape_layer = pnl.ContrastiveHebbianMechanism(
input_size=self.num_features,
hidden_size=self.indirect_layer_size,
target_size=self.num_features,
separated=True,
matrix=matrix,
integrator_mode=self.integrator_mode,
integration_rate=self.integration_rate,
max_passes=1000,
noise=self._generate_noise_function(),
learning_rate=self.indirect_learning_rate,
enable_learning=self.indirect_learning_rate != 0)
self.indirect_target_input = pnl.TransferMechanism(
size=self.num_features, name='chl-target-input')
def _generate_processes(self):
self.input_output_processes = []
for (input_index,
output_index) in itertools.product(range(self.num_dimensions),
range(self.num_dimensions)):
# proc = pnl.Process(pathway=[self.input_layers[input_index],
# (pnl.random_matrix(self.num_features, self.hidden_layer_size, 2,
# -1) * self.weight_init_scale, pnl.LEARNING),
# self.hidden_layer,
# (pnl.random_matrix(self.hidden_layer_size, self.num_features, 2,
# -1) * self.weight_init_scale, pnl.LEARNING),
# self.output_layers[output_index]],
# name='input-{i}-output-{o}-proc'.format(i=input_index,
# o=output_index))
# #learning=pnl.LEARNING))
#
# proc = pnl.Process(pathway=[self.input_layers[input_index],
# pnl.MappingProjection(matrix=(pnl.random_matrix(self.num_features, self.hidden_layer_size, 2,
# -1) * self.weight_init_scale, pnl.LEARNING_PROJECTION)),
# self.hidden_layer,
# pnl.MappingProjection(matrix=(pnl.random_matrix(self.hidden_layer_size, self.num_features, 2,
# -1) * self.weight_init_scale, pnl.LEARNING_PROJECTION)),
# self.output_layers[output_index]],
# name='input-{i}-output-{o}-proc'.format(i=input_index,
# o=output_index),
# learning=pnl.LEARNING)
input_to_hidden = pnl.MappingProjection(
name='input-{i}-to-hidden'.format(i=input_index),
sender=self.input_layers[input_index],
receiver=self.hidden_layer,
matrix=pnl.random_matrix(self.num_features,
self.hidden_layer_size, 2, -1) *
self.weight_init_scale)
hidden_to_output = pnl.MappingProjection(
name='hidden-to-output-{o}'.format(o=output_index),
sender=self.hidden_layer,
receiver=self.output_layers[output_index],
matrix=pnl.random_matrix(self.hidden_layer_size,
self.num_features, 2, -1) *
self.weight_init_scale)
proc = pnl.Process(pathway=[
self.input_layers[input_index], input_to_hidden,
self.hidden_layer, hidden_to_output,
self.output_layers[output_index]
],
name='input-{i}-output-{o}-proc'.format(
i=input_index, o=output_index),
learning=pnl.ENABLED)
self.input_output_processes.append(proc)
self.task_hidden_processes = []
self.task_output_processes = []
self.output_bias_processes = []
for output_index in range(self.num_dimensions):
self.task_hidden_processes.append(
pnl.Process(
pathway=[
self.task_layer,
# pnl.random_matrix(self.num_tasks, self.hidden_layer_size, 2,
# -1) * self.weight_init_scale,
self.hidden_layer,
# pnl.random_matrix(self.hidden_layer_size, self.num_features, 2,
# -1) * self.weight_init_scale,
self.output_layers[output_index]
],
name='task-hidden-proc-{o}'.format(o=output_index),
learning=pnl.LEARNING))
self.task_output_processes.append(
pnl.Process(
pathway=[
self.task_layer,
# pnl.random_matrix(self.num_tasks, self.num_features, 2,
# -1) * self.weight_init_scale,
self.output_layers[output_index]
],
name='task-output-proc-{o}'.format(o=output_index),
learning=pnl.LEARNING))
self.output_bias_processes.append(
pnl.Process(
pathway=[
self.output_biases[output_index],
self.output_layers[output_index]
],
name='output-bias-proc-{o}'.format(o=output_index)))
self.hidden_bias_process = pnl.Process(
pathway=[self.hidden_bias, self.hidden_layer],
name='hidden-bias-proc')
def create_weights(in_size=2, out_size=2):
return pnl.random_matrix(in_size, out_size, 2, -1) * 0.1