def get_stats_from_batch(self):
# Get the neuron-wise I/O for this layer.
layer = kgraph.copy_layer_wo_activation(self.layer,
keep_bias=False,
reuse_symbolic_tensors=False)
# Readjust the layer nodes.
for i in range(kgraph.get_layer_inbound_count(self.layer)):
layer(self.layer.get_input_at(i))
Xs, Ys = get_active_neuron_io(layer, self._active_node_indices)
if len(Ys) != 1:
raise ValueError("Assume that kernel layer have only one output.")
X, Y = Xs[0], Ys[0]
# Create layers that keep a running mean for the desired stats.
self.mean_x = ilayers.RunningMeans()
self.mean_y = ilayers.RunningMeans()
self.mean_xy = ilayers.RunningMeans()
# Compute mask and active neuron counts.
mask = ilayers.AsFloatX()(self._get_neuron_mask())
Y_masked = keras.layers.multiply([Y, mask])
count = ilayers.CountNonZero(axis=0)(mask)
count_all = ilayers.Sum(axis=0)(ilayers.OnesLike()(mask))
# Get means ...
def norm(x, count):
return ilayers.SafeDivide(factor=1)([x, count])
# ... along active neurons.
mean_x = norm(ilayers.Dot()([ilayers.Transpose()(X), mask]), count)
mean_xy = norm(ilayers.Dot()([ilayers.Transpose()(X), Y_masked]),
count)
_, a = self.mean_x([mean_x, count])
_, b = self.mean_xy([mean_xy, count])
# ... along all neurons.
mean_y = norm(ilayers.Sum(axis=0)(Y), count_all)
_, c = self.mean_y([mean_y, count_all])
# Create a dummy output to have a connected graph.
# Needs to have the shape (mb_size, 1)
dummy = keras.layers.Average()([a, b, c])
return ilayers.Sum(axis=None)(dummy)
def get_stats_from_batch(self):
Xs, Ys = get_active_neuron_io(self.layer, self._active_node_indices)
self.mean_x = ilayers.RunningMeans()
count = ilayers.CountNonZero(axis=0)(Ys[0])
sum_x = ilayers.Dot()([ilayers.Transpose()(Xs[0]), Ys[0]])
mean_x, count_x = self.mean_x([sum_x, count])
# Return dummy output to have connected graph!
return ilayers.Sum(axis=None)(count_x)
def _create_computers(self):
"""
Creates pattern objects and Keras models that are used to collect
statistics and compute patterns.
We compute the patterns by first computing statistics within
the Keras framework, which are then used to compute the patterns.
This is based on a workaround. We connect the stats computation
via dummy outputs to a model's output and then iterate over the
dataset to compute statistics.
"""
# Create a broadcasting function that is used to connect
# the dummy outputs.
# Broadcaster has shape (mini_batch_size, 1)
reduce_axes = list(range(len(K.int_shape(self.model.inputs[0]))))[1:]
dummy_broadcaster = ilayers.Sum(axis=reduce_axes,
keepdims=True)(self.model.inputs[0])
def broadcast(x):
return ilayers.Broadcast()([dummy_broadcaster, x])
# Collect all tensors that are part of a model's execution.
layers, execution_list, _ = kgraph.trace_model_execution(self.model)
model_tensors = set()
for _, input_tensors, output_tensors in execution_list:
for t in input_tensors + output_tensors:
model_tensors.add(t)
# Create pattern instances and collect the dummy outputs.
self._pattern_instances = {k: [] for k in self.pattern_types}
computer_outputs = []
for _layer_id, layer in enumerate(layers):
# This does not work with containers!
# They should be replaced by trace_model_execution.
if kchecks.is_network(layer):
raise Exception("Network in network is not suppored!")
for pattern_type, clazz in six.iteritems(self.pattern_types):
pinstance = clazz(
self.model,
layer,
model_tensors=model_tensors,
execution_list=execution_list,
)
if pinstance.has_pattern() is False:
continue
self._pattern_instances[pattern_type].append(pinstance)
dummy_output = pinstance.get_stats_from_batch()
# Broadcast dummy_output to right shape.
computer_outputs += iutils.to_list(broadcast(dummy_output))
# Now we create one or more Keras models to train the patterns.
self._n_computer_outputs = len(computer_outputs)
if self.compute_layers_in_parallel is True:
self._computers = [
keras.models.Model(inputs=self.model.inputs,
outputs=computer_outputs)
]
else:
self._computers = [
keras.models.Model(inputs=self.model.inputs,
outputs=computer_output)
for computer_output in computer_outputs
]
# Distribute computation on more gpus.
if self.gpus is not None and self.gpus > 1:
raise NotImplementedError("Not supported yet.")