def predict(self, *inputs):
"""
Using current tensorflow session this method propagates
input throught the network and returns output from it.
"""
session = tensorflow_session()
# We cache it in order to avoid graph creation
# every time user calls prediction.
cache_key = (session, id(self))
if cache_key not in self.computation_cache:
# It's important to initialize parameters when for cases when
# prediction requested for the network that wasn't trained or
# loaded from the storage.
variables = find_variables(self.layers)
initialize_uninitialized_variables(variables)
input_variables = create_input_variables(self.input_layers)
with self.disable_training_state():
self.computation_cache[cache_key] = {
'inputs': input_variables,
'outputs': self.output(*input_variables),
}
graph = self.computation_cache[cache_key]
feed_dict = dict(zip(graph['inputs'], inputs))
return session.run(graph['outputs'], feed_dict=feed_dict)
def saliency_map_graph(network):
"""
Returns tensorflow variables for saliency map.
Parameters
----------
network : network
image : ndarray
"""
session = tensorflow_session()
if not hasattr(saliency_map_graph, 'cache'):
saliency_map_graph.cache = {}
if session in saliency_map_graph.cache:
return saliency_map_graph.cache[session]
inputs = network.inputs
prediction = network.outputs
output_class = tf.argmax(prediction[0])
saliency, = tf.gradients(tf.reduce_max(prediction), inputs)
# Caching will ensure that we won't build
# tensorflow graph every time we generate
saliency_map_graph.cache[session] = inputs, saliency, output_class
return inputs, saliency, output_class
def target_function(network, x, y):
weight = network.layers[1].weight
new_weight = np.array([[x], [y]])
session = tensorflow_session()
weight.load(asfloat(new_weight), session)
return network.prediction_error(input_data, target_data)
def function(inputs, outputs, updates=None, name=None):
if updates is None:
updates = []
session = tensorflow_session()
tensorflow_updates = []
# Ensure that all new values has been computed. Absence of these
# checks might lead to the non-deterministic update behaviour.
new_values = [val[1] for val in updates if isinstance(val, (list, tuple))]
# Make sure that all outputs has been computed
with tf.control_dependencies(as_tuple(outputs, new_values)):
for update in updates:
if isinstance(update, (list, tuple)):
old_value, new_value = update
update = old_value.assign(new_value)
tensorflow_updates.append(update)
# Group variables in order to avoid output for the updates
tensorflow_updates = tf.group(*tensorflow_updates)
@wraps(function)
def wrapper(*input_values):
feed_dict = dict(zip(inputs, input_values))
result, _ = session.run(
[outputs, tensorflow_updates],
feed_dict=feed_dict,
)
return result
return wrapper
def one_training_update(self, X_train, y_train):
if self.errors.train:
last_error = self.errors.train[-1]
self.variables.last_error.load(last_error, tensorflow_session())
return super(LevenbergMarquardt,
self).one_training_update(X_train, y_train)
def train_epoch(self, input_train, target_train):
train_epoch = self.methods.train_epoch
prediction_error = self.methods.prediction_error
session = tensorflow_session()
params = [param for param, _ in self.init_train_updates()]
param_defaults = [session.run(param) for param in params]
def setup_new_step(new_step):
for param_default, param in zip(param_defaults, params):
param.load(param_default, session)
self.variables.step.load(asfloat(new_step), session)
train_epoch(input_train, target_train)
# Train epoch returns neural network error that was before
# training epoch step, that's why we need to compute
# it second time.
error = prediction_error(input_train, target_train)
return np.where(np.isnan(error), np.inf, error)
options = {'xtol': self.tol}
if self.search_method == 'brent':
options['maxiter'] = self.maxiter
res = minimize_scalar(
setup_new_step,
tol=self.tol,
method=self.search_method,
options=options,
)
return setup_new_step(res.x)
def test_hessian_computation(self):
x = tf.placeholder(name='x', dtype=tf.float32, shape=(1, ))
y = tf.placeholder(name='y', dtype=tf.float32, shape=(1, ))
f = x**2 + y**3 + 7 * x * y
# Gradient function:
# [2 * x + 7 * y,
# 3 * y ** 2 + 7 * x]
# Hessian function:
# [[2, 7 ]
# [7, 6 * y]]
hessian, gradient = find_hessian_and_gradient(f, [x, y])
session = tensorflow_session()
hessian_output, gradient_output = session.run([hessian, gradient],
feed_dict={
x: [1],
y: [2]
})
np.testing.assert_array_equal(gradient_output, np.array([16, 19]))
np.testing.assert_array_equal(hessian_output,
np.array([
[2, 7],
[7, 12],
]))
def saliency_map_graph(connection):
"""
Returns tensorflow variables for saliency map.
Parameters
----------
connection : connection
image : ndarray
"""
session = tensorflow_session()
if session in saliency_map_graph.cache:
return saliency_map_graph.cache[session]
x = tf.placeholder(
shape=as_tuple(None, connection.input_shape),
name='saliency-map/input',
dtype=tf.float32,
)
with connection.disable_training_state():
prediction = connection.output(x)
output_class = tf.argmax(prediction[0])
saliency, = tf.gradients(tf.reduce_max(prediction), x)
# Caching will ensure that we won't build tensorflow graph every time
# we generate
saliency_map_graph.cache[session] = x, saliency, output_class
return x, saliency, output_class
def on_epoch_end(network):
if network.last_epoch in steps:
print("Saving pre-trained VIN model...")
storage.save(network, env['pretrained_network_file'])
new_step = steps[network.last_epoch]
session = tensorflow_session()
network.variables.step.load(new_step, session)
def target_function(optimizer, x, y):
weight = optimizer.network.layers[1].weight
new_weight = np.array([[x], [y]])
session = tensorflow_session()
weight.load(asfloat(new_weight), session)
return optimizer.score(X_train, y_train)
def one_training_update(self, X_train, y_train):
if len(self.errors.train) >= 2:
previous_error, last_error = self.errors.train[-2:]
session = tensorflow_session()
self.variables.last_error.load(last_error, session)
self.variables.previous_error.load(previous_error, session)
return super(IRPROPPlus, self).one_training_update(X_train, y_train)
def on_epoch_start_update(self, epoch):
super(ErrDiffStepUpdate, self).on_epoch_start_update(epoch)
previous_error = self.errors.previous()
if previous_error:
session = tensorflow_session()
last_error = self.errors.last()
self.variables.last_error.load(last_error, session)
self.variables.previous_error.load(previous_error, session)
def on_epoch_start_update(self, epoch):
super(IRPROPPlus, self).on_epoch_start_update(epoch)
previous_error = self.errors.previous()
if previous_error:
last_error = self.errors.last()
session = tensorflow_session()
self.variables.last_error.load(last_error, session)
self.variables.previous_error.load(previous_error, session)
def on_epoch_start_update(self, epoch):
"""
Function will be triggered before the training epoch procedure.
Parameters
----------
epoch : int
Current epoch number.
"""
super(ConstructibleNetwork, self).on_epoch_start_update(epoch)
self.variables.epoch.load(epoch, tensorflow_session())
def save_epoch_weight(optimizer):
"""
Signal processor which save weight update for every
epoch.
"""
global weights
global current_epoch
session = tensorflow_session()
input_layer_weight = session.run(optimizer.network.layers[1].weight)
weights[:, current_epoch + 1:current_epoch + 2] = input_layer_weight
def plot_rbm_components(rbm_network):
session = tensorflow_session()
weight = session.run(rbm_network.weight)
plt.figure(figsize=(10, 10))
plt.suptitle('RBM componenets', size=16)
for index, image in enumerate(weight.T, start=1):
plt.subplot(10, 10, index)
plt.imshow(image.reshape((28, 28)), cmap=plt.cm.gray)
plt.xticks([])
plt.yticks([])
plt.show()
def test_clip_gradient(self):
session = tensorflow_session()
x = tf.Variable(asfloat(1), dtype=tf.float32)
x_clipped = clip_gradient(x, 1.5)
y = x_clipped**2
gradient, = tf.gradients(y, x)
self.assertAlmostEqual(self.eval(gradient), 1.5)
x.load(asfloat(0.1), session)
self.assertAlmostEqual(self.eval(gradient), 0.2)
x.load(asfloat(-0.1), session)
self.assertAlmostEqual(self.eval(gradient), -0.2)
x.load(asfloat(-2), session)
self.assertAlmostEqual(self.eval(gradient), -1.5)
def load_layer_parameter(layer, layer_data):
"""
Set layer parameters to the values specified in the
stored data
"""
session = tensorflow_session()
for param_name, param_data in layer_data['parameters'].items():
parameter = getattr(layer, param_name)
if not isinstance(parameter, tf.Variable):
raise ParameterLoaderError(
"The `{}` parameter from the `{}` layer expected to be "
"instance of the tf.Variable, but current value equal to {}. "
"Layer: {}".format(param_name, layer.name, parameter, layer))
parameter.load(asfloat(param_data['value']), session)
def clip_gradient(value, clip_value):
if not hasattr(clip_gradient, 'added_gradients'):
clip_gradient.added_gradients = set()
session = tensorflow_session()
graph = session.graph
operation_name = "ClipGradient-" + str(clip_value)
if operation_name not in clip_gradient.added_gradients:
# Make sure that we won't create the same operation twise.
# Otherwise tensorflow will trigger an exception.
@tf.RegisterGradient(operation_name)
def clip_gradient_grad(op, grad):
return tf.clip_by_value(grad, -clip_value, clip_value)
clip_gradient.added_gradients.add(operation_name)
with graph.gradient_override_map({"Identity": operation_name}):
return tf.identity(value)
def setUp(self):
tf.reset_default_graph()
if self.single_thread:
sess = tensorflow_session()
sess.close()
config = tf.ConfigProto(
allow_soft_placement=True,
intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1,
)
tensorflow_session.cache = tf.Session(config=config)
if not self.verbose:
logging.disable(logging.CRITICAL)
# Clean identifiers map for each test
layers.BaseLayer.global_identifiers_map = {}
environment.reproducible(seed=self.random_seed)
def setUp(self):
tf.reset_default_graph()
if self.single_thread:
sess = tensorflow_session()
sess.close()
config = tf.ConfigProto(
allow_soft_placement=True,
intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1,
)
tensorflow_session.cache = tf.Session(config=config)
if not self.verbose:
logging.disable(logging.CRITICAL)
# Clean identifiers map for each test
if hasattr(format_name_if_specified_as_pattern, 'counters'):
del format_name_if_specified_as_pattern.counters
utils.reproducible(seed=self.random_seed)
def on_epoch_start_update(self, epoch):
super(LevenbergMarquardt, self).on_epoch_start_update(epoch)
last_error = self.errors.last()
if last_error is not None:
self.variables.last_error.load(last_error, tensorflow_session())
def save_dict(connection):
"""
Save network into the dictionary.
Parameters
----------
connection : network, list of layer or connection
Returns
-------
dict
Saved parameters and information about network in dictionary
using specific format. Learn more about the NeuPy's storage
format in the official documentation.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> layers_data = storage.save_dict(connection)
>>>
>>> layers_data.keys()
['layers', 'graph', 'metadata']
"""
connection = extract_connection(connection)
session = tensorflow_session()
initialize_uninitialized_variables()
data = {
'metadata': {
'language': 'python',
'library': 'neupy',
'version': neupy.__version__,
'created': strftime("%a, %d %b %Y %H:%M:%S %Z", gmtime()),
# TODO: Remove in case if we won't need this field
# 'theano_float': theano.config.floatX,
},
# Make it as a list in order to save the right order
# of paramters, otherwise it can be convert to the dictionary.
'graph': connection.graph.layer_names_only(),
'layers': [],
}
for layer in connection:
parameters = {}
configs = {}
for attrname, parameter in layer.parameters.items():
parameters[attrname] = {
'value': asfloat(session.run(parameter)),
'trainable': parameter.is_trainable,
}
for option_name in layer.options:
if option_name not in parameters:
configs[option_name] = getattr(layer, option_name)
data['layers'].append({
'class_name': layer.__class__.__name__,
'input_shape': layer.input_shape,
'output_shape': layer.output_shape,
'name': layer.name,
'parameters': parameters,
'configs': configs,
})
return data
def tearDown(self):
sess = tensorflow_session()
sess.close()
def saliency_map(network,
image,
mode='heatmap',
sigma=8,
ax=None,
show=True,
**kwargs):
"""
Saliency Map plot.
Parameters
----------
network : network
Network based on which will be computed saliency map.
image : 3D array-like tensor
Image based on which will be computed saliency map.
mode : {``raw``, ``heatmap``}
- ``raw``
Visualize raw gradient. White color on the plot
defines high gradient values.
- ``heatmap``
Applies gaussian filter to the gradient and visualize
as a heatmap plot.
Defaults to ``heatmap``.
sigma : float
Standard deviation for kernel in Gaussian filter.
It is used only when ``mode='heatmap'``. Defaults to ``8``.
ax : object or None
Matplotlib axis object. ``None`` values means that axis equal
to the current axes instance (the same as ``ax = plt.gca()``).
Defaults to ``None``.
show : bool
If parameter is equal to ``True`` then plot will be
displayed. Defaults to ``True``.
**kwargs
Arguments for ``plt.imshow`` function.
Returns
-------
object
Matplotlib axis instance.
Examples
--------
>>> from neupy import layers, plots
>>>
>>> network = layers.join(
... layers.Input((3, 28, 28)),
... layers.Convolution((32, 3, 3)) >> layers.Relu(),
... layers.Reshape(),
... layers.Softmax(10),
... )
>>>
>>> dog_image = load_dog_image()
>>> plots.saliency_map(network, dog_image)
"""
if image.ndim == 3:
image = np.expand_dims(image, axis=0)
if image.ndim != 4:
raise ValueError("Invalid image shape. Image expected to be 3D, "
"got {}D image".format(image.ndim))
valid_modes = ('raw', 'heatmap')
if mode not in valid_modes:
raise ValueError("{!r} is invalid value for mode argument. Valid "
"mode values are: {!r}".format(mode, valid_modes))
if isinstance(network, BaseOptimizer):
network = network.network
if len(network.output_layers) != 1:
raise InvalidConnection(
"Cannot build saliency map for the network that "
"has more than one output layer.")
if len(network.input_layers) != 1:
raise InvalidConnection(
"Cannot build saliency map for the network that "
"has more than one input layer.")
if len(network.input_shape) != 4:
raise InvalidConnection(
"Input layer has to be 4 dimensions, but network expects "
"{} dimensional input".format(len(network.input_shape)))
if ax is None:
ax = plt.gca()
x, saliency, output_class = saliency_map_graph(network)
session = tensorflow_session()
saliency, output = session.run([saliency, output_class],
feed_dict={x: image})
saliency = saliency[0].max(axis=-1)
if mode == 'heatmap':
saliency = gaussian_filter(saliency, sigma=sigma)
elif mode == 'raw':
kwargs.setdefault('cmap', 'gray')
ax.set_title('Predicted output #{} (0-based indeces)'.format(output))
ax.imshow(saliency, **kwargs)
if show:
plt.show()
return ax
