def _step_fn(ctx, inputs):
"""A step fn that returns update ops."""
inputs, targets = inputs
_build_model(strategy, model, mode, inputs, targets)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = strategy.extended.call_for_each_replica(
_per_device_execution_function,
args=(distributed_training_utils.get_distributed_model(model, mode),
mode))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_' + str(mode) + '_function',
**all_session_args)
for label, output in zip(output_labels, combined_fn.outputs):
if label == 'loss':
reduce_op = ds_reduce_util.ReduceOp.SUM
else:
# We reduce all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
reduce_op = ds_reduce_util.ReduceOp.MEAN
ctx.set_last_step_output(label, output, reduce_op)
# TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn:
# feed_dict, session kwargs, run options, run_metadata for now. These should
# be handled appropriately
return combined_fn.updates_op
def _make_eager_execution_function(model, mode):
"""Makes function to run one step of distributed model eager execution."""
strategy = model._distribution_strategy
if not model._grouped_model:
clone_model_on_replicas(
model, strategy, make_callback_model=(mode == 'train'))
def _per_device_function(model):
f = model._make_execution_function(mode)
return (f.inputs, f.outputs)
# NOTE(priyag): Try creating a new FuncGraph within DS scope instead of using
# the global one.
with K.get_graph().as_default(), strategy.scope():
# Create train ops on each of the devices when we call
# `_per_device_fit_function`.
(grouped_inputs, grouped_outputs) = strategy.call_for_each_replica(
_per_device_function, args=(model._grouped_model,))
# Unwrap all the per device values returned from `call_for_each_replica`.
# Unwrapping per device values gives you a list of values that can be
# used to construct a new train function that is composed of inptus/outputs
# on all the devices over which the model is distributed.
(all_inputs, all_outputs, _, _) = distributed_training_utils.unwrap_values(
strategy,
grouped_inputs,
grouped_outputs,
with_loss_tensor=(mode != 'predict'))
return K.function(
all_inputs,
all_outputs,
name='eager_distributed_{}_function'.format(mode))
def step_fn(ctx, inputs):
"""Clones the model and calls make_predict_function."""
if model._compile_distribution:
distributed_training_utils.clone_model_on_replicas(
model, current_strategy, mode, inputs=inputs)
else:
distributed_training_utils._build_distributed_network(
model, current_strategy, mode, inputs)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.extended.call_for_each_replica(
_per_device_predict_function,
args=(distributed_training_utils.get_distributed_model(
model, ModeKeys.PREDICT),))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(
all_inputs, all_outputs,
updates=all_updates,
name='distributed_predict_function',
**all_session_args)
for label, output in zip(model.output_names, combined_fn.outputs):
ctx.set_last_step_output(label, output)
return combined_fn.updates_op
def step_fn(ctx, *inputs):
"""Clones the model and calls make_predict_function."""
# TODO(priyag, sourabhbajaj): The model gets cloned every time
# fit/test/predict is called. We should look into caching this keyed on
# input shapes.
clone_model_on_replicas(
model,
current_strategy,
make_callback_model=False,
inputs=inputs,
mode=_Mode.PREDICT)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.call_for_each_replica(
_per_device_predict_function, args=(model._grouped_model_predict,))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(
all_inputs, all_outputs,
updates=all_updates,
name='distributed_predict_function',
**all_session_args)
for label, output in zip(model.output_names, combined_fn.outputs):
ctx.set_last_step_output(label, output)
return combined_fn.updates_op
def __init__(self,
model,
inputs,
targets,
weights,
path=None,
freq=100,
hessian=False):
super(LossGradientHistory, self).__init__()
# limit number of samples for performance concerns.
if inputs[0].shape[0] > 20000:
sample_ids = np.random.choice(inputs[0].shape[0],
20000,
replace=False)
self.inputs = [x[sample_ids] for x in inputs]
self.targets = [y[sample_ids] for y in targets]
self.weights = [w[sample_ids] for w in weights]
else:
self.inputs = inputs
self.targets = targets
self.weights = weights
self.loss_grads = []
self.loss_hessians = [] if hessian else None
for i in range(len(model.outputs)):
yp = model.outputs[i]
ys = self.targets[i]
ws = self.weights[i]
f = tf.reduce_mean(model.loss_functions[i](ys,
yp,
sample_weight=ws))
gf = tf_gradients(f,
model.trainable_weights,
unconnected_gradients='zero')
self.loss_grads.append(K.function(model.inputs, gf))
if hessian:
g2f = []
for gfi in gf:
g2fi = tf_gradients(gfi,
model.trainable_weights,
unconnected_gradients='zero')
g2f.append(K.function(model.inputs, g2fi))
self.loss_hessians.append(g2f)
self.path = get_log_path(path, 'loss-')
self.freq = 0 if isinstance(freq, bool) else freq
def example():
image_path = 'D:\\Onepredict_MK\\LG CNS\\cat.jpg'
img = image.load_img(image_path, target_size=(224, 224))
img_input = preprocess_input(np.expand_dims(img, 0))
resnet = ResNet50(input_shape=(224, 224, 3),
weights='imagenet',
include_top=True)
probs = resnet.predict(img_input)
pred = np.argmax(probs[0])
activation_layer = resnet.layers[-3].name
inp = resnet.input
# for idx in range(1000):
y_c = resnet.output.op.inputs[0][:, pred]
A_k = resnet.get_layer(activation_layer).output
grads = K.gradients(y_c, A_k)[0]
# Model(inputs=[inp], outputs=[A_k, grads, resnet.output])
get_output = K.function(inputs=[inp], outputs=[A_k, grads, resnet.output])
[conv_output, grad_val, model_output] = get_output([img_input])
conv_output = conv_output[0]
grad_val = grad_val[0]
weights = np.mean(grad_val, axis=(0, 1))
grad_cam = np.zeros(dtype=np.float32, shape=conv_output.shape[0:2])
for k, w in enumerate(weights):
grad_cam += w * conv_output[:, :, k]
# RELU
grad_cam = np.maximum(grad_cam, 0)
grad_cam = cv2.resize(grad_cam, (224, 224))
# Guided grad-CAM
register_gradient()
guided_model, activation_layer = modify_backprop(resnet, 'GuidedBackProp',
args.checkpoint_path,
args.main_name)
saliency_fn = compile_saliency_function(guided_model, activation_layer)
saliency = saliency_fn([img_input, 0])
gradcam = saliency[0] * grad_cam[..., np.newaxis]
gradcam = deprocess_image(gradcam)
# grad_cam = ndimage.zoom(grad_cam, (32, 32), order=1)
plt.subplot(1, 2, 1)
plt.imshow(img, alpha=0.8)
plt.imshow(grad_cam, cmap='jet', alpha=0.5)
plt.axis('off')
plt.subplot(1, 2, 2)
plt.imshow(gradcam, cmap='jet', alpha=0.5)
plt.axis('off')
plt.show()
def test_elu(self): x = backend.placeholder(ndim=2) f = backend.function([x], [activations.elu(x, 0.5)]) test_values = np.random.random((2, 5)) result = f([test_values])[0] self.assertAllClose(result, test_values, rtol=1e-05) negative_values = np.array([[-1, -2]], dtype=backend.floatx()) result = f([negative_values])[0] true_result = (np.exp(negative_values) - 1) / 2 self.assertAllClose(result, true_result)
def test_softsign(self):
def softsign(x):
return np.divide(x, np.ones_like(x) + np.absolute(x))
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.softsign(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = softsign(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def FBM(model, cut, test):
inp = model.input # input placeholder
outputs = [layer.output for layer in model.layers] # all layer outputs
functors = [
backend.function([inp, backend.learning_phase()], [out])
for out in outputs
] # evaluation functions
layer_outs = [func([test, 1.]) for func in functors]
prediction = layer_outs[cut][0]
return prediction
def write_outfile(pixels, numarray, autoencoder, imageset, outputname,
parameters):
if pixels is None:
pixels, numarray = numpy_from_dataset(imageset, 4, False)
newpixels = np.reshape(pixels, (-1, numarray[2], numarray[3]))
embedding = list((K.function([autoencoder.input],
[layer.output])(newpixels)
for layer in autoencoder.layers
if layer.output_shape == (None, parameters[5])))
newlst = normalization(embedding)
write_output(newlst, len(embedding[0][0]), outputname)
def on_batch_end(self, batch, logs=None):
if batch == 0: return
if batch % self.args.debug_step != 0: return
# 随机取3张
# logger.debug("self.validation_data:%r",self.validation_data)
# 先重排一下验证数据
np.random.shuffle(self.validation_data.data_list)
data = self.validation_data.data_list[:9]
images, labels = self.validation_data.load_image_label(data)
e_outputs = self.model.get_layer('attention_layer').output[
1] #注意力层返回的是：return c_outputs, e_outputs
functor = K.function(
[self.model.input[0], self.model.input[1],
K.learning_phase()], [e_outputs, self.model.output])
# 调试用
# import pdb;
# pdb.set_trace()
# 返回注意力分布[B,Seq,W/4]和识别概率[B,Seq,Charset]
e_outputs_data, output_prob = functor(
[images, labels[:, :-1, :], True])
logger.debug("预测结果：images shape = %r,e_outputs_data=%r", images.shape,
e_outputs_data.shape)
writer = tf.summary.FileWriter(self.tboard_dir)
for i in range(len(images)):
# 对一张图片
image = images[i]
label = labels[i]
label = label_utils.prob2str(label, self.charset)
pred = label_utils.prob2str(output_prob[i], self.charset)
logger.debug("label字符串:%r", label)
logger.debug("pred字符串 :%r", pred)
# logger.debug("image.shape:%r,e_output.shape:%r",image.shape,e_output.shape)
tf_img = self.make_image(image, e_outputs_data[i], label, pred)
summary = tf.Summary(value=[
tf.Summary.Value(tag="{}/{}".format(self.tag, i), image=tf_img)
])
writer.add_summary(summary)
writer.close()
return
def _create_keras_history_helper(tensors, processed_ops, created_layers):
"""Helper method for `create_keras_history`.
Arguments:
tensors: A structure of Tensors for which to create Keras metadata.
processed_ops: Set. TensorFlow operations that have already been wrapped in
`TensorFlowOpLayer` instances.
created_layers: List. The `TensorFlowOpLayer` instances created.
Returns:
Tuple. First element is the updated set of TensorFlow Operations that
have been wrapped in `TensorFlowOpLayer` instances. Second element is
a list of the `TensorFlowOpLayer` instances created.
"""
# Import of `base_layer` needed in order to create `TensorFlowOpLayer`.
# Cannot be imported at top because of circular dependencies.
# TODO(omalleyt): Resolve circular dependency.
from tensorflow.python.keras.engine import base_layer # pylint: disable=g-import-not-at-top
tensor_list = nest.flatten(tensors)
for tensor in tensor_list:
if getattr(tensor, '_keras_history', None) is not None:
continue
op = tensor.op # The Op that created this Tensor.
if op not in processed_ops:
# Recursively set `_keras_history`.
op_inputs = list(op.inputs)
constants = {}
layer_inputs = []
for i, op_input in enumerate(op_inputs):
if uses_keras_history(op_input):
layer_inputs.append(op_input)
else:
# Treat any value not originating from a `keras.Input` as
# a constant. Variables cannot be supported.
if (distribution_strategy_context.in_cross_replica_context(
) and not ops.executing_eagerly_outside_functions()):
# In Legacy Graph mode, evaluating here makes Session be
# configured improperly.
constants[i] = op_input
else:
with ops.init_scope():
constants[i] = backend.function([], op_input)([])
processed_ops, created_layers = _create_keras_history_helper(
layer_inputs, processed_ops, created_layers)
name = op.name
node_def = op.node_def.SerializeToString()
op_layer = base_layer.TensorFlowOpLayer(node_def,
constants=constants,
name=name)
created_layers.append(op_layer)
op_layer._add_inbound_node( # pylint: disable=protected-access
layer_inputs, op.outputs)
processed_ops.update([op])
return processed_ops, created_layers
def build(self, a_image, ap_image, b_image, output_shape):
self.output_shape = output_shape
loss = self.build_loss(a_image, ap_image, b_image)
# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, self.net_input)
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
self.f_outputs = K.function([self.net_input], outputs)
def test_relu(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.relu(x)])
positive_values = np.random.random((2, 5))
result = f([positive_values])[0]
self.assertAllClose(result, positive_values, rtol=1e-05)
negative_values = np.random.uniform(-1, 0, (2, 5))
result = f([negative_values])[0]
expected = np.zeros((2, 5))
self.assertAllClose(result, expected, rtol=1e-05)
def __init__(self,
input_tensor,
losses,
input_range=(0, 255),
wrt_tensor=None,
norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
if self.input_tensor is self.wrt_tensor:
self.wrt_tensor_is_input_tensor = True
self.wrt_tensor = K.identity(self.wrt_tensor)
else:
self.wrt_tensor_is_input_tensor = False
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
if self.wrt_tensor_is_input_tensor:
grads = K.gradients(overall_loss, self.input_tensor)[0]
else:
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = K.l2_normalize(grads)
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function(
[self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def test_softmax_2d_axis0(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.softmax(x, axis=0)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = np.zeros((2, 5))
for i in range(5):
expected[:, i] = _ref_softmax(test_values[:, i])
self.assertAllClose(result, expected, rtol=1e-05)
def test_softmax(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.softmax(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = _ref_softmax(test_values[0])
self.assertAllClose(result[0], expected, rtol=1e-05)
with self.assertRaises(ValueError):
x = backend.placeholder(ndim=1)
activations.softmax(x)
def test_hard_sigmoid(self):
def ref_hard_sigmoid(x):
x = (x * 0.2) + 0.5
z = 0.0 if x <= 0 else (1.0 if x >= 1 else x)
return z
hard_sigmoid = np.vectorize(ref_hard_sigmoid)
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.hard_sigmoid(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = hard_sigmoid(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def test_gelu(self):
def gelu(x, approximate=False):
if approximate:
return 0.5 * x * (1.0 + np.tanh(
np.sqrt(2.0 / np.pi) * (x + 0.044715 * np.power(x, 3))))
else:
from scipy.stats import norm # pylint: disable=g-import-not-at-top
return x * norm.cdf(x)
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.gelu(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = gelu(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
f = backend.function([x], [activations.gelu(x, True)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = gelu(test_values, True)
self.assertAllClose(result, expected, rtol=1e-05)
def _create_keras_history_helper(tensors, processed_ops, created_layers):
"""Helper method for `create_keras_history`.
Arguments:
tensors: A structure of Tensors for which to create Keras metadata.
processed_ops: Set. TensorFlow operations that have already been wrapped in
`TensorFlowOpLayer` instances.
created_layers: List. The `TensorFlowOpLayer` instances created.
Returns:
Tuple. First element is the updated set of TensorFlow Operations that
have been wrapped in `TensorFlowOpLayer` instances. Second element is
a list of the `TensorFlowOpLayer` instances created.
"""
# Import of `base_layer` needed in order to create `TensorFlowOpLayer`.
# Cannot be imported at top because of circular dependencies.
# TODO(omalleyt): Resolve circular dependency.
from tensorflow.python.keras.engine import base_layer # pylint: disable=g-import-not-at-top
tensor_list = nest.flatten(tensors)
for tensor in tensor_list:
if getattr(tensor, '_keras_history', None) is not None:
continue
op = tensor.op # The Op that created this Tensor.
if op not in processed_ops:
# Recursively set `_keras_history`.
op_inputs = list(op.inputs)
constants = {}
layer_inputs = []
for i, op_input in enumerate(op_inputs):
if uses_keras_history(op_input):
layer_inputs.append(op_input)
else:
# Treat any value not originating from a `keras.Input` as
# a constant. Variables cannot be supported.
if (distribution_strategy_context.in_cross_replica_context() and
not ops.executing_eagerly_outside_functions()):
# In Legacy Graph mode, evaluating here makes Session be
# configured improperly.
constants[i] = op_input
else:
constants[i] = backend.function([], op_input)([])
processed_ops, created_layers = _create_keras_history_helper(
layer_inputs, processed_ops, created_layers)
name = op.name
node_def = op.node_def.SerializeToString()
op_layer = base_layer.TensorFlowOpLayer(
node_def, constants=constants, name=name)
created_layers.append(op_layer)
op_layer._add_inbound_node( # pylint: disable=protected-access
layer_inputs, op.outputs)
processed_ops.update([op])
return processed_ops, created_layers
def _make_train_function(self):
# TODO(tanzheny): This is a direct copy from super to make it work
# refactor it so that common logic can be shared.
has_recompiled = self._recompile_weights_loss_and_weighted_metrics()
self._check_trainable_weights_consistency()
# If we have re-compiled the loss/weighted metric sub-graphs then create
# train function even if one exists already. This is because
# `_feed_sample_weights` list has been updated on re-compile.
if getattr(self, 'train_function', None) is None or has_recompiled:
# Restore the compiled trainable state.
current_trainable_state = self._get_trainable_state()
self._set_trainable_state(self._compiled_trainable_state)
inputs = (
self._feed_inputs + self._feed_targets + self._feed_sample_weights)
if not isinstance(K.symbolic_learning_phase(), int):
inputs += [K.symbolic_learning_phase()]
linear_optimizer, dnn_optimizer = self._get_optimizers()
with K.get_graph().as_default():
with K.name_scope('training'):
# Training updates
updates = []
linear_updates = linear_optimizer.get_updates(
params=self.linear_model.trainable_weights, # pylint: disable=protected-access
loss=self.total_loss)
updates += linear_updates
dnn_updates = dnn_optimizer.get_updates(
params=self.dnn_model.trainable_weights, # pylint: disable=protected-access
loss=self.total_loss)
updates += dnn_updates
# Unconditional updates
updates += self.get_updates_for(None)
# Conditional updates relevant to this model
updates += self.get_updates_for(self.inputs)
metrics = self._get_training_eval_metrics()
metrics_tensors = [
m._call_result for m in metrics if hasattr(m, '_call_result') # pylint: disable=protected-access
]
with K.name_scope('training'):
# Gets loss and metrics. Updates weights at each call.
fn = K.function(
inputs, [self.total_loss] + metrics_tensors,
updates=updates,
name='train_function',
**self._function_kwargs)
setattr(self, 'train_function', fn)
# Restore the current trainable state
self._set_trainable_state(current_trainable_state)
def get_layer_output(model, input_pic, adjust_255=True):
input_pic_ = input_pic.copy()
input_pic_ = input_pic_.reshape((512, 512, 1))
input_pic_ = np.array([input_pic_])
# print(input_pic_.shape)
layer_info = get_layer_info(model)
for idx in range(1, len(layer_info)):
get_mid_output = K.function([model.layers[0].input], [model.layers[idx].output])
mid_output = get_mid_output([input_pic_])
if adjust_255:
mid_output *= 255
layer_info[idx]['output'] = mid_output[0][0,:,:,0]
return layer_info
def _make_execution_function(model, mode):
"""Makes function to run one step of distributed model execution."""
if context.executing_eagerly():
return _make_eager_execution_function(model, mode)
strategy = model._distribution_strategy
if not model._distributed_model:
if model._compile_distribution:
clone_model_on_replicas(model,
strategy,
make_callback_model=(mode == 'train'))
else:
_build_distributed_network(model, strategy)
def _per_device_function(model):
f = model._make_execution_function(mode)
return (f.inputs, f.outputs, f.updates_op, f.session_kwargs)
with strategy.scope():
# Create train ops on each of the devices when we call
# `_per_device_fit_function`.
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = strategy.extended.call_for_each_replica(
_per_device_function, args=(model._distributed_model, ))
if mode == 'train':
# Initialize the variables in the replicated model. This is necessary for
# multi-worker training because on some workers, initialization is not
# needed. This method does initialization or waiting for initialization
# according to the context object of distribute coordinator.
distributed_training_utils.init_restore_or_wait_for_variables()
# Unwrap all the per device values returned from `call_for_each_replica`.
# Unwrapping per device values gives you a list of values that can be
# used to construct a new train function that is composed of update ops on
# all the devices over which the model is distributed.
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
strategy,
grouped_inputs,
grouped_outputs,
grouped_updates,
grouped_session_args,
with_loss_tensor=(mode != 'predict'))
return K.function(all_inputs,
all_outputs,
updates=all_updates,
name='distributed_{}_function'.format(mode),
**all_session_args)
def run_model(image,
model,
win_x=30,
win_y=30,
std=False,
split=True,
process=True):
# image = np.pad(image, pad_width = ((0,0), (0,0), (win_x, win_x),(win_y,win_y)), mode = 'constant', constant_values = 0)
if process:
for j in xrange(image.shape[1]):
image[0, j, :, :] = process_image(image[0, j, :, :], win_x, win_y,
std)
if split:
image_size_x = image.shape[2] / 2
image_size_y = image.shape[3] / 2
else:
image_size_x = image.shape[2]
image_size_y = image.shape[3]
evaluate_model = K.function(
[model.layers[0].input, K.learning_phase()], [model.layers[-1].output])
n_features = model.layers[-1].output_shape[1]
if split:
model_output = np.zeros((n_features, 2 * image_size_x - win_x * 2,
2 * image_size_y - win_y * 2),
dtype='float32')
img_0 = image[:, :, 0:image_size_x + win_x, 0:image_size_y + win_y]
img_1 = image[:, :, 0:image_size_x + win_x, image_size_y - win_y:]
img_2 = image[:, :, image_size_x - win_x:, 0:image_size_y + win_y]
img_3 = image[:, :, image_size_x - win_x:, image_size_y - win_y:]
model_output[:, 0:image_size_x - win_x,
0:image_size_y - win_y] = evaluate_model([img_0, 0])[0]
model_output[:, 0:image_size_x - win_x,
image_size_y - win_y:] = evaluate_model([img_1, 0])[0]
model_output[:, image_size_x - win_x:,
0:image_size_y - win_y] = evaluate_model([img_2, 0])[0]
model_output[:, image_size_x - win_x:,
image_size_y - win_y:] = evaluate_model([img_3, 0])[0]
else:
model_output = evaluate_model([image, 0])[0]
model_output = model_output[0, :, :, :]
return model_output
def build(self, a_image, ap_image, b_image, output_shape):
self.output_shape = output_shape
loss = self.build_loss(a_image, ap_image, b_image)
# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, self.net_input)
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
f_inputs = [self.net_input]
for nnf in self.feature_nnfs:
f_inputs.append(nnf.placeholder)
self.f_outputs = K.function(f_inputs, outputs)
def test_selu(self):
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.selu(x)])
alpha = 1.6732632423543772848170429916717
scale = 1.0507009873554804934193349852946
positive_values = np.array([[1, 2]], dtype=backend.floatx())
result = f([positive_values])[0]
self.assertAllClose(result, positive_values * scale, rtol=1e-05)
negative_values = np.array([[-1, -2]], dtype=backend.floatx())
result = f([negative_values])[0]
true_result = (np.exp(negative_values) - 1) * scale * alpha
self.assertAllClose(result, true_result)
def __init__(self,
restore=None,
session=None,
use_log=False,
use_brelu=False):
def bounded_relu(x):
return K.relu(x, max_value=1)
if use_brelu:
activation = bounded_relu
else:
activation = 'relu'
self.num_channels = 1
self.image_size = 28
self.num_labels = 10
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(28, 28, 1)))
model.add(Activation(activation))
model.add(Conv2D(32, (3, 3)))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation(activation))
model.add(Conv2D(64, (3, 3)))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(200))
model.add(Activation(activation))
model.add(Dense(200))
model.add(Activation(activation))
model.add(Dense(10))
# output log probability, used for black-box attack
if use_log:
model.add(Activation('softmax'))
if restore:
model.load_weights(restore)
layer_outputs = []
for layer in model.layers:
if isinstance(layer, Conv2D) or isinstance(layer, Dense):
layer_outputs.append(
K.function([model.layers[0].input], [layer.output]))
self.model = model
self.layer_outputs = layer_outputs
def predict_with_uncertainty(model, testdata, ci=0.95, n_iter=100):
func = K.function([model.input], [model.output])
with eager_learning_phase_scope(value=1):
result = []
for i in range(n_iter):
print
result.append(func([testdata]))
result = np.array(result)
predmean = result.mean(axis=0).reshape(-1, )
predsd = result.std(axis=0).reshape(-1, )
lowerCI = predmean - scipy.stats.norm.ppf(1 - 0.5 * (1 - ci)) * predsd
upperCI = predmean + scipy.stats.norm.ppf(1 - 0.5 * (1 - ci)) * predsd
return np.exp(predmean), np.exp(lowerCI), np.exp(upperCI)
def test_get_predictions_jacobian(input_layer, machine_class, batch_size):
with DEFAULT_TF_GRAPH.as_default():
machine = machine_class(input_layer)
model = Model(inputs=[input_layer], outputs=machine.predictions)
if tensorflow.__version__ >= '1.14':
optimizer = ComplexValuesOptimizer(model,
machine.predictions_jacobian,
name='optimizer')
else:
optimizer = ComplexValuesOptimizer(model,
machine.predictions_jacobian)
jacobian_function = K.function(
inputs=[input_layer],
outputs=[optimizer.get_predictions_jacobian()])
manual_jacobian_function = K.function(
inputs=[input_layer], outputs=[machine.manual_jacobian])
sample = numpy.random.choice(
2, (batch_size, ) + K.int_shape(input_layer)[1:]) * 2 - 1
jacobian = jacobian_function([sample])[0]
manual_jacobian = manual_jacobian_function([sample])[0]
diff_norm = numpy.linalg.norm(jacobian - manual_jacobian, 'fro')
jacobian_norm = numpy.linalg.norm(manual_jacobian, 'fro')
assert (diff_norm / jacobian_norm) < 1e-5
def prune_low_gradient_neurons(group, percentages, dataset):
main_layer = group.main_layer
# weights = main_layer.weights.get()
model = group.full_wrapper.to_model()
# w = weights[0]
# b = weights[1]
(x_train, y_train), (x_test, y_test) = dataset
layer = model.get_layer(main_layer.name)
# weight_tensors = layer.trainable_weights # weight tensors
gradients = model.optimizer.get_gradients(model.total_loss,
layer.output) # gradient tensors
input_tensors = model.inputs + model.sample_weights + \
model._targets + [K.learning_phase()]
infer_fn = K.function(inputs=input_tensors,
outputs=model.get_layer(main_layer.name).output)
grad_fn = K.function(inputs=input_tensors, outputs=gradients)
inputs = [x_test, None, y_test, 0]
activations = infer_fn(inputs)
# while(activations.ndim > 1):
# activations = activations.sum(axis=activations.ndim - 2)
grads = grad_fn(inputs)
grads = grads[0]
print(grads.shape)
print(activations.shape)
sums = grads * activations
sums = np.abs(sums)
while (sums.ndim > 1):
sums = sums.sum(axis=sums.ndim - 2)
indices = np.argsort(sums)
_prune_neurons(group, percentages, indices)
def test_sigmoid(self):
def ref_sigmoid(x):
if x >= 0:
return 1 / (1 + np.exp(-x))
else:
z = np.exp(x)
return z / (1 + z)
sigmoid = np.vectorize(ref_sigmoid)
x = backend.placeholder(ndim=2)
f = backend.function([x], [activations.sigmoid(x)])
test_values = np.random.random((2, 5))
result = f([test_values])[0]
expected = sigmoid(test_values)
self.assertAllClose(result, expected, rtol=1e-05)
def test_get_complex_value_gradients(input_layer, batch_size,
conjugate_gradients):
with DEFAULT_TF_GRAPH.as_default():
machine = Linear(input_layer)
model = Model(inputs=[input_layer], outputs=machine.predictions)
if tensorflow.__version__ >= '1.14':
optimizer = ComplexValuesOptimizer(model,
machine.predictions_jacobian,
name='optimizer')
else:
optimizer = ComplexValuesOptimizer(model,
machine.predictions_jacobian)
loss = Multiply()([machine.predictions, machine.predictions])
manual_gradients_layer = Lambda(
lambda x: tensorflow.reshape(tensorflow.reduce_sum(2.0 * x[0] * x[1], axis=0),
machine.dense_layer.kernel.shape)) \
([machine.predictions, machine.manual_jacobian])
if conjugate_gradients:
manual_gradients_layer = Lambda(lambda x: tensorflow.conj(x))(
manual_gradients_layer)
manual_gradients_function = K.function(
inputs=[input_layer], outputs=[manual_gradients_layer])
complex_value_gradients_layer = Lambda(
lambda x: optimizer.get_model_parameters_complex_value_gradients(
tensorflow.real(x), conjugate_gradients=conjugate_gradients))(
loss)
complex_value_gradients_function = K.function(
inputs=[input_layer], outputs=[complex_value_gradients_layer])
sample = numpy.random.choice(
2, (batch_size, ) + K.int_shape(input_layer)[1:]) * 2 - 1
complex_value_gradients = complex_value_gradients_function([sample])[0]
manual_gradients = manual_gradients_function([sample])[0]
diff_norm = numpy.linalg.norm(complex_value_gradients -
manual_gradients)
gradients_norm = numpy.linalg.norm(manual_gradients)
assert (diff_norm / gradients_norm) < 1e-5
def getLayerFuncByName(model, layername):
"""
Get function pointer for layer outputs from input
Args:
model: keras model
layername: str, name of layer
Returns:
get_layer_outputs: function pointer
"""
for idx, layer in enumerate(model.layers):
if layer.name == layername:
get_layer_outputs = K.function([model.layers[0].input],
[model.layers[idx].output])
return get_layer_outputs
def CAM(self, img):
pred = self.net.predict(img).flatten()
width = int(pred[0] * util.IMG_WIDTH)
height = int(abs(pred[1] * util.IMG_HEIGHT - util.IMG_HEIGHT))
print('PREDICTION: (%s, %s)' % (width, height))
x_out = self.net.output[:, 0]
y_out = self.net.output[:, 1]
last_conv_layer = self.net.get_layer('conv2d_6')
grads = K.gradients(x_out, last_conv_layer.output)[0]
pooled_grads = K.mean(grads, axis=(0, 1, 2))
iterate = K.function([self.net.input],
[pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([img])
for i in range(256):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
heatmap = np.mean(conv_layer_output_value, axis=-1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
plt.figure(figsize=(10, 8))
plt.matshow(heatmap)
plt.show()
img = np.array(img * 255, dtype=np.uint8)
img = np.reshape(
img, (util.IMG_HEIGHT, util.IMG_WIDTH, util.INPUT_CHANNELS))
gray_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
print(img.shape)
plt.figure(figsize=(10, 8))
plt.imshow(img)
plt.show()
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = heatmap * 0.8 + img
superimposed_img = cv2.circle(superimposed_img, (width, height), 6,
(255, 0, 0), -1)
plt.figure(figsize=(10, 8))
plt.imshow(superimposed_img)
plt.show()
return
def step_fn(ctx, inputs):
"""A step fn that returns update ops."""
if mode == ModeKeys.PREDICT:
targets = None
else:
inputs, targets = inputs
if model._compile_distribution:
distributed_training_utils.clone_model_on_replicas(model,
strategy,
mode,
inputs=inputs,
targets=targets)
else:
distributed_training_utils._build_distributed_network(
model, strategy, mode, inputs, targets)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = strategy.extended.call_for_each_replica(
_per_device_execution_function,
args=(distributed_training_utils.get_distributed_model(
model, mode), ))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(all_inputs,
all_outputs,
updates=all_updates,
name='distributed_' + str(mode) + '_function',
**all_session_args)
for label, output in zip(output_labels, combined_fn.outputs):
if mode == ModeKeys.PREDICT:
ctx.set_last_step_output(label, output)
else:
if label == 'loss':
reduce_op = ds_reduce_util.ReduceOp.SUM
else:
# We reduce all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
reduce_op = ds_reduce_util.ReduceOp.MEAN
ctx.set_last_step_output(label, output, reduce_op)
# TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn:
# feed_dict, session kwargs, run options, run_metadata for now. These should
# be handled appropriately
return combined_fn.updates_op
def _make_execution_function(model, mode):
"""Makes function to run one step of distributed model execution."""
if context.executing_eagerly():
return _make_eager_execution_function(model, mode)
strategy = model._distribution_strategy
if not model._grouped_model:
clone_model_on_replicas(
model, strategy, make_callback_model=(mode == 'train'))
def _per_device_function(model):
f = model._make_execution_function(mode)
return (f.inputs, f.outputs, f.updates_op, f.session_kwargs)
with strategy.scope():
# Create train ops on each of the devices when we call
# `_per_device_fit_function`.
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = strategy.extended.call_for_each_replica(
_per_device_function, args=(model._grouped_model,))
if mode == 'train':
# Initialize the variables in the replicated model. This is necessary for
# multi-worker training because on some workers, initialization is not
# needed. This method does initialization or waiting for initialization
# according to the context object of distribute coordinator.
distributed_training_utils.init_restore_or_wait_for_variables()
# Unwrap all the per device values returned from `call_for_each_replica`.
# Unwrapping per device values gives you a list of values that can be
# used to construct a new train function that is composed of update ops on
# all the devices over which the model is distributed.
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
strategy,
grouped_inputs,
grouped_outputs,
grouped_updates,
grouped_session_args,
with_loss_tensor=(mode != 'predict'))
return K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_{}_function'.format(mode),
**all_session_args)
def _step_fn(ctx, inputs):
"""A step fn that returns update ops."""
if isinstance(inputs, (tuple, list)) and len(inputs) == 2:
inputs, targets = inputs
else:
targets = None
# When input feature is a dictionary of tensors, dictionary is flattended
# to an array and passed as a model input. This results in input mismatch
# when model input layer names are not sorted in alphabetical order as
# `nest.flatten()`sorts dictioary elements by keys. As so, transform input
# tensors into an array and order it along `model._feed_input_names`.
if isinstance(inputs, dict):
inputs = [inputs[input_name] for input_name in model._feed_input_names]
_build_model(strategy, model, mode, inputs, targets)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = strategy.extended.call_for_each_replica(
_per_replica_execution_function,
args=(distributed_training_utils.get_distributed_model(model, mode),
mode))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_' + str(mode) + '_function',
**all_session_args)
for label, output in zip(output_labels, combined_fn.outputs):
if label == 'loss':
reduce_op = ds_reduce_util.ReduceOp.SUM
else:
# We reduce all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
reduce_op = ds_reduce_util.ReduceOp.MEAN
ctx.set_last_step_output(label, output, reduce_op)
# TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn:
# feed_dict, session kwargs, run options, run_metadata for now. These should
# be handled appropriately
return combined_fn.updates_op
def _create_keras_history_helper(tensors, processed_ops=None):
"""Helper method for `create_keras_history`.
Arguments:
tensors: A structure of Tensors for which to create Keras metadata.
processed_ops: Set. TensorFlow operations that have already been wrapped
in `TensorFlowOpLayer` instances.
Returns:
The updated set of TensorFlow Operations that have been wrapped
in `TensorFlowOpLayer` instances.
"""
# Import of `base_layer` needed in order to create `TensorFlowOpLayer`.
# Cannot be imported at top because of circular dependencies.
# TODO(omalleyt): Resolve circular dependency.
from tensorflow.python.keras.engine import base_layer # pylint: disable=g-import-not-at-top
tensor_list = nest.flatten(tensors)
for tensor in tensor_list:
if getattr(tensor, '_keras_history', None) is not None:
continue
op = tensor.op # The Op that created this Tensor.
if op not in processed_ops:
# Recursively set `_keras_history`.
op_inputs = list(op.inputs)
constants = {}
layer_inputs = []
for i, op_input in enumerate(op_inputs):
if uses_keras_history(op_input):
layer_inputs.append(op_input)
else:
# Treat any value not originating from a `keras.Input` as
# a constant (Variables currently have `Placeholder` op type
# when originating from an eager context
# so can't be supported.
constants[i] = backend.function([], op_input)([])
processed_ops = _create_keras_history_helper(layer_inputs, processed_ops)
name = op.name
node_def = op.node_def.SerializeToString()
op_layer = base_layer.TensorFlowOpLayer(
node_def, constants=constants, name=name)
op_layer._add_inbound_node( # pylint: disable=protected-access
layer_inputs, op.outputs)
processed_ops.update([op])
return processed_ops
def step_fn(ctx, inputs):
"""Clones the model and calls make_fit_function."""
# TODO(priyag, sourabhbajaj): The model gets cloned every time
# fit/test/predict is called. We should look into caching this keyed on
# input shapes.
inputs, targets = inputs
clone_model_on_replicas(
model,
current_strategy,
make_callback_model=True,
inputs=inputs,
targets=targets,
mode=_Mode.TRAIN)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.call_for_each_replica(
_per_device_fit_function, args=(model._grouped_model_train,))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs,
grouped_updates, grouped_session_args)
combined_fn = K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_fit_function',
**all_session_args)
for label, output in zip(out_labels, combined_fn.outputs):
if label == 'loss':
reduce_op = distribute_lib.get_loss_reduction()
else:
# We reduce all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
reduce_op = ds_reduce_util.ReduceOp.MEAN
ctx.set_last_step_output(label, output, reduce_op)
# TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn:
# feed_dict, session kwargs, run options, run_metadata for now. These should
# be handled appropriately
return combined_fn.updates_op
def _make_graph_execution_function(model, mode):
"""Makes function to run one step of distributed model in graph mode."""
def _per_replica_function(model):
f = model._make_execution_function(mode)
return (f.inputs, f.outputs, f.updates_op, f.session_kwargs)
strategy = model._distribution_strategy
with strategy.scope():
# Create train ops on each of the devices when we call
# `_per_replica_fit_function`.
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = strategy.extended.call_for_each_replica(
_per_replica_function, args=(get_distributed_model(model, mode),))
# Initialize the variables in the replicated model. This is necessary for
# multi-worker training because on some workers, initialization is not
# needed. This method does initialization or waiting for initialization
# according to the context object of distribute coordinator.
init_restore_or_wait_for_variables()
# Unwrap all the per device values returned from `call_for_each_replica`.
# Unwrapping per device values gives you a list of values that can be
# used to construct a new train function that is composed of update ops on
# all the devices over which the model is distributed.
(all_inputs, all_outputs, all_updates, all_session_args) = unwrap_values(
strategy,
grouped_inputs,
grouped_outputs,
grouped_updates,
grouped_session_args,
with_loss_tensor=(mode != ModeKeys.PREDICT))
return K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_{}_function'.format(mode),
**all_session_args)
def step_fn(ctx, inputs):
"""Clones the model and calls make_fit_function."""
inputs, targets = inputs
if model._compile_distribution:
distributed_training_utils.clone_model_on_replicas(
model, current_strategy, mode, inputs=inputs, targets=targets)
else:
distributed_training_utils._build_distributed_network(
model, current_strategy, mode, inputs, targets)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.extended.call_for_each_replica(
_per_device_fit_function,
args=(distributed_training_utils.get_distributed_model(
model, ModeKeys.TRAIN),))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs,
grouped_updates, grouped_session_args)
combined_fn = K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_fit_function',
**all_session_args)
for label, output in zip(out_labels, combined_fn.outputs):
if label == 'loss':
reduce_op = ds_reduce_util.ReduceOp.SUM
else:
# We reduce all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
reduce_op = ds_reduce_util.ReduceOp.MEAN
ctx.set_last_step_output(label, output, reduce_op)
# TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn:
# feed_dict, session kwargs, run options, run_metadata for now. These should
# be handled appropriately
return combined_fn.updates_op
def _make_eager_execution_function(model, mode):
"""Makes function to run one step of distributed model eager execution."""
def _per_device_function(model):
f = model._make_execution_function(mode)
return (f.inputs, f.outputs)
# NOTE(priyag): Try creating a new FuncGraph within DS scope instead of using
# the global one.
strategy = model._distribution_strategy
global_graph = K.get_graph()
with global_graph.as_default(), strategy.scope():
# First we gather the relevant portions of the model across all replicas.
# `K._scratch_graph(global_graph)` signals to Keras that it should not
# lift to a separate graph when creating the per-replica functions.
with K._scratch_graph(global_graph):
# Create train ops on each of the devices when we call
# `_per_device_fit_function`.
grouped = strategy.extended.call_for_each_replica(
_per_device_function, args=(get_distributed_model(model, mode),))
grouped_inputs, grouped_outputs = grouped
# Unwrap all the per device values returned from `call_for_each_replica`.
# Unwrapping per device values gives you a list of values that can be
# used to construct a new train function that is composed of
# inputs/outputs on all the devices over which the model is distributed.
(all_inputs, all_outputs, _, _) = unwrap_values(
strategy,
grouped_inputs,
grouped_outputs,
with_loss_tensor=(mode != ModeKeys.PREDICT))
# Finally, a joint Keras function is created; this one will be created in
# a separate FuncGraph.
return K.function(
all_inputs,
all_outputs,
name='eager_distributed_{}_function'.format(mode))
def step_fn(ctx, inputs):
"""Clones the model and calls make_eval_function."""
inputs, targets = inputs
if model._compile_distribution:
distributed_training_utils. clone_model_on_replicas(
model, current_strategy,
make_callback_model=False, inputs=inputs,
targets=targets, mode=distributed_training_utils.ModeKeys.TEST)
else:
distributed_training_utils._build_distributed_network(
model, current_strategy, inputs, targets,
mode=distributed_training_utils.ModeKeys.TEST)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.extended.call_for_each_replica(
_per_device_eval_function, args=(model._distributed_model_test,))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(
all_inputs, all_outputs,
updates=all_updates,
name='distributed_test_function',
**all_session_args)
for label, output in zip(model.metrics_names, combined_fn.outputs):
if label == 'loss':
reduce_op = distribute_lib.get_loss_reduction()
else:
# We reduce all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
reduce_op = ds_reduce_util.ReduceOp.MEAN
ctx.set_last_step_output(label, output, reduce_op)
return combined_fn.updates_op
def step_fn(ctx, inputs, targets):
"""Clones the model and calls make_eval_function."""
# TODO(priyag, sourabhbajaj): The model gets cloned every time
# fit/test/predict is called. We should look into caching this keyed on
# input shapes.
clone_model_on_replicas(
model,
current_strategy,
make_callback_model=False,
inputs=inputs,
targets=targets,
mode=_Mode.TEST)
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.call_for_each_replica(
_per_device_eval_function, args=(model._grouped_model_test,))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
combined_fn = K.function(
all_inputs, all_outputs,
updates=all_updates,
name='distributed_test_function',
**all_session_args)
for label, output in zip(model.metrics_names, combined_fn.outputs):
if label == 'loss':
aggregation = distribute_lib.get_loss_reduction()
else:
# We aggregate all other metrics using mean for now. This is temporary
# workaround until new metrics are in place.
aggregation = variable_scope.VariableAggregation.MEAN
ctx.set_last_step_output(label, output, aggregation)
return combined_fn.updates_op
def test_loop(model, iterator, verbose=0, steps=None):
"""Test loop for evaluating with DistributionStrategy.
Arguments:
model: Keras Model instance.
iterator: Iterator for input data.
verbose: Integer, Verbosity mode 0 or 1.
steps: Total number of steps (batches of samples)
before declaring predictions finished.
Ignored with the default value of `None`.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the outputs.
"""
current_strategy = model._distribution_strategy
# TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged.
if current_strategy.__class__.__name__ == 'TPUStrategy':
return _experimental_test_loop(model, iterator, verbose, steps)
if not model._grouped_model:
clone_model_on_replicas(model, current_strategy)
def _per_device_eval_function(model):
model._make_eval_function()
return (model._eval_function.inputs, model._eval_function.outputs,
model._eval_function.updates_op,
model._eval_function.session_kwargs)
inputs, targets, sample_weights = _get_input_from_iterator(iterator, model)
with current_strategy.scope():
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.call_for_each_replica(
_per_device_eval_function, args=(model._grouped_model,))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args, with_loss_tensor=True)
dataset_inputs = distributed_training_utils.flatten_perdevice_values(
current_strategy, inputs)
dataset_targets = distributed_training_utils.flatten_perdevice_values(
current_strategy, targets)
distributed_test_function = K.function(
all_inputs, all_outputs,
updates=all_updates,
name='distributed_test_function',
**all_session_args)
# We need to set sample_weights to None since there are sample weight
# placeholders that are created with default values.
sample_weights = [None for _ in range(
len(model.outputs) * current_strategy.num_replicas_in_sync)]
if not isinstance(K.learning_phase(), int):
ins = dataset_inputs + dataset_targets + sample_weights + [0]
else:
ins = dataset_inputs + dataset_targets
for m in model.stateful_metric_functions:
m.reset_states()
outs = []
if verbose == 1:
progbar = Progbar(target=steps)
# Copy the weights from the original model to each of the replicated models.
orig_model_weights = model.get_weights()
distributed_model = current_strategy.unwrap(model._grouped_model)[0]
distributed_training_utils.set_weights(
current_strategy, distributed_model, orig_model_weights)
assert steps is not None
for step in range(steps):
batch_outs = distributed_test_function(ins)
if isinstance(batch_outs, list):
if step == 0:
outs = [0.] * len(batch_outs)
outs[0] += batch_outs[0] # index 0 = 'loss'
outs[1:] = batch_outs[1:]
else:
if step == 0:
outs.append(0.)
outs[0] += batch_outs # index 0 = 'loss'
if verbose >= 1:
progbar.update(step + 1)
outs[0] /= steps # index 0 = 'loss'
if len(outs) == 1:
return outs[0]
return outs
def fit_loop(
model,
iterator,
epochs=100,
verbose=1,
callbacks=None,
val_iterator=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Fit loop for training with DistributionStrategy.
Arguments:
model: Keras Model instance.
iterator: Iterator for input data.
epochs: Number of times to iterate over the data
verbose: Integer, Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_iterator: Iterator for validation data.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for
(only if doing validation from data tensors).
Ignored with the default value of `None`.
Returns:
`History` object.
Raises:
ValueError: in case of invalid arguments.
"""
current_strategy = model._distribution_strategy
# TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged.
if current_strategy.__class__.__name__ == 'TPUStrategy':
return _experimental_fit_loop(
model, iterator, epochs, verbose, callbacks, initial_epoch,
steps_per_epoch, val_iterator, validation_steps)
if not model._grouped_model:
clone_model_on_replicas(model, current_strategy, make_callback_model=True)
def _per_device_fit_function(model):
model._make_fit_function()
return (model._fit_function.inputs, model._fit_function.outputs,
model._fit_function.updates_op, model._fit_function.session_kwargs)
inputs, targets, sample_weights = _get_input_from_iterator(iterator, model)
with current_strategy.scope():
# Create train ops on each of the devices when we call
# `_per_device_fit_function`.
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.call_for_each_replica(
_per_device_fit_function, args=(model._grouped_model,))
# Unwrap all the per device values returned from `call_for_each_replica`.
# Unwrapping per device values gives you a list of values that can be
# used to construct a new train function that is composed of update ops on
# all the devices over which the model is distributed.
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs,
grouped_updates, grouped_session_args, with_loss_tensor=True)
# Dataset inputs and targets are also per devices values that need to be
# unwrapped.
dataset_inputs = distributed_training_utils.flatten_perdevice_values(
current_strategy, inputs)
dataset_targets = distributed_training_utils.flatten_perdevice_values(
current_strategy, targets)
# Create a train function that is composed of all the parameters above.
distributed_fit_function = K.function(
all_inputs,
all_outputs,
updates=all_updates,
name='distributed_fit_function',
**all_session_args)
# We need to set sample_weights to None since there are sample weight
# placeholders that are created with default values.
sample_weights = [None for _ in range(
len(model.outputs) * current_strategy.num_replicas_in_sync)]
if not isinstance(K.learning_phase(), int):
ins = dataset_inputs + dataset_targets + sample_weights + [1]
else:
ins = dataset_inputs + dataset_targets
do_validation = False
if validation_steps:
do_validation = True
# Copy the weights from the original model to each of the replicated models.
orig_model_weights = model.get_weights()
distributed_model = current_strategy.unwrap(model._grouped_model)[0]
distributed_training_utils.set_weights(
current_strategy, distributed_model, orig_model_weights)
callbacks = cbks.configure_callbacks(
callbacks,
model,
do_validation=do_validation,
val_inputs=None,
val_targets=None,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
verbose=verbose)
out_labels = model.metrics_names or []
callbacks.on_train_begin()
assert steps_per_epoch is not None
for epoch in range(initial_epoch, epochs):
# Reset stateful metrics
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
for step_index in range(steps_per_epoch):
batch_logs = {'batch': step_index, 'size': 1}
callbacks.on_batch_begin(step_index, batch_logs)
try:
outs = distributed_fit_function(ins)
except errors.OutOfRangeError:
logging.warning('Your dataset iterator ran out of data; '
'interrupting training. Make sure that your dataset '
'can generate at least `steps_per_epoch * epochs` '
'batches (in this case, %d batches).' %
steps_per_epoch * epochs)
break
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(step_index, batch_logs)
if callbacks.model.stop_training:
break
if do_validation:
val_outs = test_loop(
model,
val_iterator,
steps=validation_steps,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callbacks.model.stop_training:
break
callbacks.on_train_end()
# Copy the weights back from the replicated model to the original model.
updated_weights = current_strategy.unwrap(
model._grouped_model)[0].get_weights()
model.set_weights(updated_weights)
return model.history
def predict_loop(model, iterator, verbose=0, steps=None):
"""Predict loop for predicting with DistributionStrategy.
Arguments:
model: Keras Model instance.
iterator: Iterator for input data.
verbose: Integer, Verbosity mode 0 or 1.
steps: Total number of steps (batches of samples)
before declaring `_predict_loop` finished.
Ignored with the default value of `None`.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions
(if the model has multiple outputs).
"""
current_strategy = model._distribution_strategy
# TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged.
if current_strategy.__class__.__name__ == 'TPUStrategy':
return _experimental_predict_loop(model, iterator, verbose, steps)
if not model._grouped_model:
clone_model_on_replicas(model, current_strategy)
def _per_device_predict_function(model):
model._make_predict_function()
return (model.predict_function.inputs,
model.predict_function.outputs,
model.predict_function.updates_op,
model.predict_function.session_kwargs)
inputs, _, _ = _get_input_from_iterator(iterator, model)
with current_strategy.scope():
(grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args) = current_strategy.call_for_each_replica(
_per_device_predict_function, args=(model._grouped_model,))
(all_inputs, all_outputs, all_updates,
all_session_args) = distributed_training_utils.unwrap_values(
current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
grouped_session_args)
dataset_inputs = distributed_training_utils.flatten_perdevice_values(
current_strategy, inputs)
distributed_predict_function = K.function(
all_inputs, all_outputs,
updates=all_updates,
name='distributed_predict_function',
**all_session_args)
if not isinstance(K.learning_phase(), int):
ins = dataset_inputs + [0]
else:
ins = dataset_inputs
if verbose == 1:
progbar = Progbar(target=steps)
# Copy the weights from the original model to each of the replicated models.
orig_model_weights = model.get_weights()
distributed_model = current_strategy.unwrap(model._grouped_model)[0]
distributed_training_utils.set_weights(
current_strategy, distributed_model, orig_model_weights)
num_replicas = current_strategy.num_replicas_in_sync
# Since we do not know how many samples we will see, we cannot
# pre-allocate the returned Numpy arrays. Instead, we store one array per
# batch seen and concatenate them upon returning.
unconcatenated_outs = []
assert steps is not None
for step in range(steps):
batch_outs = distributed_predict_function(ins)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if step == 0:
# batch_outs gives you the number of model outputs. In the distributed
# case this will be number of model_outputs * num_replicas.
for _ in range(len(model.outputs)):
unconcatenated_outs.append([])
for i in range(len(model.outputs)):
nested_outs = batch_outs[i * num_replicas:
i * num_replicas + num_replicas]
outs = nest.flatten(nested_outs)
unconcatenated_outs[i].extend(outs)
if verbose >= 1:
progbar.update(step + 1)
if len(unconcatenated_outs) == 1:
return np.concatenate(unconcatenated_outs[0], axis=0)
return [
np.concatenate(unconcatenated_outs[i], axis=0)
for i in range(len(unconcatenated_outs))
]