def __call__(self, inputs, state, scope=None):
"""Long short-term memory cell (LSTM).
Args:
inputs: (batch,n) tensor
state: the states and hidden unit of the two cells
Returns:
new_state, new_inputs
"""
with vs.variable_scope(scope or type(self).__name__):
c1, c2, h1, h2 = state
# change bias argument to False since LN will add bias via shift
concat = _linear([inputs, h1, h2], 5 * self._num_units, False)
i, j, f1, f2, o = array_ops.split(concat, 5, 1)
# add layer normalization to each gate
i = ln(i, scope='i/')
j = ln(j, scope='j/')
f1 = ln(f1, scope='f1/')
f2 = ln(f2, scope='f2/')
o = ln(o, scope='o/')
new_c = (c1 * nn.sigmoid(f1 + self._forget_bias) +
c2 * nn.sigmoid(f2 + self._forget_bias) +
nn.sigmoid(i) * self._activation(j))
# add layer_normalization in calculation of new hidden state
new_h = self._activation(ln(new_c, scope='new_h/')) * nn.sigmoid(o)
new_state = rnn.LSTMStateTuple(new_c, new_h)
return new_h, new_state
def step_gru(cell_inputs, cell_state, kernel, recurrent_kernel, input_bias,
recurrent_bias):
"""Step function that will be used by Keras RNN backend."""
h_tm1 = cell_state
# inputs projected by all gate matrices at once
matrix_x = K.dot(cell_inputs, kernel)
matrix_x = K.bias_add(matrix_x, input_bias)
x_z, x_r, x_h = array_ops.split(matrix_x, 3, axis=1)
# hidden state projected by all gate matrices at once
matrix_inner = K.dot(h_tm1, recurrent_kernel)
matrix_inner = K.bias_add(matrix_inner, recurrent_bias)
recurrent_z, recurrent_r, recurrent_h = array_ops.split(matrix_inner,
3,
axis=1)
z = nn.sigmoid(x_z + recurrent_z)
r = nn.sigmoid(x_r + recurrent_r)
hh = nn.tanh(x_h + r * recurrent_h)
# previous and candidate state mixed by update gate
h = z * h_tm1 + (1 - z) * hh
return h, [h]
def sigmoid(x):
"""Sigmoid activation function.
Applies the sigmoid activation function. The sigmoid function is defined as
1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed
version of the Heaviside (Unit Step Function) function. For small values
(<-5) the sigmoid returns a value close to zero and for larger values (>5)
the result of the function gets close to 1.
Sigmoid is equivalent to a 2-element Softmax, where the second element is
assumed to be zero.
For example:
>>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32)
>>> b = tf.keras.activations.sigmoid(a)
>>> b.numpy()
array([0. , 0.26894143, 0.5 , 0.7310586 , 1. ],
dtype=float32)
Arguments:
x: Input tensor.
Returns:
Tensor with the sigmoid activation: `(1.0 / (1.0 + exp(-x)))`.
Tensor will be of same shape and dtype of input `x`.
"""
return nn.sigmoid(x)
def sigmoid(x):
"""Sigmoid activation function, `sigmoid(x) = 1 / (1 + exp(-x))`.
Applies the sigmoid activation function. For small values (<-5),
`sigmoid` returns a value close to zero, and for large values (>5)
the result of the function gets close to 1.
Sigmoid is equivalent to a 2-element Softmax, where the second element is
assumed to be zero. The sigmoid function always returns a value between
0 and 1.
For example:
>>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32)
>>> b = tf.keras.activations.sigmoid(a)
>>> b.numpy() >= 0.0
array([ True, True, True, True, True])
Arguments:
x: Input tensor.
Returns:
Tensor with the sigmoid activation: `1 / (1 + exp(-x))`.
"""
return nn.sigmoid(x)
def sigmoid(x):
"""Sigmoid activation function, `sigmoid(x) = 1 / (1 + exp(-x))`.
Applies the sigmoid activation function. For small values (<-5),
`sigmoid` returns a value close to zero, and for large values (>5)
the result of the function gets close to 1.
Sigmoid is equivalent to a 2-element Softmax, where the second element is
assumed to be zero. The sigmoid function always returns a value between
0 and 1.
For example:
>>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32)
>>> b = tf.keras.activations.sigmoid(a)
>>> b.numpy()
array([2.0611537e-09, 2.6894143e-01, 5.0000000e-01, 7.3105860e-01,
1.0000000e+00], dtype=float32)
Args:
x: Input tensor.
Returns:
Tensor with the sigmoid activation: `1 / (1 + exp(-x))`.
"""
output = nn.sigmoid(x)
# Cache the logits to use for crossentropy loss.
output._keras_logits = x # pylint: disable=protected-access
return output
def sigmoid(x):
"""Sigmoid activation function.
Arguments:
x: Input tensor.
Returns:
The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`.
"""
return nn.sigmoid(x)
def sigmoid(x):
"""Element-wise sigmoid.
Args
x: A tensor or variable.
Returns:
A tensor.
"""
return nn.sigmoid(x)
def __init__(self,
p=None,
dtype=dtypes.int32,
validate_args=False,
allow_nan_stats=True,
name="BernoulliWithSigmoidP"):
with ops.name_scope(name) as ns:
super(BernoulliWithSigmoidP, self).__init__(
p=nn.sigmoid(p),
dtype=dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
def __init__(self,
p=None,
dtype=dtypes.int32,
validate_args=False,
allow_nan_stats=True,
name="BernoulliWithSigmoidP"):
with ops.name_scope(name) as ns:
super(BernoulliWithSigmoidP, self).__init__(
p=nn.sigmoid(p),
dtype=dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
def __init__(self,
logits=None,
dtype=dtypes.int32,
validate_args=False,
allow_nan_stats=True,
name="BernoulliWithSigmoidProbs"):
parameters = locals()
with ops.name_scope(name) as ns:
super(BernoulliWithSigmoidProbs, self).__init__(
probs=nn.sigmoid(logits, name="sigmoid_probs"),
dtype=dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
self._parameters = parameters
def __init__(self,
logits=None,
dtype=dtypes.int32,
validate_args=False,
allow_nan_stats=True,
name="BernoulliWithSigmoidProbs"):
parameters = locals()
with ops.name_scope(name):
super(BernoulliWithSigmoidProbs, self).__init__(
probs=nn.sigmoid(logits, name="sigmoid_probs"),
dtype=dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=name)
self._parameters = parameters
def __init__(self,
p=None,
dtype=dtypes.int32,
validate_args=False,
allow_nan_stats=True,
name="BernoulliWithSigmoidP"):
parameters = locals()
parameters.pop("self")
with ops.name_scope(name) as ns:
super(BernoulliWithSigmoidP, self).__init__(
p=nn.sigmoid(p, name="sigmoid_p"),
dtype=dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
self._parameters = parameters
def __init__(self,
p=None,
dtype=dtypes.int32,
validate_args=False,
allow_nan_stats=True,
name="BernoulliWithSigmoidP"):
parameters = locals()
parameters.pop("self")
with ops.name_scope(name) as ns:
super(BernoulliWithSigmoidP, self).__init__(
p=nn.sigmoid(p, name="sigmoid_p"),
dtype=dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
self._parameters = parameters
def sigmoid(x):
"""Sigmoid.
Applies the sigmoid activation function. The sigmoid function is defined as
1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed
version of the Heaviside (Unit Step Function) function. For small values
(<-5) the sigmoid returns a value close to zero and for larger values (>5)
the result of the function gets close to 1.
Arguments:
x: A tensor or variable.
Returns:
A tensor.
Sigmoid activation function.
Arguments:
x: Input tensor.
Returns:
The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`.
"""
return nn.sigmoid(x)
def sigmoid(x):
"""Sigmoid.
Applies the sigmoid activation function. The sigmoid function is defined as
1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed
version of the Heaviside (Unit Step Function) function. For small values
(<-5) the sigmoid returns a value close to zero and for larger values (>5)
the result of the function gets close to 1.
Arguments:
x: A tensor or variable.
Returns:
A tensor.
Sigmoid activation function.
Arguments:
x: Input tensor.
Returns:
The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`.
"""
return nn.sigmoid(x)
def sigmoid(x):
"""Sigmoid.
Applies the sigmoid activation function. The sigmoid function is defined as
1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed
version of the Heaviside function. For small values (<-5) the sigmoid returns
a value close to zero and for larger values (>5) the result of the funtion gets
close to 1.
Arguments:
x: A tensor or variable.
Returns:
A tensor.
"""
return nn.sigmoid(x)
@keras_export('keras.activations.exponential')
def exponential(x):
return math_ops.exp(x)
@keras_export('keras.activations.hard_sigmoid')
def hard_sigmoid(x):
"""Hard sigmoid activation function.
Faster to compute than sigmoid activation.
Arguments:
x: Input tensor.
def sigmoid(x):
return nn.sigmoid(x)
def test_train_one_head(self):
head1 = multi_label_head.MultiLabelHead(n_classes=2, name='head1')
multi_head = multi_head_lib.MultiHead([head1])
logits = {
'head1': np.array([[-10., 10.], [-15., 10.]], dtype=np.float32)
}
expected_probabilities = {
'head1': nn.sigmoid(logits['head1']),
}
labels = {'head1': np.array([[1, 0], [1, 1]], dtype=np.int64)}
features = {'x': np.array(((42, ), ), dtype=np.int32)}
# For large logits, sigmoid cross entropy loss is approximated as:
# loss = labels * (logits < 0) * (-logits) +
# (1 - labels) * (logits > 0) * logits =>
# expected_unweighted_loss = [[10., 10.], [15., 0.]]
# loss = ((10 + 10) / 2 + (15 + 0) / 2) / 2 = 8.75
expected_loss = 8.75
tol = 1e-3
loss = multi_head.loss(logits=logits,
labels=labels,
features=features,
mode=model_fn.ModeKeys.TRAIN)
self.assertAllClose(expected_loss,
self.evaluate(loss),
rtol=tol,
atol=tol)
if context.executing_eagerly():
return
expected_train_result = 'my_train_op'
def _train_op_fn(loss):
return string_ops.string_join([
constant_op.constant(expected_train_result),
string_ops.as_string(loss, precision=3)
])
spec = multi_head.create_estimator_spec(features=features,
mode=model_fn.ModeKeys.TRAIN,
logits=logits,
labels=labels,
train_op_fn=_train_op_fn)
self.assertIsNotNone(spec.loss)
self.assertEqual({}, spec.eval_metric_ops)
self.assertIsNotNone(spec.train_op)
self.assertIsNone(spec.export_outputs)
test_lib._assert_no_hooks(self, spec)
# Assert predictions, loss, train_op, and summaries.
with self.cached_session() as sess:
test_lib._initialize_variables(self, spec.scaffold)
self.assertIsNotNone(spec.scaffold.summary_op)
loss, train_result, summary_str, predictions = sess.run(
(spec.loss, spec.train_op, spec.scaffold.summary_op,
spec.predictions))
self.assertAllClose(
logits['head1'],
predictions[('head1', prediction_keys.PredictionKeys.LOGITS)])
self.assertAllClose(
expected_probabilities['head1'],
predictions[('head1',
prediction_keys.PredictionKeys.PROBABILITIES)])
self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol)
self.assertEqual(
six.b('{0:s}{1:.3f}'.format(expected_train_result,
expected_loss)), train_result)
test_lib._assert_simple_summaries(
self, {
metric_keys.MetricKeys.LOSS: expected_loss,
metric_keys.MetricKeys.LOSS + '/head1': expected_loss,
}, summary_str, tol)
def test_predict_two_heads_logits_dict(self):
"""Tests predict with logits as dict."""
head1 = multi_label_head.MultiLabelHead(n_classes=2, name='head1')
head2 = multi_label_head.MultiLabelHead(n_classes=3, name='head2')
multi_head = multi_head_lib.MultiHead([head1, head2])
logits = {
'head1': np.array([[-1., 1.], [-1.5, 1.]], dtype=np.float32),
'head2': np.array([[2., -2., 2.], [-3., 2., -2.]], dtype=np.float32)
}
expected_probabilities = {
'head1': nn.sigmoid(logits['head1']),
'head2': nn.sigmoid(logits['head2']),
}
pred_keys = prediction_keys.PredictionKeys
predictions = multi_head.predictions(logits)
self.assertAllClose(
logits['head1'],
self.evaluate(predictions[('head1', pred_keys.LOGITS)]))
self.assertAllClose(
logits['head2'],
self.evaluate(predictions[('head2', pred_keys.LOGITS)]))
self.assertAllClose(
expected_probabilities['head1'],
self.evaluate(predictions[('head1', pred_keys.PROBABILITIES)]))
self.assertAllClose(
expected_probabilities['head2'],
self.evaluate(predictions[('head2', pred_keys.PROBABILITIES)]))
if context.executing_eagerly():
return
spec = multi_head.create_estimator_spec(
features={'x': np.array(((42,),), dtype=np.int32)},
mode=model_fn.ModeKeys.PREDICT,
logits=logits)
self.assertItemsEqual(
(test_lib._DEFAULT_SERVING_KEY, 'predict', 'head1',
'head1/classification', 'head1/predict', 'head2',
'head2/classification', 'head2/predict'), spec.export_outputs.keys())
# Assert predictions and export_outputs.
with self.cached_session() as sess:
test_lib._initialize_variables(self, spec.scaffold)
self.assertIsNone(spec.scaffold.summary_op)
predictions = sess.run(spec.predictions)
self.assertAllClose(
logits['head1'],
predictions[('head1', pred_keys.LOGITS)])
self.assertAllClose(
logits['head2'],
predictions[('head2', pred_keys.LOGITS)])
self.assertAllClose(
expected_probabilities['head1'],
predictions[('head1', pred_keys.PROBABILITIES)])
self.assertAllClose(
expected_probabilities['head2'],
predictions[('head2', pred_keys.PROBABILITIES)])
self.assertAllClose(
expected_probabilities['head1'],
sess.run(spec.export_outputs[test_lib._DEFAULT_SERVING_KEY].scores))
self.assertAllClose(
expected_probabilities['head1'],
sess.run(spec.export_outputs['head1'].scores))
self.assertAllClose(
expected_probabilities['head2'],
sess.run(spec.export_outputs['head2'].scores))
self.assertAllClose(
expected_probabilities['head1'],
sess.run(
spec.export_outputs['predict'].outputs['head1/probabilities']))
self.assertAllClose(
expected_probabilities['head2'],
sess.run(
spec.export_outputs['predict'].outputs['head2/probabilities']))
self.assertAllClose(
expected_probabilities['head1'],
sess.run(
spec.export_outputs['head1/predict'].outputs['probabilities']))
self.assertAllClose(
expected_probabilities['head2'],
sess.run(
spec.export_outputs['head2/predict'].outputs['probabilities']))
b0 = tf.Variable(tf.zeros([32]))
# Down sample
conv_0 = tf.nn.relu(
tf.nn.conv2d(in_data, w0, [1, 2, 2, 1], padding='SAME') + b0)
print("Convolution 0:", conv_0)
# Up sample 1. Upscale to 100 x 100 x 24
wt1 = tf.Variable(tf.truncated_normal([3, 3, 24, 32]))
end1 = time.time()
print("Checkpoint model", end1 - start)
start_time = time.time()
import check
convt_1 = sigmoid(
conv2d_transpose(conv_0,
filter=wt1,
output_shape=[batch_size, 100, 100, 24],
strides=[1, 1, 1, 1]))
print("Deconvolution 1:", convt_1)
# Up sample 2. Upscale to 256 x 256 x 2
wt2 = tf.Variable(tf.truncated_normal([3, 3, 2, 24]))
convt_2 = sigmoid(
conv2d_transpose(convt_1,
filter=wt2,
output_shape=[batch_size, IMAGE_HEIGHT, IMAGE_WIDTH, 2],
strides=[1, 1, 1, 1]))
print("Deconvolution 2:", convt_2)
# Loss computation
logits = tf.reshape(convt_2, [-1, num_labels])
def sigmoid(x): return nn.sigmoid(x)
def test_train_with_regularization_losses(self):
head1 = multi_label_head.MultiLabelHead(n_classes=2, name='head1')
head2 = multi_label_head.MultiLabelHead(n_classes=3, name='head2')
multi_head = multi_head_lib.MultiHead([head1, head2],
head_weights=[1., 2.])
logits = {
'head1':
np.array([[-10., 10.], [-15., 10.]], dtype=np.float32),
'head2':
np.array([[20., -20., 20.], [-30., 20., -20.]], dtype=np.float32),
}
expected_probabilities = {
'head1': nn.sigmoid(logits['head1']),
'head2': nn.sigmoid(logits['head2']),
}
labels = {
'head1': np.array([[1, 0], [1, 1]], dtype=np.int64),
'head2': np.array([[0, 1, 0], [1, 1, 0]], dtype=np.int64),
}
features = {'x': np.array(((42, ), ), dtype=np.int32)}
regularization_losses = [1.5, 0.5]
# For large logits, sigmoid cross entropy loss is approximated as:
# loss = labels * (logits < 0) * (-logits) +
# (1 - labels) * (logits > 0) * logits =>
# head1: expected_unweighted_loss = [[10., 10.], [15., 0.]]
# loss1 = ((10 + 10) / 2 + (15 + 0) / 2) / 2 = 8.75
# head2: expected_unweighted_loss = [[20., 20., 20.], [30., 0., 0]]
# loss2 = ((20 + 20 + 20) / 3 + (30 + 0 + 0) / 3) / 2 = 15
# Average over classes, weighted sum over batch and heads.
# weights = [1., 2.]
# merged_training_loss = 1. * loss1 + 2. * loss2
# training_loss = merged_training_loss + regularization_loss
# = 1. * loss1 + 2. * loss2 + sum([1.5, 0.5])
expected_loss_head1 = 8.75
expected_loss_head2 = 15.0
expected_regularization_loss = 2.
# training loss.
expected_loss = (1. * expected_loss_head1 + 2. * expected_loss_head2 +
expected_regularization_loss)
tol = 1e-3
loss = multi_head.loss(logits=logits,
labels=labels,
features=features,
mode=model_fn.ModeKeys.TRAIN,
regularization_losses=regularization_losses)
self.assertAllClose(expected_loss,
self.evaluate(loss),
rtol=tol,
atol=tol)
if context.executing_eagerly():
return
keys = metric_keys.MetricKeys
expected_train_result = 'my_train_op'
def _train_op_fn(loss):
return string_ops.string_join([
constant_op.constant(expected_train_result),
string_ops.as_string(loss, precision=3)
])
spec = multi_head.create_estimator_spec(
features=features,
mode=model_fn.ModeKeys.TRAIN,
logits=logits,
labels=labels,
train_op_fn=_train_op_fn,
regularization_losses=regularization_losses)
self.assertIsNotNone(spec.loss)
self.assertEqual({}, spec.eval_metric_ops)
self.assertIsNotNone(spec.train_op)
self.assertIsNone(spec.export_outputs)
test_lib._assert_no_hooks(self, spec)
# Assert predictions, loss, train_op, and summaries.
with self.cached_session() as sess:
test_lib._initialize_variables(self, spec.scaffold)
self.assertIsNotNone(spec.scaffold.summary_op)
loss, train_result, summary_str, predictions = sess.run(
(spec.loss, spec.train_op, spec.scaffold.summary_op,
spec.predictions))
self.assertAllClose(
logits['head1'],
predictions[('head1', prediction_keys.PredictionKeys.LOGITS)])
self.assertAllClose(
expected_probabilities['head1'],
predictions[('head1',
prediction_keys.PredictionKeys.PROBABILITIES)])
self.assertAllClose(
logits['head2'],
predictions[('head2', prediction_keys.PredictionKeys.LOGITS)])
self.assertAllClose(
expected_probabilities['head2'],
predictions[('head2',
prediction_keys.PredictionKeys.PROBABILITIES)])
self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol)
self.assertEqual(
six.b('{0:s}{1:.3f}'.format(expected_train_result,
expected_loss)), train_result)
test_lib._assert_simple_summaries(
self, {
keys.LOSS_REGULARIZATION: expected_regularization_loss,
keys.LOSS: expected_loss,
keys.LOSS + '/head1': expected_loss_head1,
keys.LOSS + '/head2': expected_loss_head2,
}, summary_str, tol)
