def __call__(self, inputs, state, scope=None): """Long short-term memory cell (LSTM). Args: inputs: (batch,n) tensor state: the states and hidden unit of the two cells Returns: new_state, new_inputs """ with vs.variable_scope(scope or type(self).__name__): c1, c2, h1, h2 = state # change bias argument to False since LN will add bias via shift concat = _linear([inputs, h1, h2], 5 * self._num_units, False) i, j, f1, f2, o = array_ops.split(concat, 5, 1) # add layer normalization to each gate i = ln(i, scope='i/') j = ln(j, scope='j/') f1 = ln(f1, scope='f1/') f2 = ln(f2, scope='f2/') o = ln(o, scope='o/') new_c = (c1 * nn.sigmoid(f1 + self._forget_bias) + c2 * nn.sigmoid(f2 + self._forget_bias) + nn.sigmoid(i) * self._activation(j)) # add layer_normalization in calculation of new hidden state new_h = self._activation(ln(new_c, scope='new_h/')) * nn.sigmoid(o) new_state = rnn.LSTMStateTuple(new_c, new_h) return new_h, new_state
def step_gru(cell_inputs, cell_state, kernel, recurrent_kernel, input_bias, recurrent_bias): """Step function that will be used by Keras RNN backend.""" h_tm1 = cell_state # inputs projected by all gate matrices at once matrix_x = K.dot(cell_inputs, kernel) matrix_x = K.bias_add(matrix_x, input_bias) x_z, x_r, x_h = array_ops.split(matrix_x, 3, axis=1) # hidden state projected by all gate matrices at once matrix_inner = K.dot(h_tm1, recurrent_kernel) matrix_inner = K.bias_add(matrix_inner, recurrent_bias) recurrent_z, recurrent_r, recurrent_h = array_ops.split(matrix_inner, 3, axis=1) z = nn.sigmoid(x_z + recurrent_z) r = nn.sigmoid(x_r + recurrent_r) hh = nn.tanh(x_h + r * recurrent_h) # previous and candidate state mixed by update gate h = z * h_tm1 + (1 - z) * hh return h, [h]
def sigmoid(x): """Sigmoid activation function. Applies the sigmoid activation function. The sigmoid function is defined as 1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed version of the Heaviside (Unit Step Function) function. For small values (<-5) the sigmoid returns a value close to zero and for larger values (>5) the result of the function gets close to 1. Sigmoid is equivalent to a 2-element Softmax, where the second element is assumed to be zero. For example: >>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32) >>> b = tf.keras.activations.sigmoid(a) >>> b.numpy() array([0. , 0.26894143, 0.5 , 0.7310586 , 1. ], dtype=float32) Arguments: x: Input tensor. Returns: Tensor with the sigmoid activation: `(1.0 / (1.0 + exp(-x)))`. Tensor will be of same shape and dtype of input `x`. """ return nn.sigmoid(x)
def sigmoid(x): """Sigmoid activation function, `sigmoid(x) = 1 / (1 + exp(-x))`. Applies the sigmoid activation function. For small values (<-5), `sigmoid` returns a value close to zero, and for large values (>5) the result of the function gets close to 1. Sigmoid is equivalent to a 2-element Softmax, where the second element is assumed to be zero. The sigmoid function always returns a value between 0 and 1. For example: >>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32) >>> b = tf.keras.activations.sigmoid(a) >>> b.numpy() >= 0.0 array([ True, True, True, True, True]) Arguments: x: Input tensor. Returns: Tensor with the sigmoid activation: `1 / (1 + exp(-x))`. """ return nn.sigmoid(x)
def sigmoid(x): """Sigmoid activation function, `sigmoid(x) = 1 / (1 + exp(-x))`. Applies the sigmoid activation function. For small values (<-5), `sigmoid` returns a value close to zero, and for large values (>5) the result of the function gets close to 1. Sigmoid is equivalent to a 2-element Softmax, where the second element is assumed to be zero. The sigmoid function always returns a value between 0 and 1. For example: >>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32) >>> b = tf.keras.activations.sigmoid(a) >>> b.numpy() array([2.0611537e-09, 2.6894143e-01, 5.0000000e-01, 7.3105860e-01, 1.0000000e+00], dtype=float32) Args: x: Input tensor. Returns: Tensor with the sigmoid activation: `1 / (1 + exp(-x))`. """ output = nn.sigmoid(x) # Cache the logits to use for crossentropy loss. output._keras_logits = x # pylint: disable=protected-access return output
def sigmoid(x): """Sigmoid activation function. Arguments: x: Input tensor. Returns: The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`. """ return nn.sigmoid(x)
def sigmoid(x): """Element-wise sigmoid. Args x: A tensor or variable. Returns: A tensor. """ return nn.sigmoid(x)
def __init__(self, p=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name="BernoulliWithSigmoidP"): with ops.name_scope(name) as ns: super(BernoulliWithSigmoidP, self).__init__( p=nn.sigmoid(p), dtype=dtype, validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns)
def __init__(self, p=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name="BernoulliWithSigmoidP"): with ops.name_scope(name) as ns: super(BernoulliWithSigmoidP, self).__init__( p=nn.sigmoid(p), dtype=dtype, validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns)
def __init__(self, logits=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name="BernoulliWithSigmoidProbs"): parameters = locals() with ops.name_scope(name) as ns: super(BernoulliWithSigmoidProbs, self).__init__( probs=nn.sigmoid(logits, name="sigmoid_probs"), dtype=dtype, validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns) self._parameters = parameters
def __init__(self, logits=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name="BernoulliWithSigmoidProbs"): parameters = locals() with ops.name_scope(name): super(BernoulliWithSigmoidProbs, self).__init__( probs=nn.sigmoid(logits, name="sigmoid_probs"), dtype=dtype, validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=name) self._parameters = parameters
def __init__(self, p=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name="BernoulliWithSigmoidP"): parameters = locals() parameters.pop("self") with ops.name_scope(name) as ns: super(BernoulliWithSigmoidP, self).__init__( p=nn.sigmoid(p, name="sigmoid_p"), dtype=dtype, validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns) self._parameters = parameters
def __init__(self, p=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name="BernoulliWithSigmoidP"): parameters = locals() parameters.pop("self") with ops.name_scope(name) as ns: super(BernoulliWithSigmoidP, self).__init__( p=nn.sigmoid(p, name="sigmoid_p"), dtype=dtype, validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns) self._parameters = parameters
def sigmoid(x): """Sigmoid. Applies the sigmoid activation function. The sigmoid function is defined as 1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed version of the Heaviside (Unit Step Function) function. For small values (<-5) the sigmoid returns a value close to zero and for larger values (>5) the result of the function gets close to 1. Arguments: x: A tensor or variable. Returns: A tensor. Sigmoid activation function. Arguments: x: Input tensor. Returns: The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`. """ return nn.sigmoid(x)
def sigmoid(x): """Sigmoid. Applies the sigmoid activation function. The sigmoid function is defined as 1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed version of the Heaviside (Unit Step Function) function. For small values (<-5) the sigmoid returns a value close to zero and for larger values (>5) the result of the function gets close to 1. Arguments: x: A tensor or variable. Returns: A tensor. Sigmoid activation function. Arguments: x: Input tensor. Returns: The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`. """ return nn.sigmoid(x)
def sigmoid(x): """Sigmoid. Applies the sigmoid activation function. The sigmoid function is defined as 1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed version of the Heaviside function. For small values (<-5) the sigmoid returns a value close to zero and for larger values (>5) the result of the funtion gets close to 1. Arguments: x: A tensor or variable. Returns: A tensor. """ return nn.sigmoid(x) @keras_export('keras.activations.exponential') def exponential(x): return math_ops.exp(x) @keras_export('keras.activations.hard_sigmoid') def hard_sigmoid(x): """Hard sigmoid activation function. Faster to compute than sigmoid activation. Arguments: x: Input tensor.
def sigmoid(x): return nn.sigmoid(x)
def test_train_one_head(self): head1 = multi_label_head.MultiLabelHead(n_classes=2, name='head1') multi_head = multi_head_lib.MultiHead([head1]) logits = { 'head1': np.array([[-10., 10.], [-15., 10.]], dtype=np.float32) } expected_probabilities = { 'head1': nn.sigmoid(logits['head1']), } labels = {'head1': np.array([[1, 0], [1, 1]], dtype=np.int64)} features = {'x': np.array(((42, ), ), dtype=np.int32)} # For large logits, sigmoid cross entropy loss is approximated as: # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits => # expected_unweighted_loss = [[10., 10.], [15., 0.]] # loss = ((10 + 10) / 2 + (15 + 0) / 2) / 2 = 8.75 expected_loss = 8.75 tol = 1e-3 loss = multi_head.loss(logits=logits, labels=labels, features=features, mode=model_fn.ModeKeys.TRAIN) self.assertAllClose(expected_loss, self.evaluate(loss), rtol=tol, atol=tol) if context.executing_eagerly(): return expected_train_result = 'my_train_op' def _train_op_fn(loss): return string_ops.string_join([ constant_op.constant(expected_train_result), string_ops.as_string(loss, precision=3) ]) spec = multi_head.create_estimator_spec(features=features, mode=model_fn.ModeKeys.TRAIN, logits=logits, labels=labels, train_op_fn=_train_op_fn) self.assertIsNotNone(spec.loss) self.assertEqual({}, spec.eval_metric_ops) self.assertIsNotNone(spec.train_op) self.assertIsNone(spec.export_outputs) test_lib._assert_no_hooks(self, spec) # Assert predictions, loss, train_op, and summaries. with self.cached_session() as sess: test_lib._initialize_variables(self, spec.scaffold) self.assertIsNotNone(spec.scaffold.summary_op) loss, train_result, summary_str, predictions = sess.run( (spec.loss, spec.train_op, spec.scaffold.summary_op, spec.predictions)) self.assertAllClose( logits['head1'], predictions[('head1', prediction_keys.PredictionKeys.LOGITS)]) self.assertAllClose( expected_probabilities['head1'], predictions[('head1', prediction_keys.PredictionKeys.PROBABILITIES)]) self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol) self.assertEqual( six.b('{0:s}{1:.3f}'.format(expected_train_result, expected_loss)), train_result) test_lib._assert_simple_summaries( self, { metric_keys.MetricKeys.LOSS: expected_loss, metric_keys.MetricKeys.LOSS + '/head1': expected_loss, }, summary_str, tol)
def test_predict_two_heads_logits_dict(self): """Tests predict with logits as dict.""" head1 = multi_label_head.MultiLabelHead(n_classes=2, name='head1') head2 = multi_label_head.MultiLabelHead(n_classes=3, name='head2') multi_head = multi_head_lib.MultiHead([head1, head2]) logits = { 'head1': np.array([[-1., 1.], [-1.5, 1.]], dtype=np.float32), 'head2': np.array([[2., -2., 2.], [-3., 2., -2.]], dtype=np.float32) } expected_probabilities = { 'head1': nn.sigmoid(logits['head1']), 'head2': nn.sigmoid(logits['head2']), } pred_keys = prediction_keys.PredictionKeys predictions = multi_head.predictions(logits) self.assertAllClose( logits['head1'], self.evaluate(predictions[('head1', pred_keys.LOGITS)])) self.assertAllClose( logits['head2'], self.evaluate(predictions[('head2', pred_keys.LOGITS)])) self.assertAllClose( expected_probabilities['head1'], self.evaluate(predictions[('head1', pred_keys.PROBABILITIES)])) self.assertAllClose( expected_probabilities['head2'], self.evaluate(predictions[('head2', pred_keys.PROBABILITIES)])) if context.executing_eagerly(): return spec = multi_head.create_estimator_spec( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.PREDICT, logits=logits) self.assertItemsEqual( (test_lib._DEFAULT_SERVING_KEY, 'predict', 'head1', 'head1/classification', 'head1/predict', 'head2', 'head2/classification', 'head2/predict'), spec.export_outputs.keys()) # Assert predictions and export_outputs. with self.cached_session() as sess: test_lib._initialize_variables(self, spec.scaffold) self.assertIsNone(spec.scaffold.summary_op) predictions = sess.run(spec.predictions) self.assertAllClose( logits['head1'], predictions[('head1', pred_keys.LOGITS)]) self.assertAllClose( logits['head2'], predictions[('head2', pred_keys.LOGITS)]) self.assertAllClose( expected_probabilities['head1'], predictions[('head1', pred_keys.PROBABILITIES)]) self.assertAllClose( expected_probabilities['head2'], predictions[('head2', pred_keys.PROBABILITIES)]) self.assertAllClose( expected_probabilities['head1'], sess.run(spec.export_outputs[test_lib._DEFAULT_SERVING_KEY].scores)) self.assertAllClose( expected_probabilities['head1'], sess.run(spec.export_outputs['head1'].scores)) self.assertAllClose( expected_probabilities['head2'], sess.run(spec.export_outputs['head2'].scores)) self.assertAllClose( expected_probabilities['head1'], sess.run( spec.export_outputs['predict'].outputs['head1/probabilities'])) self.assertAllClose( expected_probabilities['head2'], sess.run( spec.export_outputs['predict'].outputs['head2/probabilities'])) self.assertAllClose( expected_probabilities['head1'], sess.run( spec.export_outputs['head1/predict'].outputs['probabilities'])) self.assertAllClose( expected_probabilities['head2'], sess.run( spec.export_outputs['head2/predict'].outputs['probabilities']))
b0 = tf.Variable(tf.zeros([32])) # Down sample conv_0 = tf.nn.relu( tf.nn.conv2d(in_data, w0, [1, 2, 2, 1], padding='SAME') + b0) print("Convolution 0:", conv_0) # Up sample 1. Upscale to 100 x 100 x 24 wt1 = tf.Variable(tf.truncated_normal([3, 3, 24, 32])) end1 = time.time() print("Checkpoint model", end1 - start) start_time = time.time() import check convt_1 = sigmoid( conv2d_transpose(conv_0, filter=wt1, output_shape=[batch_size, 100, 100, 24], strides=[1, 1, 1, 1])) print("Deconvolution 1:", convt_1) # Up sample 2. Upscale to 256 x 256 x 2 wt2 = tf.Variable(tf.truncated_normal([3, 3, 2, 24])) convt_2 = sigmoid( conv2d_transpose(convt_1, filter=wt2, output_shape=[batch_size, IMAGE_HEIGHT, IMAGE_WIDTH, 2], strides=[1, 1, 1, 1])) print("Deconvolution 2:", convt_2) # Loss computation logits = tf.reshape(convt_2, [-1, num_labels])
def sigmoid(x): return nn.sigmoid(x)
def test_train_with_regularization_losses(self): head1 = multi_label_head.MultiLabelHead(n_classes=2, name='head1') head2 = multi_label_head.MultiLabelHead(n_classes=3, name='head2') multi_head = multi_head_lib.MultiHead([head1, head2], head_weights=[1., 2.]) logits = { 'head1': np.array([[-10., 10.], [-15., 10.]], dtype=np.float32), 'head2': np.array([[20., -20., 20.], [-30., 20., -20.]], dtype=np.float32), } expected_probabilities = { 'head1': nn.sigmoid(logits['head1']), 'head2': nn.sigmoid(logits['head2']), } labels = { 'head1': np.array([[1, 0], [1, 1]], dtype=np.int64), 'head2': np.array([[0, 1, 0], [1, 1, 0]], dtype=np.int64), } features = {'x': np.array(((42, ), ), dtype=np.int32)} regularization_losses = [1.5, 0.5] # For large logits, sigmoid cross entropy loss is approximated as: # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits => # head1: expected_unweighted_loss = [[10., 10.], [15., 0.]] # loss1 = ((10 + 10) / 2 + (15 + 0) / 2) / 2 = 8.75 # head2: expected_unweighted_loss = [[20., 20., 20.], [30., 0., 0]] # loss2 = ((20 + 20 + 20) / 3 + (30 + 0 + 0) / 3) / 2 = 15 # Average over classes, weighted sum over batch and heads. # weights = [1., 2.] # merged_training_loss = 1. * loss1 + 2. * loss2 # training_loss = merged_training_loss + regularization_loss # = 1. * loss1 + 2. * loss2 + sum([1.5, 0.5]) expected_loss_head1 = 8.75 expected_loss_head2 = 15.0 expected_regularization_loss = 2. # training loss. expected_loss = (1. * expected_loss_head1 + 2. * expected_loss_head2 + expected_regularization_loss) tol = 1e-3 loss = multi_head.loss(logits=logits, labels=labels, features=features, mode=model_fn.ModeKeys.TRAIN, regularization_losses=regularization_losses) self.assertAllClose(expected_loss, self.evaluate(loss), rtol=tol, atol=tol) if context.executing_eagerly(): return keys = metric_keys.MetricKeys expected_train_result = 'my_train_op' def _train_op_fn(loss): return string_ops.string_join([ constant_op.constant(expected_train_result), string_ops.as_string(loss, precision=3) ]) spec = multi_head.create_estimator_spec( features=features, mode=model_fn.ModeKeys.TRAIN, logits=logits, labels=labels, train_op_fn=_train_op_fn, regularization_losses=regularization_losses) self.assertIsNotNone(spec.loss) self.assertEqual({}, spec.eval_metric_ops) self.assertIsNotNone(spec.train_op) self.assertIsNone(spec.export_outputs) test_lib._assert_no_hooks(self, spec) # Assert predictions, loss, train_op, and summaries. with self.cached_session() as sess: test_lib._initialize_variables(self, spec.scaffold) self.assertIsNotNone(spec.scaffold.summary_op) loss, train_result, summary_str, predictions = sess.run( (spec.loss, spec.train_op, spec.scaffold.summary_op, spec.predictions)) self.assertAllClose( logits['head1'], predictions[('head1', prediction_keys.PredictionKeys.LOGITS)]) self.assertAllClose( expected_probabilities['head1'], predictions[('head1', prediction_keys.PredictionKeys.PROBABILITIES)]) self.assertAllClose( logits['head2'], predictions[('head2', prediction_keys.PredictionKeys.LOGITS)]) self.assertAllClose( expected_probabilities['head2'], predictions[('head2', prediction_keys.PredictionKeys.PROBABILITIES)]) self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol) self.assertEqual( six.b('{0:s}{1:.3f}'.format(expected_train_result, expected_loss)), train_result) test_lib._assert_simple_summaries( self, { keys.LOSS_REGULARIZATION: expected_regularization_loss, keys.LOSS: expected_loss, keys.LOSS + '/head1': expected_loss_head1, keys.LOSS + '/head2': expected_loss_head2, }, summary_str, tol)