def _read_variable_op(self): if hasattr(self, "_trainable") and self._trainable: tape.watch(self._handle) return read_variable_op(self._handle, dtype=self._dtype) else: return gen_resource_variable_ops.read_variable_op( self._handle, self._dtype)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or list of Tensors. Raises: ValueError: if it encounters something that is not a tensor. """ for t in nest.flatten(tensor): if not (_pywrap_utils.IsTensor(t) or _pywrap_utils.IsVariable(t)): raise ValueError( "Passed in object of type {}, not tf.Tensor".format( type(t))) if not backprop_util.IsTrainable(t): logging.log_first_n( logging.WARN, "The dtype of the watched tensor must be " "floating (e.g. tf.float32), got %r", 5, t.dtype) if hasattr(t, "handle"): # There are many variable-like objects, all of them currently have # `handle` attribute that points to a tensor. If this changes, internals # of watch_variable need to change as well. tape.watch_variable(self._tape, t) else: tape.watch(self._tape, t)
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() sources = [] args = [ ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] for i in range(len(args)) ] args = _ensure_unique_tensor_objects(parameter_positions, args) for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) t = tape.pop_tape() def vjp(dy=None): return imperative_grad.imperative_grad( _default_vspace, t, nest.flatten(result), sources, output_gradients=nest.flatten(dy) if dy is not None else None) return result, vjp
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" parameter_positions = _get_arg_spec(f, params, args) assert not kwds, "The gradient function can't take keyword arguments." this_tape = tape.push_new_tape(persistent=persistent) try: sources = [] args = [ ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] for i in range(len(args)) ] args = _ensure_unique_tensor_objects(parameter_positions, args) for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) if result is None: raise ValueError("Cannot differentiate a function that returns None; " "did you forget to return a value from {}?".format( f.__name__)) flat_result = nest.flatten(result) flat_result = [gen_array_ops.identity(x) for x in flat_result] result = nest.pack_sequence_as(result, flat_result) finally: tape.pop_tape(this_tape) def vjp(dy=None): if dy is not None: dy = [ops.convert_to_tensor(x) for x in nest.flatten(dy)] return imperative_grad.imperative_grad( _default_vspace, this_tape, nest.flatten(result), sources, output_gradients=dy) return result, vjp
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" parameter_positions = _get_arg_spec(f, params, args) assert not kwds, "The gradient function can't take keyword arguments." this_tape = tape.push_new_tape(persistent=persistent) try: sources = [] args = [ ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] for i in range(len(args)) ] args = _ensure_unique_tensor_objects(parameter_positions, args) for i in parameter_positions: sources.append(args[i]) tape.watch(this_tape, args[i]) result = f(*args) if result is None: raise ValueError("Cannot differentiate a function that returns None; " "did you forget to return a value from {}?".format( f.__name__)) flat_result = nest.flatten(result) flat_result = [gen_array_ops.identity(x) for x in flat_result] result = nest.pack_sequence_as(result, flat_result) finally: tape.pop_tape(this_tape) def vjp(dy=None): if dy is not None: dy = [ops.convert_to_tensor(x) for x in nest.flatten(dy)] return imperative_grad.imperative_grad( this_tape, nest.flatten(result), sources, output_gradients=dy) return result, vjp
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" parameter_positions = _get_arg_spec(f, params, args) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() try: sources = [] args = [ ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] for i in range(len(args)) ] args = _ensure_unique_tensor_objects(parameter_positions, args) for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) flat_result = nest.flatten(result) flat_result = [gen_array_ops.identity(x) for x in flat_result] result = nest.pack_sequence_as(result, flat_result) finally: t = tape.pop_tape() def vjp(dy=None): if dy is not None: dy = [ops.convert_to_tensor(x) for x in nest.flatten(dy)] return imperative_grad.imperative_grad(_default_vspace, t, nest.flatten(result), sources, output_gradients=dy) return result, vjp
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or list of Tensors. """ for t in nest.flatten(tensor): tape.watch(_handle_or_self(t))
def sparse_read(self, indices, name=None): """Reads the value of this variable sparsely, using `gather`.""" with ops.name_scope("Gather" if name is None else name) as name: if self._trainable: tape.watch(self._handle) value = resource_gather(self._handle, indices, dtype=self._dtype, name=name) return array_ops.identity(value)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or Variable a list of Tensors or Variables. """ for t in nest.flatten(tensor): if isinstance(t, resource_variable_ops.ResourceVariable): t = t.handle tape.watch(t)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or Variable a list of Tensors or Variables. """ for t in nest.flatten(tensor): if isinstance(t, resource_variable_ops.ResourceVariable): t = t.handle tape.watch(t)
def testFastpathExecute_TapeWrite(self): ctx = context.context() with backprop.GradientTape(persistent=True) as tape: a_2_by_2 = constant_op.constant(1.0, shape=[2, 2]) tape.watch(a_2_by_2) z = pywrap_tensorflow.TFE_Py_FastPathExecute( ctx._handle, ctx.device_name, "MatMul", None, None, a_2_by_2, a_2_by_2, "transpose_a", False, "transpose_b", False) dz_dy = tape.gradient(z, [a_2_by_2])[0] self.assertAllEqual(dz_dy.numpy(), constant_op.constant(4.0, shape=[2, 2]).numpy())
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" dy = kwds.pop("dy", None) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() sources = [] args = list(args) for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) return result, imperative_grad(result, sources, output_gradients=dy)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or list of Tensors. """ for t in nest.flatten(tensor): if hasattr(t, "handle"): # There are many variable-like objects, all of them currently have # `handle` attribute that points to a tensor. If this changes, internals # of watch_variable need to change as well. tape.watch_variable(self._tape, t) else: tape.watch(self._tape, t)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or list of Tensors. """ for t in nest.flatten(tensor): if hasattr(t, "handle"): # There are many variable-like objects, all of them currently have # `handle` attribute that points to a tensor. If this changes, internals # of watch_variable need to change as well. tape.watch_variable(self._tape, t) else: tape.watch(self._tape, t)
def testFastpathExecute_IdentityNTapeWrite(self): ctx = context.context() a_2_by_2 = random_ops.random_uniform((2, 2)) b_2_by_2 = random_ops.random_uniform((2, 2)) with backprop.GradientTape(persistent=True) as tape: tape.watch(a_2_by_2) tape.watch(b_2_by_2) z1 = pywrap_tensorflow.TFE_Py_FastPathExecute( ctx._handle, ctx.device_name, "IdentityN", None, None, [a_2_by_2, b_2_by_2]) z2 = array_ops.identity_n([a_2_by_2, b_2_by_2]) dz1_dy = tape.gradient(z1[0], [a_2_by_2])[0] dz2_dy = tape.gradient(z2[0], [a_2_by_2])[0] self.assertAllEqual(dz1_dy.numpy(), dz2_dy.numpy())
def get_grad(): with ops.Graph().as_default(), self.test_session(): t = constant_op.constant(1, dtype=dtypes.float32, shape=(10, 4)) x = constant_op.constant(2, dtype=dtypes.float32, shape=(10, 4)) with backprop.GradientTape() as gt: tape.watch(x) x1, _ = array_ops.split(x, num_or_size_splits=2, axis=1) y1 = x1**2 y = array_ops.concat([y1, t], axis=1) return self.evaluate(gt.gradient(y, x))
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" dy = kwds.pop("dy", None) if dy is not None: dy = ops.convert_to_tensor(dy) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() sources = [] args = [ops.convert_to_tensor(x) for x in args] for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) return result, imperative_grad( result, sources, output_gradients=dy)
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" dy = kwds.pop("dy", None) if dy is not None: dy = ops.convert_to_tensor(dy) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() sources = [] args = [ ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] for i in range(len(args)) ] for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) return result, imperative_grad(result, sources, output_gradients=dy)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or list of Tensors. """ for t in nest.flatten(tensor): if not t.dtype.is_floating: logging.log_first_n( logging.WARN, "The dtype of the watched tensor must be " "floating (e.g. tf.float32), got %r", 5, t.dtype) if hasattr(t, "handle"): # There are many variable-like objects, all of them currently have # `handle` attribute that points to a tensor. If this changes, internals # of watch_variable need to change as well. tape.watch_variable(self._tape, t) else: tape.watch(self._tape, t)
def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. Args: tensor: a Tensor or list of Tensors. """ for t in nest.flatten(tensor): if not t.dtype.is_floating: logging.log_first_n( logging.WARN, "The dtype of the watched tensor must be " "floating (e.g. tf.float32), got %r", 5, t.dtype) if hasattr(t, "handle"): # There are many variable-like objects, all of them currently have # `handle` attribute that points to a tensor. If this changes, internals # of watch_variable need to change as well. tape.watch_variable(self._tape, t) else: tape.watch(self._tape, t)
def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" dy = kwds.pop("dy", None) if dy is not None: dy = ops.convert_to_tensor(dy) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() sources = [] args = [ ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] for i in range(len(args)) ] args = _ensure_unique_tensor_objects(parameter_positions, args) for i in parameter_positions: sources.append(args[i]) tape.watch(args[i]) result = f(*args) return result, imperative_grad.imperative_grad( _default_vspace, nest.flatten(result), sources, output_gradients=nest.flatten(dy) if dy is not None else None)
def testFastpathExecute_MixedPrecisionVariableTapeWrite(self): ctx = context.context() with backprop.GradientTape(persistent=True) as tape: a_2_by_2 = constant_op.constant([[1.0, 2.0], [3.0, 4.0]], dtype=dtypes.float32) a_2_by_2_fp16 = math_ops.cast(a_2_by_2, dtype=dtypes.float16) m1 = resource_variable_ops.ResourceVariable(a_2_by_2) m2 = resource_variable_ops._MixedPrecisionVariable( m1, read_dtype=dtypes.float16) tape.watch(m2) z = pywrap_tensorflow.TFE_Py_FastPathExecute( ctx._handle, ctx.device_name, "MatMul", None, None, a_2_by_2_fp16, m2, "transpose_a", False, "transpose_b", False) dz_dy = tape.gradient(z, [m2])[0] self.assertEqual(dz_dy.dtype, dtypes.float16) expected_grads = math_ops.matmul( array_ops.transpose(a_2_by_2_fp16), constant_op.constant(1., shape=[2, 2], dtype=dtypes.float16)).numpy() self.assertAllEqual(dz_dy.numpy(), expected_grads)
def g(x): tape.watch(three) return f(x)
def f(): tape.watch(embedding.handle) embedded_x = embedding_ops.embedding_lookup(embedding, x) return tensor.Tensor(1.0, dtypes.float32) - embedded_x
def f(): with context.device('gpu:0'): tape.watch(v.handle) return v.read_value()
def fn(): tape.watch(x.handle) b = tensor.Tensor(2.0) c = math_ops.add(x.value(), b) return math_ops.add(c, tensor.Tensor(3.0))
def f(): with context.device('gpu:0'): tape.watch(v.handle) return v.read_value()
def f(): x = constant_op.constant(1.0) tape.watch(x) x = gradient_is_constant(x) x = gradient_is_constant(x) x = gradient_is_constant(x)
def _read_variable_op(self): if context.in_eager_mode() and self._trainable: tape.watch(self._handle) return read_variable_op(self._handle, dtype=self._dtype)
def f(): x = constant_op.constant(1.0) tape.watch(x) x = gradient_is_constant(x) x = gradient_is_constant(x) x = gradient_is_constant(x)
def inner(): tape.watch(v.handle) return v * v
def fn(): tape.watch(x.handle) b = tensor.Tensor(2.0) c = math_ops.add(x.value(), b) return math_ops.add(c, tensor.Tensor(3.0))
def f(): tape.watch(embedding.handle) embedded_x = embedding_ops.embedding_lookup(embedding, x) return tensor.Tensor(1.0, dtypes.float32) - embedded_x
def g(x): tape.watch(three) return f(x)