def args_to_matching_eager(l, ctx, default_dtype=None): """Convert sequence `l` to eager same-type Tensors.""" # TODO(josh11b): Could we do a better job if we also passed in the # allowed dtypes when that was known? # Is some input already a Tensor with a dtype? dtype = None for t in l: if isinstance(t, ops.EagerTensor): dtype = t.dtype break if dtype is None: # Infer a dtype based on the first value, and use that dtype for the # remaining values. ret = [] for t in l: ret.append(ops.internal_convert_to_tensor( t, dtype, preferred_dtype=default_dtype, ctx=ctx)) if dtype is None: dtype = ret[-1].dtype else: ret = [ops.internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l] return dtype, ret
def unregularized_loss(self, examples): """Add operations to compute the loss (without the regularization loss). Args: examples: Examples to compute unregularized loss on. Returns: An Operation that computes mean (unregularized) loss for given set of examples. Raises: ValueError: if examples are not well defined. """ self._assertSpecified([ 'example_labels', 'example_weights', 'sparse_features', 'dense_features' ], examples) self._assertList(['sparse_features', 'dense_features'], examples) with name_scope('sdca/unregularized_loss'): predictions = math_ops.cast(self._linear_predictions(examples), dtypes.float64) labels = math_ops.cast( internal_convert_to_tensor(examples['example_labels']), dtypes.float64) weights = math_ops.cast( internal_convert_to_tensor(examples['example_weights']), dtypes.float64) if self._options['loss_type'] == 'logistic_loss': return math_ops.reduce_sum( math_ops.multiply( sigmoid_cross_entropy_with_logits(labels=labels, logits=predictions), weights)) / math_ops.reduce_sum(weights) if self._options['loss_type'] in [ 'hinge_loss', 'smooth_hinge_loss' ]: # hinge_loss = max{0, 1 - y_i w*x} where y_i \in {-1, 1}. So, we need to # first convert 0/1 labels into -1/1 labels. all_ones = array_ops.ones_like(predictions) adjusted_labels = math_ops.subtract(2 * labels, all_ones) # Tensor that contains (unweighted) error (hinge loss) per # example. error = nn_ops.relu( math_ops.subtract( all_ones, math_ops.multiply(adjusted_labels, predictions))) weighted_error = math_ops.multiply(error, weights) return math_ops.reduce_sum( weighted_error) / math_ops.reduce_sum(weights) # squared loss err = math_ops.subtract(labels, predictions) weighted_squared_err = math_ops.multiply(math_ops.square(err), weights) # SDCA squared loss function is sum(err^2) / (2*sum(weights)) return (math_ops.reduce_sum(weighted_squared_err) / (2.0 * math_ops.reduce_sum(weights)))
def __init__(self, example_indices, feature_indices, feature_values): """Creates a `_SparseFeatureColumn` representation. Args: example_indices: A 1-D int64 tensor of shape `[N]`. Also, accepts python lists, or numpy arrays. feature_indices: A 1-D int64 tensor of shape `[N]`. Also, accepts python lists, or numpy arrays. feature_values: An optional 1-D tensor float tensor of shape `[N]`. Also, accepts python lists, or numpy arrays. Returns: A `_SparseFeatureColumn` """ with name_scope(None, 'SparseFeatureColumn', [example_indices, feature_indices]): self._example_indices = internal_convert_to_tensor( example_indices, name='example_indices', dtype=dtypes.int64) self._feature_indices = internal_convert_to_tensor( feature_indices, name='feature_indices', dtype=dtypes.int64) self._feature_values = None if feature_values is not None: with name_scope(None, 'SparseFeatureColumn', [feature_values]): self._feature_values = internal_convert_to_tensor( feature_values, name='feature_values', dtype=dtypes.float32)
def __init__(self, example_indices, feature_indices, feature_values): """Creates a `SparseFeatureColumn` representation. Args: example_indices: A 1-D int64 tensor of shape `[N]`. Also, accepts python lists, or numpy arrays. feature_indices: A 1-D int64 tensor of shape `[N]`. Also, accepts python lists, or numpy arrays. feature_values: An optional 1-D tensor float tensor of shape `[N]`. Also, accepts python lists, or numpy arrays. Returns: A `SparseFeatureColumn` """ with name_scope(None, 'SparseFeatureColumn', [example_indices, feature_indices]): self._example_indices = internal_convert_to_tensor( example_indices, name='example_indices', dtype=dtypes.int64) self._feature_indices = internal_convert_to_tensor( feature_indices, name='feature_indices', dtype=dtypes.int64) self._feature_values = None if feature_values is not None: with name_scope(None, 'SparseFeatureColumn', [feature_values]): self._feature_values = internal_convert_to_tensor( feature_values, name='feature_values', dtype=dtypes.float32)
def unregularized_loss(self, examples): """Add operations to compute the loss (without the regularization loss). Args: examples: Examples to compute unregularized loss on. Returns: An Operation that computes mean (unregularized) loss for given set of examples. Raises: ValueError: if examples are not well defined. """ self._assertSpecified([ 'example_labels', 'example_weights', 'sparse_features', 'dense_features' ], examples) self._assertList(['sparse_features', 'dense_features'], examples) with name_scope('sdca/unregularized_loss'): predictions = math_ops.cast( self._linear_predictions(examples), dtypes.float64) labels = math_ops.cast( internal_convert_to_tensor(examples['example_labels']), dtypes.float64) weights = math_ops.cast( internal_convert_to_tensor(examples['example_weights']), dtypes.float64) if self._options['loss_type'] == 'logistic_loss': return math_ops.reduce_sum(math_ops.multiply( sigmoid_cross_entropy_with_logits(labels=labels, logits=predictions), weights)) / math_ops.reduce_sum(weights) if self._options['loss_type'] == 'poisson_loss': return math_ops.reduce_sum(math_ops.multiply( log_poisson_loss(targets=labels, log_input=predictions), weights)) / math_ops.reduce_sum(weights) if self._options['loss_type'] in ['hinge_loss', 'smooth_hinge_loss']: # hinge_loss = max{0, 1 - y_i w*x} where y_i \in {-1, 1}. So, we need to # first convert 0/1 labels into -1/1 labels. all_ones = array_ops.ones_like(predictions) adjusted_labels = math_ops.subtract(2 * labels, all_ones) # Tensor that contains (unweighted) error (hinge loss) per # example. error = nn_ops.relu( math_ops.subtract(all_ones, math_ops.multiply(adjusted_labels, predictions))) weighted_error = math_ops.multiply(error, weights) return math_ops.reduce_sum(weighted_error) / math_ops.reduce_sum( weights) # squared loss err = math_ops.subtract(labels, predictions) weighted_squared_err = math_ops.multiply(math_ops.square(err), weights) # SDCA squared loss function is sum(err^2) / (2*sum(weights)) return (math_ops.reduce_sum(weighted_squared_err) / (2.0 * math_ops.reduce_sum(weights)))
def testTensorConversion(self): with context.graph_mode(): _, tower_local = _make_tower_local("sum") converted = ops.internal_convert_to_tensor(tower_local, as_ref=False) self.assertIsInstance(converted, ops.Tensor) self.assertEqual(converted.dtype, tower_local.dtype) converted = ops.internal_convert_to_tensor(tower_local, as_ref=True) # Resources variable are converted to tensors as well when as_ref is True. self.assertIsInstance(converted, ops.Tensor) self.assertEqual(converted.dtype, tower_local.dtype)
def testTensorConversion(self): with context.graph_mode(): _, tower_local = _make_tower_local(variable_scope.VariableAggregation.SUM) converted = ops.internal_convert_to_tensor(tower_local, as_ref=False) self.assertIsInstance(converted, ops.Tensor) self.assertEqual(converted.dtype, tower_local.dtype) converted = ops.internal_convert_to_tensor(tower_local, as_ref=True) # Resources variable are converted to tensors as well when as_ref is True. self.assertIsInstance(converted, ops.Tensor) self.assertEqual(converted.dtype, tower_local.dtype)
def testTensorConversion(self): with context.graph_mode(): _, replica_local = _make_replica_local( variable_scope.VariableAggregation.SUM) converted = ops.internal_convert_to_tensor(replica_local, as_ref=False) self.assertIsInstance(converted, ops.Tensor) self.assertEqual(converted.dtype, replica_local.dtype) converted = ops.internal_convert_to_tensor(replica_local, as_ref=True) # Resources variable are converted to tensors as well when as_ref is True. self.assertIsInstance(converted, ops.Tensor) self.assertEqual(converted.dtype, replica_local.dtype)
def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False): """Converts this variable to a tensor.""" if not self._should_cast(): return ops.internal_convert_to_tensor(self._variable, dtype, name, as_ref) # TODO(reedwm): Support as_ref? assert not as_ref if dtype is not None and not dtype.is_compatible_with(self.dtype): raise ValueError( 'Incompatible type conversion requested to type {!r} for variable ' 'of type {!r}'.format(dtype.name, self.dtype.name)) val = ops.internal_convert_to_tensor(self._variable, self._variable.dtype, name, as_ref=False) return math_ops.cast(val, self.dtype)
def __init__(self, indices, values, dense_shape): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of shape `[N, ndims]`. values: A 1-D tensor of any type and shape `[N]`. dense_shape: A 1-D int64 tensor of shape `[ndims]`. """ with ops.name_scope(None, "SparseTensor", [indices, values, dense_shape]): indices = ops.convert_to_tensor( indices, name="indices", dtype=dtypes.int64) # Always pass as_ref=True because we want to be able to update # values later if it is a VariableOp. # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor( values, name="values", as_ref=True) dense_shape = ops.convert_to_tensor( dense_shape, name="dense_shape", dtype=dtypes.int64) self._indices = indices self._values = values self._dense_shape = dense_shape indices_shape = indices.get_shape().with_rank(2) values_shape = values.get_shape().with_rank(1) dense_shape_shape = dense_shape.get_shape().with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape[0].merge_with(values_shape[0]) # Assert number of columns in indices matches the number of elements in # dense_shape. indices_shape[1].merge_with(dense_shape_shape[0])
def convert_to_tensor_or_sparse_tensor(value, dtype=None, name=None): """Converts value to a `SparseTensor` or `Tensor`. Args: value: A `SparseTensor`, `SparseTensorValue`, or an object whose type has a registered `Tensor` conversion function. dtype: Optional element type for the returned tensor. If missing, the type is inferred from the type of `value`. name: Optional name to use if a new `Tensor` is created. Returns: A `SparseTensor` or `Tensor` based on `value`. Raises: RuntimeError: If result type is incompatible with `dtype`. """ if dtype is not None: dtype = dtypes.as_dtype(dtype) if isinstance(value, sparse_tensor.SparseTensorValue): value = sparse_tensor.SparseTensor.from_value(value) if isinstance(value, sparse_tensor.SparseTensor): if dtype and not dtype.is_compatible_with(value.dtype): raise RuntimeError( 'Sparse dtype: requested = %s, actual = %s' % ( dtype.name, value.dtype.name)) return value return ops.internal_convert_to_tensor( value, dtype=dtype, name=name)
def __init__(self, indices, values, dense_shape): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of shape `[N, ndims]`. values: A 1-D tensor of any type and shape `[N]`. dense_shape: A 1-D int64 tensor of shape `[ndims]`. """ with ops.name_scope(None, "SparseTensor", [indices, values, dense_shape]): indices = ops.convert_to_tensor(indices, name="indices", dtype=dtypes.int64) # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor(values, name="values") dense_shape = ops.convert_to_tensor(dense_shape, name="dense_shape", dtype=dtypes.int64) self._indices = indices self._values = values self._dense_shape = dense_shape indices_shape = indices.get_shape().with_rank(2) values_shape = values.get_shape().with_rank(1) dense_shape_shape = dense_shape.get_shape().with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape.dims[0].merge_with(values_shape.dims[0]) # Assert number of columns in indices matches the number of elements in # dense_shape. indices_shape.dims[1].merge_with(dense_shape_shape.dims[0])
def convert_to_tensor_or_sparse_tensor(value, dtype=None, name=None): """Converts value to a `SparseTensor` or `Tensor`. Args: value: A `SparseTensor`, `SparseTensorValue`, or an object whose type has a registered `Tensor` conversion function. dtype: Optional element type for the returned tensor. If missing, the type is inferred from the type of `value`. name: Optional name to use if a new `Tensor` is created. Returns: A `SparseTensor` or `Tensor` based on `value`. Raises: RuntimeError: If result type is incompatible with `dtype`. """ if dtype is not None: dtype = dtypes.as_dtype(dtype) if isinstance(value, SparseTensorValue): value = SparseTensor.from_value(value) if isinstance(value, SparseTensor): if dtype and not dtype.is_compatible_with(value.dtype): raise RuntimeError("Sparse dtype: requested = %s, actual = %s" % (dtype.name, value.dtype.name)) return value return ops.internal_convert_to_tensor(value, dtype=dtype, name=name)
def regularized_loss(self, examples): """Add operations to compute the loss with regularization loss included. Args: examples: Examples to compute loss on. Returns: An Operation that computes mean (regularized) loss for given set of examples. Raises: ValueError: if examples are not well defined. """ self._assertSpecified([ 'example_labels', 'example_weights', 'sparse_features', 'dense_features' ], examples) self._assertList(['sparse_features', 'dense_features'], examples) with name_scope('sdca/regularized_loss'): weights = internal_convert_to_tensor(examples['example_weights']) return (( self._l1_loss() + # Note that here we are using the raw regularization # (as specified by the user) and *not* # self._symmetric_l2_regularization(). self._l2_loss(self._options['symmetric_l2_regularization'])) / math_ops.reduce_sum(math_ops.cast(weights, dtypes.float64)) + self.unregularized_loss(examples))
def __init__(self, path): """Record the full path to the asset.""" # The init_scope prevents functions from capturing `path` in an # initialization graph, since it is transient and should not end up in a # serialized function body. with ops.init_scope(), ops.device("CPU"): self._path = ops.internal_convert_to_tensor(path, dtype=dtypes.string, name="asset_path")
def _tensor_conversion_mirrored(var, dtype=None, name=None, as_ref=False): # Try to avoid assignments to and other mutations of MirroredVariable # state except through a DistributionStrategy.update() call. assert not as_ref return ops.internal_convert_to_tensor(var.get(), dtype=dtype, name=name, as_ref=as_ref)
def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False): """Converts this variable to a tensor.""" if not self._should_cast(): return ops.internal_convert_to_tensor(self._variable, dtype, name, as_ref) # TODO(reedwm): Support as_ref? assert not as_ref if dtype is not None and not dtype.is_compatible_with(self.dtype): raise ValueError( 'Incompatible type conversion requested to type {!r} for variable ' 'of type {!r}'.format(dtype.name, self.dtype.name)) val = ops.internal_convert_to_tensor(self._variable, self._variable.dtype, name, as_ref=False) with ops.colocate_with(None, ignore_existing=True): with ops.device(val.device): return math_ops.cast(val, self.dtype)
def partitioned_call(args, f, tout=None, executing_eagerly=None): """Executes a function while respecting device annotations. Currently, only those functions that execute within the same address space can be executed. Args: args: The arguments of the function, including captured inputs. f: The function to execute; an instance of `_DefinedFunction` or `_EagerDefinedFunction`. tout: a list containing the output dtypes enums; if `None`, inferred from the signature of `f`. executing_eagerly: (Optional) A boolean indicating whether the context is executing eagerly. If `None`, fetched from the global context. Returns: The list of `Tensor`s returned by invoking `f(args)`. If the function does not return anything, then returns `None` if eager execution is enabled, or the `Operation` if not. """ if tout is None: tout = tuple(x.type for x in f.definition.signature.output_arg) if executing_eagerly is None: executing_eagerly = context.executing_eagerly() if executing_eagerly or len(tout): if f.stateful_ops: outputs = gen_functional_ops.stateful_partitioned_call( args=args, Tout=tout, f=f) else: outputs = gen_functional_ops.partitioned_call(args=args, Tout=tout, f=f) return outputs if outputs else None # The generated binding returns an empty list for functions that don't # return any Tensors, hence the need to use `create_op` directly. args = [ops.internal_convert_to_tensor(x) for x in args] tin_attr = attr_value_pb2.AttrValue( list=attr_value_pb2.AttrValue.ListValue( type=[x.dtype.as_datatype_enum for x in args])) tout_attr = attr_value_pb2.AttrValue( list=attr_value_pb2.AttrValue.ListValue(type=tout)) func_attr = attr_value_pb2.AttrValue( func=attr_value_pb2.NameAttrList(name=f.name)) graph = ops.get_default_graph() f.add_to_graph(graph) op_name = "StatefulPartitionedCall" if f.stateful_ops else "PartitionedCall" op = graph.create_op( op_name, args, tout, compute_shapes=False, name="PartitionedFunctionCall", attrs={"Tin": tin_attr, "Tout": tout_attr, "f": func_attr}) outputs = op.outputs return outputs if outputs else op
def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False): """Converts this variable to a tensor.""" if self.dtype == self.true_dtype: return ops.internal_convert_to_tensor(self._variable, dtype, name, as_ref) # TODO(reedwm): Support as_ref? assert not as_ref if dtype is not None and not dtype.is_compatible_with(self.dtype): raise ValueError( 'Incompatible type conversion requested to type {!r} for variable ' 'of type {!r}'.format(dtype.name, self.dtype.name)) val = ops.internal_convert_to_tensor(self._variable, self._variable.dtype, name, as_ref=False) with ops.colocate_with(None, ignore_existing=True): with ops.device(val.device): return math_ops.cast(val, self.dtype)
def args_to_mixed_eager_tensors(lists, ctx): """Converts a list of same-length lists of values to eager tensors.""" assert len(lists) > 1 # Generate an error if len(lists[i]) is not the same for all i. lists_ret = [] for l in lists[1:]: if len(l) != len(lists[0]): raise ValueError( "Expected list arguments to be the same length: %d != %d (%r vs. %r)." % (len(lists[0]), len(l), lists[0], l)) lists_ret.append([]) # Convert the first element of each list first, then the second element, etc. types = [] for i in range(len(lists[0])): dtype = None # If any list has a Tensor, use that dtype for l in lists: if isinstance(l[i], ops.EagerTensor): dtype = l[i].dtype break if dtype is None: # Convert the first one and use its dtype. lists_ret[0].append( ops.internal_convert_to_tensor(lists[0][i], ctx=ctx)) dtype = lists_ret[0][i].dtype for j in range(1, len(lists)): lists_ret[j].append( ops.internal_convert_to_tensor(lists[j][i], dtype=dtype, ctx=ctx)) else: # Convert everything to the found dtype. for j in range(len(lists)): lists_ret[j].append( ops.internal_convert_to_tensor(lists[j][i], dtype=dtype, ctx=ctx)) types.append(dtype.as_datatype_enum) return types, lists_ret
def _record_gradient(op_name, inputs, attrs, results, ctx, name): """Records gradients for a TensorFlow operation. Args: op_name: Name of the TensorFlow operation (see REGISTER_OP in C++ code) to execute. inputs: A flat list of Tensor object inputs to the operation. attrs: A tuple with alternating string attr names and attr values for this operation. results: The results of the operation (as a flat list). ctx: The value of context.context(). name: Customized name for the operation. Returns: A list of maybe-wrapped results. Either Tensors or TensorNodes. Raises: An exception on error. """ if not tape.could_possibly_record(): return if op_name in _ops_which_dont_need_outputs: op_outputs = None else: # TODO(apassos) this line creates a weak circular reference where the # backprop function keeps an output alive which in turn keeps the tape entry # alive which keeps the backprop function alive. Figure out how to break # this up without breaking second derivatives of ops like Exp whose # gradients depend only on the outputs. op_outputs = results if op_name in _ops_which_dont_need_inputs: op_inputs = None else: op_inputs = inputs num_inputs = len(inputs) def grad_fn(*orig_outputs): """Generated gradient function.""" result = _magic_gradient_function(op_name, attrs, num_inputs, op_inputs, op_outputs, orig_outputs) if _tracing: print("Gradient for", (name if name else op_name), "inputs", op_inputs, "output_grads", orig_outputs, "gradients", result) return result inputs = [ops.internal_convert_to_tensor(x, ctx=ctx) for x in inputs] tape.record_operation(op_name, results, inputs, [], grad_fn) if _tracing: print("Computed op", (name if name else op_name), "inputs", inputs, "outputs", results)
def var_to_tensors(var): if resource_variable_ops.is_resource_variable(var): if tf.control_flow_v2_enabled(): # TODO(b/143690035): Note that the "captures" property relies on an # API which might change. captures = layer_collection.graph.captures return [h for vh, h in captures if vh is var.handle] else: return [var.handle] if utils.is_reference_variable(var): return [tf_ops.internal_convert_to_tensor(var, as_ref=True)] raise ValueError('%s is not a recognized variable type.' % str(var))
def _l2_loss(self, l2): """Computes the (un-normalized) l2 loss of the model.""" with name_scope('sdca/l2_loss'): sums = [] for name in ['sparse_features_weights', 'dense_features_weights']: for var in self._variables[name]: for v in self._var_to_list(var): weights = internal_convert_to_tensor(v) with ops.device(weights.device): sums.append(math_ops.reduce_sum(math_ops.square(math_ops.cast( weights, dtypes.float64)))) # SDCA L2 regularization cost is: l2 * sum(weights^2) / 2 return l2 * math_ops.add_n(sums) / 2.0
def _l2_loss(self): """Computes the (un-normalized) l2 loss of the model.""" with name_scope('sdca/l2_loss'): sums = [] for name in ['sparse_features_weights', 'dense_features_weights']: for var in self._variables[name]: for v in self._var_to_list(var): weights = internal_convert_to_tensor(v) with ops.device(weights.device): sums.append(math_ops.reduce_sum(math_ops.square(math_ops.cast( weights, dtypes.float64)))) # SDCA L2 regularization cost is: l2 * sum(weights^2) / 2 return self._symmetric_l2_regularization() * math_ops.add_n(sums) / 2.0
def _l1_loss(self): """Computes the (un-normalized) l1 loss of the model.""" with name_scope('sdca/l1_loss'): sums = [] for name in ['sparse_features_weights', 'dense_features_weights']: for var in self._variables[name]: for v in self._var_to_list(var): weights = internal_convert_to_tensor(v) with ops.device(weights.device): sums.append( math_ops.reduce_sum( math_ops.abs(math_ops.cast(weights, dtypes.float64)))) # SDCA L1 regularization cost is: l1 * sum(|weights|) return self._options['symmetric_l1_regularization'] * math_ops.add_n(sums)
def __init__(self, indices, values, dense_shape=None, shape=None): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of dense_shape `[N, ndims]`. values: A 1-D tensor of any type and dense_shape `[N]`. dense_shape: A 1-D int64 tensor of dense_shape `[ndims]`. shape: Temporary. Legacy naming of dense_shape. Only one of `shape` or `dense_shape` must be provided. Returns: A `SparseTensor`. Raises: ValueError: if both `shape` and `dense_shape` are provided. """ with ops.name_scope(None, "SparseTensor", [indices, values, shape, dense_shape]): indices = ops.convert_to_tensor(indices, name="indices", dtype=dtypes.int64) # Always pass as_ref=True because we want to be able to update # values later if it is a VariableOp. # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor(values, name="values", as_ref=True) if shape is not None and dense_shape is not None: raise ValueError( "Only one of shape or dense_shape must be provided, " "but saw %s and %s" % (shape, dense_shape)) dense_shape = shape if shape is not None else dense_shape dense_shape = ops.convert_to_tensor(dense_shape, name="dense_shape", dtype=dtypes.int64) self._indices = indices self._values = values self._dense_shape = dense_shape indices_shape = indices.get_shape().with_rank(2) values_shape = values.get_shape().with_rank(1) dense_shape_shape = dense_shape.get_shape().with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape[0].merge_with(values_shape[0]) # Assert number of columns in indices matches the number of elements in # dense_shape. indices_shape[1].merge_with(dense_shape_shape[0])
def _convert_n_to_tensor(self, input_list, as_ref=False): """Converts input list to a set of tensors.""" # input_list can be a list of Variables (that are implicitly partitioned), # in which case the underlying logic in internal_convert_to_tensor will not # concatenate the partitions together. This method takes care of the # concatenating (we only allow partitioning on the first axis). output_list = [] for x in input_list: tensor_to_convert = x if isinstance(x, list) or isinstance(x, var_ops.PartitionedVariable): # We only allow for partitioning on the first axis. tensor_to_convert = array_ops.concat(x, axis=0) output_list.append(internal_convert_to_tensor( tensor_to_convert, as_ref=as_ref)) return output_list
def args_to_mixed_eager_tensors(lists, ctx): """Converts a list of same-length lists of values to eager tensors.""" assert len(lists) > 1 # Generate an error if len(lists[i]) is not the same for all i. lists_ret = [] for l in lists[1:]: if len(l) != len(lists[0]): raise ValueError( "Expected list arguments to be the same length: %d != %d (%r vs. %r)." % (len(lists[0]), len(l), lists[0], l)) lists_ret.append([]) # Convert the first element of each list first, then the second element, etc. types = [] for i in range(len(lists[0])): dtype = None # If any list has a Tensor, use that dtype for l in lists: if isinstance(l[i], ops.EagerTensor): dtype = l[i].dtype break if dtype is None: # Convert the first one and use its dtype. lists_ret[0].append(ops.internal_convert_to_tensor(lists[0][i], ctx=ctx)) dtype = lists_ret[0][i].dtype for j in range(1, len(lists)): lists_ret[j].append( ops.internal_convert_to_tensor(lists[j][i], dtype=dtype, ctx=ctx)) else: # Convert everything to the found dtype. for j in range(len(lists)): lists_ret[j].append( ops.internal_convert_to_tensor(lists[j][i], dtype=dtype, ctx=ctx)) types.append(dtype) return types, lists_ret
def __call__(self, *args): """Executes the passed function in eager mode.""" for v in self._variables: if v._trainable: # pylint: disable=protected-access tape.watch_variable(v) tensor_inputs = [ x for x in nest.flatten(args) if isinstance(x, ops.Tensor) ] if tape.should_record(tensor_inputs) or tape.should_record( self._extra_inputs): if not self._has_backprop: self._compute_backprop() return self._backprop_call(tensor_inputs) ctx = context.context() if ctx.in_graph_mode(): g = ops.get_default_graph() if self._function_def.name not in g._functions: # pylint: disable=protected-access g._add_function(self._function_def) # pylint: disable=protected-access for f in self._graph._functions.values(): # pylint: disable=protected-access if f.name not in g._functions: # pylint: disable=protected-access g._add_function(f) # pylint: disable=protected-access signature = self._function_def.definition.signature args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args], tuple( dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) result = op.outputs if not result: return op for i, s in enumerate(self._output_shapes): result[i].set_shape(s) else: result = execute.execute(str(self._func_name), num_outputs=self._num_outputs, inputs=tensor_inputs + self._extra_inputs, attrs=None, ctx=ctx) return self._build_call_outputs(result)
def __init__(self, indices, values, dense_shape=None, shape=None): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of dense_shape `[N, ndims]`. values: A 1-D tensor of any type and dense_shape `[N]`. dense_shape: A 1-D int64 tensor of dense_shape `[ndims]`. shape: Temporary. Legacy naming of dense_shape. Only one of `shape` or `dense_shape` must be provided. Returns: A `SparseTensor`. Raises: ValueError: if both `shape` and `dense_shape` are provided. """ with ops.name_scope(None, "SparseTensor", [indices, values, shape, dense_shape]): indices = ops.convert_to_tensor( indices, name="indices", dtype=dtypes.int64) # Always pass as_ref=True because we want to be able to update # values later if it is a VariableOp. # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor( values, name="values", as_ref=True) if shape is not None and dense_shape is not None: raise ValueError("Only one of shape or dense_shape must be provided, " "but saw %s and %s" % (shape, dense_shape)) dense_shape = shape if shape is not None else dense_shape dense_shape = ops.convert_to_tensor( dense_shape, name="dense_shape", dtype=dtypes.int64) self._indices = indices self._values = values self._dense_shape = dense_shape indices_shape = indices.get_shape().with_rank(2) values_shape = values.get_shape().with_rank(1) dense_shape_shape = dense_shape.get_shape().with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape[0].merge_with(values_shape[0]) # Assert number of columns in indices matches the number of elements in # dense_shape. indices_shape[1].merge_with(dense_shape_shape[0])
def __init__(self, indices, values, dense_shape): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of shape `[N, ndims]`. values: A 1-D tensor of any type and shape `[N]`. dense_shape: A 1-D int64 tensor of shape `[ndims]`. """ if isinstance(indices, tensor_spec.TensorSpec): # TODO(b/133606651) Remove this code path -- replaced by TypeSpec. if not isinstance(values, tensor_spec.TensorSpec): raise TypeError("Expected values to be a TensorSpec") if not isinstance(dense_shape, tensor_spec.TensorSpec): raise TypeError("Expected dense_shape to be a TensorSpec") if indices.dtype != dtypes.int64 or dense_shape.dtype != dtypes.int64: raise TypeError( "indices and dense_shape must have dtype=int64") else: with ops.name_scope(None, "SparseTensor", [indices, values, dense_shape]): indices = ops.convert_to_tensor(indices, name="indices", dtype=dtypes.int64) # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor(values, name="values") dense_shape = ops.convert_to_tensor(dense_shape, name="dense_shape", dtype=dtypes.int64) self._indices = indices self._values = values self._dense_shape = dense_shape indices_shape = indices.shape.with_rank(2) values_shape = values.shape.with_rank(1) dense_shape_shape = dense_shape.shape.with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape.dims[0].merge_with(values_shape.dims[0]) # Assert number of columns in indices matches the number of elements in # dense_shape. indices_shape.dims[1].merge_with(dense_shape_shape.dims[0])
def __call__(self, *args): """Executes the passed function in eager mode.""" for v in self._variables: if v._trainable: # pylint: disable=protected-access tape.watch_variable(v) tensor_inputs = [x for x in nest.flatten(args) if isinstance(x, ops.Tensor)] if tape.should_record(tensor_inputs) or tape.should_record( self._extra_inputs): if not self._has_backprop: self._compute_backprop() return self._backprop_call(tensor_inputs) ctx = context.context() if ctx.in_graph_mode(): g = ops.get_default_graph() self.add_to_graph(g) signature = self._function_def.definition.signature args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args], tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) result = op.outputs if not result: return op for i, s in enumerate(self._output_shapes): result[i].set_shape(s) else: result = execute.execute( str(self._func_name), num_outputs=self._num_outputs, inputs=tensor_inputs + self._extra_inputs, attrs=None, ctx=ctx) return self._build_call_outputs(result)
def _backprop_call(self, args): """Calls the wrapped function and records the result on a tape.""" all_args = args + self._extra_inputs signature = self._forward_fdef.signature ctx = context.context() if ctx.in_graph_mode(): g = ops.get_default_graph() g._add_function(self._forward_fdef) # pylint: disable=protected-access op = g.create_op( signature.name, [ops.internal_convert_to_tensor(x, ctx=ctx) for x in all_args], tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) outputs = op.outputs outputs = [outputs] if isinstance( outputs, (ops.Tensor, type(None))) else list(outputs) for i, s in enumerate(self._output_shapes): outputs[i].set_shape(s) else: outputs = execute.execute( str(signature.name), num_outputs=len(signature.output_arg), inputs=all_args, attrs=None, ctx=ctx) real_outputs = outputs[:len(self._returns)] side_outputs = outputs[len(self._returns):] def backward_function(*args): return self._backward_function(*(list(args) + side_outputs)) # pylint: disable=not-callable tape.record_operation( signature.name, real_outputs, (args + self._extra_inputs), backward_function) return self._build_call_outputs(real_outputs)
def _backprop_call(self, args): """Calls the wrapped function and records the result on a tape.""" all_args = args + self._extra_inputs signature = self._forward_fdef.signature ctx = context.context() if ctx.executing_eagerly(): outputs = execute.execute( str(signature.name), num_outputs=len(signature.output_arg), inputs=all_args, attrs=None, ctx=ctx) else: g = ops.get_default_graph() g._add_function(self._forward_fdef) # pylint: disable=protected-access op = g.create_op( signature.name, [ops.internal_convert_to_tensor(x, ctx=ctx) for x in all_args], tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) outputs = op.outputs outputs = [outputs] if isinstance( outputs, (ops.Tensor, type(None))) else list(outputs) for i, s in enumerate(self._output_shapes): outputs[i].set_shape(s) real_outputs = outputs[:len(self._returns)] side_outputs = outputs[len(self._returns):] def backward_function(*args): return self._backward_function(*(list(args) + side_outputs)) # pylint: disable=not-callable tape.record_operation( signature.name, real_outputs, (args + self._extra_inputs), backward_function) return self._build_call_outputs(real_outputs)
def __call__(self, *args): """Executes the passed function in eager mode.""" for v in self._variables: if v.trainable: tape.watch_variable(v) tensor_inputs = [x for x in nest.flatten(args) if isinstance(x, ops.Tensor)] if tape.should_record(tensor_inputs) or tape.should_record( self._extra_inputs): if self._backward_function is None: self._construct_backprop_function() return self._backprop_call(tensor_inputs) ctx = context.context() if ctx.executing_eagerly(): result = execute.execute( str(self._func_name), num_outputs=self._num_outputs, inputs=tensor_inputs + self._extra_inputs, attrs=None, ctx=ctx) else: g = ops.get_default_graph() self.add_to_graph(g) signature = self._function_def.definition.signature args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args], tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) result = op.outputs if not result: return op for i, s in enumerate(self._output_shapes): result[i].set_shape(s) return self._build_call_outputs(result)
def __init__(self, indices, values, shape): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of shape `[N, ndims]`. values: A 1-D tensor of any type and shape `[N]`. shape: A 1-D int64 tensor of shape `[ndims]`. Returns: A `SparseTensor` """ with ops.name_scope(None, "SparseTensor", [indices, values, shape]): indices = ops.convert_to_tensor(indices, name="indices", dtype=dtypes.int64) # Always pass as_ref=True because we want to be able to update # values later if it is a VariableOp. # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor(values, name="values", as_ref=True) shape = ops.convert_to_tensor(shape, name="shape", dtype=dtypes.int64) self._indices = indices self._values = values self._shape = shape indices_shape = indices.get_shape().with_rank(2) values_shape = values.get_shape().with_rank(1) shape_shape = shape.get_shape().with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape[0].merge_with(values_shape[0]) # Assert number of columns in indices matches the number of elements in # shape. indices_shape[1].merge_with(shape_shape[0])
def __init__(self, indices, values, dense_shape): """Creates a `SparseTensor`. Args: indices: A 2-D int64 tensor of shape `[N, ndims]`. values: A 1-D tensor of any type and shape `[N]`. dense_shape: A 1-D int64 tensor of shape `[ndims]`. """ if isinstance(indices, tensor_spec.TensorSpec): if not isinstance(values, tensor_spec.TensorSpec): raise TypeError("Expected values to be a TensorSpec") if not isinstance(dense_shape, tensor_spec.TensorSpec): raise TypeError("Expected dense_shape to be a TensorSpec") if indices.dtype != dtypes.int64 or dense_shape.dtype != dtypes.int64: raise TypeError("indices and dense_shape must have dtype=int64") else: with ops.name_scope(None, "SparseTensor", [indices, values, dense_shape]): indices = ops.convert_to_tensor( indices, name="indices", dtype=dtypes.int64) # TODO(touts): Consider adding mutable_values() when 'values' # is a VariableOp and updating users of SparseTensor. values = ops.internal_convert_to_tensor(values, name="values") dense_shape = ops.convert_to_tensor( dense_shape, name="dense_shape", dtype=dtypes.int64) self._indices = indices self._values = values self._dense_shape = dense_shape indices_shape = indices.shape.with_rank(2) values_shape = values.shape.with_rank(1) dense_shape_shape = dense_shape.shape.with_rank(1) # Assert number of rows in indices match the number of elements in values. indices_shape.dims[0].merge_with(values_shape.dims[0]) # Assert number of columns in indices matches the number of elements in # dense_shape. indices_shape.dims[1].merge_with(dense_shape_shape.dims[0])
def convert_to_mixed_eager_tensors(values, ctx): v = [ops.internal_convert_to_tensor(t, ctx=ctx) for t in values] types = [t._datatype_enum() for t in v] # pylint: disable=protected-access return types, v
def minimize(self, global_step=None, name=None): """Add operations to train a linear model by minimizing the loss function. Args: global_step: Optional `Variable` to increment by one after the variables have been updated. name: Optional name for the returned operation. Returns: An Operation that updates the variables passed in the constructor. """ # Technically, the op depends on a lot more than the variables, # but we'll keep the list short. with name_scope(name, 'sdca/minimize'): sparse_example_indices = [] sparse_feature_indices = [] sparse_features_values = [] for sf in self._examples['sparse_features']: sparse_example_indices.append(sf.example_indices) sparse_feature_indices.append(sf.feature_indices) # If feature values are missing, sdca assumes a value of 1.0f. if sf.feature_values is not None: sparse_features_values.append(sf.feature_values) # pylint: disable=protected-access example_ids_hashed = gen_sdca_ops.sdca_fprint( internal_convert_to_tensor(self._examples['example_ids'])) # pylint: enable=protected-access example_state_data = self._hashtable.lookup(example_ids_hashed) # Solver returns example_state_update, new delta sparse_feature_weights # and delta dense_feature_weights. sparse_weights = [] sparse_indices = [] # If we have partitioned variables, keep a few dictionaries of Tensors # around that we need for the assign_add after the op call to # gen_sdca_ops.sdca_optimizer(). These are keyed because we may have a # mix of partitioned and un-partitioned variables. num_partitions_by_var = {} p_assignments_by_var = {} gather_ids_by_var = {} for v_num, (w, i) in enumerate( zip(self._slots['unshrinked_sparse_features_weights'], sparse_feature_indices)): # Append the sparse_indices (in full-variable space). sparse_idx = math_ops.cast( array_ops.unique(math_ops.cast(i, dtypes.int32))[0], dtypes.int64) sparse_indices.append(sparse_idx) if isinstance(w, list) or isinstance( w, var_ops.PartitionedVariable): num_partitions = len(w) flat_ids = array_ops.reshape(sparse_idx, [-1]) # We use div partitioning, which is easiest to support downstream. # Compute num_total_ids as the sum of dim-0 of w, then assign # to partitions based on a constant number of ids per partition. # Optimize if we already know the full shape statically. dim_0_size = self._get_first_dimension_size_statically( w, num_partitions) if tensor_shape.dimension_value(dim_0_size): num_total_ids = constant_op.constant( tensor_shape.dimension_value(dim_0_size), flat_ids.dtype) else: dim_0_sizes = [] for p in range(num_partitions): if tensor_shape.dimension_value( w[p].shape[0]) is not None: dim_0_sizes.append( tensor_shape.dimension_value( w[p].shape[0])) else: with ops.colocate_with(w[p]): dim_0_sizes.append( array_ops.shape(w[p])[0]) num_total_ids = math_ops.reduce_sum( math_ops.cast(array_ops.stack(dim_0_sizes), flat_ids.dtype)) ids_per_partition = num_total_ids // num_partitions extras = num_total_ids % num_partitions p_assignments = math_ops.maximum( flat_ids // (ids_per_partition + 1), (flat_ids - extras) // ids_per_partition) # Emulate a conditional using a boolean indicator tensor new_ids = array_ops.where( p_assignments < extras, flat_ids % (ids_per_partition + 1), (flat_ids - extras) % ids_per_partition) # Cast partition assignments to int32 for use in dynamic_partition. # There really should not be more than 2^32 partitions. p_assignments = math_ops.cast(p_assignments, dtypes.int32) # Partition list of ids based on assignments into num_partitions # separate lists. gather_ids = data_flow_ops.dynamic_partition( new_ids, p_assignments, num_partitions) # Add these into the dictionaries for use in the later update. num_partitions_by_var[v_num] = num_partitions p_assignments_by_var[v_num] = p_assignments gather_ids_by_var[v_num] = gather_ids # Gather the weights from each partition. partition_gathered_weights = [] for p in range(num_partitions): with ops.colocate_with(w[p]): partition_gathered_weights.append( array_ops.gather(w[p], gather_ids[p])) # Stitch the weights back together in the same order they were before # we dynamic_partitioned them. condition_indices = data_flow_ops.dynamic_partition( math_ops.range(array_ops.shape(new_ids)[0]), p_assignments, num_partitions) batch_gathered_weights = data_flow_ops.dynamic_stitch( condition_indices, partition_gathered_weights) else: w_as_tensor = internal_convert_to_tensor(w) with ops.device(w_as_tensor.device): batch_gathered_weights = array_ops.gather( w_as_tensor, sparse_idx) sparse_weights.append(batch_gathered_weights) # pylint: disable=protected-access if compat.forward_compatible(year=2018, month=10, day=30): esu, sfw, dfw = gen_sdca_ops.sdca_optimizer_v2( sparse_example_indices, sparse_feature_indices, sparse_features_values, self._convert_n_to_tensor( self._examples['dense_features']), internal_convert_to_tensor( self._examples['example_weights']), internal_convert_to_tensor( self._examples['example_labels']), sparse_indices, sparse_weights, self._convert_n_to_tensor( self._slots['unshrinked_dense_features_weights']), example_state_data, loss_type=self._options['loss_type'], l1=self._options['symmetric_l1_regularization'], l2=self._symmetric_l2_regularization(), num_loss_partitions=self._num_loss_partitions(), num_inner_iterations=1, adaptive=self._adaptive()) else: esu, sfw, dfw = gen_sdca_ops.sdca_optimizer( sparse_example_indices, sparse_feature_indices, sparse_features_values, self._convert_n_to_tensor( self._examples['dense_features']), internal_convert_to_tensor( self._examples['example_weights']), internal_convert_to_tensor( self._examples['example_labels']), sparse_indices, sparse_weights, self._convert_n_to_tensor( self._slots['unshrinked_dense_features_weights']), example_state_data, loss_type=self._options['loss_type'], l1=self._options['symmetric_l1_regularization'], l2=self._symmetric_l2_regularization(), num_loss_partitions=self._num_loss_partitions(), num_inner_iterations=1, adaptative=self._adaptive()) # pylint: enable=protected-access with ops.control_dependencies([esu]): update_ops = [self._hashtable.insert(example_ids_hashed, esu)] # Update the weights before the proximal step. for v_num, (w, i, u) in enumerate( zip(self._slots['unshrinked_sparse_features_weights'], sparse_indices, sfw)): if (isinstance(w, var_ops.PartitionedVariable) or isinstance(w, list)): update_ops += self._get_partitioned_update_ops( v_num, num_partitions_by_var, p_assignments_by_var, gather_ids_by_var, w, u, p_assignments, num_partitions) else: update_ops.append(state_ops.scatter_add(w, i, u)) for w, u in zip( self._slots['unshrinked_dense_features_weights'], dfw): if (isinstance(w, var_ops.PartitionedVariable) or isinstance(w, list)): split_updates = array_ops.split( u, num_or_size_splits=[ v.shape.as_list()[0] for v in w ]) for v, split_update in zip(w, split_updates): update_ops.append( state_ops.assign_add(v, split_update)) else: update_ops.append(state_ops.assign_add(w, u)) if not global_step: return control_flow_ops.group(*update_ops) with ops.control_dependencies(update_ops): return state_ops.assign_add(global_step, 1, name=name).op
def _apply_op_helper(self, op_type_name, name=None, **keywords): """Implementation of apply_op that returns output_structure, op.""" op_info = self._ops.get(op_type_name, None) if op_info is None: raise RuntimeError("Unrecognized Op name " + op_type_name) op_def = op_info.op_def # Determine the graph context. try: # Need to flatten all the arguments into a list. # pylint: disable=protected-access g = ops._get_graph_from_inputs(_Flatten(keywords.values())) # pylint: enable=protected-access except AssertionError as e: raise RuntimeError( "Cannot determine graph for Op '%s' due to: %s" % (op_type_name, e.message)) # Default name if not specified. if name is None: name = op_type_name # Check for deprecation deprecation_version = op_def.deprecation.version if deprecation_version: producer = g.graph_def_versions.producer if producer >= deprecation_version: raise NotImplementedError( ("Op %s is not available in GraphDef version %d. " "It has been removed in version %d. %s.") % (op_type_name, producer, deprecation_version, op_def.deprecation.explanation)) # Fill in the list of default types for all "type" attrs. This # will be used to choose a preferred dtype to convert to in the # absence of input type information. # # TODO(b/31302892): Currently the defaults don't work in the right # way if you have two inputs, one of whose type resolution depends # on the other. Handling this will require restructuring this code # significantly. default_type_attr_map = {} for attr_def in op_def.attr: if attr_def.type != "type": continue key = attr_def.name if attr_def.HasField("default_value"): default_type_attr_map[key] = dtypes.as_dtype( attr_def.default_value.type) # Requires that op_def has passed validation (using the C++ # ValidateOpDef() from ../framework/op_def_util.h). attrs = {} inputs = [] input_types = [] with g.as_default(), ops.name_scope(name) as scope: # Perform input type inference inferred_from = {} for input_arg in op_def.input_arg: input_name = input_arg.name if input_name in keywords: values = keywords.pop(input_name) elif input_name + "_" in keywords: # Handle the case where the name is a keyword or built-in # for Python so we use the name + _ instead. input_name += "_" values = keywords.pop(input_name) else: raise TypeError("No argument for input " + input_name) # Goals: # * Convert values to Tensors if it contains constants. # * Verify that values is a list if that matches the input_arg's # type. # * If the input_arg's type is determined by attrs, either set # those attrs and validate those attr values are legal (if # they have not yet been set) or validate the input matches # the type indicated by the attrs (if they have already been # inferred via an earlier input). # * If the input_arg has an explicit type, make sure the input # conforms. if _IsListParameter(input_arg): if not _IsListValue(values): raise TypeError( "Expected list for '%s' argument to '%s' Op, not %s." % (input_name, op_type_name, values)) # In cases where we expect all elements of the list to have the # same dtype, try to cast non-Tensor elements to that type. dtype = None default_dtype = None if input_arg.type != types_pb2.DT_INVALID: dtype = input_arg.type elif input_arg.number_attr: if input_arg.type_attr in attrs: dtype = attrs[input_arg.type_attr] else: for t in values: if isinstance(t, ops.Tensor): dtype = t.dtype break # dtype still not found, prefer using the default dtype # from the attr. if dtype is None and input_arg.type_attr in default_type_attr_map: default_dtype = default_type_attr_map[ input_arg.type_attr] try: if not input_arg.is_ref and dtype: dtype = dtypes.as_dtype(dtype).base_dtype values = ops.internal_convert_n_to_tensor( values, name=input_arg.name, dtype=dtype if dtype else None, preferred_dtype=default_dtype, as_ref=input_arg.is_ref) if input_arg.number_attr and len( set(v.dtype.base_dtype for v in values)) > 1: raise TypeError() # All types should match. except (TypeError, ValueError): # What types does the conversion function think values have? observed_types = [] for value in values: try: converted_value = ops.internal_convert_to_tensor( value, as_ref=input_arg.is_ref) observed_types.append( converted_value.dtype.base_dtype.name) except (TypeError, ValueError): observed_types.append( "<NOT CONVERTIBLE TO TENSOR>") observed = ", ".join(observed_types) prefix = ( "Tensors in list passed to '%s' of '%s' Op have types [%s]" % (input_name, op_type_name, observed)) if input_arg.number_attr: if input_arg.type != types_pb2.DT_INVALID: raise TypeError( "%s that do not match expected type %s." % (prefix, dtype.name)) elif input_arg.type_attr in attrs: raise TypeError( "%s that do not match type %s inferred from " "earlier arguments." % (prefix, dtype.name)) else: raise TypeError("%s that don't all match." % prefix) else: raise TypeError( "%s that are invalid. Tensors: %s" % (prefix, values)) types = [x.dtype for x in values] inputs.extend(values) else: # In cases where we have an expected type, try to convert non-Tensor # arguments to that type. dtype = None default_dtype = None if input_arg.type != types_pb2.DT_INVALID: dtype = input_arg.type elif input_arg.type_attr in attrs: dtype = attrs[input_arg.type_attr] elif input_arg.type_attr in default_type_attr_map: # The dtype could not be inferred solely from the inputs, # so we prefer the attr's default, so code that adds a new attr # with a default is backwards compatible. default_dtype = default_type_attr_map[ input_arg.type_attr] try: values = ops.internal_convert_to_tensor( values, name=input_arg.name, dtype=dtype, as_ref=input_arg.is_ref, preferred_dtype=default_dtype) except TypeError as err: if dtype is None: raise err else: raise TypeError( "Expected %s passed to parameter '%s' of op '%s', got %s of " "type '%s' instead." % (dtypes.as_dtype(dtype).name, input_arg.name, op_type_name, repr(values), type(values).__name__)) except ValueError: # What type does convert_to_tensor think it has? try: observed = ops.internal_convert_to_tensor( values, as_ref=input_arg.is_ref).dtype.name except ValueError as err: raise ValueError( "Tried to convert '%s' to a tensor and failed. Error: %s" % (input_name, err)) prefix = ( "Input '%s' of '%s' Op has type %s that does not match" % (input_name, op_type_name, observed)) if input_arg.type != types_pb2.DT_INVALID: raise TypeError( "%s expected type of %s." % (prefix, dtypes.as_dtype(input_arg.type).name)) else: # Update the maps with the default, if needed. k = input_arg.type_attr if k in default_type_attr_map: if k not in attrs: attrs[k] = default_type_attr_map[k] if k not in inferred_from: inferred_from[k] = "Default in OpDef" raise TypeError( "%s type %s of argument '%s'." % (prefix, dtypes.as_dtype( attrs[input_arg.type_attr]).name, inferred_from[input_arg.type_attr])) types = [values.dtype] inputs.append(values) base_types = [x.base_dtype for x in types] if input_arg.number_attr: # <number-attr> * <type> or <number-attr> * <type-attr> if input_arg.number_attr in attrs: if len(values) != attrs[input_arg.number_attr]: raise ValueError( "List argument '%s' to '%s' Op with length %d must match " "length %d of argument '%s'." % (input_name, op_type_name, len(values), attrs[input_arg.number_attr], inferred_from[input_arg.number_attr])) else: attrs[input_arg.number_attr] = len(values) inferred_from[input_arg.number_attr] = input_name num_attr = _Attr(op_def, input_arg.number_attr) if num_attr.has_minimum and len( values) < num_attr.minimum: raise ValueError( "List argument '%s' to '%s' Op with length %d shorter " "than minimum length %d." % (input_name, op_type_name, len(values), num_attr.minimum)) # All tensors must have the same base type. if any(bt != base_types[0] for bt in base_types): raise TypeError( "All tensors passed to '%s' of '%s' Op " "must have the same type." % (input_name, op_type_name)) if input_arg.type != types_pb2.DT_INVALID: # <number-attr> * <type> case if base_types and base_types[0] != input_arg.type: assert False, "Unreachable" elif input_arg.type_attr in attrs: # <number-attr> * <type-attr> case, where <type-attr> already # has an inferred value. if base_types and base_types[0] != attrs[ input_arg.type_attr]: assert False, "Unreachable" else: # <number-attr> * <type-attr> case, where we are now setting # the <type-attr> based on this input if not base_types: raise TypeError( "Don't know how to infer type variable from empty input " "list passed to input '%s' of '%s' Op." % (input_name, op_type_name)) attrs[input_arg.type_attr] = base_types[0] inferred_from[input_arg.type_attr] = input_name type_attr = _Attr(op_def, input_arg.type_attr) _SatisfiesTypeConstraint(base_types[0], type_attr, param_name=input_name) elif input_arg.type_attr: # <type-attr> attr_value = base_types[0] if input_arg.type_attr in attrs: if attrs[input_arg.type_attr] != attr_value: assert False, "Unreachable" else: for base_type in base_types: _SatisfiesTypeConstraint(base_type, _Attr( op_def, input_arg.type_attr), param_name=input_name) attrs[input_arg.type_attr] = attr_value inferred_from[input_arg.type_attr] = input_name elif input_arg.type_list_attr: # <type-list-attr> attr_value = base_types if input_arg.type_list_attr in attrs: if attrs[input_arg.type_list_attr] != attr_value: raise TypeError( "Input '%s' of '%s' Op has type list of %s that does not " "match type list %s of argument '%s'." % (input_name, op_type_name, ", ".join( dtypes.as_dtype(x).name for x in attr_value), ", ".join( dtypes.as_dtype(x).name for x in attrs[input_arg.type_list_attr]), inferred_from[input_arg.type_list_attr])) else: for base_type in base_types: _SatisfiesTypeConstraint( base_type, _Attr(op_def, input_arg.type_list_attr), param_name=input_name) attrs[input_arg.type_list_attr] = attr_value inferred_from[input_arg.type_list_attr] = input_name else: # single Tensor with specified type if base_types[0] != input_arg.type: assert False, "Unreachable" if input_arg.is_ref: if not all(x._is_ref_dtype for x in types): # pylint: disable=protected-access raise TypeError(( "'%s' Op requires that input '%s' be a mutable tensor " "(e.g.: a tf.Variable)") % (op_type_name, input_name)) input_types.extend(types) else: input_types.extend(base_types) # Process remaining attrs for attr in op_def.attr: # Skip attrs that have already had their values inferred if attr.name in attrs: if attr.name in keywords: raise TypeError( "Should not specify value for inferred attr '%s'." % attr.name) continue if attr.name in keywords: attrs[attr.name] = keywords.pop(attr.name) elif attr.name + "_" in keywords: # Attrs whose names match Python keywords have an extra '_' # appended, so we must check for that as well. attrs[attr.name] = keywords.pop(attr.name + "_") else: raise TypeError("No argument for attr " + attr.name) # Convert attr values to AttrValue protos. attr_protos = {} for attr_def in op_def.attr: key = attr_def.name value = attrs[key] attr_value = attr_value_pb2.AttrValue() if attr_def.HasField("default_value") and value is None: attr_value.CopyFrom(attr_def.default_value) attr_protos[key] = attr_value continue if attr_def.type.startswith("list("): if not _IsListValue(value): raise TypeError("Expected list for attr " + key) if attr_def.has_minimum: if len(value) < attr_def.minimum: raise ValueError( "Attr '%s' of '%s' Op passed list of length %d " "less than minimum %d." % (key, op_type_name, len(value), attr_def.minimum)) attr_value.list.SetInParent() if attr_def.type == "string": attr_value.s = _MakeStr(value, key) if attr_def.HasField("allowed_values"): if attr_value.s not in attr_def.allowed_values.list.s: raise ValueError( "Attr '%s' of '%s' Op passed string '%s' not in: \"%s\"." % (key, op_type_name, compat.as_text(attr_value.s), '", "'.join( map(compat.as_text, attr_def.allowed_values.list.s)))) elif attr_def.type == "list(string)": attr_value.list.s.extend([_MakeStr(x, key) for x in value]) if attr_def.HasField("allowed_values"): for x in attr_value.list.s: if x not in attr_def.allowed_values.list.s: raise ValueError( "Attr '%s' of '%s' Op passed string '%s' not in: \"%s\"." % (key, op_type_name, compat.as_text(x), '", "'.join( map(compat.as_text, attr_def.allowed_values.list.s)))) elif attr_def.type == "int": attr_value.i = _MakeInt(value, key) if attr_def.has_minimum: if attr_value.i < attr_def.minimum: raise ValueError( "Attr '%s' of '%s' Op passed %d less than minimum %d." % (key, op_type_name, attr_value.i, attr_def.minimum)) elif attr_def.type == "list(int)": attr_value.list.i.extend([_MakeInt(x, key) for x in value]) elif attr_def.type == "float": attr_value.f = _MakeFloat(value, key) elif attr_def.type == "list(float)": attr_value.list.f.extend( [_MakeFloat(x, key) for x in value]) elif attr_def.type == "bool": attr_value.b = _MakeBool(value, key) elif attr_def.type == "list(bool)": attr_value.list.b.extend( [_MakeBool(x, key) for x in value]) elif attr_def.type == "type": attr_value.type = _MakeType(value, attr_def) elif attr_def.type == "list(type)": attr_value.list.type.extend( [_MakeType(x, attr_def) for x in value]) elif attr_def.type == "shape": attr_value.shape.CopyFrom(_MakeShape(value, key)) elif attr_def.type == "list(shape)": attr_value.list.shape.extend( [_MakeShape(x, key) for x in value]) elif attr_def.type == "tensor": attr_value.tensor.CopyFrom(_MakeTensor(value, key)) elif attr_def.type == "list(tensor)": attr_value.list.tensor.extend( [_MakeTensor(x, key) for x in value]) elif attr_def.type == "func": attr_value.func.CopyFrom(_MakeFunc(value, key)) elif attr_def.type == "list(func)": attr_value.list.func.extend( [_MakeFunc(x, key) for x in value]) else: raise TypeError("Unrecognized Attr type " + attr_def.type) attr_protos[key] = attr_value del attrs # attrs is no longer authoritative, use attr_protos instead # Determine output types (possibly using attrs) output_types = [] output_structure = [] for arg in op_def.output_arg: types = [] if arg.number_attr: n = _AttrValue(attr_protos, arg.number_attr).i if arg.type_attr: types = [_AttrValue(attr_protos, arg.type_attr).type ] * n else: types = [arg.type] * n output_structure.append(n) elif arg.type_attr: t = _AttrValue(attr_protos, arg.type_attr) types = [t.type] output_structure.append(None) elif arg.type_list_attr: t = _AttrValue(attr_protos, arg.type_list_attr) types = t.list.type output_structure.append(len(types)) else: types = [arg.type] output_structure.append(None) if arg.is_ref: types = [dtypes.as_dtype(x)._as_ref for x in types] # pylint: disable=protected-access output_types.extend(types) if keywords: raise TypeError( "apply_op() got unexpected keyword arguments: " + ", ".join(sorted(keywords.keys()))) # NOTE(mrry): We add an explicit colocation constraint between # the newly created op and any of its reference-typed inputs. must_colocate_inputs = [ val for arg, val in zip(op_def.input_arg, inputs) if arg.is_ref ] with _MaybeColocateWith(must_colocate_inputs): # Add Op to graph op = g.create_op(op_type_name, inputs, output_types, name=scope, input_types=input_types, attrs=attr_protos, op_def=op_def) return output_structure, op_def.is_stateful, op
def make_tensor(x): if isinstance(x, ops.Tensor): return x return ops.internal_convert_to_tensor(x, ctx=ctx)
def minimize(self, global_step=None, name=None): """Add operations to train a linear model by minimizing the loss function. Args: global_step: Optional `Variable` to increment by one after the variables have been updated. name: Optional name for the returned operation. Returns: An Operation that updates the variables passed in the constructor. """ # Technically, the op depends on a lot more than the variables, # but we'll keep the list short. with name_scope(name, 'sdca/minimize'): sparse_example_indices = [] sparse_feature_indices = [] sparse_features_values = [] for sf in self._examples['sparse_features']: sparse_example_indices.append(sf.example_indices) sparse_feature_indices.append(sf.feature_indices) # If feature values are missing, sdca assumes a value of 1.0f. if sf.feature_values is not None: sparse_features_values.append(sf.feature_values) # pylint: disable=protected-access example_ids_hashed = gen_sdca_ops.sdca_fprint( internal_convert_to_tensor(self._examples['example_ids'])) # pylint: enable=protected-access example_state_data = self._hashtable.lookup(example_ids_hashed) # Solver returns example_state_update, new delta sparse_feature_weights # and delta dense_feature_weights. sparse_weights = [] sparse_indices = [] # If we have partitioned variables, keep a few dictionaries of Tensors # around that we need for the assign_add after the op call to # gen_sdca_ops.sdca_optimizer(). These are keyed because we may have a # mix of partitioned and un-partitioned variables. num_partitions_by_var = {} p_assignments_by_var = {} gather_ids_by_var = {} for v_num, (w, i) in enumerate( zip(self._slots['unshrinked_sparse_features_weights'], sparse_feature_indices)): # Append the sparse_indices (in full-variable space). sparse_idx = math_ops.cast( array_ops.unique(math_ops.cast(i, dtypes.int32))[0], dtypes.int64) sparse_indices.append(sparse_idx) if isinstance(w, list) or isinstance(w, var_ops.PartitionedVariable): num_partitions = len(w) flat_ids = array_ops.reshape(sparse_idx, [-1]) # We use div partitioning, which is easiest to support downstream. # Compute num_total_ids as the sum of dim-0 of w, then assign # to partitions based on a constant number of ids per partition. # Optimize if we already know the full shape statically. dim_0_size = self._get_first_dimension_size_statically( w, num_partitions) if tensor_shape.dimension_value(dim_0_size): num_total_ids = constant_op.constant( tensor_shape.dimension_value(dim_0_size), flat_ids.dtype) else: dim_0_sizes = [] for p in range(num_partitions): if tensor_shape.dimension_value(w[p].shape[0]) is not None: dim_0_sizes.append(tensor_shape.dimension_value(w[p].shape[0])) else: with ops.colocate_with(w[p]): dim_0_sizes.append(array_ops.shape(w[p])[0]) num_total_ids = math_ops.reduce_sum( math_ops.cast(array_ops.stack(dim_0_sizes), flat_ids.dtype)) ids_per_partition = num_total_ids // num_partitions extras = num_total_ids % num_partitions p_assignments = math_ops.maximum( flat_ids // (ids_per_partition + 1), (flat_ids - extras) // ids_per_partition) # Emulate a conditional using a boolean indicator tensor new_ids = array_ops.where(p_assignments < extras, flat_ids % (ids_per_partition + 1), (flat_ids - extras) % ids_per_partition) # Cast partition assignments to int32 for use in dynamic_partition. # There really should not be more than 2^32 partitions. p_assignments = math_ops.cast(p_assignments, dtypes.int32) # Partition list of ids based on assignments into num_partitions # separate lists. gather_ids = data_flow_ops.dynamic_partition(new_ids, p_assignments, num_partitions) # Add these into the dictionaries for use in the later update. num_partitions_by_var[v_num] = num_partitions p_assignments_by_var[v_num] = p_assignments gather_ids_by_var[v_num] = gather_ids # Gather the weights from each partition. partition_gathered_weights = [] for p in range(num_partitions): with ops.colocate_with(w[p]): partition_gathered_weights.append( array_ops.gather(w[p], gather_ids[p])) # Stitch the weights back together in the same order they were before # we dynamic_partitioned them. condition_indices = data_flow_ops.dynamic_partition( math_ops.range(array_ops.shape(new_ids)[0]), p_assignments, num_partitions) batch_gathered_weights = data_flow_ops.dynamic_stitch( condition_indices, partition_gathered_weights) else: w_as_tensor = internal_convert_to_tensor(w) with ops.device(w_as_tensor.device): batch_gathered_weights = array_ops.gather( w_as_tensor, sparse_idx) sparse_weights.append(batch_gathered_weights) # pylint: disable=protected-access if compat.forward_compatible(year=2018, month=10, day=30): esu, sfw, dfw = gen_sdca_ops.sdca_optimizer_v2( sparse_example_indices, sparse_feature_indices, sparse_features_values, self._convert_n_to_tensor(self._examples['dense_features']), internal_convert_to_tensor(self._examples['example_weights']), internal_convert_to_tensor(self._examples['example_labels']), sparse_indices, sparse_weights, self._convert_n_to_tensor(self._slots[ 'unshrinked_dense_features_weights']), example_state_data, loss_type=self._options['loss_type'], l1=self._options['symmetric_l1_regularization'], l2=self._symmetric_l2_regularization(), num_loss_partitions=self._num_loss_partitions(), num_inner_iterations=1, adaptive=self._adaptive()) else: esu, sfw, dfw = gen_sdca_ops.sdca_optimizer( sparse_example_indices, sparse_feature_indices, sparse_features_values, self._convert_n_to_tensor(self._examples['dense_features']), internal_convert_to_tensor(self._examples['example_weights']), internal_convert_to_tensor(self._examples['example_labels']), sparse_indices, sparse_weights, self._convert_n_to_tensor(self._slots[ 'unshrinked_dense_features_weights']), example_state_data, loss_type=self._options['loss_type'], l1=self._options['symmetric_l1_regularization'], l2=self._symmetric_l2_regularization(), num_loss_partitions=self._num_loss_partitions(), num_inner_iterations=1, adaptative=self._adaptive()) # pylint: enable=protected-access with ops.control_dependencies([esu]): update_ops = [self._hashtable.insert(example_ids_hashed, esu)] # Update the weights before the proximal step. for v_num, (w, i, u) in enumerate( zip(self._slots['unshrinked_sparse_features_weights'], sparse_indices, sfw)): if (isinstance(w, var_ops.PartitionedVariable) or isinstance(w, list)): update_ops += self._get_partitioned_update_ops( v_num, num_partitions_by_var, p_assignments_by_var, gather_ids_by_var, w, u, p_assignments, num_partitions) else: update_ops.append(state_ops.scatter_add(w, i, u)) for w, u in zip(self._slots['unshrinked_dense_features_weights'], dfw): if (isinstance(w, var_ops.PartitionedVariable) or isinstance(w, list)): split_updates = array_ops.split( u, num_or_size_splits=[v.shape.as_list()[0] for v in w]) for v, split_update in zip(w, split_updates): update_ops.append(state_ops.assign_add(v, split_update)) else: update_ops.append(state_ops.assign_add(w, u)) if not global_step: return control_flow_ops.group(*update_ops) with ops.control_dependencies(update_ops): return state_ops.assign_add(global_step, 1, name=name).op
def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None, executor_type=None): """Executes a function while respecting device annotations. Currently, only those functions that execute within the same address space can be executed. Args: args: The arguments of the function, including captured inputs. f: The function to execute; an instance of `_DefinedFunction` or `_EagerDefinedFunction`. tout: a list containing the output dtypes enums; if `None`, inferred from the signature of `f`. executing_eagerly: (Optional) A boolean indicating whether the context is executing eagerly. If `None`, fetched from the global context. config: (Optional) A `tensorflow::ConfigProto` proto, serialized. If `None`, all optimizations are disabled. Currently only handled for eager defined functions. executor_type: (Optional) A string for the name of the executor to be used in the function call. If not set, or set to an empty string, the default tensorflow executor will be used. Returns: The list of `Tensor`s returned by invoking `f(args)`. If the function does not return anything, then returns `None` if eager execution is enabled, or the `Operation` if not. """ if tout is None: tout = tuple(x.type for x in f.definition.signature.output_arg) if executing_eagerly is None: executing_eagerly = context.executing_eagerly() if config is None: config = function_utils.get_disabled_rewriter_config() if executor_type is None: executor_type = "" if executing_eagerly or len(tout): if f.stateful_ops: outputs = gen_functional_ops.stateful_partitioned_call( args=args, Tout=tout, f=f, config_proto=config, executor_type=executor_type) else: outputs = gen_functional_ops.partitioned_call( args=args, Tout=tout, f=f, config_proto=config, executor_type=executor_type) return outputs if outputs else None # The generated binding returns an empty list for functions that don't # return any Tensors, hence the need to use `create_op` directly. args = [ops.internal_convert_to_tensor(x) for x in args] tin_attr = attr_value_pb2.AttrValue( list=attr_value_pb2.AttrValue.ListValue( type=[x.dtype.as_datatype_enum for x in args])) tout_attr = attr_value_pb2.AttrValue( list=attr_value_pb2.AttrValue.ListValue(type=tout)) func_attr = attr_value_pb2.AttrValue( func=attr_value_pb2.NameAttrList(name=f.name)) executor_type_attr = attr_value_pb2.AttrValue( s=compat.as_bytes(executor_type)) # When running in graph mode, the graph and function graphs are optimized # (i.e. run through grappler) per the session options, so we can disable any # eager-specific rewriting. config_proto = attr_value_pb2.AttrValue( s=function_utils.get_disabled_rewriter_config()) graph = ops.get_default_graph() f.add_to_graph(graph) op_name = "StatefulPartitionedCall" if f.stateful_ops else "PartitionedCall" op = graph.create_op( op_name, args, tout, compute_shapes=False, name="PartitionedFunctionCall", attrs={ "Tin": tin_attr, "Tout": tout_attr, "f": func_attr, "config_proto": config_proto, "executor_type": executor_type_attr, }) outputs = op.outputs return outputs if outputs else op
def _tensor_conversion(var, dtype=None, name=None, as_ref=False): # Try to avoid assignments to and other mutations of MirroredVariable # state except through a DistributionStrategy.update() call. assert not as_ref return ops.internal_convert_to_tensor( var.get(), dtype=dtype, name=name, as_ref=as_ref)
"""Base class for asset files which need to be tracked.""" def __init__(self, path): """Record the full path to the asset.""" # We use a variable here so that @tf.functions do not capture a literal # value. The init_scope prevents functions from capturing `path` in an # initialization graph, since it is transient and should not end up in a # serialized function body. When serialized in a SavedModel, the variable # will be set during the loading process to its location in the assets/ # directory. with ops.init_scope(): if context.executing_eagerly(): self._path = self._no_dependency( resource_variable_ops.ResourceVariable( path, dtype=dtypes.string, name="asset_path")) else: # Adding a variable is too disruptive when v1-style graph building, # since things may get fed and local variable initializers would then # need to be run. self._path = path @property def asset_path(self): """Fetch the current asset path.""" return self._path ops.register_tensor_conversion_function( TrackableAsset, lambda asset, **kw: ops.internal_convert_to_tensor(asset.asset_path, **kw))
def _convert_labeled_tensor_mock_to_tensor(value, *args, **kwargs): return ops.internal_convert_to_tensor(value.tensor, *args, **kwargs)
def op_list_to_dict(op_list, convert_variable_to_tensor=True): """Create a dictionary of names to operation lists. Args: op_list: A list, tuple, or set of Variables or SaveableObjects. convert_variable_to_tensor: Whether or not to convert single Variables with no slice info into Tensors. Returns: A dictionary of names to the operations that must be saved under that name. Variables with save_slice_info are grouped together under the same key in no particular order. Raises: TypeError: If the type of op_list or its elements is not supported. ValueError: If at least two saveables share the same name. """ if not isinstance(op_list, (list, tuple, set)): raise TypeError("Variables to save should be passed in a dict or a " "list: %s" % op_list) # When ResourceVariables are converted to Tensors, read ops are added to the # graph. Sorting the op_list ensures that the resulting graph is always # constructed in a deterministic way: op_list = sorted(op_list, key=lambda x: x.name) names_to_saveables = {} # pylint: disable=protected-access for var in op_list: resource_or_ref_variable = ( isinstance(var, resource_variable_ops.ResourceVariable) or isinstance(var, variables.RefVariable)) if isinstance(var, saveable_object.SaveableObject): names_to_saveables[var.name] = var elif isinstance(var, variables.PartitionedVariable): if var.name in names_to_saveables: raise ValueError("At least two variables have the same name: %s" % var.name) names_to_saveables[var.name] = var elif isinstance(var, variables.Variable) and var._save_slice_info: name = var._save_slice_info.full_name if name in names_to_saveables: if not isinstance(names_to_saveables[name], list): raise ValueError("Mixing slices and non-slices with the same name: " "%s" % name) names_to_saveables[name].append(var) else: names_to_saveables[name] = [var] elif isinstance(var, trackable.Trackable) and not resource_or_ref_variable: trackable_saveables = [ (factory() if callable(factory) else factory) for factory in var._gather_saveables_for_checkpoint().values()] names_to_saveables.update( op_list_to_dict(trackable_saveables)) else: # Variables (reference and resource) have an _in_graph_mode property # indicating whether they were created in a graph building context. We # also get Tensors when graph building, which do not have this property. if not getattr(var, "_in_graph_mode", True): if not isinstance(var, resource_variable_ops.ResourceVariable): raise ValueError( "Can only save/restore ResourceVariables when eager execution " "is enabled, type: %s." % type(var)) set_var = names_to_saveables.setdefault(var._shared_name, var) if set_var is not var: raise ValueError( ("Two different ResourceVariable objects with the same " "shared_name '%s' were passed to the Saver. This likely means " "that they were created in different Graphs or isolation " "contexts, and may not be checkpointed together.") % (var._shared_name,)) else: if convert_variable_to_tensor: if isinstance(var, resource_variable_ops.ResourceVariable): var = var._graph_element # pylint: disable=protected-access else: var = ops.internal_convert_to_tensor(var, as_ref=True) if not _tensor_comes_from_variable(var): raise TypeError("Variable to save is not a Variable: %s" % var) if var.op.type == "ReadVariableOp": name = var.op.inputs[0].op.name else: name = var.op.name if name in names_to_saveables: raise ValueError("At least two variables have the same name: %s" % name) names_to_saveables[name] = var # pylint: enable=protected-access return names_to_saveables
def apply_op(self, op_type_name, name=None, **keywords): # pylint: disable=g-doc-args """Add a node invoking a registered Op to a graph. Example usage: # input1 and input2 can be Tensors or anything ops.convert_to_tensor() # will convert to a Tensor. op_def_library.apply_op("op", input1=input1, input2=input2) # Can specify a node name. op_def_library.apply_op("op", input1=input1, name="node_name") # Must use keyword arguments, with the names specified in the OpDef. op_def_library.apply_op("op", input_name=input, attr_name=attr) All attrs must either be inferred from an input or specified. (If inferred, the attr must not be specified.) If an attr has a default value specified in the Op's OpDef, then you may pass None as the value of that attr to get the default. Args: op_type_name: string. Must match the name field of a registered Op. name: string. Optional name of the created op. **keywords: input Tensor and attr arguments specified by name, and optional parameters to pass when constructing the Operation. Returns: The Tensor(s) representing the output of the operation, or the Operation itself if there are no outputs. Raises: RuntimeError: On some errors. TypeError: On some errors. ValueError: On some errors. """ op_info = self._ops.get(op_type_name, None) if op_info is None: raise RuntimeError("Unrecognized Op name " + op_type_name) op_def = op_info.op_def # Determine the graph context. try: # Need to flatten all the arguments into a list. # pylint: disable=protected-access g = ops._get_graph_from_inputs(_Flatten(keywords.values())) # pyline: enable=protected-access except AssertionError as e: raise RuntimeError( "Cannot determine graph for Op '%s' due to: %s" % (op_type_name, e.message)) # Default name if not specified. if name is None: name = op_type_name # Check for deprecation deprecation_version = op_def.deprecation.version if deprecation_version: producer = g.graph_def_versions.producer if producer >= deprecation_version: raise NotImplementedError( ("Op %s is not available in GraphDef version %d. " "It has been removed in version %d. %s.") % (op_type_name, producer, deprecation_version, op_def.deprecation.explanation)) # Fill in the list of default types for all "type" attrs. This # will be used to choose a preferred dtype to convert to in the # absence of input type information. # # TODO(b/31302892): Currently the defaults don't work in the right # way if you have two inputs, one of whose type resolution depends # on the other. Handling this will require restructuring this code # significantly. default_type_attr_map = {} for attr_def in op_def.attr: if attr_def.type != "type": continue key = attr_def.name if attr_def.HasField("default_value"): default_type_attr_map[key] = dtypes.as_dtype( attr_def.default_value.type) # Requires that op_def has passed validation (using the C++ # ValidateOpDef() from ../framework/op_def_util.h). attrs = {} inputs = [] input_types = [] with g.as_default(), ops.name_scope(name) as scope: # Perform input type inference inferred_from = {} for input_arg in op_def.input_arg: input_name = input_arg.name if input_name in keywords: values = keywords.pop(input_name) elif input_name + "_" in keywords: # Handle the case where the name is a keyword or built-in # for Python so we use the name + _ instead. input_name += "_" values = keywords.pop(input_name) else: raise TypeError("No argument for input " + input_name) # Goals: # * Convert values to Tensors if it contains constants. # * Verify that values is a list if that matches the input_arg's # type. # * If the input_arg's type is determined by attrs, either set # those attrs and validate those attr values are legal (if # they have not yet been set) or validate the input matches # the type indicated by the attrs (if they have already been # inferred via an earlier input). # * If the input_arg has an explicit type, make sure the input # conforms. if _IsListParameter(input_arg): if not _IsListValue(values): raise TypeError( "Expected list for '%s' argument to '%s' Op, not %s." % (input_name, op_type_name, values)) # In cases where we expect all elements of the list to have the # same dtype, try to cast non-Tensor elements to that type. dtype = None default_dtype = None if input_arg.type != types_pb2.DT_INVALID: dtype = input_arg.type elif input_arg.number_attr: if input_arg.type_attr in attrs: dtype = attrs[input_arg.type_attr] else: for t in values: if isinstance(t, ops.Tensor): dtype = t.dtype break # dtype still not found, prefer using the default dtype # from the attr. if dtype is None and input_arg.type_attr in default_type_attr_map: default_dtype = default_type_attr_map[input_arg.type_attr] try: if not input_arg.is_ref and dtype: dtype = dtypes.as_dtype(dtype).base_dtype values = ops.internal_convert_n_to_tensor( values, name=input_arg.name, dtype=dtype if dtype else None, preferred_dtype=default_dtype, as_ref=input_arg.is_ref) if input_arg.number_attr and len( set(v.dtype.base_dtype for v in values)) > 1: raise TypeError() # All types should match. except (TypeError, ValueError): # What types does the conversion function think values have? observed_types = [] for value in values: try: converted_value = ops.internal_convert_to_tensor( value, as_ref=input_arg.is_ref) observed_types.append(converted_value.dtype.base_dtype.name) except (TypeError, ValueError): observed_types.append("<NOT CONVERTIBLE TO TENSOR>") observed = ", ".join(observed_types) prefix = ( "Tensors in list passed to '%s' of '%s' Op have types [%s]" % (input_name, op_type_name, observed)) if input_arg.number_attr: if input_arg.type != types_pb2.DT_INVALID: raise TypeError("%s that do not match expected type %s." % (prefix, dtype.name)) elif input_arg.type_attr in attrs: raise TypeError("%s that do not match type %s inferred from " "earlier arguments." % (prefix, dtype.name)) else: raise TypeError("%s that don't all match." % prefix) else: raise TypeError("%s that are invalid." % prefix) types = [x.dtype for x in values] inputs.extend(values) else: # In cases where we have an expected type, try to convert non-Tensor # arguments to that type. dtype = None default_dtype = None if input_arg.type != types_pb2.DT_INVALID: dtype = input_arg.type elif input_arg.type_attr in attrs: dtype = attrs[input_arg.type_attr] elif input_arg.type_attr in default_type_attr_map: # The dtype could not be inferred solely from the inputs, # so we prefer the attr's default, so code that adds a new attr # with a default is backwards compatible. default_dtype = default_type_attr_map[input_arg.type_attr] try: values = ops.internal_convert_to_tensor( values, name=input_arg.name, dtype=dtype, as_ref=input_arg.is_ref, preferred_dtype=default_dtype) except TypeError as err: if dtype is None: raise err else: raise TypeError( "Expected %s passed to parameter '%s' of op '%s', got %s of " "type '%s' instead." % (dtypes.as_dtype(dtype).name, input_arg.name, op_type_name, repr(values), type(values).__name__)) except ValueError: # What type does convert_to_tensor think it has? observed = ops.internal_convert_to_tensor( values, as_ref=input_arg.is_ref).dtype.name prefix = ("Input '%s' of '%s' Op has type %s that does not match" % (input_name, op_type_name, observed)) if input_arg.type != types_pb2.DT_INVALID: raise TypeError("%s expected type of %s." % (prefix, dtypes.as_dtype(input_arg.type).name)) else: # Update the maps with the default, if needed. k = input_arg.type_attr if k in default_type_attr_map: if k not in attrs: attrs[k] = default_type_attr_map[k] if k not in inferred_from: inferred_from[k] = "Default in OpDef" raise TypeError( "%s type %s of argument '%s'." % (prefix, dtypes.as_dtype(attrs[input_arg.type_attr]).name, inferred_from[input_arg.type_attr])) types = [values.dtype] inputs.append(values) base_types = [x.base_dtype for x in types] if input_arg.number_attr: # <number-attr> * <type> or <number-attr> * <type-attr> if input_arg.number_attr in attrs: if len(values) != attrs[input_arg.number_attr]: raise ValueError( "List argument '%s' to '%s' Op with length %d must match " "length %d of argument '%s'." % (input_name, op_type_name, len(values), attrs[input_arg.number_attr], inferred_from[input_arg.number_attr])) else: attrs[input_arg.number_attr] = len(values) inferred_from[input_arg.number_attr] = input_name num_attr = _Attr(op_def, input_arg.number_attr) if num_attr.has_minimum and len(values) < num_attr.minimum: raise ValueError( "List argument '%s' to '%s' Op with length %d shorter " "than minimum length %d." % (input_name, op_type_name, len(values), num_attr.minimum)) # All tensors must have the same base type. if any([bt != base_types[0] for bt in base_types]): raise TypeError( "All tensors passed to '%s' of '%s' Op " "must have the same type." % (input_name, op_type_name)) if input_arg.type != types_pb2.DT_INVALID: # <number-attr> * <type> case if base_types and base_types[0] != input_arg.type: assert False, "Unreachable" elif input_arg.type_attr in attrs: # <number-attr> * <type-attr> case, where <type-attr> already # has an inferred value. if base_types and base_types[0] != attrs[input_arg.type_attr]: assert False, "Unreachable" else: # <number-attr> * <type-attr> case, where we are now setting # the <type-attr> based on this input if not base_types: raise TypeError( "Don't know how to infer type variable from empty input " "list passed to input '%s' of '%s' Op." % (input_name, op_type_name)) attrs[input_arg.type_attr] = base_types[0] inferred_from[input_arg.type_attr] = input_name type_attr = _Attr(op_def, input_arg.type_attr) _SatisfiesTypeConstraint(base_types[0], type_attr) elif input_arg.type_attr: # <type-attr> attr_value = base_types[0] if input_arg.type_attr in attrs: if attrs[input_arg.type_attr] != attr_value: assert False, "Unreachable" else: for base_type in base_types: _SatisfiesTypeConstraint(base_type, _Attr(op_def, input_arg.type_attr)) attrs[input_arg.type_attr] = attr_value inferred_from[input_arg.type_attr] = input_name elif input_arg.type_list_attr: # <type-list-attr> attr_value = base_types if input_arg.type_list_attr in attrs: if attrs[input_arg.type_list_attr] != attr_value: raise TypeError( "Input '%s' of '%s' Op has type list of %s that does not " "match type list %s of argument '%s'." % (input_name, op_type_name, ", ".join(dtypes.as_dtype(x).name for x in attr_value), ", ".join(dtypes.as_dtype(x).name for x in attrs[input_arg.type_list_attr]), inferred_from[input_arg.type_list_attr])) else: for base_type in base_types: _SatisfiesTypeConstraint(base_type, _Attr(op_def, input_arg.type_list_attr)) attrs[input_arg.type_list_attr] = attr_value inferred_from[input_arg.type_list_attr] = input_name else: # single Tensor with specified type if base_types[0] != input_arg.type: assert False, "Unreachable" if input_arg.is_ref: if not all(x._is_ref_dtype for x in types): # pylint: disable=protected-access raise TypeError( "Input '%s' of '%s' Op requires l-value input" % (input_name, op_type_name)) input_types.extend(types) else: input_types.extend(base_types) # Process remaining attrs for attr in op_def.attr: # Skip attrs that have already had their values inferred if attr.name in attrs: if attr.name in keywords: raise TypeError( "Should not specify value for inferred attr '%s'." % attr.name) continue if attr.name in keywords: attrs[attr.name] = keywords.pop(attr.name) elif attr.name + "_" in keywords: # Attrs whose names match Python keywords have an extra '_' # appended, so we must check for that as well. attrs[attr.name] = keywords.pop(attr.name + "_") else: raise TypeError("No argument for attr " + attr.name) # Convert attr values to AttrValue protos. attr_protos = {} for attr_def in op_def.attr: key = attr_def.name value = attrs[key] attr_value = attr_value_pb2.AttrValue() if attr_def.HasField("default_value") and value is None: attr_value.CopyFrom(attr_def.default_value) attr_protos[key] = attr_value continue if attr_def.type.startswith("list("): if not _IsListValue(value): raise TypeError("Expected list for attr " + key) if attr_def.has_minimum: if len(value) < attr_def.minimum: raise ValueError("Attr '%s' of '%s' Op passed list of length %d " "less than minimum %d." % (key, op_type_name, len(value), attr_def.minimum)) attr_value.list.SetInParent() if attr_def.type == "string": attr_value.s = _MakeStr(value, key) if attr_def.HasField("allowed_values"): if attr_value.s not in attr_def.allowed_values.list.s: raise ValueError( "Attr '%s' of '%s' Op passed string '%s' not in: \"%s\"." % (key, op_type_name, compat.as_text(attr_value.s), '", "'.join(map(compat.as_text, attr_def.allowed_values.list.s)))) elif attr_def.type == "list(string)": attr_value.list.s.extend([_MakeStr(x, key) for x in value]) if attr_def.HasField("allowed_values"): for x in attr_value.list.s: if x not in attr_def.allowed_values.list.s: raise ValueError( "Attr '%s' of '%s' Op passed string '%s' not in: \"%s\"." % (key, op_type_name, compat.as_text(x), '", "'.join(map(compat.as_text, attr_def.allowed_values.list.s)))) elif attr_def.type == "int": attr_value.i = _MakeInt(value, key) if attr_def.has_minimum: if attr_value.i < attr_def.minimum: raise ValueError( "Attr '%s' of '%s' Op passed %d less than minimum %d." % (key, op_type_name, attr_value.i, attr_def.minimum)) elif attr_def.type == "list(int)": attr_value.list.i.extend([_MakeInt(x, key) for x in value]) elif attr_def.type == "float": attr_value.f = _MakeFloat(value, key) elif attr_def.type == "list(float)": attr_value.list.f.extend([_MakeFloat(x, key) for x in value]) elif attr_def.type == "bool": attr_value.b = _MakeBool(value, key) elif attr_def.type == "list(bool)": attr_value.list.b.extend([_MakeBool(x, key) for x in value]) elif attr_def.type == "type": attr_value.type = _MakeType(value, attr_def) elif attr_def.type == "list(type)": attr_value.list.type.extend( [_MakeType(x, attr_def) for x in value]) elif attr_def.type == "shape": attr_value.shape.CopyFrom(_MakeShape(value, key)) elif attr_def.type == "list(shape)": attr_value.list.shape.extend( [_MakeShape(x, key) for x in value]) elif attr_def.type == "tensor": attr_value.tensor.CopyFrom(_MakeTensor(value, key)) elif attr_def.type == "list(tensor)": attr_value.list.tensor.extend( [_MakeTensor(x, key) for x in value]) elif attr_def.type == "func": if isinstance(value, attr_value_pb2.NameAttrList): attr_value.func.CopyFrom(value) elif isinstance(value, compat.bytes_or_text_types): attr_value.func.name = value else: value.add_to_graph(ops.get_default_graph()) attr_value.func.name = value.name else: raise TypeError("Unrecognized Attr type " + attr_def.type) attr_protos[key] = attr_value del attrs # attrs is no longer authoritative, use attr_protos instead # Determine output types (possibly using attrs) output_types = [] output_structure = [] for arg in op_def.output_arg: types = [] if arg.number_attr: n = _AttrValue(attr_protos, arg.number_attr).i if arg.type_attr: types = [_AttrValue(attr_protos, arg.type_attr).type] * n else: types = [arg.type] * n output_structure.append(n) elif arg.type_attr: t = _AttrValue(attr_protos, arg.type_attr) types = [t.type] output_structure.append(None) elif arg.type_list_attr: t = _AttrValue(attr_protos, arg.type_list_attr) types = t.list.type output_structure.append(len(types)) else: types = [arg.type] output_structure.append(None) if arg.is_ref: types = [dtypes.as_dtype(x)._as_ref for x in types] # pylint: disable=protected-access output_types.extend(types) if keywords: raise TypeError("apply_op() got unexpected keyword arguments: " + ", ".join(sorted(keywords.keys()))) # NOTE(mrry): We add an explicit colocation constraint between # the newly created op and any of its reference-typed inputs. must_colocate_inputs = [val for arg, val in zip(op_def.input_arg, inputs) if arg.is_ref] with _MaybeColocateWith(must_colocate_inputs): # Add Op to graph op = g.create_op(op_type_name, inputs, output_types, name=scope, input_types=input_types, attrs=attr_protos, op_def=op_def) if output_structure: outputs = op.outputs res = _Restructure(ops.convert_n_to_tensor(outputs), output_structure) if isinstance(res, list) and not res and op_def.is_stateful: return op else: return res else: return op
return property(Cache().__getitem__) ``` However, that implementation holds class instances as keys, and as a result blocks garbage collection. (And modifying it to use weakref's as keys raises the lookup overhead to ~0.4 us) As a result, the WeakKeyDictionary implementation below turns out to be more prudent. Args: f: The function to cache. Returns: f decorated with simple caching behavior. """ cache = weakref.WeakKeyDictionary() @functools.wraps(f) def wrapped(item): output = cache.get(item) if output is None: cache[item] = output = f(item) return output return wrapped ops.register_tensor_conversion_function( TrackableAsset, lambda asset, **kw: ops.internal_convert_to_tensor(asset.asset_path, **kw))
def _tensor_conversion_tower_local(var, dtype=None, name=None, as_ref=False): return ops.internal_convert_to_tensor( var.get(), dtype=dtype, name=name, as_ref=as_ref)
def _tensor_conversion_tower_local(var, dtype=None, name=None, as_ref=False): return ops.internal_convert_to_tensor(var.get(), dtype=dtype, name=name, as_ref=as_ref)
def _convert_labeled_tensor_to_tensor(value, *args, **kwargs): # call ops.convert_to_tensor to handle optional arguments appropriately return ops.internal_convert_to_tensor(value.tensor, *args, **kwargs)