def build(self, input_shape): dtype = dtypes.as_dtype(self.dtype or K.floatx()) if not (dtype.is_floating or dtype.is_complex): raise TypeError('Unable to build `Dense` layer with non-floating point ' 'dtype %s' % (dtype,)) input_shape = tensor_shape.TensorShape(input_shape) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') last_dim = tensor_shape.dimension_value(input_shape[-1]) self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim}) self.kernel = self.add_weight( 'kernel', shape=[last_dim, self.units], initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint, dtype=self.dtype, trainable=True) if self.use_bias: self.bias = self.add_weight( 'bias', shape=[self.units,], initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint, dtype=self.dtype, trainable=True) else: self.bias = None self.built = True
def _min_matrix_dim(self): """Minimum of domain/range dimension, if statically available, else None.""" domain_dim = tensor_shape.dimension_value(self.domain_dimension) range_dim = tensor_shape.dimension_value(self.range_dimension) if domain_dim is None or range_dim is None: return None return min(domain_dim, range_dim)
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') last_dim = tensor_shape.dimension_value(input_shape[-1]) self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim}) self.kernel = self.add_weight( 'kernel', shape=[last_dim, self.units], initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint, dtype=self.dtype, trainable=True) if self.use_bias: self.bias = self.add_weight( 'bias', shape=[self.units,], initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint, dtype=self.dtype, trainable=True) else: self.bias = None self.built = True
def maybe_check_quadrature_param(param, name, validate_args): """Helper which checks validity of `loc` and `scale` init args.""" with ops.name_scope(name="check_" + name, values=[param]): assertions = [] if param.shape.ndims is not None: if param.shape.ndims == 0: raise ValueError("Mixing params must be a (batch of) vector; " "{}.rank={} is not at least one.".format( name, param.shape.ndims)) elif validate_args: assertions.append(check_ops.assert_rank_at_least( param, 1, message=("Mixing params must be a (batch of) vector; " "{}.rank is not at least one.".format( name)))) # TODO(jvdillon): Remove once we support k-mixtures. if param.shape.with_rank_at_least(1)[-1] is not None: if tensor_shape.dimension_value(param.shape[-1]) != 1: raise NotImplementedError("Currently only bimixtures are supported; " "{}.shape[-1]={} is not 1.".format( name, tensor_shape.dimension_value( param.shape[-1]))) elif validate_args: assertions.append(check_ops.assert_equal( array_ops.shape(param)[-1], 1, message=("Currently only bimixtures are supported; " "{}.shape[-1] is not 1.".format(name)))) if assertions: return control_flow_ops.with_dependencies(assertions, param) return param
def convert_legacy_structure(output_types, output_shapes, output_classes): """Returns a `Structure` that represents the given legacy structure. This method provides a way to convert from the existing `Dataset` and `Iterator` structure-related properties to a `Structure` object. A "legacy" structure is represented by the `tf.data.Dataset.output_types`, `tf.data.Dataset.output_shapes`, and `tf.data.Dataset.output_classes` properties. TODO(b/110122868): Remove this function once `Structure` is used throughout `tf.data`. Args: output_types: A nested structure of `tf.DType` objects corresponding to each component of a structured value. output_shapes: A nested structure of `tf.TensorShape` objects corresponding to each component a structured value. output_classes: A nested structure of Python `type` objects corresponding to each component of a structured value. Returns: A `Structure`. Raises: TypeError: If a structure cannot be built from the arguments, because one of the component classes in `output_classes` is not supported. """ flat_types = nest.flatten(output_types) flat_shapes = nest.flatten(output_shapes) flat_classes = nest.flatten(output_classes) flat_ret = [] for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes, flat_classes): if isinstance(flat_class, Structure): flat_ret.append(flat_class) elif issubclass(flat_class, sparse_tensor_lib.SparseTensor): flat_ret.append(SparseTensorStructure(flat_type, flat_shape)) elif issubclass(flat_class, ops.Tensor): flat_ret.append(TensorStructure(flat_type, flat_shape)) elif issubclass(flat_class, tensor_array_ops.TensorArray): # We sneaked the dynamic_size and infer_shape into the legacy shape. flat_ret.append( TensorArrayStructure( flat_type, flat_shape[2:], dynamic_size=tensor_shape.dimension_value(flat_shape[0]), infer_shape=tensor_shape.dimension_value(flat_shape[1]))) else: # NOTE(mrry): Since legacy structures produced by iterators only # comprise Tensors, SparseTensors, and nests, we do not need to # support all structure types here. raise TypeError( "Could not build a structure for output class %r" % (flat_class,)) ret = nest.pack_sequence_as(output_classes, flat_ret) if isinstance(ret, Structure): return ret else: return NestedStructure(ret)
def _inverse_event_shape(self, output_shape): batch_shape, n1, n2 = (output_shape[:-2], tensor_shape.dimension_value(output_shape[-2]), tensor_shape.dimension_value(output_shape[-1])) if n1 is None or n2 is None: m = None elif n1 != n2: raise ValueError("Matrix must be square. (saw [{}, {}])".format(n1, n2)) else: m = n1 * (n1 + 1) / 2 return batch_shape.concatenate([m])
def _forward(self, x): if self._unroll_loop: event_size = tensor_shape.dimension_value( x.shape.with_rank_at_least(1)[-1]) if event_size is None: raise ValueError( "The final dimension of `x` must be known at graph construction " "time if `unroll_loop=True`. `x.shape: %r`" % x.shape) y = array_ops.zeros_like(x, name="y0") for _ in range(event_size): shift, log_scale = self._shift_and_log_scale_fn(y) # next_y = scale * x + shift next_y = x if log_scale is not None: next_y *= math_ops.exp(log_scale) if shift is not None: next_y += shift y = next_y return y event_size = array_ops.shape(x)[-1] # If the event size is available at graph construction time, we can inform # the graph compiler of the maximum number of steps. If not, # static_event_size will be None, and the maximum_iterations argument will # have no effect. static_event_size = tensor_shape.dimension_value( x.shape.with_rank_at_least(1)[-1]) y0 = array_ops.zeros_like(x, name="y0") # call the template once to ensure creation _ = self._shift_and_log_scale_fn(y0) def _loop_body(index, y0): """While-loop body for autoregression calculation.""" # Set caching device to avoid re-getting the tf.Variable for every while # loop iteration. with variable_scope_lib.variable_scope( variable_scope_lib.get_variable_scope()) as vs: if vs.caching_device is None: vs.set_caching_device(lambda op: op.device) shift, log_scale = self._shift_and_log_scale_fn(y0) y = x if log_scale is not None: y *= math_ops.exp(log_scale) if shift is not None: y += shift return index + 1, y _, y = control_flow_ops.while_loop( cond=lambda index, _: index < event_size, body=_loop_body, loop_vars=(0, y0), maximum_iterations=static_event_size) return y
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') self.input_spec = base.InputSpec( min_ndim=2, axes={-1: tensor_shape.dimension_value(input_shape[-1])}) self.kernel = self.add_variable( 'kernel', shape=[tensor_shape.dimension_value(input_shape[-1]), self.units], initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, dtype=self.dtype, trainable=True) self.mask = self.add_variable( name='mask', shape=[tensor_shape.dimension_value(input_shape[-1]), self.units], initializer=init_ops.ones_initializer(), trainable=False, dtype=self.dtype) self.threshold = self.add_variable( name='threshold', shape=[], initializer=init_ops.zeros_initializer(), trainable=False, dtype=self.dtype) # Add masked_weights in the weights namescope so as to make it easier # for the quantization library to add quant ops. self.masked_kernel = math_ops.multiply(self.mask, self.kernel, MASKED_WEIGHT_NAME) ops.add_to_collection(MASK_COLLECTION, self.mask) ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel) ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold) ops.add_to_collection(WEIGHT_COLLECTION, self.kernel) if self.use_bias: self.bias = self.add_variable( 'bias', shape=[ self.units, ], initializer=self.bias_initializer, regularizer=self.bias_regularizer, dtype=self.dtype, trainable=True) else: self.bias = None self.built = True
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) channel_axis = 1 if self.data_format == 'channels_first' else -1 if tensor_shape.dimension_value(input_shape[channel_axis]) is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') input_dim = tensor_shape.dimension_value(input_shape[channel_axis]) kernel_shape = self.kernel_size + (input_dim, self.filters) self.mask = self.add_variable( name='mask', shape=kernel_shape, initializer=init_ops.ones_initializer(), trainable=False, dtype=self.dtype) self.kernel = self.add_variable( name='kernel', shape=kernel_shape, initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, trainable=True, dtype=self.dtype) self.threshold = self.add_variable( name='threshold', shape=[], initializer=init_ops.zeros_initializer(), trainable=False, dtype=self.dtype) # Add masked_weights in the weights namescope so as to make it easier # for the quantization library to add quant ops. self.masked_kernel = math_ops.multiply(self.mask, self.kernel, MASKED_WEIGHT_NAME) ops.add_to_collection(MASK_COLLECTION, self.mask) ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel) ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold) ops.add_to_collection(WEIGHT_COLLECTION, self.kernel) if self.use_bias: self.bias = self.add_variable( name='bias', shape=(self.filters,), initializer=self.bias_initializer, regularizer=self.bias_regularizer, trainable=True, dtype=self.dtype) else: self.bias = None self.input_spec = base.InputSpec( ndim=self.rank + 2, axes={channel_axis: input_dim}) self.built = True
def recalculate_output_shapes(output_shapes): """Recalculates the output_shapes after dividing it by num_workers.""" if len(output_shapes) < 1: raise ValueError("Input shape should have at least one dimension.") if (tensor_shape.dimension_value(output_shapes[0]) and tensor_shape.dimension_value(output_shapes[0]) % num_workers != 0): raise errors.InvalidArgumentError( None, None, "First dim of input shape: %d is not divisible by num_workers: %d" % (output_shapes[0], num_workers)) output_dims = [d for d in output_shapes.dims] output_dims[0] = output_dims[0] // num_workers return tensor_shape.TensorShape(output_dims)
def _set_diag_operators(self, diag_update, is_diag_update_positive): """Set attributes self._diag_update and self._diag_operator.""" if diag_update is not None: self._diag_operator = linear_operator_diag.LinearOperatorDiag( self._diag_update, is_positive_definite=is_diag_update_positive) self._diag_inv_operator = linear_operator_diag.LinearOperatorDiag( 1. / self._diag_update, is_positive_definite=is_diag_update_positive) else: if tensor_shape.dimension_value(self.u.shape[-1]) is not None: r = tensor_shape.dimension_value(self.u.shape[-1]) else: r = array_ops.shape(self.u)[-1] self._diag_operator = linear_operator_identity.LinearOperatorIdentity( num_rows=r, dtype=self.dtype) self._diag_inv_operator = self._diag_operator
def crf_log_likelihood(inputs, tag_indices, sequence_lengths, transition_params=None): """Computes the log-likelihood of tag sequences in a CRF. Args: inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials to use as input to the CRF layer. tag_indices: A [batch_size, max_seq_len] matrix of tag indices for which we compute the log-likelihood. sequence_lengths: A [batch_size] vector of true sequence lengths. transition_params: A [num_tags, num_tags] transition matrix, if available. Returns: log_likelihood: A [batch_size] `Tensor` containing the log-likelihood of each example, given the sequence of tag indices. transition_params: A [num_tags, num_tags] transition matrix. This is either provided by the caller or created in this function. """ # Get shape information. num_tags = tensor_shape.dimension_value(inputs.shape[2]) # Get the transition matrix if not provided. if transition_params is None: transition_params = vs.get_variable("transitions", [num_tags, num_tags]) sequence_scores = crf_sequence_score(inputs, tag_indices, sequence_lengths, transition_params) log_norm = crf_log_norm(inputs, sequence_lengths, transition_params) # Normalize the scores to get the log-likelihood per example. log_likelihood = sequence_scores - log_norm return log_likelihood, transition_params
def _create_vars(self, var_list, state): # Construct ordered dictionary for variable dimensions, sorted by name. shape_dict = {} for v in var_list: shape_dict[v.name] = tensor_shape.dimension_value(np.prod(v.get_shape())) self.shape_dict = collections.OrderedDict( sorted(shape_dict.items(), key=lambda t: t[0])) # Assign each variable its location in flat_grad. The locations are based on # the order of sorted names. idx = 0 for v_name, v_dim in self.shape_dict.items(): self.index_dict[v_name] = idx idx += v_dim state.create_non_slot( initial_value=math_ops.cast(0., dtype=var_list[0].dtype.base_dtype), name="global_step") # Buffer for keeping past gradients. window = state.get_hyper("window") grad_buffer_init = array_ops.zeros( [window, idx], dtype=var_list[0].dtype.base_dtype) state.create_non_slot(initial_value=grad_buffer_init, name="grad_buffer") state.create_non_slot( initial_value=array_ops.zeros( (idx,), dtype=var_list[0].dtype.base_dtype), name="moment1") # Flattened gradient that contains gradients for all variables in the model. state.create_non_slot( initial_value=array_ops.zeros( (idx,), dtype=var_list[0].dtype.base_dtype), name="flat_grad")
def rank(self): """The number of dimensions in this shape, or None if unknown.""" inner_ndims = tensor_shape.dimension_value(self._inner_dim_sizes.shape[0]) if inner_ndims is None: return None else: return len(self._partitioned_dim_sizes) + inner_ndims
def testUnknownIndices(self): params = constant_op.constant([[0, 1, 2]]) indices = array_ops.placeholder(dtypes.int32) gather_nd_t = array_ops.gather_nd(params, indices) shape = gather_nd_t.get_shape() self.assertEqual(None, shape.ndims) self.assertEqual(None, tensor_shape.dimension_value(shape[0]))
def row_splits_to_segment_ids(splits, name=None): """Generates the segmentation corresponding to a RaggedTensor `row_splits`. Returns an integer vector `segment_ids`, where `segment_ids[i] == j` if `splits[j] <= i < splits[j+1]`. Example: ```python >>> ragged.row_splits_to_segment_ids([0, 3, 3, 5, 6, 9]).eval() [ 0 0 0 2 2 3 4 4 4 ] ``` Args: splits: A sorted 1-D int64 Tensor. `splits[0]` must be zero. name: A name prefix for the returned tensor (optional). Returns: A sorted 1-D int64 Tensor, with `shape=[splits[-1]]` Raises: ValueError: If `splits` is invalid. """ with ops.name_scope(name, "RaggedSplitsToSegmentIds", [splits]) as name: splits = ops.convert_to_tensor(splits, dtype=dtypes.int64, name="splits") splits.shape.assert_has_rank(1) if tensor_shape.dimension_value(splits.shape[0]) == 0: raise ValueError("Invalid row_splits: []") row_lengths = splits[1:] - splits[:-1] nrows = array_ops.shape(splits, out_type=dtypes.int64)[-1] - 1 indices = math_ops.range(nrows) return ragged_util.repeat(indices, repeats=row_lengths, axis=0)
def _forward_event_shape(self, input_shape): batch_shape, d = (input_shape[:-1], tensor_shape.dimension_value(input_shape[-1])) if d is None: n = None else: n = vector_size_to_square_matrix_size(d, self.validate_args) return batch_shape.concatenate([n, n])
def _get_final_shape(qs): """Helper to build `TensorShape`.""" bs = dist.batch_shape.with_rank_at_least(1) num_components = tensor_shape.dimension_value(bs[-1]) if num_components is not None: num_components += 1 tail = tensor_shape.TensorShape([num_components, qs]) return bs[:-1].concatenate(tail)
def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) input_shape = input_shape.with_rank_at_least(2) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError( 'The innermost dimension of input_shape must be defined, but saw: %s' % input_shape) return input_shape[:-1].concatenate(self.units)
def _batch_shape_tensor(self): with ops.control_dependencies(self._runtime_assertions): batch_shape = self.distribution.batch_shape_tensor() dim0 = tensor_shape.dimension_value( batch_shape.shape.with_rank_at_least(1)[0]) batch_ndims = (dim0 if dim0 is not None else array_ops.shape(batch_shape)[0]) return batch_shape[:batch_ndims - self.reinterpreted_batch_ndims]
def crf_log_norm(inputs, sequence_lengths, transition_params): """Computes the normalization for a CRF. Args: inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials to use as input to the CRF layer. sequence_lengths: A [batch_size] vector of true sequence lengths. transition_params: A [num_tags, num_tags] transition matrix. Returns: log_norm: A [batch_size] vector of normalizers for a CRF. """ # Split up the first and rest of the inputs in preparation for the forward # algorithm. first_input = array_ops.slice(inputs, [0, 0, 0], [-1, 1, -1]) first_input = array_ops.squeeze(first_input, [1]) # If max_seq_len is 1, we skip the algorithm and simply reduce_logsumexp over # the "initial state" (the unary potentials). def _single_seq_fn(): log_norm = math_ops.reduce_logsumexp(first_input, [1]) # Mask `log_norm` of the sequences with length <= zero. log_norm = array_ops.where(math_ops.less_equal(sequence_lengths, 0), array_ops.zeros_like(log_norm), log_norm) return log_norm def _multi_seq_fn(): """Forward computation of alpha values.""" rest_of_input = array_ops.slice(inputs, [0, 1, 0], [-1, -1, -1]) # Compute the alpha values in the forward algorithm in order to get the # partition function. forward_cell = CrfForwardRnnCell(transition_params) # Sequence length is not allowed to be less than zero. sequence_lengths_less_one = math_ops.maximum( constant_op.constant(0, dtype=sequence_lengths.dtype), sequence_lengths - 1) _, alphas = rnn.dynamic_rnn( cell=forward_cell, inputs=rest_of_input, sequence_length=sequence_lengths_less_one, initial_state=first_input, dtype=dtypes.float32) log_norm = math_ops.reduce_logsumexp(alphas, [1]) # Mask `log_norm` of the sequences with length <= zero. log_norm = array_ops.where(math_ops.less_equal(sequence_lengths, 0), array_ops.zeros_like(log_norm), log_norm) return log_norm return utils.smart_cond( pred=math_ops.equal( tensor_shape.dimension_value( inputs.shape[1]) or array_ops.shape(inputs)[1], 1), true_fn=_single_seq_fn, false_fn=_multi_seq_fn)
def pad_if_necessary(t, name, last_dim_padding): n = tensor_shape.dimension_value(t.shape[-1]) if not n or n == m: return t if n == m - 1: paddings = ([[0, 0] for _ in range(len(t.shape) - 1)] + [last_dim_padding]) return array_ops.pad(t, paddings) raise ValueError('Expected {} to be have length {} or {}, got {}.'.format( name, m, m - 1, n))
def __init__(self, transition_params): """Initialize the CrfForwardRnnCell. Args: transition_params: A [num_tags, num_tags] matrix of binary potentials. This matrix is expanded into a [1, num_tags, num_tags] in preparation for the broadcast summation occurring within the cell. """ self._transition_params = array_ops.expand_dims(transition_params, 0) self._num_tags = tensor_shape.dimension_value(transition_params.shape[0])
def _cache_input_depth(self, x): if self._input_depth is None: self._input_depth = tensor_shape.dimension_value( x.shape.with_rank_at_least(1)[-1]) if self._input_depth is None: raise NotImplementedError( "Rightmost dimension must be known prior to graph execution.") if self._num_masked >= self._input_depth: raise ValueError( "Number of masked units must be smaller than the event size.")
def make_padded_shapes(shapes, none_filler=None): padded = [] for shape in nest.flatten(shapes): shape = tensor_shape.TensorShape(shape) shape = [ none_filler if tensor_shape.dimension_value(d) is None else d for d in shape ] padded.append(shape) return nest.pack_sequence_as(shapes, padded)
def _shape_invariant_to_components(self, shape=None): if shape is None: shape = self.dense_shape.shape # TODO(edloper): What if shape.dense_shape.shape.ndims is None? if shape.ndims != 1: raise ValueError("Shape invariant for SparseTensor must have the form " "TensorShape([r]), got %r" % shape) rank = tensor_shape.dimension_value(shape[0]) return [tensor_shape.TensorShape([None, rank]), # indices tensor_shape.TensorShape([None]), # values tensor_shape.TensorShape([rank])] # dense_shape
def testSlice(self): known = tensor_shape.TensorShape([0, 1, 2, 3, 4]) self.assertEqual(tensor_shape.Dimension(2), known[2]) tensor_shape.TensorShape([1, 2, 3]).assert_is_compatible_with(known[1:4]) unknown = tensor_shape.TensorShape(None) self.assertEqual( tensor_shape.Dimension(None).value, tensor_shape.dimension_value(unknown[2])) tensor_shape.TensorShape( [None, None, None]).assert_is_compatible_with(unknown[1:4])
def _static_check_for_same_dimensions(operators): """ValueError if operators determined to have different dimensions.""" if len(operators) < 2: return domain_dimensions = [ (op.name, tensor_shape.dimension_value(op.domain_dimension)) for op in operators if tensor_shape.dimension_value(op.domain_dimension) is not None] if len(set(value for name, value in domain_dimensions)) > 1: raise ValueError("Operators must have the same domain dimension. Found: %s" % domain_dimensions) range_dimensions = [ (op.name, tensor_shape.dimension_value(op.range_dimension)) for op in operators if tensor_shape.dimension_value(op.range_dimension) is not None] if len(set(value for name, value in range_dimensions)) > 1: raise ValueError("Operators must have the same range dimension. Found: %s" % range_dimensions)
def _project_input(self, inputs, c_prev, m_prev, with_c): """Fills in c_prev and m_prev with projected input, for input dimensions. Args: inputs: inputs tensor c_prev: cell value m_prev: previous output with_c: boolean; whether to include project_c. Raises: ValueError: if len(self._config.input) != len(inputs) """ conf = self._config if (inputs is not None and tensor_shape.dimension_value(inputs.shape.with_rank(2)[1]) > 0 and conf.inputs): if isinstance(inputs, tuple): if len(conf.inputs) != len(inputs): raise ValueError('Expect inputs as a tuple of {} ' 'tensors'.format(len(conf.inputs))) input_splits = inputs else: input_splits = array_ops.split( value=inputs, num_or_size_splits=len(conf.inputs), axis=1) input_sz = tensor_shape.dimension_value( input_splits[0].shape.with_rank(2)[1]) for i, j in enumerate(conf.inputs): input_project_m = vs.get_variable( 'project_m_{}'.format(j), [input_sz, conf.num_units], dtype=inputs.dtype) m_prev[j] = math_ops.matmul(input_splits[i], input_project_m) if with_c: input_project_c = vs.get_variable( 'project_c_{}'.format(j), [input_sz, conf.num_units], dtype=inputs.dtype) c_prev[j] = math_ops.matmul(input_splits[i], input_project_c)
def interpolate_scale(grid, scale): """Helper which interpolates between two scales.""" if len(scale) != 2: raise NotImplementedError("Currently only bimixtures are supported; " "len(scale)={} is not 2.".format(len(scale))) deg = tensor_shape.dimension_value(grid.shape.with_rank_at_least(1)[-1]) if deg is None: raise ValueError("Num quadrature grid points must be known prior " "to graph execution.") with ops.name_scope("interpolate_scale", values=[grid]): return [linop_add_lib.add_operators([ linop_scale(grid[..., k, q], s) for k, s in enumerate(scale) ])[0] for q in range(deg)]
def _step(self, actions): """Returns a TensorFlow op to step the environment. Args: actions: A Tensor, or a nested dict, list or tuple of Tensors corresponding to `action_spec()`. Returns: A `TimeStep` tuple of: step_type: A scalar int32 tensor representing the `StepType` value. reward: A float32 tensor representing the reward at this time_step. discount: A scalar float32 tensor representing the discount [0, 1]. observation: A Tensor, or a nested dict, list or tuple of Tensors corresponding to `observation_spec()`. Raises: ValueError: If any of the actions are scalars or their major axis is known and is not equal to `self.batch_size`. """ def _step_py(*flattened_actions): with _check_not_called_concurrently(self._lock): packed = tf.nest.pack_sequence_as( structure=self.action_spec(), flat_sequence=flattened_actions) self._time_step = self._env.step(packed) return tf.nest.flatten(self._time_step) def _isolated_step_py(*flattened_actions): return self._execute(_step_py, *flattened_actions) with tf.name_scope('step'): flat_actions = [tf.identity(x) for x in tf.nest.flatten(actions)] if self._check_dims: for action in flat_actions: dim_value = tensor_shape.dimension_value(action.shape[0]) if (action.shape.rank == 0 or (dim_value is not None and dim_value != self.batch_size)): raise ValueError( 'Expected actions whose major dimension is batch_size (%d), ' 'but saw action with shape %s:\n %s' % (self.batch_size, action.shape, action)) outputs = tf.numpy_function(_isolated_step_py, flat_actions, self._time_step_dtypes, name='step_py_func') return self._time_step_from_numpy_function_outputs(outputs)
def crf_multitag_sequence_score(inputs, tag_bitmap, sequence_lengths, transition_params): """Computes the unnormalized score of all tag sequences matching tag_bitmap. tag_bitmap enables more than one tag to be considered correct at each time step. This is useful when an observed output at a given time step is consistent with more than one tag, and thus the log likelihood of that observation must take into account all possible consistent tags. Using one-hot vectors in tag_bitmap gives results identical to crf_sequence_score. Args: inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials to use as input to the CRF layer. tag_bitmap: A [batch_size, max_seq_len, num_tags] boolean tensor representing all active tags at each index for which to calculate the unnormalized score. sequence_lengths: A [batch_size] vector of true sequence lengths. transition_params: A [num_tags, num_tags] transition matrix. Returns: sequence_scores: A [batch_size] vector of unnormalized sequence scores. """ # If max_seq_len is 1, we skip the score calculation and simply gather the # unary potentials of all active tags. def _single_seq_fn(): filtered_inputs = array_ops.where( tag_bitmap, inputs, array_ops.fill(array_ops.shape(inputs), float("-inf"))) return math_ops.reduce_logsumexp(filtered_inputs, axis=[1, 2], keepdims=False) def _multi_seq_fn(): # Compute the logsumexp of all scores of sequences matching the given tags. filtered_inputs = array_ops.where( tag_bitmap, inputs, array_ops.fill(array_ops.shape(inputs), float("-inf"))) return crf_log_norm(inputs=filtered_inputs, sequence_lengths=sequence_lengths, transition_params=transition_params) return utils.smart_cond(pred=math_ops.equal( tensor_shape.dimension_value(inputs.shape[1]) or array_ops.shape(inputs)[1], 1), true_fn=_single_seq_fn, false_fn=_multi_seq_fn)
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) last_dim = tensor_shape.dimension_value(input_shape[-1]) self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim}) self.kernel = self.add_weight( 'kernel', shape=[last_dim, self.units], initializer=self.kernel_initializer, dtype=self.dtype, trainable=True) def call(self, input): self.add_loss(self.rate * tf.reduce_sum(inputs)) rank = inputs.shape.rank if rank is not None and rank > 2: outputs = standard_ops.tensordot(inputs, self.kernel, [[rank - 1], [0]]) if not context.executing_eagerly(): shape = inputs.shape.as_list() output_shape = shape[:-1] + [self.units] outputs.set_shape(output_shape) else: inputs = math_ops.cast(inputs, self._compute_dtype) if K.is_sparse(inputs): outputs = sparse_ops.sparse_tensor_dense_matmul(inputs, self.kernel) else: outputs = gen_math_ops.mat_mul(inputs, self.kernel) if self.activation is not None: return self.activation(outputs) return outputs def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) input_shape = input_shape.with_rank_at_least(2) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError( 'The innermost dimension of input_shape must be defined, but saw: %s' % input_shape) return input_shape[:-1].concatenate(self.units) def get_config(self): config = { 'units': self.units, 'activation': activations.serialize(self.activation), 'kernel_initializer': initializers.serialize(self.kernel_initializer), } base_config = super(SNNDense, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def _to_dense(self): """Generic and often inefficient implementation. Override often.""" logging.warn("Using (possibly slow) default implementation of to_dense." " Converts by self.matmul(identity).") if self.batch_shape.is_fully_defined(): batch_shape = self.batch_shape else: batch_shape = self.batch_shape_tensor() dim_value = tensor_shape.dimension_value(self.domain_dimension) if dim_value is not None: n = dim_value else: n = self.domain_dimension_tensor() eye = linalg_ops.eye(num_rows=n, batch_shape=batch_shape, dtype=self.dtype) return self.matmul(eye)
def _add_zero_flow_controls_at_boundary(control_point_locations, control_point_flows, image_height, image_width, boundary_points_per_edge): """Add control points for zero-flow boundary conditions. Augment the set of control points with extra points on the boundary of the image that have zero flow. Args: control_point_locations: input control points control_point_flows: their flows image_height: image height image_width: image width boundary_points_per_edge: number of points to add in the middle of each edge (not including the corners). The total number of points added is 4 + 4*(boundary_points_per_edge). Returns: merged_control_point_locations: augmented set of control point locations merged_control_point_flows: augmented set of control point flows """ batch_size = tensor_shape.dimension_value(control_point_locations.shape[0]) boundary_point_locations = _get_boundary_locations( image_height, image_width, boundary_points_per_edge) boundary_point_flows = np.zeros([boundary_point_locations.shape[0], 2]) type_to_use = control_point_locations.dtype boundary_point_locations = constant_op.constant(_expand_to_minibatch( boundary_point_locations, batch_size), dtype=type_to_use) boundary_point_flows = constant_op.constant(_expand_to_minibatch( boundary_point_flows, batch_size), dtype=type_to_use) merged_control_point_locations = array_ops.concat( [control_point_locations, boundary_point_locations], 1) merged_control_point_flows = array_ops.concat( [control_point_flows, boundary_point_flows], 1) return merged_control_point_locations, merged_control_point_flows
def _from_compatible_tensor_list(self, flat_value): if self._ragged_rank <= 0: raise ValueError( "ragged_rank must be greater than zero. Found ragged_rank: %d" % self._ragged_rank) result = ragged_tensor.RaggedTensor._from_variant( flat_value[0], dtype=self._dtype, output_ragged_rank=self._ragged_rank) if self._shape.ndims is not None: outer_dim = tensor_shape.dimension_value(self._shape[0]) if outer_dim is not None: result.row_splits.set_shape([outer_dim + 1]) result.flat_values.set_shape( tensor_shape.TensorShape([None]).concatenate( self._shape[1 + self._ragged_rank:])) return result
def _to_dense(self): """Generic and often inefficient implementation. Override often.""" if self.batch_shape.is_fully_defined(): batch_shape = self.batch_shape else: batch_shape = self.batch_shape_tensor() dim_value = tensor_shape.dimension_value(self.domain_dimension) if dim_value is not None: n = dim_value else: n = self.domain_dimension_tensor() eye = linalg_ops.eye(num_rows=n, batch_shape=batch_shape, dtype=self.dtype) return self.matmul(eye)
def testPartiallyDefinedShape(self): s = tensor_shape.TensorShape([tensor_shape.Dimension( 3), tensor_shape.Dimension(None), tensor_shape.Dimension(7)]) # pylint: disable=g-error-prone-assert-raises with self.assertRaisesRegex(ValueError, "Shape .+ is not fully defined"): s.assert_is_fully_defined() # pylint: enable=g-error-prone-assert-raises self.assertEqual(s.rank, 3) self.assertLen(s, 3) self.assertTrue(s) s.assert_has_rank(3) self.assertEqual(tensor_shape.Dimension(3), s[0]) self.assertEqual(tensor_shape.Dimension(None).value, s.dims[1].value) self.assertEqual(tensor_shape.Dimension(7), s.dims[2]) s.assert_same_rank([6, 3, 7]) for d1, d2 in zip(s, [3, None, 7]): assert tensor_shape.dimension_value(d1) == d2
def _multi_seq_fn(): """Decoding of highest scoring sequence.""" # For simplicity, in shape comments, denote: # 'batch_size' by 'B', 'max_seq_len' by 'T' , 'num_tags' by 'O' (output). num_tags = tensor_shape.dimension_value(potentials.shape[2]) # Computes forward decoding. Get last score and backpointers. crf_fwd_cell = CrfDecodeForwardRnnCell(transition_params) initial_state = array_ops.slice(potentials, [0, 0, 0], [-1, 1, -1]) initial_state = array_ops.squeeze(initial_state, axis=[1]) # [B, O] inputs = array_ops.slice(potentials, [0, 1, 0], [-1, -1, -1]) # [B, T-1, O] # Sequence length is not allowed to be less than zero. sequence_length_less_one = math_ops.maximum( constant_op.constant(0, dtype=sequence_length.dtype), sequence_length - 1) backpointers, last_score = rnn.dynamic_rnn( # [B, T - 1, O], [B, O] crf_fwd_cell, inputs=inputs, sequence_length=sequence_length_less_one, initial_state=initial_state, time_major=False, dtype=dtypes.int32) backpointers = gen_array_ops.reverse_sequence( # [B, T - 1, O] backpointers, sequence_length_less_one, seq_dim=1) # Computes backward decoding. Extract tag indices from backpointers. crf_bwd_cell = CrfDecodeBackwardRnnCell(num_tags) initial_state = math_ops.cast(math_ops.argmax(last_score, axis=1), # [B] dtype=dtypes.int32) initial_state = array_ops.expand_dims(initial_state, axis=-1) # [B, 1] decode_tags, _ = rnn.dynamic_rnn( # [B, T - 1, 1] crf_bwd_cell, inputs=backpointers, sequence_length=sequence_length_less_one, initial_state=initial_state, time_major=False, dtype=dtypes.int32) decode_tags = array_ops.squeeze(decode_tags, axis=[2]) # [B, T - 1] decode_tags = array_ops.concat([initial_state, decode_tags], # [B, T] axis=1) decode_tags = gen_array_ops.reverse_sequence( # [B, T] decode_tags, sequence_length, seq_dim=1) best_score = math_ops.reduce_max(last_score, axis=1) # [B] return decode_tags, best_score
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) last_dim = tensor_shape.dimension_value(input_shape[-1]) if last_dim is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim}) shape = len(input_shape.as_list()) * [1] shape[-1] = last_dim self.kernel = self.add_weight(shape=shape, initializer=self.kernel_initializer, name='kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint, dtype=self.dtype, trainable=True) super(LearnableNoise, self).build(input_shape) self.built = True
def testPartiallyDefinedShape(self): s = tensor_shape.TensorShape([ tensor_shape.Dimension(3), tensor_shape.Dimension(None), tensor_shape.Dimension(7) ]) with self.assertRaises(ValueError): s.assert_is_fully_defined() self.assertEqual(3, s.rank) self.assertEqual(3, len(s)) self.assertTrue(s) s.assert_has_rank(3) self.assertEqual(tensor_shape.Dimension(3), s[0]) self.assertEqual(tensor_shape.Dimension(None).value, s.dims[1].value) self.assertEqual(tensor_shape.Dimension(7), s.dims[2]) s.assert_same_rank([6, 3, 7]) for d1, d2 in zip(s, [3, None, 7]): assert tensor_shape.dimension_value(d1) == d2
def build(self, input_shape): """GRU cell.""" input_size = tensor_shape.dimension_value(input_shape[1]) if input_size is None: raise ValueError("Expecting input_size to be set.") self._gate_kernel = self.add_variable( "gates/kernel", [input_size + self._cell_size, self._cell_size * 2]) self._gate_bias = self.add_variable( "gates/bias", [self._cell_size * 2], initializer=init_ops.constant_initializer(1.0)) self._candidate_kernel = self.add_variable( "candidate/kernel", [input_size + self._cell_size, self._cell_size]) self._candidate_bias = self.add_variable( "candidate/bias", [self._cell_size], initializer=init_ops.constant_initializer(0.0)) self.built = True
def build(self, input_shape): # Check if the input size exist. input_size = tensor_shape.dimension_value(input_shape[1]) if input_size is None: raise ValueError("Expecting input_size to be set.") self._gate_kernel = self.add_variable( "w_ru", [input_size + self._cell_size, self._cell_size * 2]) self._gate_bias = self.add_variable( "b_ru", [self._cell_size * 2], initializer=init_ops.constant_initializer(1.0)) self._candidate_kernel = self.add_variable( "w_c", [input_size + self._cell_size, self._cell_size]) self._candidate_bias = self.add_variable( "b_c", [self._cell_size], initializer=init_ops.constant_initializer(0.0)) self.built = True
def _maybe_mask(m, seq_len_mask): rank = m.get_shape().ndims rank = rank if rank is not None else array_ops.rank(m) extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32) m_batch_size = tensor_shape.dimension_value( m.shape[0]) or array_ops.shape(m)[0] if memory_sequence_length is not None: message = ("memory_sequence_length and memory tensor batch sizes do not " "match.") with ops.control_dependencies([ check_ops.assert_equal( seq_len_batch_size, m_batch_size, message=message)]): seq_len_mask = array_ops.reshape( seq_len_mask, array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0)) return m * seq_len_mask else: return m
def build(self, input_shape): self.last_dim = tensor_shape.dimension_value(input_shape[-1]) if self.weight_reparam: self.kernel_w = self.add_weight( "w", shape=[self.last_dim, self.d_out], initializer=tf.keras.initializers.he_normal(seed=None)) self.kernel_g = self.add_weight("g", shape=[1, self.d_out], initializer=tf.ones_initializer()) else: self.kernel = self.add_weight( "w", shape=[self.last_dim, self.d_out], initializer=tf.initializers.GlorotNormal()) self.bias = self.add_weight("b", shape=[self.d_out], initializer=tf.zeros_initializer())
def testFullyDefinedShape(self): s = tensor_shape.TensorShape([tensor_shape.Dimension( 3), tensor_shape.Dimension(4), tensor_shape.Dimension(7)]) s.assert_is_fully_defined() self.assertEqual(s.rank, 3) self.assertLen(s, 3) self.assertTrue(s) s.assert_has_rank(3) self.assertEqual([tensor_shape.Dimension(3), tensor_shape.Dimension(4), tensor_shape.Dimension(7)], s.dims) self.assertEqual(tensor_shape.Dimension(3), s[0]) self.assertEqual(tensor_shape.Dimension(4), s[1]) self.assertEqual(tensor_shape.Dimension(7), s[2]) self.assertEqual([3, 4, 7], s.as_list()) s.assert_is_compatible_with([3, 4, 7]) s.assert_same_rank([6, 3, 7]) for d1, d2 in zip(s, [3, 4, 7]): assert tensor_shape.dimension_value(d1) == d2
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) last_dim = tensor_shape.dimension_value(input_shape[-1]) if last_dim is None: raise ValueError('The last dimension of the inputs to `PBPLayer` ' 'should be defined. Found `None`.') self.input_spec = tf.keras.layers.InputSpec(min_ndim=2, axes={-1: last_dim}) self.inv_sqrtV1 = tf.cast(1.0 / tf.math.sqrt(1.0 * last_dim + 1), dtype=self.dtype) self.inv_V1 = tf.math.square(self.inv_sqrtV1) over_gamma = ReciprocalGammaInitializer(6.0, 6.0) self.kernel_m = self.add_weight( "kernel_mean", shape=[last_dim, self.units], initializer=tf.keras.initializers.HeNormal(), dtype=self.dtype, trainable=True) self.kernel_v = self.add_weight("kernel_variance", shape=[last_dim, self.units], initializer=over_gamma, dtype=self.dtype, trainable=True) self.bias_m = self.add_weight( "bias_mean", shape=[ self.units, ], initializer=tf.keras.initializers.HeNormal(), dtype=self.dtype, trainable=True) self.bias_v = self.add_weight("bias_variance", shape=[ self.units, ], initializer=over_gamma, dtype=self.dtype, trainable=True) self.Normal = tfp.distributions.Normal( loc=tf.constant(0.0, dtype=self.dtype), scale=tf.constant(1.0, dtype=self.dtype)) self.built = True
def crf_sequence_score(inputs, tag_indices, sequence_lengths, transition_params): """Computes the unnormalized score for a tag sequence. Args: inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials to use as input to the CRF layer. tag_indices: A [batch_size, max_seq_len] matrix of tag indices for which we compute the unnormalized score. sequence_lengths: A [batch_size] vector of true sequence lengths. transition_params: A [num_tags, num_tags] transition matrix. Returns: sequence_scores: A [batch_size] vector of unnormalized sequence scores. """ # If max_seq_len is 1, we skip the score calculation and simply gather the # unary potentials of the single tag. def _single_seq_fn(): batch_size = array_ops.shape(inputs, out_type=tag_indices.dtype)[0] example_inds = array_ops.reshape( math_ops.range(batch_size, dtype=tag_indices.dtype), [-1, 1]) sequence_scores = array_ops.gather_nd( array_ops.squeeze(inputs, [1]), array_ops.concat([example_inds, tag_indices], axis=1)) sequence_scores = array_ops.where(math_ops.less_equal(sequence_lengths, 0), array_ops.zeros_like(sequence_scores), sequence_scores) return sequence_scores def _multi_seq_fn(): # Compute the scores of the given tag sequence. unary_scores = crf_unary_score(tag_indices, sequence_lengths, inputs) binary_scores = crf_binary_score(tag_indices, sequence_lengths, transition_params) sequence_scores = unary_scores + binary_scores return sequence_scores return utils.smart_cond( pred=math_ops.equal( tensor_shape.dimension_value( inputs.shape[1]) or array_ops.shape(inputs)[1], 1), true_fn=_single_seq_fn, false_fn=_multi_seq_fn)
def build(self, input_shape): print(input_shape) print(self.units) input_len = tensor_shape.dimension_value(input_shape[-1]) self.kernel = self.add_weight('kernel', shape=[input_len, self.units], initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint, dtype=self.dtype, trainable=True) self.bias = self.add_weight('bias', shape=[self.units, input_len], initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint, dtype=self.dtype, trainable=True) self.built = True
def build(self, input_shape): # super(IndependentDense, self).build(input_shape) self.last_dim = tensor_shape.dimension_value(input_shape[-1]) self.input_spec = InputSpec(min_ndim=2, axes={-1: self.last_dim}) self.mu_init = tf.random_uniform_initializer(-((3 / self.last_dim) ** 0.5), (3 / self.last_dim) ** 0.5) self.sigma_init = tf.constant_initializer(0.017) self.w_mu = self.add_weight("w_mu", [self.last_dim, self.units], initializer=self.mu_init, dtype=self.dtype, trainable=True) self.w_sigma = self.add_weight("w_sigma", [self.last_dim, self.units], initializer=self.sigma_init, dtype=self.dtype, trainable=True) if self.use_bias: self.b_mu = self.add_weight("b_mu", [self.units, ], initializer=self.mu_init, dtype=self.dtype, trainable=True) self.b_sigma = self.add_weight("b_sigma", [self.units, ], initializer=self.sigma_init, dtype=self.dtype, trainable=True) self.built = True
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) for i in range(1, len(input_shape)): if tensor_shape.dimension_value(input_shape[i]) is None: raise ValueError( 'The input shape [1:] should be defined, but found element `None`.' ) if self.use_kernel: varName = 'kernel' elif self.use_bias: varName = 'bias' get_in = input_shape.as_list()[1:] self.get_var = self.add_weight(varName, shape=get_in, initializer=self.var_initializer, regularizer=self.var_regularizer, constraint=self.var_constraint, dtype=self.dtype, trainable=True) super(Ghost, self).build(input_shape)
def __init__(self, transition_params): """Initialize the CrfDecodeForwardRnnCell. Args: transition_params: 1. A [num_tags, num_tags] matrix of binary potentials. This matrix is expanded into a [1, num_tags, num_tags] in preparation for the broadcast summation occurring within the cell. 2. or a [Batch, num_tags, num_tags] """ if K.ndim(transition_params) == 2: transition_params = array_ops.expand_dims(transition_params, 0) assert K.ndim(transition_params ) == 3, "transition_params should have 2 or 3 dims" self._transition_params = transition_params self._num_tags = tensor_shape.dimension_value( transition_params.shape[1])
def row_splits_to_segment_ids(splits, name=None, out_type=None): """Generates the segmentation corresponding to a RaggedTensor `row_splits`. Returns an integer vector `segment_ids`, where `segment_ids[i] == j` if `splits[j] <= i < splits[j+1]`. Example: ```python >>> ragged.row_splits_to_segment_ids([0, 3, 3, 5, 6, 9]).eval() [ 0 0 0 2 2 3 4 4 4 ] ``` Args: splits: A sorted 1-D integer Tensor. `splits[0]` must be zero. name: A name prefix for the returned tensor (optional). out_type: The dtype for the return value. Defaults to `splits.dtype`, or `tf.int64` if `splits` does not have a dtype. Returns: A sorted 1-D integer Tensor, with `shape=[splits[-1]]` Raises: ValueError: If `splits` is invalid. """ with ops.name_scope(name, "RaggedSplitsToSegmentIds", [splits]) as name: splits = ops.convert_to_tensor(splits, name="splits", preferred_dtype=dtypes.int64) if splits.dtype not in (dtypes.int32, dtypes.int64): raise ValueError("splits must have dtype int32 or int64") splits.shape.assert_has_rank(1) if tensor_shape.dimension_value(splits.shape[0]) == 0: raise ValueError("Invalid row_splits: []") if out_type is None: out_type = splits.dtype else: out_type = dtypes.as_dtype(out_type) row_lengths = splits[1:] - splits[:-1] nrows = array_ops.shape(splits, out_type=out_type)[-1] - 1 indices = math_ops.range(nrows) return ragged_util.repeat(indices, repeats=row_lengths, axis=0)
def range_dimension_tensor(self, name="range_dimension_tensor"): """Dimension (in the sense of vector spaces) of the range of this operator. Determined at runtime. If this operator acts like the batch matrix `A` with `A.shape = [B1,...,Bb, M, N]`, then this returns `M`. Args: name: A name for this `Op`. Returns: `int32` `Tensor` """ # Derived classes get this "for free" once .shape() is implemented. with self._name_scope(name): # Prefer to use statically defined shape if available. dim_value = tensor_shape.dimension_value(self.range_dimension) if dim_value is not None: return ops.convert_to_tensor(dim_value) else: return self.shape_tensor()[-2]
def testBuildRingGatherPassStructure(self): # 1 worker, 1 device input_tensors, device_names = self._buildInput(1, 1) pred_by_c_d, rank_by_c_d = ar._ring_permutations(1, 1, [0]) output_tensors = ar._build_ring_gather(input_tensors, device_names, 1, pred_by_c_d, rank_by_c_d, math_ops.add) self.assertEqual(output_tensors, input_tensors) # 1 worker, 4 devices, 2 subchunks input_tensors, device_names = self._buildInput(1, 4) pred_by_c_d, rank_by_c_d = ar._ring_permutations(1, 2, [0, 1, 2, 3]) output_tensors, pad_len = ar._build_ring_gather( input_tensors, device_names, 2, pred_by_c_d, rank_by_c_d, math_ops.add) self.assertEqual(0, pad_len) # same number outputs as inputs self.assertEqual(len(output_tensors), len(input_tensors)) num_chunks = 2 * len(input_tensors) tlen = tensor_shape.dimension_value(input_tensors[0].shape[0]) for otl in output_tensors: self.assertEqual(len(otl), num_chunks) for ot in otl: self.assertEqual(ot.shape, [tlen/num_chunks])
def get_state_transition_noise_covariance( self, minimum_initial_variance=1e-5): # Most state space models use only an explicit observation noise term to # model deviations from expectations, and so a low initial transition noise # parameter is helpful there. Since deviations from expectations are also # modeled as transition noise in VARMA, we set its initial value based on a # slight over-estimate empirical observation noise. if self._input_statistics is not None: feature_variance = self._scale_variance( self._input_statistics.series_start_moments.variance) initial_transition_noise_scale = math_ops.log( math_ops.maximum( math_ops.reduce_mean(feature_variance), minimum_initial_variance)) else: initial_transition_noise_scale = 0. state_noise_transform = ops.convert_to_tensor( self.get_noise_transform(), dtype=self.dtype) state_noise_dimension = tensor_shape.dimension_value( state_noise_transform.shape[1]) return math_utils.variable_covariance_matrix( state_noise_dimension, "state_transition_noise", dtype=self.dtype, initial_overall_scale_log=initial_transition_noise_scale)
def _fn(x): """MADE parameterized via `masked_autoregressive_default_template`.""" # TODO(b/67594795): Better support of dynamic shape. input_depth = tensor_shape.dimension_value( x.shape.with_rank_at_least(1)[-1]) if input_depth is None: raise NotImplementedError( "Rightmost dimension must be known prior to graph execution." ) input_shape = (np.int32(x.shape.as_list()) if x.shape.is_fully_defined() else array_ops.shape(x)) for i, units in enumerate(hidden_layers): x = masked_dense(inputs=x, units=units, num_blocks=input_depth, exclusive=True if i == 0 else False, activation=activation, *args, **kwargs) x = masked_dense(inputs=x, units=(1 if shift_only else 2) * input_depth, num_blocks=input_depth, activation=None, *args, **kwargs) if shift_only: x = array_ops.reshape(x, shape=input_shape) return x, None x = array_ops.reshape(x, shape=array_ops.concat([input_shape, [2]], axis=0)) shift, log_scale = array_ops.unstack(x, num=2, axis=-1) which_clip = (math_ops.clip_by_value if log_scale_clip_gradient else _clip_by_value_preserve_grad) log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip) return shift, log_scale
def __init__(self, input_shape, units, gain=2.0, use_weight_scaling=True, use_bias=True, activation=None, **kwargs): warnings.warn( f"CustomDense is deprecated. Use tf.keras.layers.Dense wrapped in WeightScalingWrapper instead.", DeprecationWarning, 2) if 'bias_initializer' in kwargs: logging.warning( f"{self.__class__.__name__} ignores bias_initializer={kwargs['bias_initializer']}" ) del kwargs['bias_initializer'] if 'kernel_initializer' in kwargs: logging.warning( f"{self.__class__.__name__} ignores kernel_initializer={kwargs['kernel_initializer']}" ) del kwargs['kernel_initializer'] super(CustomDense, self).__init__(units=units, use_bias=use_bias, **kwargs) self.bias_initializer = tf.keras.initializers.zeros() self.gain = gain self.use_weight_scaling = use_weight_scaling self._wrapper_use_bias = use_bias self._wrapper_activation = activations.get(activation) self._argument_input_shape = input_shape # compute kernel shape input_shape = tensor_shape.TensorShape(input_shape) last_dim = tensor_shape.dimension_value(input_shape[-1]) kernel_shape = (last_dim, self.units) self.op_scale, self.kernel_initializer = he_kernel_initializer( kernel_shape, self.gain, self.use_weight_scaling)
def interpolate_loc(grid, loc): """Helper which interpolates between two locs.""" if len(loc) != 2: raise NotImplementedError("Currently only bimixtures are supported; " "len(scale)={} is not 2.".format(len(loc))) deg = tensor_shape.dimension_value(grid.shape.with_rank_at_least(1)[-1]) if deg is None: raise ValueError("Num quadrature grid points must be known prior " "to graph execution.") with ops.name_scope("interpolate_loc", values=[grid, loc]): if loc is None or loc[0] is None and loc[1] is None: return [None]*deg # shape: [B, 1, k, deg] w = grid[..., array_ops.newaxis, :, :] loc = [x[..., array_ops.newaxis] # shape: [B, e, 1] if x is not None else None for x in loc] if loc[0] is None: x = w[..., 1, :] * loc[1] # shape: [B, e, deg] elif loc[1] is None: x = w[..., 0, :] * loc[0] # shape: [B, e, deg] else: delta = loc[0] - loc[1] x = w[..., 0, :] * delta + loc[1] # shape: [B, e, deg] return [x[..., k] for k in range(deg)] # list(shape:[B, e])
def build(self, input_shape): super().build(input_shape) self.build = False self.last_dim = tensor_shape.dimension_value(input_shape[-1]) self.noisy_w = self.add_weight( 'noise_kernel', shape=[self.last_dim, self.units], initializer=tf.random_normal_initializer(0.0, .1), regularizer=self.kernel_regularizer, constraint=self.kernel_constraint, dtype=self.dtype, trainable=True) if self.use_bias: self.noisy_b = self.add_weight( 'noise_bias', shape=[self.units, ], initializer=tf.constant_initializer(self.noise_sigma / (self.units**0.5)), regularizer=self.bias_regularizer, constraint=self.bias_constraint, dtype=self.dtype, trainable=True) else: self.bias = None self.build = True