def build(self, input_shape):
dtype = dtypes.as_dtype(self.dtype or K.floatx())
if not (dtype.is_floating or dtype.is_complex):
raise TypeError('Unable to build `Dense` layer with non-floating point '
'dtype %s' % (dtype,))
input_shape = tensor_shape.TensorShape(input_shape)
if tensor_shape.dimension_value(input_shape[-1]) is None:
raise ValueError('The last dimension of the inputs to `Dense` '
'should be defined. Found `None`.')
last_dim = tensor_shape.dimension_value(input_shape[-1])
self.input_spec = InputSpec(min_ndim=2,
axes={-1: last_dim})
self.kernel = self.add_weight(
'kernel',
shape=[last_dim, self.units],
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
dtype=self.dtype,
trainable=True)
if self.use_bias:
self.bias = self.add_weight(
'bias',
shape=[self.units,],
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
dtype=self.dtype,
trainable=True)
else:
self.bias = None
self.built = True
def _min_matrix_dim(self):
"""Minimum of domain/range dimension, if statically available, else None."""
domain_dim = tensor_shape.dimension_value(self.domain_dimension)
range_dim = tensor_shape.dimension_value(self.range_dimension)
if domain_dim is None or range_dim is None:
return None
return min(domain_dim, range_dim)
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if tensor_shape.dimension_value(input_shape[-1]) is None:
raise ValueError('The last dimension of the inputs to `Dense` '
'should be defined. Found `None`.')
last_dim = tensor_shape.dimension_value(input_shape[-1])
self.input_spec = InputSpec(min_ndim=2,
axes={-1: last_dim})
self.kernel = self.add_weight(
'kernel',
shape=[last_dim, self.units],
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
dtype=self.dtype,
trainable=True)
if self.use_bias:
self.bias = self.add_weight(
'bias',
shape=[self.units,],
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
dtype=self.dtype,
trainable=True)
else:
self.bias = None
self.built = True
def maybe_check_quadrature_param(param, name, validate_args):
"""Helper which checks validity of `loc` and `scale` init args."""
with ops.name_scope(name="check_" + name, values=[param]):
assertions = []
if param.shape.ndims is not None:
if param.shape.ndims == 0:
raise ValueError("Mixing params must be a (batch of) vector; "
"{}.rank={} is not at least one.".format(
name, param.shape.ndims))
elif validate_args:
assertions.append(check_ops.assert_rank_at_least(
param, 1,
message=("Mixing params must be a (batch of) vector; "
"{}.rank is not at least one.".format(
name))))
# TODO(jvdillon): Remove once we support k-mixtures.
if param.shape.with_rank_at_least(1)[-1] is not None:
if tensor_shape.dimension_value(param.shape[-1]) != 1:
raise NotImplementedError("Currently only bimixtures are supported; "
"{}.shape[-1]={} is not 1.".format(
name,
tensor_shape.dimension_value(
param.shape[-1])))
elif validate_args:
assertions.append(check_ops.assert_equal(
array_ops.shape(param)[-1], 1,
message=("Currently only bimixtures are supported; "
"{}.shape[-1] is not 1.".format(name))))
if assertions:
return control_flow_ops.with_dependencies(assertions, param)
return param
def convert_legacy_structure(output_types, output_shapes, output_classes):
"""Returns a `Structure` that represents the given legacy structure.
This method provides a way to convert from the existing `Dataset` and
`Iterator` structure-related properties to a `Structure` object. A "legacy"
structure is represented by the `tf.data.Dataset.output_types`,
`tf.data.Dataset.output_shapes`, and `tf.data.Dataset.output_classes`
properties.
TODO(b/110122868): Remove this function once `Structure` is used throughout
`tf.data`.
Args:
output_types: A nested structure of `tf.DType` objects corresponding to
each component of a structured value.
output_shapes: A nested structure of `tf.TensorShape` objects
corresponding to each component a structured value.
output_classes: A nested structure of Python `type` objects corresponding
to each component of a structured value.
Returns:
A `Structure`.
Raises:
TypeError: If a structure cannot be built from the arguments, because one of
the component classes in `output_classes` is not supported.
"""
flat_types = nest.flatten(output_types)
flat_shapes = nest.flatten(output_shapes)
flat_classes = nest.flatten(output_classes)
flat_ret = []
for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes,
flat_classes):
if isinstance(flat_class, Structure):
flat_ret.append(flat_class)
elif issubclass(flat_class, sparse_tensor_lib.SparseTensor):
flat_ret.append(SparseTensorStructure(flat_type, flat_shape))
elif issubclass(flat_class, ops.Tensor):
flat_ret.append(TensorStructure(flat_type, flat_shape))
elif issubclass(flat_class, tensor_array_ops.TensorArray):
# We sneaked the dynamic_size and infer_shape into the legacy shape.
flat_ret.append(
TensorArrayStructure(
flat_type, flat_shape[2:],
dynamic_size=tensor_shape.dimension_value(flat_shape[0]),
infer_shape=tensor_shape.dimension_value(flat_shape[1])))
else:
# NOTE(mrry): Since legacy structures produced by iterators only
# comprise Tensors, SparseTensors, and nests, we do not need to
# support all structure types here.
raise TypeError(
"Could not build a structure for output class %r" % (flat_class,))
ret = nest.pack_sequence_as(output_classes, flat_ret)
if isinstance(ret, Structure):
return ret
else:
return NestedStructure(ret)
def _inverse_event_shape(self, output_shape):
batch_shape, n1, n2 = (output_shape[:-2],
tensor_shape.dimension_value(output_shape[-2]),
tensor_shape.dimension_value(output_shape[-1]))
if n1 is None or n2 is None:
m = None
elif n1 != n2:
raise ValueError("Matrix must be square. (saw [{}, {}])".format(n1, n2))
else:
m = n1 * (n1 + 1) / 2
return batch_shape.concatenate([m])
def _forward(self, x):
if self._unroll_loop:
event_size = tensor_shape.dimension_value(
x.shape.with_rank_at_least(1)[-1])
if event_size is None:
raise ValueError(
"The final dimension of `x` must be known at graph construction "
"time if `unroll_loop=True`. `x.shape: %r`" % x.shape)
y = array_ops.zeros_like(x, name="y0")
for _ in range(event_size):
shift, log_scale = self._shift_and_log_scale_fn(y)
# next_y = scale * x + shift
next_y = x
if log_scale is not None:
next_y *= math_ops.exp(log_scale)
if shift is not None:
next_y += shift
y = next_y
return y
event_size = array_ops.shape(x)[-1]
# If the event size is available at graph construction time, we can inform
# the graph compiler of the maximum number of steps. If not,
# static_event_size will be None, and the maximum_iterations argument will
# have no effect.
static_event_size = tensor_shape.dimension_value(
x.shape.with_rank_at_least(1)[-1])
y0 = array_ops.zeros_like(x, name="y0")
# call the template once to ensure creation
_ = self._shift_and_log_scale_fn(y0)
def _loop_body(index, y0):
"""While-loop body for autoregression calculation."""
# Set caching device to avoid re-getting the tf.Variable for every while
# loop iteration.
with variable_scope_lib.variable_scope(
variable_scope_lib.get_variable_scope()) as vs:
if vs.caching_device is None:
vs.set_caching_device(lambda op: op.device)
shift, log_scale = self._shift_and_log_scale_fn(y0)
y = x
if log_scale is not None:
y *= math_ops.exp(log_scale)
if shift is not None:
y += shift
return index + 1, y
_, y = control_flow_ops.while_loop(
cond=lambda index, _: index < event_size,
body=_loop_body,
loop_vars=(0, y0),
maximum_iterations=static_event_size)
return y
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if tensor_shape.dimension_value(input_shape[-1]) is None:
raise ValueError('The last dimension of the inputs to `Dense` '
'should be defined. Found `None`.')
self.input_spec = base.InputSpec(
min_ndim=2, axes={-1: tensor_shape.dimension_value(input_shape[-1])})
self.kernel = self.add_variable(
'kernel',
shape=[tensor_shape.dimension_value(input_shape[-1]), self.units],
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
dtype=self.dtype,
trainable=True)
self.mask = self.add_variable(
name='mask',
shape=[tensor_shape.dimension_value(input_shape[-1]), self.units],
initializer=init_ops.ones_initializer(),
trainable=False,
dtype=self.dtype)
self.threshold = self.add_variable(
name='threshold',
shape=[],
initializer=init_ops.zeros_initializer(),
trainable=False,
dtype=self.dtype)
# Add masked_weights in the weights namescope so as to make it easier
# for the quantization library to add quant ops.
self.masked_kernel = math_ops.multiply(self.mask, self.kernel,
MASKED_WEIGHT_NAME)
ops.add_to_collection(MASK_COLLECTION, self.mask)
ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel)
ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold)
ops.add_to_collection(WEIGHT_COLLECTION, self.kernel)
if self.use_bias:
self.bias = self.add_variable(
'bias',
shape=[
self.units,
],
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
dtype=self.dtype,
trainable=True)
else:
self.bias = None
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
channel_axis = 1 if self.data_format == 'channels_first' else -1
if tensor_shape.dimension_value(input_shape[channel_axis]) is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = tensor_shape.dimension_value(input_shape[channel_axis])
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.mask = self.add_variable(
name='mask',
shape=kernel_shape,
initializer=init_ops.ones_initializer(),
trainable=False,
dtype=self.dtype)
self.kernel = self.add_variable(
name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
trainable=True,
dtype=self.dtype)
self.threshold = self.add_variable(
name='threshold',
shape=[],
initializer=init_ops.zeros_initializer(),
trainable=False,
dtype=self.dtype)
# Add masked_weights in the weights namescope so as to make it easier
# for the quantization library to add quant ops.
self.masked_kernel = math_ops.multiply(self.mask, self.kernel,
MASKED_WEIGHT_NAME)
ops.add_to_collection(MASK_COLLECTION, self.mask)
ops.add_to_collection(MASKED_WEIGHT_COLLECTION, self.masked_kernel)
ops.add_to_collection(THRESHOLD_COLLECTION, self.threshold)
ops.add_to_collection(WEIGHT_COLLECTION, self.kernel)
if self.use_bias:
self.bias = self.add_variable(
name='bias',
shape=(self.filters,),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = base.InputSpec(
ndim=self.rank + 2, axes={channel_axis: input_dim})
self.built = True
def recalculate_output_shapes(output_shapes):
"""Recalculates the output_shapes after dividing it by num_workers."""
if len(output_shapes) < 1:
raise ValueError("Input shape should have at least one dimension.")
if (tensor_shape.dimension_value(output_shapes[0]) and
tensor_shape.dimension_value(output_shapes[0]) % num_workers != 0):
raise errors.InvalidArgumentError(
None, None,
"First dim of input shape: %d is not divisible by num_workers: %d" %
(output_shapes[0], num_workers))
output_dims = [d for d in output_shapes.dims]
output_dims[0] = output_dims[0] // num_workers
return tensor_shape.TensorShape(output_dims)
def _set_diag_operators(self, diag_update, is_diag_update_positive):
"""Set attributes self._diag_update and self._diag_operator."""
if diag_update is not None:
self._diag_operator = linear_operator_diag.LinearOperatorDiag(
self._diag_update, is_positive_definite=is_diag_update_positive)
self._diag_inv_operator = linear_operator_diag.LinearOperatorDiag(
1. / self._diag_update, is_positive_definite=is_diag_update_positive)
else:
if tensor_shape.dimension_value(self.u.shape[-1]) is not None:
r = tensor_shape.dimension_value(self.u.shape[-1])
else:
r = array_ops.shape(self.u)[-1]
self._diag_operator = linear_operator_identity.LinearOperatorIdentity(
num_rows=r, dtype=self.dtype)
self._diag_inv_operator = self._diag_operator
def crf_log_likelihood(inputs,
tag_indices,
sequence_lengths,
transition_params=None):
"""Computes the log-likelihood of tag sequences in a CRF.
Args:
inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials
to use as input to the CRF layer.
tag_indices: A [batch_size, max_seq_len] matrix of tag indices for which we
compute the log-likelihood.
sequence_lengths: A [batch_size] vector of true sequence lengths.
transition_params: A [num_tags, num_tags] transition matrix, if available.
Returns:
log_likelihood: A [batch_size] `Tensor` containing the log-likelihood of
each example, given the sequence of tag indices.
transition_params: A [num_tags, num_tags] transition matrix. This is either
provided by the caller or created in this function.
"""
# Get shape information.
num_tags = tensor_shape.dimension_value(inputs.shape[2])
# Get the transition matrix if not provided.
if transition_params is None:
transition_params = vs.get_variable("transitions", [num_tags, num_tags])
sequence_scores = crf_sequence_score(inputs, tag_indices, sequence_lengths,
transition_params)
log_norm = crf_log_norm(inputs, sequence_lengths, transition_params)
# Normalize the scores to get the log-likelihood per example.
log_likelihood = sequence_scores - log_norm
return log_likelihood, transition_params
def _create_vars(self, var_list, state):
# Construct ordered dictionary for variable dimensions, sorted by name.
shape_dict = {}
for v in var_list:
shape_dict[v.name] = tensor_shape.dimension_value(np.prod(v.get_shape()))
self.shape_dict = collections.OrderedDict(
sorted(shape_dict.items(), key=lambda t: t[0]))
# Assign each variable its location in flat_grad. The locations are based on
# the order of sorted names.
idx = 0
for v_name, v_dim in self.shape_dict.items():
self.index_dict[v_name] = idx
idx += v_dim
state.create_non_slot(
initial_value=math_ops.cast(0., dtype=var_list[0].dtype.base_dtype),
name="global_step")
# Buffer for keeping past gradients.
window = state.get_hyper("window")
grad_buffer_init = array_ops.zeros(
[window, idx], dtype=var_list[0].dtype.base_dtype)
state.create_non_slot(initial_value=grad_buffer_init, name="grad_buffer")
state.create_non_slot(
initial_value=array_ops.zeros(
(idx,), dtype=var_list[0].dtype.base_dtype),
name="moment1")
# Flattened gradient that contains gradients for all variables in the model.
state.create_non_slot(
initial_value=array_ops.zeros(
(idx,), dtype=var_list[0].dtype.base_dtype),
name="flat_grad")
def rank(self):
"""The number of dimensions in this shape, or None if unknown."""
inner_ndims = tensor_shape.dimension_value(self._inner_dim_sizes.shape[0])
if inner_ndims is None:
return None
else:
return len(self._partitioned_dim_sizes) + inner_ndims
def testUnknownIndices(self): params = constant_op.constant([[0, 1, 2]]) indices = array_ops.placeholder(dtypes.int32) gather_nd_t = array_ops.gather_nd(params, indices) shape = gather_nd_t.get_shape() self.assertEqual(None, shape.ndims) self.assertEqual(None, tensor_shape.dimension_value(shape[0]))
def row_splits_to_segment_ids(splits, name=None):
"""Generates the segmentation corresponding to a RaggedTensor `row_splits`.
Returns an integer vector `segment_ids`, where `segment_ids[i] == j` if
`splits[j] <= i < splits[j+1]`. Example:
```python
>>> ragged.row_splits_to_segment_ids([0, 3, 3, 5, 6, 9]).eval()
[ 0 0 0 2 2 3 4 4 4 ]
```
Args:
splits: A sorted 1-D int64 Tensor. `splits[0]` must be zero.
name: A name prefix for the returned tensor (optional).
Returns:
A sorted 1-D int64 Tensor, with `shape=[splits[-1]]`
Raises:
ValueError: If `splits` is invalid.
"""
with ops.name_scope(name, "RaggedSplitsToSegmentIds", [splits]) as name:
splits = ops.convert_to_tensor(splits, dtype=dtypes.int64, name="splits")
splits.shape.assert_has_rank(1)
if tensor_shape.dimension_value(splits.shape[0]) == 0:
raise ValueError("Invalid row_splits: []")
row_lengths = splits[1:] - splits[:-1]
nrows = array_ops.shape(splits, out_type=dtypes.int64)[-1] - 1
indices = math_ops.range(nrows)
return ragged_util.repeat(indices, repeats=row_lengths, axis=0)
def _forward_event_shape(self, input_shape):
batch_shape, d = (input_shape[:-1],
tensor_shape.dimension_value(input_shape[-1]))
if d is None:
n = None
else:
n = vector_size_to_square_matrix_size(d, self.validate_args)
return batch_shape.concatenate([n, n])
def _get_final_shape(qs):
"""Helper to build `TensorShape`."""
bs = dist.batch_shape.with_rank_at_least(1)
num_components = tensor_shape.dimension_value(bs[-1])
if num_components is not None:
num_components += 1
tail = tensor_shape.TensorShape([num_components, qs])
return bs[:-1].concatenate(tail)
def compute_output_shape(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
input_shape = input_shape.with_rank_at_least(2)
if tensor_shape.dimension_value(input_shape[-1]) is None:
raise ValueError(
'The innermost dimension of input_shape must be defined, but saw: %s'
% input_shape)
return input_shape[:-1].concatenate(self.units)
def _batch_shape_tensor(self):
with ops.control_dependencies(self._runtime_assertions):
batch_shape = self.distribution.batch_shape_tensor()
dim0 = tensor_shape.dimension_value(
batch_shape.shape.with_rank_at_least(1)[0])
batch_ndims = (dim0
if dim0 is not None
else array_ops.shape(batch_shape)[0])
return batch_shape[:batch_ndims - self.reinterpreted_batch_ndims]
def crf_log_norm(inputs, sequence_lengths, transition_params):
"""Computes the normalization for a CRF.
Args:
inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials
to use as input to the CRF layer.
sequence_lengths: A [batch_size] vector of true sequence lengths.
transition_params: A [num_tags, num_tags] transition matrix.
Returns:
log_norm: A [batch_size] vector of normalizers for a CRF.
"""
# Split up the first and rest of the inputs in preparation for the forward
# algorithm.
first_input = array_ops.slice(inputs, [0, 0, 0], [-1, 1, -1])
first_input = array_ops.squeeze(first_input, [1])
# If max_seq_len is 1, we skip the algorithm and simply reduce_logsumexp over
# the "initial state" (the unary potentials).
def _single_seq_fn():
log_norm = math_ops.reduce_logsumexp(first_input, [1])
# Mask `log_norm` of the sequences with length <= zero.
log_norm = array_ops.where(math_ops.less_equal(sequence_lengths, 0),
array_ops.zeros_like(log_norm),
log_norm)
return log_norm
def _multi_seq_fn():
"""Forward computation of alpha values."""
rest_of_input = array_ops.slice(inputs, [0, 1, 0], [-1, -1, -1])
# Compute the alpha values in the forward algorithm in order to get the
# partition function.
forward_cell = CrfForwardRnnCell(transition_params)
# Sequence length is not allowed to be less than zero.
sequence_lengths_less_one = math_ops.maximum(
constant_op.constant(0, dtype=sequence_lengths.dtype),
sequence_lengths - 1)
_, alphas = rnn.dynamic_rnn(
cell=forward_cell,
inputs=rest_of_input,
sequence_length=sequence_lengths_less_one,
initial_state=first_input,
dtype=dtypes.float32)
log_norm = math_ops.reduce_logsumexp(alphas, [1])
# Mask `log_norm` of the sequences with length <= zero.
log_norm = array_ops.where(math_ops.less_equal(sequence_lengths, 0),
array_ops.zeros_like(log_norm),
log_norm)
return log_norm
return utils.smart_cond(
pred=math_ops.equal(
tensor_shape.dimension_value(
inputs.shape[1]) or array_ops.shape(inputs)[1],
1),
true_fn=_single_seq_fn,
false_fn=_multi_seq_fn)
def pad_if_necessary(t, name, last_dim_padding):
n = tensor_shape.dimension_value(t.shape[-1])
if not n or n == m:
return t
if n == m - 1:
paddings = ([[0, 0] for _ in range(len(t.shape) - 1)] +
[last_dim_padding])
return array_ops.pad(t, paddings)
raise ValueError('Expected {} to be have length {} or {}, got {}.'.format(
name, m, m - 1, n))
def __init__(self, transition_params):
"""Initialize the CrfForwardRnnCell.
Args:
transition_params: A [num_tags, num_tags] matrix of binary potentials.
This matrix is expanded into a [1, num_tags, num_tags] in preparation
for the broadcast summation occurring within the cell.
"""
self._transition_params = array_ops.expand_dims(transition_params, 0)
self._num_tags = tensor_shape.dimension_value(transition_params.shape[0])
def _cache_input_depth(self, x):
if self._input_depth is None:
self._input_depth = tensor_shape.dimension_value(
x.shape.with_rank_at_least(1)[-1])
if self._input_depth is None:
raise NotImplementedError(
"Rightmost dimension must be known prior to graph execution.")
if self._num_masked >= self._input_depth:
raise ValueError(
"Number of masked units must be smaller than the event size.")
def make_padded_shapes(shapes, none_filler=None):
padded = []
for shape in nest.flatten(shapes):
shape = tensor_shape.TensorShape(shape)
shape = [
none_filler if tensor_shape.dimension_value(d) is None else d
for d in shape
]
padded.append(shape)
return nest.pack_sequence_as(shapes, padded)
def _shape_invariant_to_components(self, shape=None):
if shape is None:
shape = self.dense_shape.shape
# TODO(edloper): What if shape.dense_shape.shape.ndims is None?
if shape.ndims != 1:
raise ValueError("Shape invariant for SparseTensor must have the form "
"TensorShape([r]), got %r" % shape)
rank = tensor_shape.dimension_value(shape[0])
return [tensor_shape.TensorShape([None, rank]), # indices
tensor_shape.TensorShape([None]), # values
tensor_shape.TensorShape([rank])] # dense_shape
def testSlice(self):
known = tensor_shape.TensorShape([0, 1, 2, 3, 4])
self.assertEqual(tensor_shape.Dimension(2), known[2])
tensor_shape.TensorShape([1, 2, 3]).assert_is_compatible_with(known[1:4])
unknown = tensor_shape.TensorShape(None)
self.assertEqual(
tensor_shape.Dimension(None).value,
tensor_shape.dimension_value(unknown[2]))
tensor_shape.TensorShape(
[None, None, None]).assert_is_compatible_with(unknown[1:4])
def _static_check_for_same_dimensions(operators):
"""ValueError if operators determined to have different dimensions."""
if len(operators) < 2:
return
domain_dimensions = [
(op.name, tensor_shape.dimension_value(op.domain_dimension))
for op in operators
if tensor_shape.dimension_value(op.domain_dimension) is not None]
if len(set(value for name, value in domain_dimensions)) > 1:
raise ValueError("Operators must have the same domain dimension. Found: %s"
% domain_dimensions)
range_dimensions = [
(op.name, tensor_shape.dimension_value(op.range_dimension))
for op in operators
if tensor_shape.dimension_value(op.range_dimension) is not None]
if len(set(value for name, value in range_dimensions)) > 1:
raise ValueError("Operators must have the same range dimension. Found: %s" %
range_dimensions)
def _project_input(self, inputs, c_prev, m_prev, with_c):
"""Fills in c_prev and m_prev with projected input, for input dimensions.
Args:
inputs: inputs tensor
c_prev: cell value
m_prev: previous output
with_c: boolean; whether to include project_c.
Raises:
ValueError: if len(self._config.input) != len(inputs)
"""
conf = self._config
if (inputs is not None and
tensor_shape.dimension_value(inputs.shape.with_rank(2)[1]) > 0 and
conf.inputs):
if isinstance(inputs, tuple):
if len(conf.inputs) != len(inputs):
raise ValueError('Expect inputs as a tuple of {} '
'tensors'.format(len(conf.inputs)))
input_splits = inputs
else:
input_splits = array_ops.split(
value=inputs, num_or_size_splits=len(conf.inputs), axis=1)
input_sz = tensor_shape.dimension_value(
input_splits[0].shape.with_rank(2)[1])
for i, j in enumerate(conf.inputs):
input_project_m = vs.get_variable(
'project_m_{}'.format(j), [input_sz, conf.num_units],
dtype=inputs.dtype)
m_prev[j] = math_ops.matmul(input_splits[i], input_project_m)
if with_c:
input_project_c = vs.get_variable(
'project_c_{}'.format(j), [input_sz, conf.num_units],
dtype=inputs.dtype)
c_prev[j] = math_ops.matmul(input_splits[i], input_project_c)
def interpolate_scale(grid, scale):
"""Helper which interpolates between two scales."""
if len(scale) != 2:
raise NotImplementedError("Currently only bimixtures are supported; "
"len(scale)={} is not 2.".format(len(scale)))
deg = tensor_shape.dimension_value(grid.shape.with_rank_at_least(1)[-1])
if deg is None:
raise ValueError("Num quadrature grid points must be known prior "
"to graph execution.")
with ops.name_scope("interpolate_scale", values=[grid]):
return [linop_add_lib.add_operators([
linop_scale(grid[..., k, q], s)
for k, s in enumerate(scale)
])[0] for q in range(deg)]
def _step(self, actions):
"""Returns a TensorFlow op to step the environment.
Args:
actions: A Tensor, or a nested dict, list or tuple of Tensors
corresponding to `action_spec()`.
Returns:
A `TimeStep` tuple of:
step_type: A scalar int32 tensor representing the `StepType` value.
reward: A float32 tensor representing the reward at this
time_step.
discount: A scalar float32 tensor representing the discount [0, 1].
observation: A Tensor, or a nested dict, list or tuple of Tensors
corresponding to `observation_spec()`.
Raises:
ValueError: If any of the actions are scalars or their major axis is known
and is not equal to `self.batch_size`.
"""
def _step_py(*flattened_actions):
with _check_not_called_concurrently(self._lock):
packed = tf.nest.pack_sequence_as(
structure=self.action_spec(),
flat_sequence=flattened_actions)
self._time_step = self._env.step(packed)
return tf.nest.flatten(self._time_step)
def _isolated_step_py(*flattened_actions):
return self._execute(_step_py, *flattened_actions)
with tf.name_scope('step'):
flat_actions = [tf.identity(x) for x in tf.nest.flatten(actions)]
if self._check_dims:
for action in flat_actions:
dim_value = tensor_shape.dimension_value(action.shape[0])
if (action.shape.rank == 0
or (dim_value is not None
and dim_value != self.batch_size)):
raise ValueError(
'Expected actions whose major dimension is batch_size (%d), '
'but saw action with shape %s:\n %s' %
(self.batch_size, action.shape, action))
outputs = tf.numpy_function(_isolated_step_py,
flat_actions,
self._time_step_dtypes,
name='step_py_func')
return self._time_step_from_numpy_function_outputs(outputs)
def crf_multitag_sequence_score(inputs, tag_bitmap, sequence_lengths,
transition_params):
"""Computes the unnormalized score of all tag sequences matching tag_bitmap.
tag_bitmap enables more than one tag to be considered correct at each time
step. This is useful when an observed output at a given time step is
consistent with more than one tag, and thus the log likelihood of that
observation must take into account all possible consistent tags.
Using one-hot vectors in tag_bitmap gives results identical to
crf_sequence_score.
Args:
inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials
to use as input to the CRF layer.
tag_bitmap: A [batch_size, max_seq_len, num_tags] boolean tensor
representing all active tags at each index for which to calculate the
unnormalized score.
sequence_lengths: A [batch_size] vector of true sequence lengths.
transition_params: A [num_tags, num_tags] transition matrix.
Returns:
sequence_scores: A [batch_size] vector of unnormalized sequence scores.
"""
# If max_seq_len is 1, we skip the score calculation and simply gather the
# unary potentials of all active tags.
def _single_seq_fn():
filtered_inputs = array_ops.where(
tag_bitmap, inputs,
array_ops.fill(array_ops.shape(inputs), float("-inf")))
return math_ops.reduce_logsumexp(filtered_inputs,
axis=[1, 2],
keepdims=False)
def _multi_seq_fn():
# Compute the logsumexp of all scores of sequences matching the given tags.
filtered_inputs = array_ops.where(
tag_bitmap, inputs,
array_ops.fill(array_ops.shape(inputs), float("-inf")))
return crf_log_norm(inputs=filtered_inputs,
sequence_lengths=sequence_lengths,
transition_params=transition_params)
return utils.smart_cond(pred=math_ops.equal(
tensor_shape.dimension_value(inputs.shape[1])
or array_ops.shape(inputs)[1], 1),
true_fn=_single_seq_fn,
false_fn=_multi_seq_fn)
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
last_dim = tensor_shape.dimension_value(input_shape[-1])
self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim})
self.kernel = self.add_weight(
'kernel',
shape=[last_dim, self.units],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
def call(self, input):
self.add_loss(self.rate * tf.reduce_sum(inputs))
rank = inputs.shape.rank
if rank is not None and rank > 2:
outputs = standard_ops.tensordot(inputs, self.kernel, [[rank - 1], [0]])
if not context.executing_eagerly():
shape = inputs.shape.as_list()
output_shape = shape[:-1] + [self.units]
outputs.set_shape(output_shape)
else:
inputs = math_ops.cast(inputs, self._compute_dtype)
if K.is_sparse(inputs):
outputs = sparse_ops.sparse_tensor_dense_matmul(inputs, self.kernel)
else:
outputs = gen_math_ops.mat_mul(inputs, self.kernel)
if self.activation is not None:
return self.activation(outputs)
return outputs
def compute_output_shape(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
input_shape = input_shape.with_rank_at_least(2)
if tensor_shape.dimension_value(input_shape[-1]) is None:
raise ValueError(
'The innermost dimension of input_shape must be defined, but saw: %s'
% input_shape)
return input_shape[:-1].concatenate(self.units)
def get_config(self):
config = {
'units': self.units,
'activation': activations.serialize(self.activation),
'kernel_initializer': initializers.serialize(self.kernel_initializer),
}
base_config = super(SNNDense, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def _to_dense(self):
"""Generic and often inefficient implementation. Override often."""
logging.warn("Using (possibly slow) default implementation of to_dense."
" Converts by self.matmul(identity).")
if self.batch_shape.is_fully_defined():
batch_shape = self.batch_shape
else:
batch_shape = self.batch_shape_tensor()
dim_value = tensor_shape.dimension_value(self.domain_dimension)
if dim_value is not None:
n = dim_value
else:
n = self.domain_dimension_tensor()
eye = linalg_ops.eye(num_rows=n, batch_shape=batch_shape, dtype=self.dtype)
return self.matmul(eye)
def _add_zero_flow_controls_at_boundary(control_point_locations,
control_point_flows, image_height,
image_width, boundary_points_per_edge):
"""Add control points for zero-flow boundary conditions.
Augment the set of control points with extra points on the
boundary of the image that have zero flow.
Args:
control_point_locations: input control points
control_point_flows: their flows
image_height: image height
image_width: image width
boundary_points_per_edge: number of points to add in the middle of each
edge (not including the corners).
The total number of points added is
4 + 4*(boundary_points_per_edge).
Returns:
merged_control_point_locations: augmented set of control point locations
merged_control_point_flows: augmented set of control point flows
"""
batch_size = tensor_shape.dimension_value(control_point_locations.shape[0])
boundary_point_locations = _get_boundary_locations(
image_height, image_width, boundary_points_per_edge)
boundary_point_flows = np.zeros([boundary_point_locations.shape[0], 2])
type_to_use = control_point_locations.dtype
boundary_point_locations = constant_op.constant(_expand_to_minibatch(
boundary_point_locations, batch_size),
dtype=type_to_use)
boundary_point_flows = constant_op.constant(_expand_to_minibatch(
boundary_point_flows, batch_size),
dtype=type_to_use)
merged_control_point_locations = array_ops.concat(
[control_point_locations, boundary_point_locations], 1)
merged_control_point_flows = array_ops.concat(
[control_point_flows, boundary_point_flows], 1)
return merged_control_point_locations, merged_control_point_flows
def _from_compatible_tensor_list(self, flat_value):
if self._ragged_rank <= 0:
raise ValueError(
"ragged_rank must be greater than zero. Found ragged_rank: %d"
% self._ragged_rank)
result = ragged_tensor.RaggedTensor._from_variant(
flat_value[0],
dtype=self._dtype,
output_ragged_rank=self._ragged_rank)
if self._shape.ndims is not None:
outer_dim = tensor_shape.dimension_value(self._shape[0])
if outer_dim is not None:
result.row_splits.set_shape([outer_dim + 1])
result.flat_values.set_shape(
tensor_shape.TensorShape([None]).concatenate(
self._shape[1 + self._ragged_rank:]))
return result
def _to_dense(self):
"""Generic and often inefficient implementation. Override often."""
if self.batch_shape.is_fully_defined():
batch_shape = self.batch_shape
else:
batch_shape = self.batch_shape_tensor()
dim_value = tensor_shape.dimension_value(self.domain_dimension)
if dim_value is not None:
n = dim_value
else:
n = self.domain_dimension_tensor()
eye = linalg_ops.eye(num_rows=n,
batch_shape=batch_shape,
dtype=self.dtype)
return self.matmul(eye)
def testPartiallyDefinedShape(self):
s = tensor_shape.TensorShape([tensor_shape.Dimension(
3), tensor_shape.Dimension(None), tensor_shape.Dimension(7)])
# pylint: disable=g-error-prone-assert-raises
with self.assertRaisesRegex(ValueError, "Shape .+ is not fully defined"):
s.assert_is_fully_defined()
# pylint: enable=g-error-prone-assert-raises
self.assertEqual(s.rank, 3)
self.assertLen(s, 3)
self.assertTrue(s)
s.assert_has_rank(3)
self.assertEqual(tensor_shape.Dimension(3), s[0])
self.assertEqual(tensor_shape.Dimension(None).value, s.dims[1].value)
self.assertEqual(tensor_shape.Dimension(7), s.dims[2])
s.assert_same_rank([6, 3, 7])
for d1, d2 in zip(s, [3, None, 7]):
assert tensor_shape.dimension_value(d1) == d2
def _multi_seq_fn():
"""Decoding of highest scoring sequence."""
# For simplicity, in shape comments, denote:
# 'batch_size' by 'B', 'max_seq_len' by 'T' , 'num_tags' by 'O' (output).
num_tags = tensor_shape.dimension_value(potentials.shape[2])
# Computes forward decoding. Get last score and backpointers.
crf_fwd_cell = CrfDecodeForwardRnnCell(transition_params)
initial_state = array_ops.slice(potentials, [0, 0, 0], [-1, 1, -1])
initial_state = array_ops.squeeze(initial_state, axis=[1]) # [B, O]
inputs = array_ops.slice(potentials, [0, 1, 0], [-1, -1, -1]) # [B, T-1, O]
# Sequence length is not allowed to be less than zero.
sequence_length_less_one = math_ops.maximum(
constant_op.constant(0, dtype=sequence_length.dtype),
sequence_length - 1)
backpointers, last_score = rnn.dynamic_rnn( # [B, T - 1, O], [B, O]
crf_fwd_cell,
inputs=inputs,
sequence_length=sequence_length_less_one,
initial_state=initial_state,
time_major=False,
dtype=dtypes.int32)
backpointers = gen_array_ops.reverse_sequence( # [B, T - 1, O]
backpointers, sequence_length_less_one, seq_dim=1)
# Computes backward decoding. Extract tag indices from backpointers.
crf_bwd_cell = CrfDecodeBackwardRnnCell(num_tags)
initial_state = math_ops.cast(math_ops.argmax(last_score, axis=1), # [B]
dtype=dtypes.int32)
initial_state = array_ops.expand_dims(initial_state, axis=-1) # [B, 1]
decode_tags, _ = rnn.dynamic_rnn( # [B, T - 1, 1]
crf_bwd_cell,
inputs=backpointers,
sequence_length=sequence_length_less_one,
initial_state=initial_state,
time_major=False,
dtype=dtypes.int32)
decode_tags = array_ops.squeeze(decode_tags, axis=[2]) # [B, T - 1]
decode_tags = array_ops.concat([initial_state, decode_tags], # [B, T]
axis=1)
decode_tags = gen_array_ops.reverse_sequence( # [B, T]
decode_tags, sequence_length, seq_dim=1)
best_score = math_ops.reduce_max(last_score, axis=1) # [B]
return decode_tags, best_score
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
last_dim = tensor_shape.dimension_value(input_shape[-1])
if last_dim is None:
raise ValueError('The last dimension of the inputs to `Dense` '
'should be defined. Found `None`.')
self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim})
shape = len(input_shape.as_list()) * [1]
shape[-1] = last_dim
self.kernel = self.add_weight(shape=shape,
initializer=self.kernel_initializer,
name='kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
dtype=self.dtype,
trainable=True)
super(LearnableNoise, self).build(input_shape)
self.built = True
def testPartiallyDefinedShape(self):
s = tensor_shape.TensorShape([
tensor_shape.Dimension(3),
tensor_shape.Dimension(None),
tensor_shape.Dimension(7)
])
with self.assertRaises(ValueError):
s.assert_is_fully_defined()
self.assertEqual(3, s.rank)
self.assertEqual(3, len(s))
self.assertTrue(s)
s.assert_has_rank(3)
self.assertEqual(tensor_shape.Dimension(3), s[0])
self.assertEqual(tensor_shape.Dimension(None).value, s.dims[1].value)
self.assertEqual(tensor_shape.Dimension(7), s.dims[2])
s.assert_same_rank([6, 3, 7])
for d1, d2 in zip(s, [3, None, 7]):
assert tensor_shape.dimension_value(d1) == d2
def build(self, input_shape):
"""GRU cell."""
input_size = tensor_shape.dimension_value(input_shape[1])
if input_size is None:
raise ValueError("Expecting input_size to be set.")
self._gate_kernel = self.add_variable(
"gates/kernel", [input_size + self._cell_size, self._cell_size * 2])
self._gate_bias = self.add_variable(
"gates/bias", [self._cell_size * 2],
initializer=init_ops.constant_initializer(1.0))
self._candidate_kernel = self.add_variable(
"candidate/kernel", [input_size + self._cell_size, self._cell_size])
self._candidate_bias = self.add_variable(
"candidate/bias", [self._cell_size],
initializer=init_ops.constant_initializer(0.0))
self.built = True
def build(self, input_shape):
# Check if the input size exist.
input_size = tensor_shape.dimension_value(input_shape[1])
if input_size is None:
raise ValueError("Expecting input_size to be set.")
self._gate_kernel = self.add_variable(
"w_ru", [input_size + self._cell_size, self._cell_size * 2])
self._gate_bias = self.add_variable(
"b_ru", [self._cell_size * 2],
initializer=init_ops.constant_initializer(1.0))
self._candidate_kernel = self.add_variable(
"w_c", [input_size + self._cell_size, self._cell_size])
self._candidate_bias = self.add_variable(
"b_c", [self._cell_size],
initializer=init_ops.constant_initializer(0.0))
self.built = True
def _maybe_mask(m, seq_len_mask):
rank = m.get_shape().ndims
rank = rank if rank is not None else array_ops.rank(m)
extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
m_batch_size = tensor_shape.dimension_value(
m.shape[0]) or array_ops.shape(m)[0]
if memory_sequence_length is not None:
message = ("memory_sequence_length and memory tensor batch sizes do not "
"match.")
with ops.control_dependencies([
check_ops.assert_equal(
seq_len_batch_size, m_batch_size, message=message)]):
seq_len_mask = array_ops.reshape(
seq_len_mask,
array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
return m * seq_len_mask
else:
return m
def build(self, input_shape):
self.last_dim = tensor_shape.dimension_value(input_shape[-1])
if self.weight_reparam:
self.kernel_w = self.add_weight(
"w",
shape=[self.last_dim, self.d_out],
initializer=tf.keras.initializers.he_normal(seed=None))
self.kernel_g = self.add_weight("g",
shape=[1, self.d_out],
initializer=tf.ones_initializer())
else:
self.kernel = self.add_weight(
"w",
shape=[self.last_dim, self.d_out],
initializer=tf.initializers.GlorotNormal())
self.bias = self.add_weight("b",
shape=[self.d_out],
initializer=tf.zeros_initializer())
def testFullyDefinedShape(self):
s = tensor_shape.TensorShape([tensor_shape.Dimension(
3), tensor_shape.Dimension(4), tensor_shape.Dimension(7)])
s.assert_is_fully_defined()
self.assertEqual(s.rank, 3)
self.assertLen(s, 3)
self.assertTrue(s)
s.assert_has_rank(3)
self.assertEqual([tensor_shape.Dimension(3),
tensor_shape.Dimension(4),
tensor_shape.Dimension(7)], s.dims)
self.assertEqual(tensor_shape.Dimension(3), s[0])
self.assertEqual(tensor_shape.Dimension(4), s[1])
self.assertEqual(tensor_shape.Dimension(7), s[2])
self.assertEqual([3, 4, 7], s.as_list())
s.assert_is_compatible_with([3, 4, 7])
s.assert_same_rank([6, 3, 7])
for d1, d2 in zip(s, [3, 4, 7]):
assert tensor_shape.dimension_value(d1) == d2
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
last_dim = tensor_shape.dimension_value(input_shape[-1])
if last_dim is None:
raise ValueError('The last dimension of the inputs to `PBPLayer` '
'should be defined. Found `None`.')
self.input_spec = tf.keras.layers.InputSpec(min_ndim=2,
axes={-1: last_dim})
self.inv_sqrtV1 = tf.cast(1.0 / tf.math.sqrt(1.0 * last_dim + 1),
dtype=self.dtype)
self.inv_V1 = tf.math.square(self.inv_sqrtV1)
over_gamma = ReciprocalGammaInitializer(6.0, 6.0)
self.kernel_m = self.add_weight(
"kernel_mean",
shape=[last_dim, self.units],
initializer=tf.keras.initializers.HeNormal(),
dtype=self.dtype,
trainable=True)
self.kernel_v = self.add_weight("kernel_variance",
shape=[last_dim, self.units],
initializer=over_gamma,
dtype=self.dtype,
trainable=True)
self.bias_m = self.add_weight(
"bias_mean",
shape=[
self.units,
],
initializer=tf.keras.initializers.HeNormal(),
dtype=self.dtype,
trainable=True)
self.bias_v = self.add_weight("bias_variance",
shape=[
self.units,
],
initializer=over_gamma,
dtype=self.dtype,
trainable=True)
self.Normal = tfp.distributions.Normal(
loc=tf.constant(0.0, dtype=self.dtype),
scale=tf.constant(1.0, dtype=self.dtype))
self.built = True
def crf_sequence_score(inputs, tag_indices, sequence_lengths,
transition_params):
"""Computes the unnormalized score for a tag sequence.
Args:
inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials
to use as input to the CRF layer.
tag_indices: A [batch_size, max_seq_len] matrix of tag indices for which we
compute the unnormalized score.
sequence_lengths: A [batch_size] vector of true sequence lengths.
transition_params: A [num_tags, num_tags] transition matrix.
Returns:
sequence_scores: A [batch_size] vector of unnormalized sequence scores.
"""
# If max_seq_len is 1, we skip the score calculation and simply gather the
# unary potentials of the single tag.
def _single_seq_fn():
batch_size = array_ops.shape(inputs, out_type=tag_indices.dtype)[0]
example_inds = array_ops.reshape(
math_ops.range(batch_size, dtype=tag_indices.dtype), [-1, 1])
sequence_scores = array_ops.gather_nd(
array_ops.squeeze(inputs, [1]),
array_ops.concat([example_inds, tag_indices], axis=1))
sequence_scores = array_ops.where(math_ops.less_equal(sequence_lengths, 0),
array_ops.zeros_like(sequence_scores),
sequence_scores)
return sequence_scores
def _multi_seq_fn():
# Compute the scores of the given tag sequence.
unary_scores = crf_unary_score(tag_indices, sequence_lengths, inputs)
binary_scores = crf_binary_score(tag_indices, sequence_lengths,
transition_params)
sequence_scores = unary_scores + binary_scores
return sequence_scores
return utils.smart_cond(
pred=math_ops.equal(
tensor_shape.dimension_value(
inputs.shape[1]) or array_ops.shape(inputs)[1],
1),
true_fn=_single_seq_fn,
false_fn=_multi_seq_fn)
def build(self, input_shape):
print(input_shape)
print(self.units)
input_len = tensor_shape.dimension_value(input_shape[-1])
self.kernel = self.add_weight('kernel',
shape=[input_len, self.units],
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
dtype=self.dtype,
trainable=True)
self.bias = self.add_weight('bias',
shape=[self.units, input_len],
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
dtype=self.dtype,
trainable=True)
self.built = True
def build(self, input_shape):
# super(IndependentDense, self).build(input_shape)
self.last_dim = tensor_shape.dimension_value(input_shape[-1])
self.input_spec = InputSpec(min_ndim=2,
axes={-1: self.last_dim})
self.mu_init = tf.random_uniform_initializer(-((3 / self.last_dim) ** 0.5), (3 / self.last_dim) ** 0.5)
self.sigma_init = tf.constant_initializer(0.017)
self.w_mu = self.add_weight("w_mu", [self.last_dim, self.units], initializer=self.mu_init, dtype=self.dtype,
trainable=True)
self.w_sigma = self.add_weight("w_sigma", [self.last_dim, self.units], initializer=self.sigma_init,
dtype=self.dtype, trainable=True)
if self.use_bias:
self.b_mu = self.add_weight("b_mu", [self.units, ], initializer=self.mu_init, dtype=self.dtype,
trainable=True)
self.b_sigma = self.add_weight("b_sigma", [self.units, ], initializer=self.sigma_init, dtype=self.dtype,
trainable=True)
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
for i in range(1, len(input_shape)):
if tensor_shape.dimension_value(input_shape[i]) is None:
raise ValueError(
'The input shape [1:] should be defined, but found element `None`.'
)
if self.use_kernel:
varName = 'kernel'
elif self.use_bias:
varName = 'bias'
get_in = input_shape.as_list()[1:]
self.get_var = self.add_weight(varName,
shape=get_in,
initializer=self.var_initializer,
regularizer=self.var_regularizer,
constraint=self.var_constraint,
dtype=self.dtype,
trainable=True)
super(Ghost, self).build(input_shape)
def __init__(self, transition_params):
"""Initialize the CrfDecodeForwardRnnCell.
Args:
transition_params:
1. A [num_tags, num_tags] matrix of binary
potentials. This matrix is expanded into a
[1, num_tags, num_tags] in preparation for the broadcast
summation occurring within the cell.
2. or a [Batch, num_tags, num_tags]
"""
if K.ndim(transition_params) == 2:
transition_params = array_ops.expand_dims(transition_params, 0)
assert K.ndim(transition_params
) == 3, "transition_params should have 2 or 3 dims"
self._transition_params = transition_params
self._num_tags = tensor_shape.dimension_value(
transition_params.shape[1])
def row_splits_to_segment_ids(splits, name=None, out_type=None):
"""Generates the segmentation corresponding to a RaggedTensor `row_splits`.
Returns an integer vector `segment_ids`, where `segment_ids[i] == j` if
`splits[j] <= i < splits[j+1]`. Example:
```python
>>> ragged.row_splits_to_segment_ids([0, 3, 3, 5, 6, 9]).eval()
[ 0 0 0 2 2 3 4 4 4 ]
```
Args:
splits: A sorted 1-D integer Tensor. `splits[0]` must be zero.
name: A name prefix for the returned tensor (optional).
out_type: The dtype for the return value. Defaults to `splits.dtype`,
or `tf.int64` if `splits` does not have a dtype.
Returns:
A sorted 1-D integer Tensor, with `shape=[splits[-1]]`
Raises:
ValueError: If `splits` is invalid.
"""
with ops.name_scope(name, "RaggedSplitsToSegmentIds", [splits]) as name:
splits = ops.convert_to_tensor(splits,
name="splits",
preferred_dtype=dtypes.int64)
if splits.dtype not in (dtypes.int32, dtypes.int64):
raise ValueError("splits must have dtype int32 or int64")
splits.shape.assert_has_rank(1)
if tensor_shape.dimension_value(splits.shape[0]) == 0:
raise ValueError("Invalid row_splits: []")
if out_type is None:
out_type = splits.dtype
else:
out_type = dtypes.as_dtype(out_type)
row_lengths = splits[1:] - splits[:-1]
nrows = array_ops.shape(splits, out_type=out_type)[-1] - 1
indices = math_ops.range(nrows)
return ragged_util.repeat(indices, repeats=row_lengths, axis=0)
def range_dimension_tensor(self, name="range_dimension_tensor"):
"""Dimension (in the sense of vector spaces) of the range of this operator.
Determined at runtime.
If this operator acts like the batch matrix `A` with
`A.shape = [B1,...,Bb, M, N]`, then this returns `M`.
Args:
name: A name for this `Op`.
Returns:
`int32` `Tensor`
"""
# Derived classes get this "for free" once .shape() is implemented.
with self._name_scope(name):
# Prefer to use statically defined shape if available.
dim_value = tensor_shape.dimension_value(self.range_dimension)
if dim_value is not None:
return ops.convert_to_tensor(dim_value)
else:
return self.shape_tensor()[-2]
def testBuildRingGatherPassStructure(self):
# 1 worker, 1 device
input_tensors, device_names = self._buildInput(1, 1)
pred_by_c_d, rank_by_c_d = ar._ring_permutations(1, 1, [0])
output_tensors = ar._build_ring_gather(input_tensors, device_names, 1,
pred_by_c_d, rank_by_c_d,
math_ops.add)
self.assertEqual(output_tensors, input_tensors)
# 1 worker, 4 devices, 2 subchunks
input_tensors, device_names = self._buildInput(1, 4)
pred_by_c_d, rank_by_c_d = ar._ring_permutations(1, 2, [0, 1, 2, 3])
output_tensors, pad_len = ar._build_ring_gather(
input_tensors, device_names, 2, pred_by_c_d, rank_by_c_d, math_ops.add)
self.assertEqual(0, pad_len)
# same number outputs as inputs
self.assertEqual(len(output_tensors), len(input_tensors))
num_chunks = 2 * len(input_tensors)
tlen = tensor_shape.dimension_value(input_tensors[0].shape[0])
for otl in output_tensors:
self.assertEqual(len(otl), num_chunks)
for ot in otl:
self.assertEqual(ot.shape, [tlen/num_chunks])
def get_state_transition_noise_covariance(
self, minimum_initial_variance=1e-5):
# Most state space models use only an explicit observation noise term to
# model deviations from expectations, and so a low initial transition noise
# parameter is helpful there. Since deviations from expectations are also
# modeled as transition noise in VARMA, we set its initial value based on a
# slight over-estimate empirical observation noise.
if self._input_statistics is not None:
feature_variance = self._scale_variance(
self._input_statistics.series_start_moments.variance)
initial_transition_noise_scale = math_ops.log(
math_ops.maximum(
math_ops.reduce_mean(feature_variance), minimum_initial_variance))
else:
initial_transition_noise_scale = 0.
state_noise_transform = ops.convert_to_tensor(
self.get_noise_transform(), dtype=self.dtype)
state_noise_dimension = tensor_shape.dimension_value(
state_noise_transform.shape[1])
return math_utils.variable_covariance_matrix(
state_noise_dimension, "state_transition_noise",
dtype=self.dtype,
initial_overall_scale_log=initial_transition_noise_scale)
def _fn(x):
"""MADE parameterized via `masked_autoregressive_default_template`."""
# TODO(b/67594795): Better support of dynamic shape.
input_depth = tensor_shape.dimension_value(
x.shape.with_rank_at_least(1)[-1])
if input_depth is None:
raise NotImplementedError(
"Rightmost dimension must be known prior to graph execution."
)
input_shape = (np.int32(x.shape.as_list()) if
x.shape.is_fully_defined() else array_ops.shape(x))
for i, units in enumerate(hidden_layers):
x = masked_dense(inputs=x,
units=units,
num_blocks=input_depth,
exclusive=True if i == 0 else False,
activation=activation,
*args,
**kwargs)
x = masked_dense(inputs=x,
units=(1 if shift_only else 2) * input_depth,
num_blocks=input_depth,
activation=None,
*args,
**kwargs)
if shift_only:
x = array_ops.reshape(x, shape=input_shape)
return x, None
x = array_ops.reshape(x,
shape=array_ops.concat([input_shape, [2]],
axis=0))
shift, log_scale = array_ops.unstack(x, num=2, axis=-1)
which_clip = (math_ops.clip_by_value if log_scale_clip_gradient
else _clip_by_value_preserve_grad)
log_scale = which_clip(log_scale, log_scale_min_clip,
log_scale_max_clip)
return shift, log_scale
def __init__(self,
input_shape,
units,
gain=2.0,
use_weight_scaling=True,
use_bias=True,
activation=None,
**kwargs):
warnings.warn(
f"CustomDense is deprecated. Use tf.keras.layers.Dense wrapped in WeightScalingWrapper instead.",
DeprecationWarning, 2)
if 'bias_initializer' in kwargs:
logging.warning(
f"{self.__class__.__name__} ignores bias_initializer={kwargs['bias_initializer']}"
)
del kwargs['bias_initializer']
if 'kernel_initializer' in kwargs:
logging.warning(
f"{self.__class__.__name__} ignores kernel_initializer={kwargs['kernel_initializer']}"
)
del kwargs['kernel_initializer']
super(CustomDense, self).__init__(units=units,
use_bias=use_bias,
**kwargs)
self.bias_initializer = tf.keras.initializers.zeros()
self.gain = gain
self.use_weight_scaling = use_weight_scaling
self._wrapper_use_bias = use_bias
self._wrapper_activation = activations.get(activation)
self._argument_input_shape = input_shape
# compute kernel shape
input_shape = tensor_shape.TensorShape(input_shape)
last_dim = tensor_shape.dimension_value(input_shape[-1])
kernel_shape = (last_dim, self.units)
self.op_scale, self.kernel_initializer = he_kernel_initializer(
kernel_shape, self.gain, self.use_weight_scaling)
def interpolate_loc(grid, loc):
"""Helper which interpolates between two locs."""
if len(loc) != 2:
raise NotImplementedError("Currently only bimixtures are supported; "
"len(scale)={} is not 2.".format(len(loc)))
deg = tensor_shape.dimension_value(grid.shape.with_rank_at_least(1)[-1])
if deg is None:
raise ValueError("Num quadrature grid points must be known prior "
"to graph execution.")
with ops.name_scope("interpolate_loc", values=[grid, loc]):
if loc is None or loc[0] is None and loc[1] is None:
return [None]*deg
# shape: [B, 1, k, deg]
w = grid[..., array_ops.newaxis, :, :]
loc = [x[..., array_ops.newaxis] # shape: [B, e, 1]
if x is not None else None for x in loc]
if loc[0] is None:
x = w[..., 1, :] * loc[1] # shape: [B, e, deg]
elif loc[1] is None:
x = w[..., 0, :] * loc[0] # shape: [B, e, deg]
else:
delta = loc[0] - loc[1]
x = w[..., 0, :] * delta + loc[1] # shape: [B, e, deg]
return [x[..., k] for k in range(deg)] # list(shape:[B, e])
def build(self, input_shape):
super().build(input_shape)
self.build = False
self.last_dim = tensor_shape.dimension_value(input_shape[-1])
self.noisy_w = self.add_weight(
'noise_kernel',
shape=[self.last_dim, self.units],
initializer=tf.random_normal_initializer(0.0, .1),
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
dtype=self.dtype,
trainable=True)
if self.use_bias:
self.noisy_b = self.add_weight(
'noise_bias',
shape=[self.units, ],
initializer=tf.constant_initializer(self.noise_sigma / (self.units**0.5)),
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
dtype=self.dtype,
trainable=True)
else:
self.bias = None
self.build = True
