def testNegAxisCorrectness(self): x_ = np.float32([[1., 2, 3], [4, 5, 6]]) value_ = np.float32(0.25) count_ = np.int32(2) x = tf1.placeholder_with_default( x_, shape=x_.shape if self.is_static_shape else None) value = ( tf.constant(value_) if self.is_static_shape else tf1.placeholder_with_default(value_, shape=None)) count = ( tf.constant(count_) if self.is_static_shape else tf1.placeholder_with_default(count_, shape=None)) x0_front = distribution_util.pad( x, axis=-2, value=value, count=count, front=True) x0_back = distribution_util.pad( x, axis=-2, count=count, back=True) x0_both = distribution_util.pad( x, axis=-2, value=value, front=True, back=True) if self.is_static_shape: self.assertAllEqual([4, 3], x0_front.shape) self.assertAllEqual([4, 3], x0_back.shape) self.assertAllEqual([4, 3], x0_both.shape) [x0_front_, x0_back_, x0_both_] = self.evaluate([ x0_front, x0_back, x0_both]) self.assertAllClose( np.float32([[value_]*3, [value_]*3, [1, 2, 3], [4, 5, 6]]), x0_front_, atol=0., rtol=1e-6) self.assertAllClose( np.float32([[1, 2, 3], [4, 5, 6], [0.]*3, [0.]*3]), x0_back_, atol=0., rtol=1e-6) self.assertAllClose( np.float32([[value_]*3, [1, 2, 3], [4, 5, 6], [value_]*3]), x0_both_, atol=0., rtol=1e-6)
def pad_shape_right_with_ones(x, ndims): """Maybe add `ndims` ones to `x.shape` on the right. If `ndims` is zero, this is a no-op; otherwise, we will create and return a new `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the right side. If the shape of `x` is known statically, the shape of the return value will be as well. Args: x: The `Tensor` we'll return a reshaping of. ndims: Python `integer` number of ones to pad onto `x.shape`. Returns: If `ndims` is zero, `x`; otherwise, a `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the right side. If possible, returns a `Tensor` whose shape is known statically. Raises: ValueError: if `ndims` is not a Python `integer` greater than or equal to zero. """ if not (isinstance(ndims, int) and ndims >= 0): raise ValueError( '`ndims` must be a Python `integer` greater than zero. Got: {}' .format(ndims)) if ndims == 0: return x x = tf.convert_to_tensor(x) original_shape = x.shape new_shape = distribution_util.pad( tf.shape(x), axis=0, back=True, value=1, count=ndims) x = tf.reshape(x, new_shape) x.set_shape(original_shape.concatenate([1]*ndims)) return x
def _embed_l2_regularization(): """Adds synthetic observations to implement L2 regularization.""" # `tf.matrix_solve_ls` does not respect the `l2_regularization` argument # when `fast_unsafe_numerics` is `False`. This function adds synthetic # observations to the data to implement the regularization instead. # Adding observations `sqrt(l2_regularizer) * I` is mathematically # equivalent to adding the term # `-l2_regularizer ||coefficients||_2**2` to the log-likelihood. num_model_coefficients = num_cols(model_matrix) batch_shape = tf.shape(model_matrix)[:-2] if l2_regularization_penalty_factor is None: eye = tf.eye( num_model_coefficients, batch_shape=batch_shape, dtype=a.dtype) else: eye = tf.linalg.tensor_diag( tf.cast(l2_regularization_penalty_factor, dtype=a.dtype)) broadcasted_shape = prefer_static.concat( [batch_shape, [num_model_coefficients, num_model_coefficients]], axis=0) eye = tf.broadcast_to(eye, broadcasted_shape) a_ = tf.concat([a, tf.sqrt(l2_regularizer) * eye], axis=-2) b_ = distribution_util.pad( b, count=num_model_coefficients, axis=-1, back=True) # Return l2_regularizer=0 since its now embedded. l2_regularizer_ = np.array(0, dtype_util.as_numpy_dtype(a.dtype)) return a_, b_, l2_regularizer_
def quadrature_scheme_softmaxnormal_gauss_hermite( normal_loc, normal_scale, quadrature_size, validate_args=False, name=None): """Use Gauss-Hermite quadrature to form quadrature on `K - 1` simplex. A `SoftmaxNormal` random variable `Y` may be generated via ``` Y = SoftmaxCentered(X), X = Normal(normal_loc, normal_scale) ``` Note: for a given `quadrature_size`, this method is generally less accurate than `quadrature_scheme_softmaxnormal_quantiles`. Args: normal_loc: `float`-like `Tensor` with shape `[b1, ..., bB, K-1]`, B>=0. The location parameter of the Normal used to construct the SoftmaxNormal. normal_scale: `float`-like `Tensor`. Broadcastable with `normal_loc`. The scale parameter of the Normal used to construct the SoftmaxNormal. quadrature_size: Python `int` scalar representing the number of quadrature points. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect outputs. name: Python `str` name prefixed to Ops created by this class. Returns: grid: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the convex combination of affine parameters for `K` components. `grid[..., :, n]` is the `n`-th grid point, living in the `K - 1` simplex. probs: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the associated with each grid point. """ with tf.name_scope(name, "quadrature_scheme_softmaxnormal_gauss_hermite", [normal_loc, normal_scale]): normal_loc = tf.convert_to_tensor(normal_loc, name="normal_loc") dt = normal_loc.dtype.base_dtype normal_scale = tf.convert_to_tensor( normal_scale, dtype=dt, name="normal_scale") normal_scale = maybe_check_quadrature_param( normal_scale, "normal_scale", validate_args) grid, probs = np.polynomial.hermite.hermgauss(deg=quadrature_size) grid = grid.astype(dt.dtype.as_numpy_dtype) probs = probs.astype(dt.dtype.as_numpy_dtype) probs /= np.linalg.norm(probs, ord=1, keepdims=True) probs = tf.convert_to_tensor(probs, name="probs", dtype=dt) grid = softmax( -distribution_util.pad( (normal_loc[..., tf.newaxis] + np.sqrt(2.) * normal_scale[..., tf.newaxis] * grid), axis=-2, front=True), axis=-2) # shape: [B, components, deg] return grid, probs
def pad_shape_right_with_ones(x, ndims): """Maybe add `ndims` ones to `x.shape` on the right. If `ndims` is zero, this is a no-op; otherwise, we will create and return a new `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the right side. If the shape of `x` is known statically, the shape of the return value will be as well. Args: x: The `Tensor` we'll return a reshaping of. ndims: Python `integer` number of ones to pad onto `x.shape`. Returns: If `ndims` is zero, `x`; otherwise, a `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the right side. If possible, returns a `Tensor` whose shape is known statically. Raises: ValueError: if `ndims` is not a Python `integer` greater than or equal to zero. """ if not (isinstance(ndims, int) and ndims >= 0): raise ValueError( '`ndims` must be a Python `integer` greater than zero. Got: {}'. format(ndims)) if ndims == 0: return x x = tf.convert_to_tensor(x) original_shape = x.shape new_shape = distribution_util.pad(tf.shape(x), axis=0, back=True, value=1, count=ndims) x = tf.reshape(x, new_shape) x.set_shape(original_shape.concatenate([1] * ndims)) return x
def quadrature_scheme_softmaxnormal_gauss_hermite( normal_loc, normal_scale, quadrature_size, validate_args=False, name=None): """Use Gauss-Hermite quadrature to form quadrature on `K - 1` simplex. A `SoftmaxNormal` random variable `Y` may be generated via ``` Y = SoftmaxCentered(X), X = Normal(normal_loc, normal_scale) ``` Note: for a given `quadrature_size`, this method is generally less accurate than `quadrature_scheme_softmaxnormal_quantiles`. Args: normal_loc: `float`-like `Tensor` with shape `[b1, ..., bB, K-1]`, B>=0. The location parameter of the Normal used to construct the SoftmaxNormal. normal_scale: `float`-like `Tensor`. Broadcastable with `normal_loc`. The scale parameter of the Normal used to construct the SoftmaxNormal. quadrature_size: Python `int` scalar representing the number of quadrature points. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect outputs. name: Python `str` name prefixed to Ops created by this class. Returns: grid: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the convex combination of affine parameters for `K` components. `grid[..., :, n]` is the `n`-th grid point, living in the `K - 1` simplex. probs: Shape `[b1, ..., bB, K, quadrature_size]` `Tensor` representing the associated with each grid point. """ with tf.name_scope( name or "quadrature_scheme_softmaxnormal_gauss_hermite"): normal_loc = tf.convert_to_tensor(value=normal_loc, name="normal_loc") npdt = dtype_util.as_numpy_dtype(normal_loc.dtype) normal_scale = tf.convert_to_tensor( value=normal_scale, dtype=npdt, name="normal_scale") normal_scale = maybe_check_quadrature_param( normal_scale, "normal_scale", validate_args) grid, probs = np.polynomial.hermite.hermgauss(deg=quadrature_size) grid = grid.astype(npdt) probs = probs.astype(npdt) probs /= np.linalg.norm(probs, ord=1, keepdims=True) probs = tf.convert_to_tensor(value=probs, name="probs", dtype=npdt) grid = softmax( -distribution_util.pad( (normal_loc[..., tf.newaxis] + np.sqrt(2.) * normal_scale[..., tf.newaxis] * grid), axis=-2, front=True), axis=-2) # shape: [B, components, deg] return grid, probs
def testPosAxisCorrectness(self): x_ = np.float32([[1., 2, 3], [4, 5, 6]]) value_ = np.float32(0.25) count_ = np.int32(2) with self.test_session() as sess: x = tf.placeholder_with_default( x_, shape=x_.shape if self.is_static_shape else None) value = (tf.constant(value_) if self.is_static_shape else tf.placeholder_with_default(value_, shape=None)) count = (tf.constant(count_) if self.is_static_shape else tf.placeholder_with_default(count_, shape=None)) x1_front = distribution_util.pad(x, axis=1, value=value, count=count, front=True) x1_back = distribution_util.pad(x, axis=1, count=count, back=True) x1_both = distribution_util.pad(x, axis=1, value=value, front=True, back=True) if self.is_static_shape: self.assertAllEqual([2, 5], x1_front.shape) self.assertAllEqual([2, 5], x1_back.shape) self.assertAllEqual([2, 5], x1_both.shape) [x1_front_, x1_back_, x1_both_] = sess.run([x1_front, x1_back, x1_both]) self.assertAllClose(np.float32([[value_] * 2 + [1, 2, 3], [value_] * 2 + [4, 5, 6]]), x1_front_, atol=0., rtol=1e-6) self.assertAllClose(np.float32([[1, 2, 3] + [0.] * 2, [4, 5, 6] + [0.] * 2]), x1_back_, atol=0., rtol=1e-6) self.assertAllClose(np.float32([[value_, 1, 2, 3, value_], [value_, 4, 5, 6, value_]]), x1_both_, atol=0., rtol=1e-6)
def _forward(self, x): # Pad the last dim with a zeros vector. We need this because it lets us # infer the scale in the inverse function. y = distribution_util.pad(x, axis=-1, back=True) # Set shape hints. if x.shape.ndims is not None: shape = x.shape[:-1].concatenate(x.shape[-1] + 1) y.shape.assert_is_compatible_with(shape) y.set_shape(shape) return tf.nn.softmax(y)
def _forward(self, x): # Pad the last dim with a zeros vector. We need this because it lets us # infer the scale in the inverse function. y = distribution_util.pad(x, axis=-1, back=True) # Set shape hints. if x.shape.ndims is not None: shape = x.shape[:-1].concatenate(x.shape[-1] + 1) y.shape.assert_is_compatible_with(shape) y.set_shape(shape) return tf.nn.softmax(y)
def pad_shape_with_ones(x, ndims, start=-1): """Maybe add `ndims` ones to `x.shape` starting at `start`. If `ndims` is zero, this is a no-op; otherwise, we will create and return a new `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the right side. If the shape of `x` is known statically, the shape of the return value will be as well. Args: x: The `Tensor` we'll return a reshaping of. ndims: Python `integer` number of ones to pad onto `x.shape`. start: Python `integer` specifying where to start padding with ones. Must be a negative integer. For instance, a value of `-1` means to pad at the end of the shape. Default value: `-1`. Returns: If `ndims` is zero, `x`; otherwise, a `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the right side. If possible, returns a `Tensor` whose shape is known statically. Raises: ValueError: if `ndims` is not a Python `integer` greater than or equal to zero. """ if not (isinstance(ndims, int) and ndims >= 0): raise ValueError( '`ndims` must be a Python `integer` greater than zero. Got: {}'. format(ndims)) if not (isinstance(start, int) and start <= -1): raise ValueError( '`start` must be a Python `integer` less than zero. Got: {}'. format(start)) if ndims == 0: return x x = tf.convert_to_tensor(value=x) original_shape = x.shape rank = tf.rank(x) first_shape = tf.shape(x)[:rank + start + 1] second_shape = tf.shape(x)[rank + start + 1:] new_shape = distribution_util.pad(first_shape, axis=0, back=True, value=1, count=ndims) new_shape = tf.concat([new_shape, second_shape], axis=0) x = tf.reshape(x, new_shape) if start == -1: x.set_shape(original_shape.concatenate([1] * ndims)) elif original_shape.ndims is not None: x.set_shape(original_shape[:original_shape.ndims + start + 1].concatenate([1] * ndims).concatenate( original_shape[original_shape.ndims + start + 1:])) return x
def _forward(self, x): # Pad the last dim with a zeros vector. We need this because it lets us # infer the scale in the inverse function. y = distribution_util.pad(x, axis=-1, back=True) # Set shape hints. if tensorshape_util.rank(x.shape) is not None: last_dim = tf.compat.dimension_value(x.shape[-1]) shape = tensorshape_util.concatenate( x.shape[:-1], None if last_dim is None else last_dim + 1) tensorshape_util.set_shape(y, shape) return tf.math.softmax(y)
def _embed_l2_regularization(): """Adds synthetic observations to implement L2 regularization.""" # `tf.matrix_solve_ls` does not respect the `l2_regularization` argument # when `fast_unsafe_numerics` is `False`. This function adds synthetic # observations to the data to implement the regularization instead. # Adding observations `sqrt(l2_regularizer) * I` is mathematically # equivalent to adding the term # `-l2_regularizer ||coefficients||_2**2` to the log-likelihood. num_model_coefficients = num_cols(model_matrix) batch_shape = tf.shape(input=model_matrix)[:-2] eye = tf.eye( num_model_coefficients, batch_shape=batch_shape, dtype=a.dtype) a_ = tf.concat([a, tf.sqrt(l2_regularizer) * eye], axis=-2) b_ = distribution_util.pad( b, count=num_model_coefficients, axis=-1, back=True) # Return l2_regularizer=0 since its now embedded. l2_regularizer_ = np.array(0, a.dtype.as_numpy_dtype) return a_, b_, l2_regularizer_
def _embed_l2_regularization(): """Adds synthetic observations to implement L2 regularization.""" # `tf.matrix_solve_ls` does not respect the `l2_regularization` argument # when `fast_unsafe_numerics` is `False`. This function adds synthetic # observations to the data to implement the regularization instead. # Adding observations `sqrt(l2_regularizer) * I` is mathematically # equivalent to adding the term # `-l2_regularizer ||coefficients||_2**2` to the log-likelihood. num_model_coefficients = num_cols(model_matrix) batch_shape = tf.shape(model_matrix)[:-2] eye = tf.eye( num_model_coefficients, batch_shape=batch_shape, dtype=a.dtype) a_ = tf.concat([a, tf.sqrt(l2_regularizer) * eye], axis=-2) b_ = distribution_util.pad( b, count=num_model_coefficients, axis=-1, back=True) # Return l2_regularizer=0 since its now embedded. l2_regularizer_ = np.array(0, a.dtype.as_numpy_dtype) return a_, b_, l2_regularizer_
def _uniform_correlation_like_matrix(num_rows, batch_shape, dtype, seed): """Returns a uniformly random `Tensor` of "correlation-like" matrices. A "correlation-like" matrix is a symmetric square matrix with all entries between -1 and 1 (inclusive) and 1s on the main diagonal. Of these, the ones that are positive semi-definite are exactly the correlation matrices. Args: num_rows: Python `int` dimension of the correlation-like matrices. batch_shape: `Tensor` or Python `tuple` of `int` shape of the batch to return. dtype: `dtype` of the `Tensor` to return. seed: Random seed. Returns: matrices: A `Tensor` of shape `batch_shape + [num_rows, num_rows]` and dtype `dtype`. Each entry is in [-1, 1], and each matrix along the bottom two dimensions is symmetric and has 1s on the main diagonal. """ num_entries = num_rows * (num_rows + 1) / 2 ones = tf.ones(shape=[num_entries], dtype=dtype) # It seems wasteful to generate random values for the diagonal since # I am going to throw them away, but `fill_triangular` fills the # diagonal, so I probably need them. # It's not impossible that it would be more efficient to just fill # the whole matrix with random values instead of messing with # `fill_triangular`. Then would need to filter almost half out with # `matrix_band_part`. unifs = uniform.Uniform(-ones, ones).sample(batch_shape, seed=seed) tril = util.fill_triangular(unifs) symmetric = tril + tf.linalg.transpose(tril) diagonal_ones = tf.ones(shape=util.pad(batch_shape, axis=0, back=True, value=num_rows), dtype=dtype) return tf.linalg.set_diag(symmetric, diagonal_ones)
def _uniform_correlation_like_matrix(num_rows, batch_shape, dtype, seed): """Returns a uniformly random `Tensor` of "correlation-like" matrices. A "correlation-like" matrix is a symmetric square matrix with all entries between -1 and 1 (inclusive) and 1s on the main diagonal. Of these, the ones that are positive semi-definite are exactly the correlation matrices. Args: num_rows: Python `int` dimension of the correlation-like matrices. batch_shape: `Tensor` or Python `tuple` of `int` shape of the batch to return. dtype: `dtype` of the `Tensor` to return. seed: Random seed. Returns: matrices: A `Tensor` of shape `batch_shape + [num_rows, num_rows]` and dtype `dtype`. Each entry is in [-1, 1], and each matrix along the bottom two dimensions is symmetric and has 1s on the main diagonal. """ num_entries = num_rows * (num_rows + 1) / 2 ones = tf.ones(shape=[num_entries], dtype=dtype) # It seems wasteful to generate random values for the diagonal since # I am going to throw them away, but `fill_triangular` fills the # diagonal, so I probably need them. # It's not impossible that it would be more efficient to just fill # the whole matrix with random values instead of messing with # `fill_triangular`. Then would need to filter almost half out with # `matrix_band_part`. unifs = uniform.Uniform(-ones, ones).sample(batch_shape, seed=seed) tril = util.fill_triangular(unifs) symmetric = tril + tf.matrix_transpose(tril) diagonal_ones = tf.ones( shape=util.pad(batch_shape, axis=0, back=True, value=num_rows), dtype=dtype) return tf.matrix_set_diag(symmetric, diagonal_ones)
def auto_correlation(x, axis=-1, max_lags=None, center=True, normalize=True, name='auto_correlation'): """Auto correlation along one axis. Given a `1-D` wide sense stationary (WSS) sequence `X`, the auto correlation `RXX` may be defined as (with `E` expectation and `Conj` complex conjugate) ``` RXX[m] := E{ W[m] Conj(W[0]) } = E{ W[0] Conj(W[-m]) }, W[n] := (X[n] - MU) / S, MU := E{ X[0] }, S**2 := E{ (X[0] - MU) Conj(X[0] - MU) }. ``` This function takes the viewpoint that `x` is (along one axis) a finite sub-sequence of a realization of (WSS) `X`, and then uses `x` to produce an estimate of `RXX[m]` as follows: After extending `x` from length `L` to `inf` by zero padding, the auto correlation estimate `rxx[m]` is computed for `m = 0, 1, ..., max_lags` as ``` rxx[m] := (L - m)**-1 sum_n w[n + m] Conj(w[n]), w[n] := (x[n] - mu) / s, mu := L**-1 sum_n x[n], s**2 := L**-1 sum_n (x[n] - mu) Conj(x[n] - mu) ``` The error in this estimate is proportional to `1 / sqrt(len(x) - m)`, so users often set `max_lags` small enough so that the entire output is meaningful. Note that since `mu` is an imperfect estimate of `E{ X[0] }`, and we divide by `len(x) - m` rather than `len(x) - m - 1`, our estimate of auto correlation contains a slight bias, which goes to zero as `len(x) - m --> infinity`. Args: x: `float32` or `complex64` `Tensor`. axis: Python `int`. The axis number along which to compute correlation. Other dimensions index different batch members. max_lags: Positive `int` tensor. The maximum value of `m` to consider (in equation above). If `max_lags >= x.shape[axis]`, we effectively re-set `max_lags` to `x.shape[axis] - 1`. center: Python `bool`. If `False`, do not subtract the mean estimate `mu` from `x[n]` when forming `w[n]`. normalize: Python `bool`. If `False`, do not divide by the variance estimate `s**2` when forming `w[n]`. name: `String` name to prepend to created ops. Returns: `rxx`: `Tensor` of same `dtype` as `x`. `rxx.shape[i] = x.shape[i]` for `i != axis`, and `rxx.shape[axis] = max_lags + 1`. Raises: TypeError: If `x` is not a supported type. """ # Implementation details: # Extend length N / 2 1-D array x to length N by zero padding onto the end. # Then, set # F[x]_k := sum_n x_n exp{-i 2 pi k n / N }. # It is not hard to see that # F[x]_k Conj(F[x]_k) = F[R]_k, where # R_m := sum_n x_n Conj(x_{(n - m) mod N}). # One can also check that R_m / (N / 2 - m) is an unbiased estimate of RXX[m]. # Since F[x] is the DFT of x, this leads us to a zero-padding and FFT/IFFT # based version of estimating RXX. # Note that this is a special case of the Wiener-Khinchin Theorem. with tf.name_scope(name): x = tf.convert_to_tensor(x, name='x') # Rotate dimensions of x in order to put axis at the rightmost dim. # FFT op requires this. rank = ps.rank(x) if axis < 0: axis = rank + axis shift = rank - 1 - axis # Suppose x.shape[axis] = T, so there are T 'time' steps. # ==> x_rotated.shape = B + [T], # where B is x_rotated's batch shape. x_rotated = distribution_util.rotate_transpose(x, shift) if center: x_rotated = x_rotated - tf.reduce_mean( x_rotated, axis=-1, keepdims=True) # x_len = N / 2 from above explanation. The length of x along axis. # Get a value for x_len that works in all cases. x_len = ps.shape(x_rotated)[-1] # TODO(langmore) Investigate whether this zero padding helps or hurts. At # the moment is necessary so that all FFT implementations work. # Zero pad to the next power of 2 greater than 2 * x_len, which equals # 2**(ceil(Log_2(2 * x_len))). Note: Log_2(X) = Log_e(X) / Log_e(2). x_len_float64 = ps.cast(x_len, np.float64) target_length = ps.pow(np.float64(2.), ps.ceil(ps.log(x_len_float64 * 2) / np.log(2.))) pad_length = ps.cast(target_length - x_len_float64, np.int32) # We should have: # x_rotated_pad.shape = x_rotated.shape[:-1] + [T + pad_length] # = B + [T + pad_length] x_rotated_pad = distribution_util.pad(x_rotated, axis=-1, back=True, count=pad_length) dtype = x.dtype if not dtype_util.is_complex(dtype): if not dtype_util.is_floating(dtype): raise TypeError( 'Argument x must have either float or complex dtype' ' found: {}'.format(dtype)) x_rotated_pad = tf.complex( x_rotated_pad, dtype_util.as_numpy_dtype(dtype_util.real_dtype(dtype))(0.)) # Autocorrelation is IFFT of power-spectral density (up to some scaling). fft_x_rotated_pad = tf.signal.fft(x_rotated_pad) spectral_density = fft_x_rotated_pad * tf.math.conj(fft_x_rotated_pad) # shifted_product is R[m] from above detailed explanation. # It is the inner product sum_n X[n] * Conj(X[n - m]). shifted_product = tf.signal.ifft(spectral_density) # Cast back to real-valued if x was real to begin with. shifted_product = tf.cast(shifted_product, dtype) # Figure out if we can deduce the final static shape, and set max_lags. # Use x_rotated as a reference, because it has the time dimension in the far # right, and was created before we performed all sorts of crazy shape # manipulations. know_static_shape = True if not tensorshape_util.is_fully_defined(x_rotated.shape): know_static_shape = False if max_lags is None: max_lags = x_len - 1 else: max_lags = tf.convert_to_tensor(max_lags, name='max_lags') max_lags_ = tf.get_static_value(max_lags) if max_lags_ is None or not know_static_shape: know_static_shape = False max_lags = tf.minimum(x_len - 1, max_lags) else: max_lags = min(x_len - 1, max_lags_) # Chop off the padding. # We allow users to provide a huge max_lags, but cut it off here. # shifted_product_chopped.shape = x_rotated.shape[:-1] + [max_lags] shifted_product_chopped = shifted_product[..., :max_lags + 1] # If possible, set shape. if know_static_shape: chopped_shape = tensorshape_util.as_list(x_rotated.shape) chopped_shape[-1] = min(x_len, max_lags + 1) tensorshape_util.set_shape(shifted_product_chopped, chopped_shape) # Recall R[m] is a sum of N / 2 - m nonzero terms x[n] Conj(x[n - m]). The # other terms were zeros arising only due to zero padding. # `denominator = (N / 2 - m)` (defined below) is the proper term to # divide by to make this an unbiased estimate of the expectation # E[X[n] Conj(X[n - m])]. x_len = ps.cast(x_len, dtype_util.real_dtype(dtype)) max_lags = ps.cast(max_lags, dtype_util.real_dtype(dtype)) denominator = x_len - ps.range(0., max_lags + 1.) denominator = ps.cast(denominator, dtype) shifted_product_rotated = shifted_product_chopped / denominator if normalize: shifted_product_rotated /= shifted_product_rotated[..., :1] # Transpose dimensions back to those of x. return distribution_util.rotate_transpose(shifted_product_rotated, -shift)
def auto_correlation(x, axis=-1, max_lags=None, center=True, normalize=True, name='auto_correlation'): """Auto correlation along one axis. Given a `1-D` wide sense stationary (WSS) sequence `X`, the auto correlation `RXX` may be defined as (with `E` expectation and `Conj` complex conjugate) ``` RXX[m] := E{ W[m] Conj(W[0]) } = E{ W[0] Conj(W[-m]) }, W[n] := (X[n] - MU) / S, MU := E{ X[0] }, S**2 := E{ (X[0] - MU) Conj(X[0] - MU) }. ``` This function takes the viewpoint that `x` is (along one axis) a finite sub-sequence of a realization of (WSS) `X`, and then uses `x` to produce an estimate of `RXX[m]` as follows: After extending `x` from length `L` to `inf` by zero padding, the auto correlation estimate `rxx[m]` is computed for `m = 0, 1, ..., max_lags` as ``` rxx[m] := (L - m)**-1 sum_n w[n + m] Conj(w[n]), w[n] := (x[n] - mu) / s, mu := L**-1 sum_n x[n], s**2 := L**-1 sum_n (x[n] - mu) Conj(x[n] - mu) ``` The error in this estimate is proportional to `1 / sqrt(len(x) - m)`, so users often set `max_lags` small enough so that the entire output is meaningful. Note that since `mu` is an imperfect estimate of `E{ X[0] }`, and we divide by `len(x) - m` rather than `len(x) - m - 1`, our estimate of auto correlation contains a slight bias, which goes to zero as `len(x) - m --> infinity`. Args: x: `float32` or `complex64` `Tensor`. axis: Python `int`. The axis number along which to compute correlation. Other dimensions index different batch members. max_lags: Positive `int` tensor. The maximum value of `m` to consider (in equation above). If `max_lags >= x.shape[axis]`, we effectively re-set `max_lags` to `x.shape[axis] - 1`. center: Python `bool`. If `False`, do not subtract the mean estimate `mu` from `x[n]` when forming `w[n]`. normalize: Python `bool`. If `False`, do not divide by the variance estimate `s**2` when forming `w[n]`. name: `String` name to prepend to created ops. Returns: `rxx`: `Tensor` of same `dtype` as `x`. `rxx.shape[i] = x.shape[i]` for `i != axis`, and `rxx.shape[axis] = max_lags + 1`. Raises: TypeError: If `x` is not a supported type. """ # Implementation details: # Extend length N / 2 1-D array x to length N by zero padding onto the end. # Then, set # F[x]_k := sum_n x_n exp{-i 2 pi k n / N }. # It is not hard to see that # F[x]_k Conj(F[x]_k) = F[R]_k, where # R_m := sum_n x_n Conj(x_{(n - m) mod N}). # One can also check that R_m / (N / 2 - m) is an unbiased estimate of RXX[m]. # Since F[x] is the DFT of x, this leads us to a zero-padding and FFT/IFFT # based version of estimating RXX. # Note that this is a special case of the Wiener-Khinchin Theorem. with tf.name_scope(name, values=[x]): x = tf.convert_to_tensor(x, name='x') # Rotate dimensions of x in order to put axis at the rightmost dim. # FFT op requires this. rank = util.prefer_static_rank(x) if axis < 0: axis = rank + axis shift = rank - 1 - axis # Suppose x.shape[axis] = T, so there are T 'time' steps. # ==> x_rotated.shape = B + [T], # where B is x_rotated's batch shape. x_rotated = util.rotate_transpose(x, shift) if center: x_rotated -= tf.reduce_mean(x_rotated, axis=-1, keepdims=True) # x_len = N / 2 from above explanation. The length of x along axis. # Get a value for x_len that works in all cases. x_len = util.prefer_static_shape(x_rotated)[-1] # TODO(langmore) Investigate whether this zero padding helps or hurts. At # the moment is necessary so that all FFT implementations work. # Zero pad to the next power of 2 greater than 2 * x_len, which equals # 2**(ceil(Log_2(2 * x_len))). Note: Log_2(X) = Log_e(X) / Log_e(2). x_len_float64 = tf.cast(x_len, np.float64) target_length = tf.pow( np.float64(2.), tf.ceil(tf.log(x_len_float64 * 2) / np.log(2.))) pad_length = tf.cast(target_length - x_len_float64, np.int32) # We should have: # x_rotated_pad.shape = x_rotated.shape[:-1] + [T + pad_length] # = B + [T + pad_length] x_rotated_pad = util.pad(x_rotated, axis=-1, back=True, count=pad_length) dtype = x.dtype if not dtype.is_complex: if not dtype.is_floating: raise TypeError('Argument x must have either float or complex dtype' ' found: {}'.format(dtype)) x_rotated_pad = tf.complex(x_rotated_pad, dtype.real_dtype.as_numpy_dtype(0.)) # Autocorrelation is IFFT of power-spectral density (up to some scaling). fft_x_rotated_pad = tf.fft(x_rotated_pad) spectral_density = fft_x_rotated_pad * tf.conj(fft_x_rotated_pad) # shifted_product is R[m] from above detailed explanation. # It is the inner product sum_n X[n] * Conj(X[n - m]). shifted_product = tf.ifft(spectral_density) # Cast back to real-valued if x was real to begin with. shifted_product = tf.cast(shifted_product, dtype) # Figure out if we can deduce the final static shape, and set max_lags. # Use x_rotated as a reference, because it has the time dimension in the far # right, and was created before we performed all sorts of crazy shape # manipulations. know_static_shape = True if not x_rotated.shape.is_fully_defined(): know_static_shape = False if max_lags is None: max_lags = x_len - 1 else: max_lags = tf.convert_to_tensor(max_lags, name='max_lags') max_lags_ = tf.contrib.util.constant_value(max_lags) if max_lags_ is None or not know_static_shape: know_static_shape = False max_lags = tf.minimum(x_len - 1, max_lags) else: max_lags = min(x_len - 1, max_lags_) # Chop off the padding. # We allow users to provide a huge max_lags, but cut it off here. # shifted_product_chopped.shape = x_rotated.shape[:-1] + [max_lags] shifted_product_chopped = shifted_product[..., :max_lags + 1] # If possible, set shape. if know_static_shape: chopped_shape = x_rotated.shape.as_list() chopped_shape[-1] = min(x_len, max_lags + 1) shifted_product_chopped.set_shape(chopped_shape) # Recall R[m] is a sum of N / 2 - m nonzero terms x[n] Conj(x[n - m]). The # other terms were zeros arising only due to zero padding. # `denominator = (N / 2 - m)` (defined below) is the proper term to # divide by to make this an unbiased estimate of the expectation # E[X[n] Conj(X[n - m])]. x_len = tf.cast(x_len, dtype.real_dtype) max_lags = tf.cast(max_lags, dtype.real_dtype) denominator = x_len - tf.range(0., max_lags + 1.) denominator = tf.cast(denominator, dtype) shifted_product_rotated = shifted_product_chopped / denominator if normalize: shifted_product_rotated /= shifted_product_rotated[..., :1] # Transpose dimensions back to those of x. return util.rotate_transpose(shifted_product_rotated, -shift)