def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4): """Compares (I)DCT to SciPy (if available) and a NumPy implementation.""" np_dct = NP_DCT[dct_type](signals, norm) tf_dct = spectral_ops.dct(signals, type=dct_type, norm=norm).eval() self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol) np_idct = NP_IDCT[dct_type](signals, norm) tf_idct = spectral_ops.idct(signals, type=dct_type, norm=norm).eval() self.assertAllClose(np_idct, tf_idct, atol=atol, rtol=rtol) if fftpack: scipy_dct = fftpack.dct(signals, type=dct_type, norm=norm) self.assertAllClose(scipy_dct, tf_dct, atol=atol, rtol=rtol) scipy_idct = fftpack.idct(signals, type=dct_type, norm=norm) self.assertAllClose(scipy_idct, tf_idct, atol=atol, rtol=rtol) # Verify inverse(forward(s)) == s, up to a normalization factor. tf_idct_dct = spectral_ops.idct(tf_dct, type=dct_type, norm=norm).eval() tf_dct_idct = spectral_ops.dct(tf_idct, type=dct_type, norm=norm).eval() if norm is None: if dct_type == 1: tf_idct_dct *= 0.5 / (signals.shape[-1] - 1) tf_dct_idct *= 0.5 / (signals.shape[-1] - 1) else: tf_idct_dct *= 0.5 / signals.shape[-1] tf_dct_idct *= 0.5 / signals.shape[-1] self.assertAllClose(signals, tf_idct_dct, atol=atol, rtol=rtol) self.assertAllClose(signals, tf_dct_idct, atol=atol, rtol=rtol)
def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4): """Compares (I)DCT to SciPy (if available) and a NumPy implementation.""" np_dct = NP_DCT[dct_type](signals, norm) tf_dct = spectral_ops.dct(signals, type=dct_type, norm=norm).eval() self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol) np_idct = NP_IDCT[dct_type](signals, norm) tf_idct = spectral_ops.idct(signals, type=dct_type, norm=norm).eval() self.assertAllClose(np_idct, tf_idct, atol=atol, rtol=rtol) if fftpack: scipy_dct = fftpack.dct(signals, type=dct_type, norm=norm) self.assertAllClose(scipy_dct, tf_dct, atol=atol, rtol=rtol) scipy_idct = fftpack.idct(signals, type=dct_type, norm=norm) self.assertAllClose(scipy_idct, tf_idct, atol=atol, rtol=rtol) # Verify inverse(forward(s)) == s, up to a normalization factor. tf_idct_dct = spectral_ops.idct( tf_dct, type=dct_type, norm=norm).eval() tf_dct_idct = spectral_ops.dct( tf_idct, type=dct_type, norm=norm).eval() if norm is None: if dct_type == 1: tf_idct_dct *= 0.5 / (signals.shape[-1] - 1) tf_dct_idct *= 0.5 / (signals.shape[-1] - 1) else: tf_idct_dct *= 0.5 / signals.shape[-1] tf_dct_idct *= 0.5 / signals.shape[-1] self.assertAllClose(signals, tf_idct_dct, atol=atol, rtol=rtol) self.assertAllClose(signals, tf_dct_idct, atol=atol, rtol=rtol)
def stdct(signals, frame_length, frame_step, fft_length=None, window_fn=functools.partial(window_ops.hann_window, periodic=True), pad_end=False, name=None): with ops.name_scope(name, 'stdct', [signals, frame_length, frame_step]): signals = ops.convert_to_tensor(signals, name='signals') signals.shape.with_rank_at_least(1) frame_length = ops.convert_to_tensor(frame_length, name='frame_length') frame_length.shape.assert_has_rank(0) frame_step = ops.convert_to_tensor(frame_step, name='frame_step') frame_step.shape.assert_has_rank(0) if fft_length is None: fft_length = _enclosing_power_of_two(frame_length) else: fft_length = ops.convert_to_tensor(fft_length, name='fft_length') framed_signals = shape_ops.frame(signals, frame_length, frame_step, pad_end=pad_end) if window_fn is not None: window = window_fn(frame_length, dtype=framed_signals.dtype) framed_signals *= window return spectral_ops.dct(framed_signals)
def _compare(self, signals, norm, atol=5e-4, rtol=5e-4): """Compares the DCT to SciPy (if available) and a NumPy implementation.""" np_dct = self._np_dct2(signals, norm) tf_dct = spectral_ops.dct(signals, type=2, norm=norm).eval() self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol) if fftpack: scipy_dct = fftpack.dct(signals, type=2, norm=norm) self.assertAllClose(scipy_dct, tf_dct, atol=atol, rtol=rtol)
def test_error(self): signals = np.random.rand(10) # Unsupported type. with self.assertRaises(ValueError): spectral_ops.dct(signals, type=3) # Unknown normalization. with self.assertRaises(ValueError): spectral_ops.dct(signals, norm="bad") with self.assertRaises(NotImplementedError): spectral_ops.dct(signals, n=10) with self.assertRaises(NotImplementedError): spectral_ops.dct(signals, axis=0)
def mfccs_from_log_mel_spectrograms(log_mel_spectrograms, name=None): """Computes [MFCCs][mfcc] of `log_mel_spectrograms`. Implemented with GPU-compatible ops and supports gradients. [Mel-Frequency Cepstral Coefficient (MFCC)][mfcc] calculation consists of taking the DCT-II of a log-magnitude mel-scale spectrogram. [HTK][htk]'s MFCCs use a particular scaling of the DCT-II which is almost orthogonal normalization. We follow this convention. All `num_mel_bins` MFCCs are returned and it is up to the caller to select a subset of the MFCCs based on their application. For example, it is typical to only use the first few for speech recognition, as this results in an approximately pitch-invariant representation of the signal. For example: ```python sample_rate = 16000.0 # A Tensor of [batch_size, num_samples] mono PCM samples in the range [-1, 1]. pcm = tf.placeholder(tf.float32, [None, None]) # A 1024-point STFT with frames of 64 ms and 75% overlap. stfts = tf.contrib.signal.stft(pcm, frame_length=1024, frame_step=256, fft_length=1024) spectrograms = tf.abs(stfts) # Warp the linear scale spectrograms into the mel-scale. num_spectrogram_bins = stfts.shape[-1].value lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 80 linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix( num_mel_bins, num_spectrogram_bins, sample_rate, lower_edge_hertz, upper_edge_hertz) mel_spectrograms = tf.tensordot( spectrograms, linear_to_mel_weight_matrix, 1) mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate( linear_to_mel_weight_matrix.shape[-1:])) # Compute a stabilized log to get log-magnitude mel-scale spectrograms. log_mel_spectrograms = tf.log(mel_spectrograms + 1e-6) # Compute MFCCs from log_mel_spectrograms and take the first 13. mfccs = tf.contrib.signal.mfccs_from_log_mel_spectrograms( log_mel_spectrograms)[..., :13] ``` Args: log_mel_spectrograms: A `[..., num_mel_bins]` `float32` `Tensor` of log-magnitude mel-scale spectrograms. name: An optional name for the operation. Returns: A `[..., num_mel_bins]` `float32` `Tensor` of the MFCCs of `log_mel_spectrograms`. Raises: ValueError: If `num_mel_bins` is not positive. [mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum [htk]: https://en.wikipedia.org/wiki/HTK_(software) """ with ops.name_scope(name, 'mfccs_from_log_mel_spectrograms', [log_mel_spectrograms]): # Compute the DCT-II of the resulting log-magnitude mel-scale spectrogram. # The DCT used in HTK scales every basis vector by sqrt(2/N), which is the # scaling required for an "orthogonal" DCT-II *except* in the 0th bin, where # the true orthogonal DCT (as implemented by scipy) scales by sqrt(1/N). For # this reason, we don't apply orthogonal normalization and scale the DCT by # `0.5 * sqrt(2/N)` manually. log_mel_spectrograms = ops.convert_to_tensor(log_mel_spectrograms, dtype=dtypes.float32) if (log_mel_spectrograms.shape.ndims and log_mel_spectrograms.shape.dims[-1].value is not None): num_mel_bins = log_mel_spectrograms.shape.dims[-1].value if num_mel_bins == 0: raise ValueError('num_mel_bins must be positive. Got: %s' % log_mel_spectrograms) else: num_mel_bins = array_ops.shape(log_mel_spectrograms)[-1] dct2 = spectral_ops.dct(log_mel_spectrograms) return dct2 * math_ops.rsqrt(math_ops.to_float(num_mel_bins) * 2.0)
def test_error(self): signals = np.random.rand(10) # Unsupported type. with self.assertRaises(ValueError): spectral_ops.dct(signals, type=5) # DCT-I normalization not implemented. with self.assertRaises(ValueError): spectral_ops.dct(signals, type=1, norm="ortho") # DCT-I requires at least two inputs. with self.assertRaises(ValueError): spectral_ops.dct(np.random.rand(1), type=1) # Unknown normalization. with self.assertRaises(ValueError): spectral_ops.dct(signals, norm="bad") with self.assertRaises(NotImplementedError): spectral_ops.dct(signals, n=10) with self.assertRaises(NotImplementedError): spectral_ops.dct(signals, axis=0)
def mfccs_from_log_mel_spectrograms(log_mel_spectrograms, name=None): """Computes [MFCCs][mfcc] of `log_mel_spectrograms`. Implemented with GPU-compatible ops and supports gradients. [Mel-Frequency Cepstral Coefficient (MFCC)][mfcc] calculation consists of taking the DCT-II of a log-magnitude mel-scale spectrogram. [HTK][htk]'s MFCCs use a particular scaling of the DCT-II which is almost orthogonal normalization. We follow this convention. All `num_mel_bins` MFCCs are returned and it is up to the caller to select a subset of the MFCCs based on their application. For example, it is typical to only use the first few for speech recognition, as this results in an approximately pitch-invariant representation of the signal. For example: ```python sample_rate = 16000.0 # A Tensor of [batch_size, num_samples] mono PCM samples in the range [-1, 1]. pcm = tf.placeholder(tf.float32, [None, None]) # A 1024-point STFT with frames of 64 ms and 75% overlap. stfts = tf.contrib.signal.stft(pcm, frame_length=1024, frame_step=256, fft_length=1024) spectrograms = tf.abs(stfts) # Warp the linear scale spectrograms into the mel-scale. num_spectrogram_bins = stfts.shape[-1].value lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 80 linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix( num_mel_bins, num_spectrogram_bins, sample_rate, lower_edge_hertz, upper_edge_hertz) mel_spectrograms = tf.tensordot( spectrograms, linear_to_mel_weight_matrix, 1) mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate( linear_to_mel_weight_matrix.shape[-1:])) # Compute a stabilized log to get log-magnitude mel-scale spectrograms. log_mel_spectrograms = tf.log(mel_spectrograms + 1e-6) # Compute MFCCs from log_mel_spectrograms and take the first 13. mfccs = tf.contrib.signal.mfccs_from_log_mel_spectrograms( log_mel_spectrograms)[..., :13] ``` Args: log_mel_spectrograms: A `[..., num_mel_bins]` `float32` `Tensor` of log-magnitude mel-scale spectrograms. name: An optional name for the operation. Returns: A `[..., num_mel_bins]` `float32` `Tensor` of the MFCCs of `log_mel_spectrograms`. Raises: ValueError: If `num_mel_bins` is not positive. [mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum [htk]: https://en.wikipedia.org/wiki/HTK_(software) """ with ops.name_scope(name, 'mfccs_from_log_mel_spectrograms', [log_mel_spectrograms]): # Compute the DCT-II of the resulting log-magnitude mel-scale spectrogram. # The DCT used in HTK scales every basis vector by sqrt(2/N), which is the # scaling required for an "orthogonal" DCT-II *except* in the 0th bin, where # the true orthogonal DCT (as implemented by scipy) scales by sqrt(1/N). For # this reason, we don't apply orthogonal normalization and scale the DCT by # `0.5 * sqrt(2/N)` manually. log_mel_spectrograms = ops.convert_to_tensor(log_mel_spectrograms, dtype=dtypes.float32) if (log_mel_spectrograms.shape.ndims and log_mel_spectrograms.shape[-1].value is not None): num_mel_bins = log_mel_spectrograms.shape[-1].value if num_mel_bins == 0: raise ValueError('num_mel_bins must be positive. Got: %s' % log_mel_spectrograms) else: num_mel_bins = array_ops.shape(log_mel_spectrograms)[-1] dct2 = spectral_ops.dct(log_mel_spectrograms) return dct2 * math_ops.rsqrt(num_mel_bins * 2.0)