def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4):
"""Compares (I)DCT to SciPy (if available) and a NumPy implementation."""
np_dct = NP_DCT[dct_type](signals, norm)
tf_dct = spectral_ops.dct(signals, type=dct_type, norm=norm).eval()
self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol)
np_idct = NP_IDCT[dct_type](signals, norm)
tf_idct = spectral_ops.idct(signals, type=dct_type, norm=norm).eval()
self.assertAllClose(np_idct, tf_idct, atol=atol, rtol=rtol)
if fftpack:
scipy_dct = fftpack.dct(signals, type=dct_type, norm=norm)
self.assertAllClose(scipy_dct, tf_dct, atol=atol, rtol=rtol)
scipy_idct = fftpack.idct(signals, type=dct_type, norm=norm)
self.assertAllClose(scipy_idct, tf_idct, atol=atol, rtol=rtol)
# Verify inverse(forward(s)) == s, up to a normalization factor.
tf_idct_dct = spectral_ops.idct(tf_dct, type=dct_type,
norm=norm).eval()
tf_dct_idct = spectral_ops.dct(tf_idct, type=dct_type,
norm=norm).eval()
if norm is None:
if dct_type == 1:
tf_idct_dct *= 0.5 / (signals.shape[-1] - 1)
tf_dct_idct *= 0.5 / (signals.shape[-1] - 1)
else:
tf_idct_dct *= 0.5 / signals.shape[-1]
tf_dct_idct *= 0.5 / signals.shape[-1]
self.assertAllClose(signals, tf_idct_dct, atol=atol, rtol=rtol)
self.assertAllClose(signals, tf_dct_idct, atol=atol, rtol=rtol)
def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4):
"""Compares (I)DCT to SciPy (if available) and a NumPy implementation."""
np_dct = NP_DCT[dct_type](signals, norm)
tf_dct = spectral_ops.dct(signals, type=dct_type, norm=norm).eval()
self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol)
np_idct = NP_IDCT[dct_type](signals, norm)
tf_idct = spectral_ops.idct(signals, type=dct_type, norm=norm).eval()
self.assertAllClose(np_idct, tf_idct, atol=atol, rtol=rtol)
if fftpack:
scipy_dct = fftpack.dct(signals, type=dct_type, norm=norm)
self.assertAllClose(scipy_dct, tf_dct, atol=atol, rtol=rtol)
scipy_idct = fftpack.idct(signals, type=dct_type, norm=norm)
self.assertAllClose(scipy_idct, tf_idct, atol=atol, rtol=rtol)
# Verify inverse(forward(s)) == s, up to a normalization factor.
tf_idct_dct = spectral_ops.idct(
tf_dct, type=dct_type, norm=norm).eval()
tf_dct_idct = spectral_ops.dct(
tf_idct, type=dct_type, norm=norm).eval()
if norm is None:
if dct_type == 1:
tf_idct_dct *= 0.5 / (signals.shape[-1] - 1)
tf_dct_idct *= 0.5 / (signals.shape[-1] - 1)
else:
tf_idct_dct *= 0.5 / signals.shape[-1]
tf_dct_idct *= 0.5 / signals.shape[-1]
self.assertAllClose(signals, tf_idct_dct, atol=atol, rtol=rtol)
self.assertAllClose(signals, tf_dct_idct, atol=atol, rtol=rtol)
def stdct(signals,
frame_length,
frame_step,
fft_length=None,
window_fn=functools.partial(window_ops.hann_window, periodic=True),
pad_end=False,
name=None):
with ops.name_scope(name, 'stdct', [signals, frame_length, frame_step]):
signals = ops.convert_to_tensor(signals, name='signals')
signals.shape.with_rank_at_least(1)
frame_length = ops.convert_to_tensor(frame_length, name='frame_length')
frame_length.shape.assert_has_rank(0)
frame_step = ops.convert_to_tensor(frame_step, name='frame_step')
frame_step.shape.assert_has_rank(0)
if fft_length is None:
fft_length = _enclosing_power_of_two(frame_length)
else:
fft_length = ops.convert_to_tensor(fft_length, name='fft_length')
framed_signals = shape_ops.frame(signals,
frame_length,
frame_step,
pad_end=pad_end)
if window_fn is not None:
window = window_fn(frame_length, dtype=framed_signals.dtype)
framed_signals *= window
return spectral_ops.dct(framed_signals)
def _compare(self, signals, norm, atol=5e-4, rtol=5e-4):
"""Compares the DCT to SciPy (if available) and a NumPy implementation."""
np_dct = self._np_dct2(signals, norm)
tf_dct = spectral_ops.dct(signals, type=2, norm=norm).eval()
self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol)
if fftpack:
scipy_dct = fftpack.dct(signals, type=2, norm=norm)
self.assertAllClose(scipy_dct, tf_dct, atol=atol, rtol=rtol)
def test_error(self):
signals = np.random.rand(10)
# Unsupported type.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, type=3)
# Unknown normalization.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, norm="bad")
with self.assertRaises(NotImplementedError):
spectral_ops.dct(signals, n=10)
with self.assertRaises(NotImplementedError):
spectral_ops.dct(signals, axis=0)
def mfccs_from_log_mel_spectrograms(log_mel_spectrograms, name=None):
"""Computes [MFCCs][mfcc] of `log_mel_spectrograms`.
Implemented with GPU-compatible ops and supports gradients.
[Mel-Frequency Cepstral Coefficient (MFCC)][mfcc] calculation consists of
taking the DCT-II of a log-magnitude mel-scale spectrogram. [HTK][htk]'s MFCCs
use a particular scaling of the DCT-II which is almost orthogonal
normalization. We follow this convention.
All `num_mel_bins` MFCCs are returned and it is up to the caller to select
a subset of the MFCCs based on their application. For example, it is typical
to only use the first few for speech recognition, as this results in
an approximately pitch-invariant representation of the signal.
For example:
```python
sample_rate = 16000.0
# A Tensor of [batch_size, num_samples] mono PCM samples in the range [-1, 1].
pcm = tf.placeholder(tf.float32, [None, None])
# A 1024-point STFT with frames of 64 ms and 75% overlap.
stfts = tf.contrib.signal.stft(pcm, frame_length=1024, frame_step=256,
fft_length=1024)
spectrograms = tf.abs(stfts)
# Warp the linear scale spectrograms into the mel-scale.
num_spectrogram_bins = stfts.shape[-1].value
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 80
linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
num_mel_bins, num_spectrogram_bins, sample_rate, lower_edge_hertz,
upper_edge_hertz)
mel_spectrograms = tf.tensordot(
spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(
linear_to_mel_weight_matrix.shape[-1:]))
# Compute a stabilized log to get log-magnitude mel-scale spectrograms.
log_mel_spectrograms = tf.log(mel_spectrograms + 1e-6)
# Compute MFCCs from log_mel_spectrograms and take the first 13.
mfccs = tf.contrib.signal.mfccs_from_log_mel_spectrograms(
log_mel_spectrograms)[..., :13]
```
Args:
log_mel_spectrograms: A `[..., num_mel_bins]` `float32` `Tensor` of
log-magnitude mel-scale spectrograms.
name: An optional name for the operation.
Returns:
A `[..., num_mel_bins]` `float32` `Tensor` of the MFCCs of
`log_mel_spectrograms`.
Raises:
ValueError: If `num_mel_bins` is not positive.
[mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
[htk]: https://en.wikipedia.org/wiki/HTK_(software)
"""
with ops.name_scope(name, 'mfccs_from_log_mel_spectrograms',
[log_mel_spectrograms]):
# Compute the DCT-II of the resulting log-magnitude mel-scale spectrogram.
# The DCT used in HTK scales every basis vector by sqrt(2/N), which is the
# scaling required for an "orthogonal" DCT-II *except* in the 0th bin, where
# the true orthogonal DCT (as implemented by scipy) scales by sqrt(1/N). For
# this reason, we don't apply orthogonal normalization and scale the DCT by
# `0.5 * sqrt(2/N)` manually.
log_mel_spectrograms = ops.convert_to_tensor(log_mel_spectrograms,
dtype=dtypes.float32)
if (log_mel_spectrograms.shape.ndims
and log_mel_spectrograms.shape.dims[-1].value is not None):
num_mel_bins = log_mel_spectrograms.shape.dims[-1].value
if num_mel_bins == 0:
raise ValueError('num_mel_bins must be positive. Got: %s' %
log_mel_spectrograms)
else:
num_mel_bins = array_ops.shape(log_mel_spectrograms)[-1]
dct2 = spectral_ops.dct(log_mel_spectrograms)
return dct2 * math_ops.rsqrt(math_ops.to_float(num_mel_bins) * 2.0)
def test_error(self):
signals = np.random.rand(10)
# Unsupported type.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, type=5)
# DCT-I normalization not implemented.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, type=1, norm="ortho")
# DCT-I requires at least two inputs.
with self.assertRaises(ValueError):
spectral_ops.dct(np.random.rand(1), type=1)
# Unknown normalization.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, norm="bad")
with self.assertRaises(NotImplementedError):
spectral_ops.dct(signals, n=10)
with self.assertRaises(NotImplementedError):
spectral_ops.dct(signals, axis=0)
def test_error(self):
signals = np.random.rand(10)
# Unsupported type.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, type=5)
# DCT-I normalization not implemented.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, type=1, norm="ortho")
# DCT-I requires at least two inputs.
with self.assertRaises(ValueError):
spectral_ops.dct(np.random.rand(1), type=1)
# Unknown normalization.
with self.assertRaises(ValueError):
spectral_ops.dct(signals, norm="bad")
with self.assertRaises(NotImplementedError):
spectral_ops.dct(signals, n=10)
with self.assertRaises(NotImplementedError):
spectral_ops.dct(signals, axis=0)
def mfccs_from_log_mel_spectrograms(log_mel_spectrograms, name=None):
"""Computes [MFCCs][mfcc] of `log_mel_spectrograms`.
Implemented with GPU-compatible ops and supports gradients.
[Mel-Frequency Cepstral Coefficient (MFCC)][mfcc] calculation consists of
taking the DCT-II of a log-magnitude mel-scale spectrogram. [HTK][htk]'s MFCCs
use a particular scaling of the DCT-II which is almost orthogonal
normalization. We follow this convention.
All `num_mel_bins` MFCCs are returned and it is up to the caller to select
a subset of the MFCCs based on their application. For example, it is typical
to only use the first few for speech recognition, as this results in
an approximately pitch-invariant representation of the signal.
For example:
```python
sample_rate = 16000.0
# A Tensor of [batch_size, num_samples] mono PCM samples in the range [-1, 1].
pcm = tf.placeholder(tf.float32, [None, None])
# A 1024-point STFT with frames of 64 ms and 75% overlap.
stfts = tf.contrib.signal.stft(pcm, frame_length=1024, frame_step=256,
fft_length=1024)
spectrograms = tf.abs(stfts)
# Warp the linear scale spectrograms into the mel-scale.
num_spectrogram_bins = stfts.shape[-1].value
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 80
linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
num_mel_bins, num_spectrogram_bins, sample_rate, lower_edge_hertz,
upper_edge_hertz)
mel_spectrograms = tf.tensordot(
spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(
linear_to_mel_weight_matrix.shape[-1:]))
# Compute a stabilized log to get log-magnitude mel-scale spectrograms.
log_mel_spectrograms = tf.log(mel_spectrograms + 1e-6)
# Compute MFCCs from log_mel_spectrograms and take the first 13.
mfccs = tf.contrib.signal.mfccs_from_log_mel_spectrograms(
log_mel_spectrograms)[..., :13]
```
Args:
log_mel_spectrograms: A `[..., num_mel_bins]` `float32` `Tensor` of
log-magnitude mel-scale spectrograms.
name: An optional name for the operation.
Returns:
A `[..., num_mel_bins]` `float32` `Tensor` of the MFCCs of
`log_mel_spectrograms`.
Raises:
ValueError: If `num_mel_bins` is not positive.
[mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
[htk]: https://en.wikipedia.org/wiki/HTK_(software)
"""
with ops.name_scope(name, 'mfccs_from_log_mel_spectrograms',
[log_mel_spectrograms]):
# Compute the DCT-II of the resulting log-magnitude mel-scale spectrogram.
# The DCT used in HTK scales every basis vector by sqrt(2/N), which is the
# scaling required for an "orthogonal" DCT-II *except* in the 0th bin, where
# the true orthogonal DCT (as implemented by scipy) scales by sqrt(1/N). For
# this reason, we don't apply orthogonal normalization and scale the DCT by
# `0.5 * sqrt(2/N)` manually.
log_mel_spectrograms = ops.convert_to_tensor(log_mel_spectrograms,
dtype=dtypes.float32)
if (log_mel_spectrograms.shape.ndims and
log_mel_spectrograms.shape[-1].value is not None):
num_mel_bins = log_mel_spectrograms.shape[-1].value
if num_mel_bins == 0:
raise ValueError('num_mel_bins must be positive. Got: %s' %
log_mel_spectrograms)
else:
num_mel_bins = array_ops.shape(log_mel_spectrograms)[-1]
dct2 = spectral_ops.dct(log_mel_spectrograms)
return dct2 * math_ops.rsqrt(num_mel_bins * 2.0)
