def test_tf_vs_tf_direct(self):
# Compare TF implementation of Mel (based on FFT)
# vs TF direct implementation (based on FT)
feature_size = 257
num_mel_bins = 80
lower_edge_hertz = 125.0
upper_edge_hertz = 7600.0
sample_rate = 16000.0
batch_size = 1
test_utils.set_seed(1)
# generate input data
frame = np.random.rand(batch_size, feature_size)
# prepare model with TF implementation of Mel based on FFT
input1 = tf.keras.layers.Input(shape=(feature_size, ),
batch_size=batch_size,
dtype=tf.float32)
mel_spectrum = mel_spectrogram.MelSpectrogram(
mode=modes.Modes.NON_STREAM_INFERENCE,
use_tf=True,
num_mel_bins=num_mel_bins,
lower_edge_hertz=lower_edge_hertz,
upper_edge_hertz=upper_edge_hertz,
sample_rate=sample_rate)
output1 = mel_spectrum(input1)
model_tf = tf.keras.models.Model(input1, output1)
# generate mel
output_tf = model_tf.predict(frame)
# prepare model with TF implementation of Mel based on direct FT
input2 = tf.keras.layers.Input(shape=(feature_size, ),
batch_size=batch_size,
dtype=tf.float32)
mel_spectrum_direct = mel_spectrogram.MelSpectrogram(
mode=modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE,
use_tf=False,
num_mel_bins=num_mel_bins,
lower_edge_hertz=lower_edge_hertz,
upper_edge_hertz=upper_edge_hertz,
sample_rate=sample_rate)
output2 = mel_spectrum_direct(input2)
model_tf_direct = tf.keras.models.Model(input2, output2)
# generate mel
output_tf_direct = model_tf_direct.predict(frame)
self.assertAllClose(output_tf, output_tf_direct, rtol=1e-5, atol=1e-4)
def setUp(self):
super(MagnitudeRDFTmelTest, self).setUp()
test_utils.set_seed(123)
self.signal_size = 100
# input signal
self.signal = np.random.rand(1, self.signal_size)
# model parameters
self.use_tf_fft = False
self.magnitude_squared = False
self.num_mel_bins = 40
self.lower_edge_hertz = 20.0
self.upper_edge_hertz = 4000.0
self.sample_rate = 16000.0
# build rdft mel model and run it
input_signal = tf.keras.Input(shape=(self.signal_size, ), batch_size=1)
mag_rdft = magnitude_rdft.MagnitudeRDFT(
use_tf_fft=self.use_tf_fft,
magnitude_squared=self.magnitude_squared)(input_signal)
mel_spectr = mel_spectrogram.MelSpectrogram(
use_tf=False,
num_mel_bins=self.num_mel_bins,
lower_edge_hertz=self.lower_edge_hertz,
upper_edge_hertz=self.upper_edge_hertz,
sample_rate=self.sample_rate)(mag_rdft)
model_rdft_mel = tf.keras.Model(input_signal, mel_spectr)
model_rdft_mel.summary()
self.shape_rdft_mel = model_rdft_mel.layers[2].mel_weight_matrix.shape
self.rdft_mel_output = model_rdft_mel.predict(self.signal)
def test_rdft_mel_vs_merged_rdft_mel(self, mel_non_zero_only):
signal_size = 257
# input signal
signal = np.random.rand(1, signal_size)
# model parameters
mode = Modes.NON_STREAM_INFERENCE
use_tf_fft = False
magnitude_squared = False
num_mel_bins = 40
lower_edge_hertz = 20.0
upper_edge_hertz = 4000.0
sample_rate = 16000.0
# build rdft mel model and run it
input_signal = tf.keras.Input(shape=(signal_size, ), batch_size=1)
mag_rdft = magnitude_rdft.MagnitudeRDFT(
mode=mode,
use_tf_fft=use_tf_fft,
magnitude_squared=magnitude_squared)(input_signal)
mel_spectr = mel_spectrogram.MelSpectrogram(
mode=mode,
use_tf=False,
num_mel_bins=num_mel_bins,
lower_edge_hertz=lower_edge_hertz,
upper_edge_hertz=upper_edge_hertz,
sample_rate=sample_rate)(mag_rdft)
model_rdft_mel = tf.keras.Model(input_signal, mel_spectr)
model_rdft_mel.summary()
rdft_mel_output = model_rdft_mel.predict(signal)
# build merged rdft mel model and run it
merged_rdft_mel = magnitude_rdft_mel.MagnitudeRDFTmel(
use_tf_fft=use_tf_fft,
magnitude_squared=magnitude_squared,
num_mel_bins=num_mel_bins,
lower_edge_hertz=lower_edge_hertz,
upper_edge_hertz=upper_edge_hertz,
sample_rate=sample_rate,
mel_non_zero_only=mel_non_zero_only)(input_signal)
model_merged_rdft_mel = tf.keras.Model(input_signal, merged_rdft_mel)
model_merged_rdft_mel.summary()
merged_rdft_mel_output = model_merged_rdft_mel.predict(signal)
shape_rdft_mel = model_rdft_mel.layers[2].mel_weight_matrix.shape
shape_rdft_melmerged = model_merged_rdft_mel.layers[
1].mel_weight_matrix.shape
if mel_non_zero_only:
# shape of mel matrix with merged method is 2x smaller
self.assertGreater(
shape_rdft_mel[0] * shape_rdft_mel[1],
2 * shape_rdft_melmerged[0] * shape_rdft_melmerged[1])
else:
self.assertEqual(shape_rdft_mel[0] * shape_rdft_mel[1],
shape_rdft_melmerged[0] * shape_rdft_melmerged[1])
self.assertAllClose(rdft_mel_output, merged_rdft_mel_output)