def _set_params(self, use_peepholes):
seed = 123
np.random.seed(seed)
rn.seed(seed)
tf.random.set_seed(seed)
# generate input signal
self.inference_batch_size = 1
self.data_size = 32
self.feature_size = 4
self.signal = np.random.rand(self.inference_batch_size, self.data_size,
self.feature_size)
# create non streamable model
inputs = tf.keras.layers.Input(shape=(self.data_size,
self.feature_size),
batch_size=self.inference_batch_size,
dtype=tf.float32)
self.units = 3
self.num_proj = 4
outputs = lstm.LSTM(units=self.units,
return_sequences=True,
use_peepholes=use_peepholes,
num_proj=self.num_proj)(inputs)
self.model_non_streamable = tf.keras.Model(inputs, outputs)
self.output_lstm = self.model_non_streamable.predict(self.signal)
def test_streaming_inference_internal_state(self, use_peepholes):
# create streaming inference model with internal state
self._set_params(use_peepholes)
mode = Modes.STREAM_INTERNAL_STATE_INFERENCE
inputs = tf.keras.layers.Input(shape=(1, self.feature_size),
batch_size=self.inference_batch_size,
dtype=tf.float32)
outputs = lstm.LSTM(units=self.units,
mode=mode,
use_peepholes=use_peepholes,
num_proj=self.num_proj)(inputs)
model_stream = tf.keras.Model(inputs, outputs)
# set weights + states
if use_peepholes:
weights_states = self.model_non_streamable.get_weights() + [
np.zeros((self.inference_batch_size, self.units))
] + [np.zeros((self.inference_batch_size, self.num_proj))]
else:
weights_states = self.model_non_streamable.get_weights() + [
np.zeros((self.inference_batch_size, self.units))
] + [np.zeros((self.inference_batch_size, self.units))]
model_stream.set_weights(weights_states)
# compare streamable (with internal state) vs non streamable models
for i in range(self.data_size): # loop over time samples
input_stream = self.signal[:, i, :]
input_stream = np.expand_dims(input_stream, 1)
output_stream = model_stream.predict(input_stream)
self.assertAllClose(output_stream[0][0], self.output_lstm[0][i])
def model(flags):
"""LSTM model.
Similar model in papers:
Convolutional Recurrent Neural Networks for Small-Footprint Keyword Spotting
https://arxiv.org/pdf/1703.05390.pdf (with no conv layer)
Model topology is similar with "Hello Edge: Keyword Spotting on
Microcontrollers" https://arxiv.org/pdf/1711.07128.pdf
Args:
flags: data/model parameters
Returns:
Keras model for training
"""
input_audio = tf.keras.layers.Input(
shape=modes.get_input_data_shape(flags, modes.Modes.TRAINING),
batch_size=flags.batch_size)
net = input_audio
if flags.preprocess == 'raw':
# it is a self contained model, user need to feed raw audio only
net = speech_features.SpeechFeatures(
speech_features.SpeechFeatures.get_params(flags))(
net)
for units, return_sequences, num_proj in zip(
utils.parse(flags.lstm_units), utils.parse(flags.return_sequences),
utils.parse(flags.num_proj)):
net = lstm.LSTM(
units=units,
return_sequences=return_sequences,
stateful=flags.stateful,
use_peepholes=flags.use_peepholes,
num_proj=num_proj)(
net)
net = stream.Stream(cell=tf.keras.layers.Flatten())(net)
net = tf.keras.layers.Dropout(rate=flags.dropout1)(net)
for units, activation in zip(
utils.parse(flags.units1), utils.parse(flags.act1)):
net = tf.keras.layers.Dense(units=units, activation=activation)(net)
net = tf.keras.layers.Dense(units=flags.label_count)(net)
if flags.return_softmax:
net = tf.keras.layers.Activation('softmax')(net)
return tf.keras.Model(input_audio, net)
def test_streaming_inference_external_state(self, use_peepholes):
# create streaming inference model with external state
self._set_params(use_peepholes)
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
inputs = tf.keras.layers.Input(shape=(1, self.feature_size),
batch_size=self.inference_batch_size,
dtype=tf.float32)
lstm_layer = lstm.LSTM(units=self.units,
mode=mode,
use_peepholes=use_peepholes,
num_proj=self.num_proj)
outputs = lstm_layer(inputs)
model_stream = tf.keras.Model([inputs] + lstm_layer.get_input_state(),
[outputs] +
lstm_layer.get_output_state())
# set weights only
model_stream.set_weights(self.model_non_streamable.get_weights())
# input states
input_state1 = np.zeros((self.inference_batch_size, self.units))
if use_peepholes:
input_state2 = np.zeros((self.inference_batch_size, self.num_proj))
else:
input_state2 = np.zeros((self.inference_batch_size, self.units))
# compare streamable vs non streamable models
for i in range(self.data_size): # loop over time samples
input_stream = self.signal[:, i, :]
input_stream = np.expand_dims(input_stream, 1)
output_streams = model_stream.predict(
[input_stream, input_state1, input_state2])
# update input states
input_state1 = output_streams[1]
input_state2 = output_streams[2]
self.assertAllClose(output_streams[0][0][0],
self.output_lstm[0][i])
