def _set_params(self, use_peepholes): seed = 123 np.random.seed(seed) rn.seed(seed) tf.random.set_seed(seed) # generate input signal self.inference_batch_size = 1 self.data_size = 32 self.feature_size = 4 self.signal = np.random.rand(self.inference_batch_size, self.data_size, self.feature_size) # create non streamable model inputs = tf.keras.layers.Input(shape=(self.data_size, self.feature_size), batch_size=self.inference_batch_size, dtype=tf.float32) self.units = 3 self.num_proj = 4 outputs = lstm.LSTM(units=self.units, return_sequences=True, use_peepholes=use_peepholes, num_proj=self.num_proj)(inputs) self.model_non_streamable = tf.keras.Model(inputs, outputs) self.output_lstm = self.model_non_streamable.predict(self.signal)
def test_streaming_inference_internal_state(self, use_peepholes): # create streaming inference model with internal state self._set_params(use_peepholes) mode = Modes.STREAM_INTERNAL_STATE_INFERENCE inputs = tf.keras.layers.Input(shape=(1, self.feature_size), batch_size=self.inference_batch_size, dtype=tf.float32) outputs = lstm.LSTM(units=self.units, mode=mode, use_peepholes=use_peepholes, num_proj=self.num_proj)(inputs) model_stream = tf.keras.Model(inputs, outputs) # set weights + states if use_peepholes: weights_states = self.model_non_streamable.get_weights() + [ np.zeros((self.inference_batch_size, self.units)) ] + [np.zeros((self.inference_batch_size, self.num_proj))] else: weights_states = self.model_non_streamable.get_weights() + [ np.zeros((self.inference_batch_size, self.units)) ] + [np.zeros((self.inference_batch_size, self.units))] model_stream.set_weights(weights_states) # compare streamable (with internal state) vs non streamable models for i in range(self.data_size): # loop over time samples input_stream = self.signal[:, i, :] input_stream = np.expand_dims(input_stream, 1) output_stream = model_stream.predict(input_stream) self.assertAllClose(output_stream[0][0], self.output_lstm[0][i])
def model(flags): """LSTM model. Similar model in papers: Convolutional Recurrent Neural Networks for Small-Footprint Keyword Spotting https://arxiv.org/pdf/1703.05390.pdf (with no conv layer) Model topology is similar with "Hello Edge: Keyword Spotting on Microcontrollers" https://arxiv.org/pdf/1711.07128.pdf Args: flags: data/model parameters Returns: Keras model for training """ input_audio = tf.keras.layers.Input( shape=modes.get_input_data_shape(flags, modes.Modes.TRAINING), batch_size=flags.batch_size) net = input_audio if flags.preprocess == 'raw': # it is a self contained model, user need to feed raw audio only net = speech_features.SpeechFeatures( speech_features.SpeechFeatures.get_params(flags))( net) for units, return_sequences, num_proj in zip( utils.parse(flags.lstm_units), utils.parse(flags.return_sequences), utils.parse(flags.num_proj)): net = lstm.LSTM( units=units, return_sequences=return_sequences, stateful=flags.stateful, use_peepholes=flags.use_peepholes, num_proj=num_proj)( net) net = stream.Stream(cell=tf.keras.layers.Flatten())(net) net = tf.keras.layers.Dropout(rate=flags.dropout1)(net) for units, activation in zip( utils.parse(flags.units1), utils.parse(flags.act1)): net = tf.keras.layers.Dense(units=units, activation=activation)(net) net = tf.keras.layers.Dense(units=flags.label_count)(net) if flags.return_softmax: net = tf.keras.layers.Activation('softmax')(net) return tf.keras.Model(input_audio, net)
def test_streaming_inference_external_state(self, use_peepholes): # create streaming inference model with external state self._set_params(use_peepholes) mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE inputs = tf.keras.layers.Input(shape=(1, self.feature_size), batch_size=self.inference_batch_size, dtype=tf.float32) lstm_layer = lstm.LSTM(units=self.units, mode=mode, use_peepholes=use_peepholes, num_proj=self.num_proj) outputs = lstm_layer(inputs) model_stream = tf.keras.Model([inputs] + lstm_layer.get_input_state(), [outputs] + lstm_layer.get_output_state()) # set weights only model_stream.set_weights(self.model_non_streamable.get_weights()) # input states input_state1 = np.zeros((self.inference_batch_size, self.units)) if use_peepholes: input_state2 = np.zeros((self.inference_batch_size, self.num_proj)) else: input_state2 = np.zeros((self.inference_batch_size, self.units)) # compare streamable vs non streamable models for i in range(self.data_size): # loop over time samples input_stream = self.signal[:, i, :] input_stream = np.expand_dims(input_stream, 1) output_streams = model_stream.predict( [input_stream, input_state1, input_state2]) # update input states input_state1 = output_streams[1] input_state2 = output_streams[2] self.assertAllClose(output_streams[0][0][0], self.output_lstm[0][i])