def test_streaming_inference_external_state(self):
output_non_stream_np, model_tf = self._run_non_stream_model()
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(shape=(
1,
self.input_data.shape[2],
),
batch_size=self.batch_size,
name="inp1")
]
# convert non streaming trainable model to a streaming one
model_stream = utils.convert_to_inference_model(
model_tf, input_tensors, mode)
input_states_np = np.zeros(
[self.batch_size, self.memory_size, self.weights[1].shape[-1]])
# streaming emulation: loop over every element in time
for i in range(self.input_data.shape[1]):
input_batch_np = self.input_data[:, i, :]
input_batch_np = np.expand_dims(input_batch_np, 1)
output_np, output_states_np = model_stream.predict(
[input_batch_np, input_states_np])
input_states_np = output_states_np
for b in range(self.input_data.shape[0]): # loop over batch
self.assertAllClose(output_np[b][0],
output_non_stream_np[b][i])
def test_tf_non_streaming_vs_streaming_inference_external_state(self):
"""Tests non stream inference vs stream inference with external state."""
speech_params = speech_features.SpeechFeatures.get_params(self.params)
mode = modes.Modes.NON_STREAM_INFERENCE
# TF non streaming frame extraction based on tf.signal.frame
mel_speech_tf = speech_features.SpeechFeatures(
speech_params, mode, self.inference_batch_size)
# it receives all data with size: data_size
input1 = tf.keras.layers.Input(shape=(self.data_size, ),
batch_size=self.inference_batch_size,
dtype=tf.float32)
output1 = mel_speech_tf(input1)
model_tf = tf.keras.models.Model(input1, output1)
# generate frames for the whole signal (no streaming here)
output_tf = model_tf.predict(self.signal)
# input data for streaming mode
input_tensors = [
tf.keras.layers.Input(shape=(self.frame_step, ),
batch_size=self.inference_batch_size,
dtype=tf.float32)
]
# convert non streaming trainable model to
# streaming inference with external state
mode = modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE
model_stream = utils.convert_to_inference_model(
model_tf, input_tensors, mode)
# initialize state of streaming model
pre_state = self.signal[:, 0:self.frame_size - self.frame_step]
state2 = np.concatenate((np.zeros(shape=(1, self.frame_step),
dtype=np.float32), pre_state),
axis=1)
# run streaming frames extraction
start = self.frame_size - self.frame_step
end = self.frame_size
streamed_frames = []
while end <= self.data_size:
# next data update
stream_update = self.signal[:, start:end]
# get new frame from stream of data
output_frame, output_state = model_stream.predict(
[stream_update, state2])
state2 = output_state
streamed_frames.append(output_frame)
# update indexes of streamed updates
start = end
end = start + self.frame_step
# compare streaming vs non streaming frames extraction
for i in range(len(streamed_frames)):
self.assertAllClose(streamed_frames[i][0][0],
output_tf[0][i],
rtol=1e-4,
atol=1e-4)
def test_streaming_inference_external_state(self):
# test on input with [batch, time, feature1]
self.init(shape=(8, 2))
# convert non streamable trainable model to streamable with external state
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(shape=(
1,
2,
),
batch_size=self.batch_size,
name="inp1")
]
model_stream = utils.convert_to_inference_model(
self.model_train, input_tensors, mode)
input_state = np.zeros(model_stream.inputs[1].shape, dtype=np.float32)
# run streaming inference
for i in range(self.inputs.shape[1]):
input_stream = np.expand_dims(self.inputs[0][i], 0)
input_stream = np.expand_dims(input_stream, 1)
output_stream, output_state = model_stream.predict(
[input_stream, input_state])
input_state = output_state # update input state
self.assertAllEqual(output_stream, self.outputs)
def test_inference_internal_state(self):
mode = modes.Modes.STREAM_INTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(
shape=(
1,
self.feature_size,
),
batch_size=1,
name='inp_stream')
]
# convert non streaming model to streaming one
model_stream = utils.convert_to_inference_model(self.model,
input_tensors, mode)
model_stream.summary()
# confirm that it returns zeros
for i in range(self.max_counter):
i = i % self.input_non_stream_np.shape[1]
input_stream_np = self.input_non_stream_np[:, i, :]
input_stream_np = np.expand_dims(input_stream_np, 1)
self.assertAllEqual(
np.zeros_like(input_stream_np), model_stream.predict(input_stream_np))
# confirm that after self.max_counter iterations it returns input
for _ in range(self.max_counter):
self.assertAllClose(
input_stream_np, model_stream.predict(input_stream_np))
def test_inference_external_state(self, max_counter):
inputs = tf.keras.layers.Input(shape=(
self.time_size,
self.feature_size,
),
batch_size=1)
net = counter.Counter(max_counter=max_counter)(inputs)
model = tf.keras.Model(inputs, net)
mode = modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(shape=(
1,
self.feature_size,
),
batch_size=1,
name="inp_stream")
]
# convert non streaming model to streaming one
model_stream = utils.convert_to_inference_model(
model, input_tensors, mode)
model_stream.summary()
# second input to stream model is a state, so we can use its shape
input_state_np = np.zeros(model_stream.inputs[1].shape,
dtype=np.float32)
# confirm that it returns zeros in the first max_counter iterations
for i in range(max_counter):
i = i % self.input_non_stream_np.shape[1]
input_stream_np = self.input_non_stream_np[:, i, :]
input_stream_np = np.expand_dims(input_stream_np, 1)
output_stream_np, output_state_np = model_stream.predict(
[input_stream_np, input_state_np])
input_state_np = output_state_np # update input state
self.assertAllEqual(np.zeros_like(input_stream_np),
output_stream_np)
# confirm that after self.max_counter iterations it returns input tensor
for i in range(max_counter):
i = i % self.input_non_stream_np.shape[1]
input_stream_np = self.input_non_stream_np[:, i, :]
input_stream_np = np.expand_dims(input_stream_np, 1)
output_stream_np, output_state_np = model_stream.predict(
[input_stream_np, input_state_np])
input_state_np = output_state_np # update input state
self.assertAllClose(input_stream_np, output_stream_np)
def test_streaming_external_state(self):
# Streaming convolution with external state - used for inference.
# Create a non-streaming model first
mode = modes.Modes.TRAINING
layer = self._get_conv2d_layer(mode)
inputs = tf.keras.layers.Input(shape=(self.time_dim, self.feature_dim,
1),
batch_size=self.batch_size)
outputs = layer(inputs)
model = tf.keras.Model(inputs, outputs)
# Swap to streaming mode
mode = modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.Input(shape=(
1,
self.feature_dim,
1,
),
batch_size=self.batch_size,
name='inp1')
]
# Initialize the first state with zeros.
input_states = np.zeros(
[self.batch_size, self.kernel_size[0], self.feature_dim, 1])
# Use the pipeline to convert the model into streaming external state
model_stream = utils.convert_to_inference_model(
model, input_tensors, mode)
# Simulating streaming using a for loop
for i in range(self.time_dim):
input_feature = self.inputs[:, i, :, :]
input_feature = np.expand_dims(input_feature, 1)
output_np, output_states = model_stream.predict(
[input_feature, input_states])
# Propagate the output state as the input state of the next iteration.
input_states = output_states
for b in range(self.batch_size): # loop over batch
self.assertAllClose(output_np[b][0],
self.expected_base_output[b][i])
def test_tf_non_streaming_vs_streaming_external_state(self):
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(shape=(self.frame_step, ),
batch_size=1,
name="inp1")
]
# convert non streaming trainable model to a streaming one
model_stream = utils.convert_to_inference_model(
self.model_tf, input_tensors, mode)
# initialize input state of streaming data framer
pre_state = self.signal[:, 0:self.frame_size - self.frame_step]
states = np.concatenate((np.zeros(shape=(1, self.frame_step),
dtype=np.float32), pre_state),
axis=1)
start = self.frame_size - self.frame_step
end = self.frame_size
streamed_frames = []
# run streaming frames extraction
while end <= self.data_size:
# next data update
stream_update = self.signal[:, start:end]
# get new frame from stream of data
output_frame, new_states = model_stream.predict(
[stream_update, states])
# update frame states and feed it as input in the next iteration
states = new_states
streamed_frames.append(output_frame)
start = end
end = start + self.frame_step
# compare streaming vs non streaming frames extraction
for i in range(0, len(self.output_frames_tf[0])):
self.assertAllEqual(streamed_frames[i][0][0],
self.output_frames_tf[0][i])
def test_streaming_inference_external_state(self):
with tf1.Session() as sess:
output_non_stream_np, model_tf = self._run_non_stream_model()
# input data for streaming stateless model
input_tensors = [
tf.keras.layers.Input(shape=(
1,
self.input_data.shape[2],
),
batch_size=self.batch_size,
dtype=tf.float32)
]
# convert non streaming model to streaming one with external state
mode = modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE
model_stream = utils.convert_to_inference_model(
model_tf, input_tensors, mode)
# validate that model is convertable to tflite
converter = tf1.lite.TFLiteConverter.from_session(
sess, model_stream.inputs, model_stream.outputs)
self.assertTrue(converter.convert())
inputs = []
for s in range(len(model_stream.inputs)):
inputs.append(
np.zeros(model_stream.inputs[s].shape, dtype=np.float32))
# streaming emulation: loop over every element in time
for i in range(self.input_data.shape[1]):
input_batch_np = self.input_data[:, i, :]
input_batch_np = np.expand_dims(input_batch_np, 1)
inputs[0] = input_batch_np
outputs = model_stream.predict(inputs)
# input_states_np = output_states_np
for s in range(1, len(model_stream.inputs)):
inputs[s] = outputs[s]
for b in range(self.input_data.shape[0]): # loop over batch
self.assertAllClose(outputs[0][b][0],
output_non_stream_np[b][i])
def test_streaming_inference_internal_state(self):
# test on input with [batch, time, feature1, feature2]
self.init(shape=(8, 2, 1))
# convert non streamable trainable model to streamable with internal state
mode = modes.Modes.STREAM_INTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(
shape=(
1,
2,
1,
), batch_size=self.batch_size, name="inp1")
]
model_stream = utils.convert_to_inference_model(self.model_train,
input_tensors, mode)
# run streaming inference
for i in range(self.inputs.shape[1]):
input_stream = np.expand_dims(self.inputs[0][i], 0)
input_stream = np.expand_dims(input_stream, 1)
output_stream = model_stream.predict(input_stream)
self.assertAllEqual(output_stream, self.outputs)
def test_streaming_with_effective_tdim(self):
time_size = 10
feature_size = 3
batch_size = 1
time_dim = 1 # index of time dimensions
ring_buffer_size_in_time_dim = 3 # effective size of aperture in time dim
inputs = tf.keras.layers.Input(shape=(time_size, feature_size),
batch_size=batch_size,
name='inp_sequence')
mode = modes.Modes.TRAINING
# in streaming mode it will create a
# ring buffer with time dim size ring_buffer_size_in_time_dim
outputs = stream.Stream(
cell=Sum(time_dim=time_dim),
mode=mode,
ring_buffer_size_in_time_dim=ring_buffer_size_in_time_dim)(inputs)
model_train = tf.keras.Model(inputs, outputs)
model_train.summary()
mode = modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE
input_tensors = [
tf.keras.layers.Input(
shape=(
1, # time dim is size 1 in streaming mode
feature_size,
),
batch_size=batch_size,
name='inp_stream')
]
# convert non streaming model to streaming one
model_stream = utils.convert_to_inference_model(
model_train, input_tensors, mode)
model_stream.summary()
# second input tostream model is a state, so we can use its shape
input_state_np = np.zeros(model_stream.inputs[1].shape,
dtype=np.float32)
# input test data
non_stream_input = np.random.randint(1,
10,
size=(batch_size, time_size,
feature_size))
# run streaming inference
# iterate over time dim sample by sample
for i in range(input_state_np.shape[1]):
input_stream_np = np.expand_dims(non_stream_input[0][i], 0)
input_stream_np = np.expand_dims(input_stream_np, 1)
input_stream_np = input_stream_np.astype(np.float32)
output_stream_np, output_state_np = model_stream.predict(
[input_stream_np, input_state_np])
input_state_np = output_state_np # update input state
# emulate sliding window summation
target = np.sum(
non_stream_input[:,
max(0, i - ring_buffer_size_in_time_dim):i +
1],
axis=time_dim)
self.assertAllEqual(target, output_stream_np)
# validate name tag of model's state
expected_str = 'ExternalState'
self.assertAllEqual(
expected_str,
model_stream.inputs[1].name.split('/')[-1][:len(expected_str)])
