def testPreprocessStreamInferenceModeTFandTFLite(self,
preprocess,
feature_type,
model_name='gru'):
# Validate that model with different preprocessing
# can be converted to stream inference mode with TF and TFLite.
params = model_params.HOTWORD_MODEL_PARAMS[model_name]
# set parameters to test
params.preprocess = preprocess
params.feature_type = feature_type
params = model_flags.update_flags(params)
# create model
model = models.MODELS[params.model_name](params)
# convert TF non streaming model to TFLite streaming inference
# with external states
self.assertTrue(utils.model_to_tflite(
self.sess, model, params, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE))
# convert TF non streaming model to TF streaming with external states
self.assertTrue(utils.to_streaming_inference(
model, params, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE))
# convert TF non streaming model to TF streaming with internal states
self.assertTrue(utils.to_streaming_inference(
model, params, modes.Modes.STREAM_INTERNAL_STATE_INFERENCE))
def _RunGetDataTest(self, preprocess, window_size_ms):
tmp_dir = self.get_temp_dir()
wav_dir = os.path.join(tmp_dir, "wavs")
os.mkdir(wav_dir)
self._SaveWavFolders(wav_dir, ["a", "b", "c"], 100)
background_dir = os.path.join(wav_dir, "_background_noise_")
os.mkdir(background_dir)
wav_data = self._GetWavData()
for i in range(10):
file_path = os.path.join(background_dir,
"background_audio_%d.wav" % i)
self._SaveTestWavFile(file_path, wav_data)
flags = self._GetDefaultFlags()
flags.window_size_ms = window_size_ms
flags.preprocess = preprocess
flags.train_dir = tmp_dir
flags.data_dir = wav_dir
flags = model_flags.update_flags(flags)
with self.cached_session() as sess:
audio_processor = input_data.AudioProcessor(flags)
result_data, result_labels = audio_processor.get_data(
10, 0, flags, 0.3, 0.1, 100, "training", 0.0, sess)
self.assertLen(result_data, 10)
self.assertLen(result_labels, 10)
def test_model_to_saved(self, model_name='dnn'):
"""SavedModel supports both stateless and stateful graphs."""
params = model_params.HOTWORD_MODEL_PARAMS[model_name]
params = model_flags.update_flags(params)
# create model
model = models.MODELS[params.model_name](params)
utils.model_to_saved(model, params, FLAGS.test_tmpdir)
def testTrain(self, split_data):
input_flags = self._GetDefaultFlags(split_data)
input_flags = model_flags.update_flags(input_flags)
train.train(input_flags)
self.assertTrue(
tf.io.gfile.exists(os.path.join(input_flags.train_dir, 'graph.pbtxt')))
self.assertTrue(
tf.io.gfile.exists(os.path.join(input_flags.train_dir, 'labels.txt')))
self.assertTrue(
tf.io.gfile.exists(
os.path.join(input_flags.train_dir, 'accuracy_last.txt')))
def _GetDefaultFlags(self, split_data):
params = model_params.dnn_params()
params.data_dir = self._PrepareDummyTrainingData(
) if split_data == 1 else self._PrepareDummyTrainingDataSplit()
params.wanted_words = 'a,b,c'
params.split_data = split_data
params.summaries_dir = self._PrepareDummyDir('summaries' +
str(split_data))
params.train_dir = self._PrepareDummyDir('train' + str(split_data))
params.how_many_training_steps = '2'
params.learning_rate = '0.01'
params.eval_step_interval = 1
params.save_step_interval = 1
params.clip_duration_ms = 100
params.batch_size = 1
return model_flags.update_flags(params)
def testToNonStreamInferenceTFandTFLite(self, model_name='svdf'):
"""Validate that model can be converted to non stream inference mode."""
params = _HOTWORD_MODEL_PARAMS[model_name]
params = model_flags.update_flags(params)
# create model
model = models.MODELS[params.model_name](params)
# convert TF non streaming model to TF non streaming inference model
# it will disable dropouts
self.assertTrue(
utils.to_streaming_inference(model, params,
Modes.NON_STREAM_INFERENCE))
# convert TF non streaming model to TFLite non streaming inference
self.assertTrue(
utils.model_to_tflite(self.sess, model, params,
Modes.NON_STREAM_INFERENCE))
def test_external_streaming_shapes(self, model_name):
params = model_params.HOTWORD_MODEL_PARAMS[model_name]
params = model_flags.update_flags(params)
model = models.MODELS[params.model_name](params)
external_model = utils.to_streaming_inference(
model, params, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE)
# The first 'n' inputs correspond to the 'n' inputs that the model takes
# in non-streaming mode. The rest of the input tensors represent the
# internal states for each layer in the model.
inputs = [
np.zeros(shape, dtype=np.float32)
for shape in external_model.input_shapes
]
outputs = external_model.predict(inputs)
for output, expected_shape in zip(outputs,
external_model.output_shapes):
self.assertEqual(output.shape, expected_shape)
def _testTFLite(self,
preprocess='raw',
feature_type='mfcc_op',
model_name='svdf'):
params = model_params.HOTWORD_MODEL_PARAMS[model_name]
# set parameters to test
params.preprocess = preprocess
params.feature_type = feature_type
params = model_flags.update_flags(params)
# create model
model = models.MODELS[params.model_name](params)
# convert TF non streaming model to TFLite non streaming inference
self.assertTrue(
utils.model_to_tflite(self.sess, model, params,
Modes.NON_STREAM_INFERENCE))
def testToStreamInferenceModeTFandTFLite(self, model_name='gru'):
"""Validate that model can be converted to any streaming inference mode."""
params = _HOTWORD_MODEL_PARAMS[model_name]
params = model_flags.update_flags(params)
# create model
model = models.MODELS[params.model_name](params)
# convert TF non streaming model to TFLite streaming inference
# with external states
self.assertTrue(
utils.model_to_tflite(self.sess, model, params,
Modes.STREAM_EXTERNAL_STATE_INFERENCE))
# convert TF non streaming model to TF streaming with external states
self.assertTrue(
utils.to_streaming_inference(
model, params, Modes.STREAM_EXTERNAL_STATE_INFERENCE))
# convert TF non streaming model to TF streaming with internal states
self.assertTrue(
utils.to_streaming_inference(
model, params, Modes.STREAM_INTERNAL_STATE_INFERENCE))
def main(_):
# Update flags
flags = model_flags.update_flags(FLAGS)
if flags.train:
# Create model folders where logs and model will be stored
os.makedirs(flags.train_dir)
os.mkdir(flags.summaries_dir)
# Model training
train.train(flags)
# write all flags settings into json
with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f:
json.dump(flags.__dict__, f)
# convert to SavedModel
test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE)
test.convert_model_saved(flags, 'stream_state_internal',
Modes.STREAM_INTERNAL_STATE_INFERENCE)
# ---------------- non streaming model accuracy evaluation ----------------
# with TF
folder = 'tf'
test.tf_non_stream_model_accuracy(flags, folder)
# with TF.
# We can apply non stream model on stream data, by running inference
# every 200ms (for example), so that total latency will be similar with
# streaming model which is executed every 20ms.
# To measure the impact of sampling on model accuracy,
# we introduce time_shift_ms during accuracy evaluation.
# Convert milliseconds to samples:
time_shift_samples = int(
(flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND)
test.tf_non_stream_model_accuracy(
flags,
folder,
time_shift_samples,
accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt')
# with TFLite
folder = 'tflite_non_stream'
fname = 'non_stream.tflite'
mode = Modes.NON_STREAM_INFERENCE
test.convert_model_tflite(flags, folder, mode, fname)
test.tflite_non_stream_model_accuracy(flags, folder, fname)
# ---------------- TF streaming model accuracy evaluation ----------------
# Streaming model (with external state) evaluation using TF (with state reset)
folder = 'tf'
test.tf_stream_state_external_model_accuracy(
flags,
folder,
accuracy_name='stream_state_external_model_accuracy_sub_set_reset1.txt',
reset_state=True) # with state reset between test sequences
# Streaming model (with external state) evaluation using TF (no state reset)
test.tf_stream_state_external_model_accuracy(
flags,
folder,
accuracy_name='stream_state_external_model_accuracy_sub_set_reset0.txt',
reset_state=False) # without state reset
# Streaming model (with internal state) evaluation using TF (no state reset)
test.tf_stream_state_internal_model_accuracy(flags, folder)
# --------------- TFlite streaming model accuracy evaluation ---------------
# convert model to TFlite
folder = 'tflite_stream_state_external'
fname = 'stream_state_external.tflite'
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
test.convert_model_tflite(flags, folder, mode, fname)
# Streaming model accuracy evaluation with TFLite with state reset
test.tflite_stream_state_external_model_accuracy(
flags,
folder,
fname,
accuracy_name='tflite_stream_state_external_model_accuracy_reset1.txt',
reset_state=True)
# Streaming model accuracy evaluation with TFLite without state reset
test.tflite_stream_state_external_model_accuracy(
flags,
folder,
fname,
accuracy_name='tflite_stream_state_external_model_accuracy_reset0.txt',
reset_state=False)
def main(_):
# Update flags
flags = model_flags.update_flags(FLAGS)
if flags.train:
# Create model folders where logs and model will be stored
os.makedirs(flags.train_dir)
os.mkdir(flags.summaries_dir)
# Model training
train.train(flags)
else:
if not os.path.isdir(flags.train_dir):
raise ValueError(
'model is not trained set "--train 1" and retrain it')
# write all flags settings into json
with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f:
json.dump(flags.__dict__, f)
# convert to SavedModel
test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE)
try:
test.convert_model_saved(flags, 'stream_state_internal',
Modes.STREAM_INTERNAL_STATE_INFERENCE)
except (ValueError, IndexError) as e:
logging.info('FAILED to run TF streaming: %s', e)
logging.info('run TF non streaming model accuracy evaluation')
# with TF
folder_name = 'tf'
test.tf_non_stream_model_accuracy(flags, folder_name)
# with TF.
# We can apply non stream model on stream data, by running inference
# every 200ms (for example), so that total latency will be similar with
# streaming model which is executed every 20ms.
# To measure the impact of sampling on model accuracy,
# we introduce time_shift_ms during accuracy evaluation.
# Convert milliseconds to samples:
time_shift_samples = int(
(flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND)
test.tf_non_stream_model_accuracy(
flags,
folder_name,
time_shift_samples,
accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt')
name2opt = {
'': None,
'quantize_opt_for_size_': [tf.lite.Optimize.OPTIMIZE_FOR_SIZE],
}
for opt_name, optimizations in name2opt.items():
if (opt_name and flags.feature_type == 'mfcc_tf'
and flags.preprocess == 'raw'):
logging.info(
'feature type mfcc_tf needs quantization aware training '
'for quantization - it is not implemented')
continue
folder_name = opt_name + 'tflite_non_stream'
file_name = 'non_stream.tflite'
mode = Modes.NON_STREAM_INFERENCE
test.convert_model_tflite(flags,
folder_name,
mode,
file_name,
optimizations=optimizations)
test.tflite_non_stream_model_accuracy(flags, folder_name, file_name)
# these models are using bi-rnn, so they are non streamable by default
# also models using striding or pooling are not supported for streaming now
non_streamable_models = {'att_mh_rnn', 'att_rnn', 'tc_resnet'}
model_is_streamable = True
if flags.model_name in non_streamable_models:
model_is_streamable = False
# below models can use striding in time dimension,
# but this is currently unsupported
elif flags.model_name == 'cnn':
for strides in parse(flags.cnn_strides):
if strides[0] > 1:
model_is_streamable = False
break
elif flags.model_name == 'ds_cnn':
if parse(flags.cnn1_strides)[0] > 1:
model_is_streamable = False
for strides in parse(flags.dw2_strides):
if strides[0] > 1:
model_is_streamable = False
break
# if model can be streamed, then run conversion/evaluation in streaming mode
if model_is_streamable:
# ---------------- TF streaming model accuracy evaluation ----------------
# Streaming model with external state evaluation using TF with state reset
if not opt_name:
logging.info(
'run TF evalution only without optimization/quantization')
try:
folder_name = 'tf'
test.tf_stream_state_external_model_accuracy(
flags,
folder_name,
accuracy_name=
'stream_state_external_model_accuracy_sub_set_reset1.txt',
reset_state=True
) # with state reset between test sequences
# Streaming (with external state) evaluation using TF no state reset
test.tf_stream_state_external_model_accuracy(
flags,
folder_name,
accuracy_name=
'stream_state_external_model_accuracy_sub_set_reset0.txt',
reset_state=False) # without state reset
# Streaming (with internal state) evaluation using TF no state reset
test.tf_stream_state_internal_model_accuracy(
flags, folder_name)
except (ValueError, IndexError) as e:
logging.info('FAILED to run TF streaming: %s', e)
logging.info('run TFlite streaming model accuracy evaluation')
try:
# convert model to TFlite
folder_name = opt_name + 'tflite_stream_state_external'
file_name = 'stream_state_external.tflite'
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
test.convert_model_tflite(flags,
folder_name,
mode,
file_name,
optimizations=optimizations)
# Streaming model accuracy evaluation with TFLite with state reset
test.tflite_stream_state_external_model_accuracy(
flags,
folder_name,
file_name,
accuracy_name=
'tflite_stream_state_external_model_accuracy_reset1.txt',
reset_state=True)
# Streaming model accuracy evaluation with TFLite without state reset
test.tflite_stream_state_external_model_accuracy(
flags,
folder_name,
file_name,
accuracy_name=
'tflite_stream_state_external_model_accuracy_reset0.txt',
reset_state=False)
except (ValueError, IndexError) as e:
logging.info('FAILED to run TFLite streaming: %s', e)
def main(_):
# Update flags
flags = model_flags.update_flags(FLAGS)
if flags.train:
# Create model folders where logs and model will be stored
os.makedirs(flags.train_dir)
os.mkdir(flags.summaries_dir)
# Model training
train.train(flags)
# write all flags settings into json
with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f:
json.dump(flags.__dict__, f)
# convert to SavedModel
test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE)
test.convert_model_saved(flags, 'stream_state_internal',
Modes.STREAM_INTERNAL_STATE_INFERENCE)
logging.info('run TF non streaming model accuracy evaluation')
# with TF
folder_name = 'tf'
test.tf_non_stream_model_accuracy(flags, folder_name)
# with TF.
# We can apply non stream model on stream data, by running inference
# every 200ms (for example), so that total latency will be similar with
# streaming model which is executed every 20ms.
# To measure the impact of sampling on model accuracy,
# we introduce time_shift_ms during accuracy evaluation.
# Convert milliseconds to samples:
time_shift_samples = int(
(flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND)
test.tf_non_stream_model_accuracy(
flags,
folder_name,
time_shift_samples,
accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt')
name2opt = {
'': None,
'quantize_opt_for_size_': [tf.lite.Optimize.OPTIMIZE_FOR_SIZE],
}
for opt_name, optimizations in name2opt.items():
if opt_name and flags.feature_type == 'mfcc_tf':
logging.info(
'feature type mfcc_tf needs quantization aware training '
'for quantization - it is not implemented')
continue
folder_name = opt_name + 'tflite_non_stream'
file_name = 'non_stream.tflite'
mode = Modes.NON_STREAM_INFERENCE
test.convert_model_tflite(flags,
folder_name,
mode,
file_name,
optimizations=optimizations)
test.tflite_non_stream_model_accuracy(flags, folder_name, file_name)
# ---------------- TF streaming model accuracy evaluation ----------------
# Streaming model (with external state) evaluation using TF with state reset
if not opt_name:
logging.info(
'run TF evalution only without optimization/quantization')
try:
folder_name = 'tf'
test.tf_stream_state_external_model_accuracy(
flags,
folder_name,
accuracy_name=
'stream_state_external_model_accuracy_sub_set_reset1.txt',
reset_state=True
) # with state reset between test sequences
# Streaming (with external state) evaluation using TF no state reset
test.tf_stream_state_external_model_accuracy(
flags,
folder_name,
accuracy_name=
'stream_state_external_model_accuracy_sub_set_reset0.txt',
reset_state=False) # without state reset
# Streaming (with internal state) evaluation using TF no state reset
test.tf_stream_state_internal_model_accuracy(
flags, folder_name)
except ValueError as e:
logging.error('FAILED to run TF streaming: %s', e)
logging.info('run TFlite streaming model accuracy evaluation')
try:
# convert model to TFlite
folder_name = opt_name + 'tflite_stream_state_external'
file_name = 'stream_state_external.tflite'
mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE
test.convert_model_tflite(flags,
folder_name,
mode,
file_name,
optimizations=optimizations)
# Streaming model accuracy evaluation with TFLite with state reset
test.tflite_stream_state_external_model_accuracy(
flags,
folder_name,
file_name,
accuracy_name=
'tflite_stream_state_external_model_accuracy_reset1.txt',
reset_state=True)
# Streaming model accuracy evaluation with TFLite without state reset
test.tflite_stream_state_external_model_accuracy(
flags,
folder_name,
file_name,
accuracy_name=
'tflite_stream_state_external_model_accuracy_reset0.txt',
reset_state=False)
except ValueError as e:
logging.error('FAILED to run TFLite streaming: %s', e)
