def testPreprocessStreamInferenceModeTFandTFLite(self, preprocess, feature_type, model_name='gru'): # Validate that model with different preprocessing # can be converted to stream inference mode with TF and TFLite. params = model_params.HOTWORD_MODEL_PARAMS[model_name] # set parameters to test params.preprocess = preprocess params.feature_type = feature_type params = model_flags.update_flags(params) # create model model = models.MODELS[params.model_name](params) # convert TF non streaming model to TFLite streaming inference # with external states self.assertTrue(utils.model_to_tflite( self.sess, model, params, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE)) # convert TF non streaming model to TF streaming with external states self.assertTrue(utils.to_streaming_inference( model, params, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE)) # convert TF non streaming model to TF streaming with internal states self.assertTrue(utils.to_streaming_inference( model, params, modes.Modes.STREAM_INTERNAL_STATE_INFERENCE))
def _RunGetDataTest(self, preprocess, window_size_ms): tmp_dir = self.get_temp_dir() wav_dir = os.path.join(tmp_dir, "wavs") os.mkdir(wav_dir) self._SaveWavFolders(wav_dir, ["a", "b", "c"], 100) background_dir = os.path.join(wav_dir, "_background_noise_") os.mkdir(background_dir) wav_data = self._GetWavData() for i in range(10): file_path = os.path.join(background_dir, "background_audio_%d.wav" % i) self._SaveTestWavFile(file_path, wav_data) flags = self._GetDefaultFlags() flags.window_size_ms = window_size_ms flags.preprocess = preprocess flags.train_dir = tmp_dir flags.data_dir = wav_dir flags = model_flags.update_flags(flags) with self.cached_session() as sess: audio_processor = input_data.AudioProcessor(flags) result_data, result_labels = audio_processor.get_data( 10, 0, flags, 0.3, 0.1, 100, "training", 0.0, sess) self.assertLen(result_data, 10) self.assertLen(result_labels, 10)
def test_model_to_saved(self, model_name='dnn'): """SavedModel supports both stateless and stateful graphs.""" params = model_params.HOTWORD_MODEL_PARAMS[model_name] params = model_flags.update_flags(params) # create model model = models.MODELS[params.model_name](params) utils.model_to_saved(model, params, FLAGS.test_tmpdir)
def testTrain(self, split_data): input_flags = self._GetDefaultFlags(split_data) input_flags = model_flags.update_flags(input_flags) train.train(input_flags) self.assertTrue( tf.io.gfile.exists(os.path.join(input_flags.train_dir, 'graph.pbtxt'))) self.assertTrue( tf.io.gfile.exists(os.path.join(input_flags.train_dir, 'labels.txt'))) self.assertTrue( tf.io.gfile.exists( os.path.join(input_flags.train_dir, 'accuracy_last.txt')))
def _GetDefaultFlags(self, split_data): params = model_params.dnn_params() params.data_dir = self._PrepareDummyTrainingData( ) if split_data == 1 else self._PrepareDummyTrainingDataSplit() params.wanted_words = 'a,b,c' params.split_data = split_data params.summaries_dir = self._PrepareDummyDir('summaries' + str(split_data)) params.train_dir = self._PrepareDummyDir('train' + str(split_data)) params.how_many_training_steps = '2' params.learning_rate = '0.01' params.eval_step_interval = 1 params.save_step_interval = 1 params.clip_duration_ms = 100 params.batch_size = 1 return model_flags.update_flags(params)
def testToNonStreamInferenceTFandTFLite(self, model_name='svdf'): """Validate that model can be converted to non stream inference mode.""" params = _HOTWORD_MODEL_PARAMS[model_name] params = model_flags.update_flags(params) # create model model = models.MODELS[params.model_name](params) # convert TF non streaming model to TF non streaming inference model # it will disable dropouts self.assertTrue( utils.to_streaming_inference(model, params, Modes.NON_STREAM_INFERENCE)) # convert TF non streaming model to TFLite non streaming inference self.assertTrue( utils.model_to_tflite(self.sess, model, params, Modes.NON_STREAM_INFERENCE))
def test_external_streaming_shapes(self, model_name): params = model_params.HOTWORD_MODEL_PARAMS[model_name] params = model_flags.update_flags(params) model = models.MODELS[params.model_name](params) external_model = utils.to_streaming_inference( model, params, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE) # The first 'n' inputs correspond to the 'n' inputs that the model takes # in non-streaming mode. The rest of the input tensors represent the # internal states for each layer in the model. inputs = [ np.zeros(shape, dtype=np.float32) for shape in external_model.input_shapes ] outputs = external_model.predict(inputs) for output, expected_shape in zip(outputs, external_model.output_shapes): self.assertEqual(output.shape, expected_shape)
def _testTFLite(self, preprocess='raw', feature_type='mfcc_op', model_name='svdf'): params = model_params.HOTWORD_MODEL_PARAMS[model_name] # set parameters to test params.preprocess = preprocess params.feature_type = feature_type params = model_flags.update_flags(params) # create model model = models.MODELS[params.model_name](params) # convert TF non streaming model to TFLite non streaming inference self.assertTrue( utils.model_to_tflite(self.sess, model, params, Modes.NON_STREAM_INFERENCE))
def testToStreamInferenceModeTFandTFLite(self, model_name='gru'): """Validate that model can be converted to any streaming inference mode.""" params = _HOTWORD_MODEL_PARAMS[model_name] params = model_flags.update_flags(params) # create model model = models.MODELS[params.model_name](params) # convert TF non streaming model to TFLite streaming inference # with external states self.assertTrue( utils.model_to_tflite(self.sess, model, params, Modes.STREAM_EXTERNAL_STATE_INFERENCE)) # convert TF non streaming model to TF streaming with external states self.assertTrue( utils.to_streaming_inference( model, params, Modes.STREAM_EXTERNAL_STATE_INFERENCE)) # convert TF non streaming model to TF streaming with internal states self.assertTrue( utils.to_streaming_inference( model, params, Modes.STREAM_INTERNAL_STATE_INFERENCE))
def main(_): # Update flags flags = model_flags.update_flags(FLAGS) if flags.train: # Create model folders where logs and model will be stored os.makedirs(flags.train_dir) os.mkdir(flags.summaries_dir) # Model training train.train(flags) # write all flags settings into json with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f: json.dump(flags.__dict__, f) # convert to SavedModel test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE) test.convert_model_saved(flags, 'stream_state_internal', Modes.STREAM_INTERNAL_STATE_INFERENCE) # ---------------- non streaming model accuracy evaluation ---------------- # with TF folder = 'tf' test.tf_non_stream_model_accuracy(flags, folder) # with TF. # We can apply non stream model on stream data, by running inference # every 200ms (for example), so that total latency will be similar with # streaming model which is executed every 20ms. # To measure the impact of sampling on model accuracy, # we introduce time_shift_ms during accuracy evaluation. # Convert milliseconds to samples: time_shift_samples = int( (flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND) test.tf_non_stream_model_accuracy( flags, folder, time_shift_samples, accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt') # with TFLite folder = 'tflite_non_stream' fname = 'non_stream.tflite' mode = Modes.NON_STREAM_INFERENCE test.convert_model_tflite(flags, folder, mode, fname) test.tflite_non_stream_model_accuracy(flags, folder, fname) # ---------------- TF streaming model accuracy evaluation ---------------- # Streaming model (with external state) evaluation using TF (with state reset) folder = 'tf' test.tf_stream_state_external_model_accuracy( flags, folder, accuracy_name='stream_state_external_model_accuracy_sub_set_reset1.txt', reset_state=True) # with state reset between test sequences # Streaming model (with external state) evaluation using TF (no state reset) test.tf_stream_state_external_model_accuracy( flags, folder, accuracy_name='stream_state_external_model_accuracy_sub_set_reset0.txt', reset_state=False) # without state reset # Streaming model (with internal state) evaluation using TF (no state reset) test.tf_stream_state_internal_model_accuracy(flags, folder) # --------------- TFlite streaming model accuracy evaluation --------------- # convert model to TFlite folder = 'tflite_stream_state_external' fname = 'stream_state_external.tflite' mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE test.convert_model_tflite(flags, folder, mode, fname) # Streaming model accuracy evaluation with TFLite with state reset test.tflite_stream_state_external_model_accuracy( flags, folder, fname, accuracy_name='tflite_stream_state_external_model_accuracy_reset1.txt', reset_state=True) # Streaming model accuracy evaluation with TFLite without state reset test.tflite_stream_state_external_model_accuracy( flags, folder, fname, accuracy_name='tflite_stream_state_external_model_accuracy_reset0.txt', reset_state=False)
def main(_): # Update flags flags = model_flags.update_flags(FLAGS) if flags.train: # Create model folders where logs and model will be stored os.makedirs(flags.train_dir) os.mkdir(flags.summaries_dir) # Model training train.train(flags) else: if not os.path.isdir(flags.train_dir): raise ValueError( 'model is not trained set "--train 1" and retrain it') # write all flags settings into json with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f: json.dump(flags.__dict__, f) # convert to SavedModel test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE) try: test.convert_model_saved(flags, 'stream_state_internal', Modes.STREAM_INTERNAL_STATE_INFERENCE) except (ValueError, IndexError) as e: logging.info('FAILED to run TF streaming: %s', e) logging.info('run TF non streaming model accuracy evaluation') # with TF folder_name = 'tf' test.tf_non_stream_model_accuracy(flags, folder_name) # with TF. # We can apply non stream model on stream data, by running inference # every 200ms (for example), so that total latency will be similar with # streaming model which is executed every 20ms. # To measure the impact of sampling on model accuracy, # we introduce time_shift_ms during accuracy evaluation. # Convert milliseconds to samples: time_shift_samples = int( (flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND) test.tf_non_stream_model_accuracy( flags, folder_name, time_shift_samples, accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt') name2opt = { '': None, 'quantize_opt_for_size_': [tf.lite.Optimize.OPTIMIZE_FOR_SIZE], } for opt_name, optimizations in name2opt.items(): if (opt_name and flags.feature_type == 'mfcc_tf' and flags.preprocess == 'raw'): logging.info( 'feature type mfcc_tf needs quantization aware training ' 'for quantization - it is not implemented') continue folder_name = opt_name + 'tflite_non_stream' file_name = 'non_stream.tflite' mode = Modes.NON_STREAM_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) test.tflite_non_stream_model_accuracy(flags, folder_name, file_name) # these models are using bi-rnn, so they are non streamable by default # also models using striding or pooling are not supported for streaming now non_streamable_models = {'att_mh_rnn', 'att_rnn', 'tc_resnet'} model_is_streamable = True if flags.model_name in non_streamable_models: model_is_streamable = False # below models can use striding in time dimension, # but this is currently unsupported elif flags.model_name == 'cnn': for strides in parse(flags.cnn_strides): if strides[0] > 1: model_is_streamable = False break elif flags.model_name == 'ds_cnn': if parse(flags.cnn1_strides)[0] > 1: model_is_streamable = False for strides in parse(flags.dw2_strides): if strides[0] > 1: model_is_streamable = False break # if model can be streamed, then run conversion/evaluation in streaming mode if model_is_streamable: # ---------------- TF streaming model accuracy evaluation ---------------- # Streaming model with external state evaluation using TF with state reset if not opt_name: logging.info( 'run TF evalution only without optimization/quantization') try: folder_name = 'tf' test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset1.txt', reset_state=True ) # with state reset between test sequences # Streaming (with external state) evaluation using TF no state reset test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset0.txt', reset_state=False) # without state reset # Streaming (with internal state) evaluation using TF no state reset test.tf_stream_state_internal_model_accuracy( flags, folder_name) except (ValueError, IndexError) as e: logging.info('FAILED to run TF streaming: %s', e) logging.info('run TFlite streaming model accuracy evaluation') try: # convert model to TFlite folder_name = opt_name + 'tflite_stream_state_external' file_name = 'stream_state_external.tflite' mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) # Streaming model accuracy evaluation with TFLite with state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset1.txt', reset_state=True) # Streaming model accuracy evaluation with TFLite without state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset0.txt', reset_state=False) except (ValueError, IndexError) as e: logging.info('FAILED to run TFLite streaming: %s', e)
def main(_): # Update flags flags = model_flags.update_flags(FLAGS) if flags.train: # Create model folders where logs and model will be stored os.makedirs(flags.train_dir) os.mkdir(flags.summaries_dir) # Model training train.train(flags) # write all flags settings into json with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f: json.dump(flags.__dict__, f) # convert to SavedModel test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE) test.convert_model_saved(flags, 'stream_state_internal', Modes.STREAM_INTERNAL_STATE_INFERENCE) logging.info('run TF non streaming model accuracy evaluation') # with TF folder_name = 'tf' test.tf_non_stream_model_accuracy(flags, folder_name) # with TF. # We can apply non stream model on stream data, by running inference # every 200ms (for example), so that total latency will be similar with # streaming model which is executed every 20ms. # To measure the impact of sampling on model accuracy, # we introduce time_shift_ms during accuracy evaluation. # Convert milliseconds to samples: time_shift_samples = int( (flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND) test.tf_non_stream_model_accuracy( flags, folder_name, time_shift_samples, accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt') name2opt = { '': None, 'quantize_opt_for_size_': [tf.lite.Optimize.OPTIMIZE_FOR_SIZE], } for opt_name, optimizations in name2opt.items(): if opt_name and flags.feature_type == 'mfcc_tf': logging.info( 'feature type mfcc_tf needs quantization aware training ' 'for quantization - it is not implemented') continue folder_name = opt_name + 'tflite_non_stream' file_name = 'non_stream.tflite' mode = Modes.NON_STREAM_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) test.tflite_non_stream_model_accuracy(flags, folder_name, file_name) # ---------------- TF streaming model accuracy evaluation ---------------- # Streaming model (with external state) evaluation using TF with state reset if not opt_name: logging.info( 'run TF evalution only without optimization/quantization') try: folder_name = 'tf' test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset1.txt', reset_state=True ) # with state reset between test sequences # Streaming (with external state) evaluation using TF no state reset test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset0.txt', reset_state=False) # without state reset # Streaming (with internal state) evaluation using TF no state reset test.tf_stream_state_internal_model_accuracy( flags, folder_name) except ValueError as e: logging.error('FAILED to run TF streaming: %s', e) logging.info('run TFlite streaming model accuracy evaluation') try: # convert model to TFlite folder_name = opt_name + 'tflite_stream_state_external' file_name = 'stream_state_external.tflite' mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) # Streaming model accuracy evaluation with TFLite with state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset1.txt', reset_state=True) # Streaming model accuracy evaluation with TFLite without state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset0.txt', reset_state=False) except ValueError as e: logging.error('FAILED to run TFLite streaming: %s', e)