def main(_): # Update flags flags = model_flags.update_flags(FLAGS) if flags.train: # Create model folders where logs and model will be stored os.makedirs(flags.train_dir) os.mkdir(flags.summaries_dir) # Model training train.train(flags) else: if not os.path.isdir(flags.train_dir): raise ValueError( 'model is not trained set "--train 1" and retrain it') # write all flags settings into json with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f: json.dump(flags.__dict__, f) # convert to SavedModel test.convert_model_saved(flags, 'non_stream', modes.Modes.NON_STREAM_INFERENCE) try: test.convert_model_saved(flags, 'stream_state_internal', modes.Modes.STREAM_INTERNAL_STATE_INFERENCE) except (ValueError, IndexError) as e: logging.info('FAILED to run TF streaming: %s', e) logging.info('run TF non streaming model accuracy evaluation') # with TF folder_name = 'tf' test.tf_non_stream_model_accuracy(flags, folder_name) # with TF. # We can apply non stream model on stream data, by running inference # every 200ms (for example), so that total latency will be similar with # streaming model which is executed every 20ms. # To measure the impact of sampling on model accuracy, # we introduce time_shift_ms during accuracy evaluation. # Convert milliseconds to samples: time_shift_samples = int( (flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND) test.tf_non_stream_model_accuracy( flags, folder_name, time_shift_samples, accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt') name2opt = { '': None, 'quantize_opt_for_size_': [tf.lite.Optimize.DEFAULT], } for opt_name, optimizations in name2opt.items(): if (opt_name and flags.feature_type == 'mfcc_tf' and flags.preprocess == 'raw'): logging.info( 'feature type mfcc_tf needs quantization aware training ' 'for quantization - it is not implemented') continue folder_name = opt_name + 'tflite_non_stream' file_name = 'non_stream.tflite' mode = modes.Modes.NON_STREAM_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) test.tflite_non_stream_model_accuracy(flags, folder_name, file_name) # these models are using bi-rnn, so they are non streamable by default # also models using striding or pooling are not supported for streaming now non_streamable_models = {'att_mh_rnn', 'att_rnn', 'tc_resnet'} model_is_streamable = True if flags.model_name in non_streamable_models: model_is_streamable = False # below models can use striding in time dimension, # but this is currently unsupported elif flags.model_name == 'cnn': for strides in model_utils.parse(flags.cnn_strides): if strides[0] > 1: model_is_streamable = False break elif flags.model_name == 'ds_cnn': if model_utils.parse(flags.cnn1_strides)[0] > 1: model_is_streamable = False for strides in model_utils.parse(flags.dw2_strides): if strides[0] > 1: model_is_streamable = False break # set input data shape for testing inference in streaming mode flags.data_shape = modes.get_input_data_shape( flags, modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE) # if model can be streamed, then run conversion/evaluation in streaming mode if model_is_streamable: # ---------------- TF streaming model accuracy evaluation ---------------- # Streaming model with external state evaluation using TF with state reset if not opt_name: logging.info( 'run TF evalution only without optimization/quantization') try: folder_name = 'tf' test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset1.txt', reset_state=True ) # with state reset between test sequences # Streaming (with external state) evaluation using TF no state reset test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset0.txt', reset_state=False) # without state reset # Streaming (with internal state) evaluation using TF no state reset test.tf_stream_state_internal_model_accuracy( flags, folder_name) except (ValueError, IndexError) as e: logging.info('FAILED to run TF streaming: %s', e) logging.info('run TFlite streaming model accuracy evaluation') try: # convert model to TFlite folder_name = opt_name + 'tflite_stream_state_external' file_name = 'stream_state_external.tflite' mode = modes.Modes.STREAM_EXTERNAL_STATE_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) # Streaming model accuracy evaluation with TFLite with state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset1.txt', reset_state=True) # Streaming model accuracy evaluation with TFLite without state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset0.txt', reset_state=False) except (ValueError, IndexError) as e: logging.info('FAILED to run TFLite streaming: %s', e)
def main(_): # Update flags flags = model_flags.update_flags(FLAGS) if flags.train: # Create model folders where logs and model will be stored os.makedirs(flags.train_dir) os.mkdir(flags.summaries_dir) # Model training train.train(flags) # write all flags settings into json with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f: json.dump(flags.__dict__, f) # convert to SavedModel test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE) test.convert_model_saved(flags, 'stream_state_internal', Modes.STREAM_INTERNAL_STATE_INFERENCE) # ---------------- non streaming model accuracy evaluation ---------------- # with TF folder = 'tf' test.tf_non_stream_model_accuracy(flags, folder) # with TF. # We can apply non stream model on stream data, by running inference # every 200ms (for example), so that total latency will be similar with # streaming model which is executed every 20ms. # To measure the impact of sampling on model accuracy, # we introduce time_shift_ms during accuracy evaluation. # Convert milliseconds to samples: time_shift_samples = int( (flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND) test.tf_non_stream_model_accuracy( flags, folder, time_shift_samples, accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt') # with TFLite folder = 'tflite_non_stream' fname = 'non_stream.tflite' mode = Modes.NON_STREAM_INFERENCE test.convert_model_tflite(flags, folder, mode, fname) test.tflite_non_stream_model_accuracy(flags, folder, fname) # ---------------- TF streaming model accuracy evaluation ---------------- # Streaming model (with external state) evaluation using TF (with state reset) folder = 'tf' test.tf_stream_state_external_model_accuracy( flags, folder, accuracy_name='stream_state_external_model_accuracy_sub_set_reset1.txt', reset_state=True) # with state reset between test sequences # Streaming model (with external state) evaluation using TF (no state reset) test.tf_stream_state_external_model_accuracy( flags, folder, accuracy_name='stream_state_external_model_accuracy_sub_set_reset0.txt', reset_state=False) # without state reset # Streaming model (with internal state) evaluation using TF (no state reset) test.tf_stream_state_internal_model_accuracy(flags, folder) # --------------- TFlite streaming model accuracy evaluation --------------- # convert model to TFlite folder = 'tflite_stream_state_external' fname = 'stream_state_external.tflite' mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE test.convert_model_tflite(flags, folder, mode, fname) # Streaming model accuracy evaluation with TFLite with state reset test.tflite_stream_state_external_model_accuracy( flags, folder, fname, accuracy_name='tflite_stream_state_external_model_accuracy_reset1.txt', reset_state=True) # Streaming model accuracy evaluation with TFLite without state reset test.tflite_stream_state_external_model_accuracy( flags, folder, fname, accuracy_name='tflite_stream_state_external_model_accuracy_reset0.txt', reset_state=False)
def main(_): # Update flags flags = model_flags.update_flags(FLAGS) if flags.train: # Create model folders where logs and model will be stored os.makedirs(flags.train_dir) os.mkdir(flags.summaries_dir) # Model training train.train(flags) # write all flags settings into json with open(os.path.join(flags.train_dir, 'flags.json'), 'wt') as f: json.dump(flags.__dict__, f) # convert to SavedModel test.convert_model_saved(flags, 'non_stream', Modes.NON_STREAM_INFERENCE) test.convert_model_saved(flags, 'stream_state_internal', Modes.STREAM_INTERNAL_STATE_INFERENCE) logging.info('run TF non streaming model accuracy evaluation') # with TF folder_name = 'tf' test.tf_non_stream_model_accuracy(flags, folder_name) # with TF. # We can apply non stream model on stream data, by running inference # every 200ms (for example), so that total latency will be similar with # streaming model which is executed every 20ms. # To measure the impact of sampling on model accuracy, # we introduce time_shift_ms during accuracy evaluation. # Convert milliseconds to samples: time_shift_samples = int( (flags.time_shift_ms * flags.sample_rate) / model_flags.MS_PER_SECOND) test.tf_non_stream_model_accuracy( flags, folder_name, time_shift_samples, accuracy_name='tf_non_stream_model_sampling_stream_accuracy.txt') name2opt = { '': None, 'quantize_opt_for_size_': [tf.lite.Optimize.OPTIMIZE_FOR_SIZE], } for opt_name, optimizations in name2opt.items(): if opt_name and flags.feature_type == 'mfcc_tf': logging.info( 'feature type mfcc_tf needs quantization aware training ' 'for quantization - it is not implemented') continue folder_name = opt_name + 'tflite_non_stream' file_name = 'non_stream.tflite' mode = Modes.NON_STREAM_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) test.tflite_non_stream_model_accuracy(flags, folder_name, file_name) # ---------------- TF streaming model accuracy evaluation ---------------- # Streaming model (with external state) evaluation using TF with state reset if not opt_name: logging.info( 'run TF evalution only without optimization/quantization') try: folder_name = 'tf' test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset1.txt', reset_state=True ) # with state reset between test sequences # Streaming (with external state) evaluation using TF no state reset test.tf_stream_state_external_model_accuracy( flags, folder_name, accuracy_name= 'stream_state_external_model_accuracy_sub_set_reset0.txt', reset_state=False) # without state reset # Streaming (with internal state) evaluation using TF no state reset test.tf_stream_state_internal_model_accuracy( flags, folder_name) except ValueError as e: logging.error('FAILED to run TF streaming: %s', e) logging.info('run TFlite streaming model accuracy evaluation') try: # convert model to TFlite folder_name = opt_name + 'tflite_stream_state_external' file_name = 'stream_state_external.tflite' mode = Modes.STREAM_EXTERNAL_STATE_INFERENCE test.convert_model_tflite(flags, folder_name, mode, file_name, optimizations=optimizations) # Streaming model accuracy evaluation with TFLite with state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset1.txt', reset_state=True) # Streaming model accuracy evaluation with TFLite without state reset test.tflite_stream_state_external_model_accuracy( flags, folder_name, file_name, accuracy_name= 'tflite_stream_state_external_model_accuracy_reset0.txt', reset_state=False) except ValueError as e: logging.error('FAILED to run TFLite streaming: %s', e)