def create_sequence_dataset_input_fn(dataset_dir, prepare_fn, record_file_name_format, meta_data_file_name_format, bucket_boundaries): prepare_fn(dataset_dir) train_data_file = record_file_name_format.format(dataset_dir, Modes.TRAIN) eval_data_file = record_file_name_format.format(dataset_dir, Modes.EVAL) meta_data_filename = meta_data_file_name_format.format(dataset_dir) train_input_fn = create_input_data_fn( mode=Modes.TRAIN, pipeline_config=TFRecordSequencePipelineConfig( dynamic_pad=True, bucket_boundaries=bucket_boundaries, data_files=train_data_file, meta_data_file=meta_data_filename)) eval_input_fn = create_input_data_fn( mode=Modes.EVAL, pipeline_config=TFRecordSequencePipelineConfig( dynamic_pad=True, bucket_boundaries=bucket_boundaries, data_files=eval_data_file, meta_data_file=meta_data_filename)) return train_input_fn, eval_input_fn
def test_train_config(self): config_dict = { 'data_pipeline': TFRecordSequencePipelineConfig( data_files=['~/data_file'], meta_data_file='~/meta_data_file', shuffle=True, num_epochs=10, batch_size=64).to_schema(), 'steps': 100, 'hooks': [ StepLoggingTensorHookConfig(['Dense_1', 'Conv2D_4'], every_n_iter=100).to_schema() ] } config = TrainConfig.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict)
def test_tf_record_sequence_pipeline_config(self): config_dict = { 'batch_size': 64, 'num_epochs': 10, 'shuffle': True, 'data_files': ['train_data_file'], 'meta_data_file': 'meta_data_file', } config = TFRecordSequencePipelineConfig.from_dict(config_dict) config_to_dict = config.to_dict() assert config_to_dict['num_epochs'] == config_dict['num_epochs'] assert config_to_dict['shuffle'] == config_dict['shuffle'] assert config_to_dict['data_files'] == config_dict['data_files'] assert config_to_dict['meta_data_file'] == config_dict[ 'meta_data_file']
def test_eval_config(self): config_dict = { 'data_pipeline': TFRecordSequencePipelineConfig( data_files=['~/data_file'], meta_data_file='~/meta_data_file', shuffle=True, num_epochs=10, batch_size=64).to_schema(), 'steps': 10, 'hooks': [ StepLoggingTensorHookConfig(['Dense_1', 'Conv2D_4'], every_n_iter=100).to_schema() ], 'delay_secs': 0, 'continuous_eval_throttle_secs': 60, } config = EvalConfig.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict)
def create_sequence_dataset_predict_input_fn(dataset_dir, prepare_fn, record_file_name_format, meta_data_file_name_format, bucket_boundaries): prepare_fn(dataset_dir) test_data_file = record_file_name_format.format(dataset_dir, Modes.PREDICT) meta_data_filename = meta_data_file_name_format.format(dataset_dir) test_input_fn = create_input_data_fn( mode=Modes.PREDICT, pipeline_config=TFRecordSequencePipelineConfig( dynamic_pad=True, num_epochs=1, batch_size=4, min_after_dequeue=0, bucket_boundaries=bucket_boundaries, data_files=test_data_file, meta_data_file=meta_data_filename)) return test_input_fn