def create_sequence_dataset_input_fn(dataset_dir, prepare_fn,
record_file_name_format,
meta_data_file_name_format,
bucket_boundaries):
prepare_fn(dataset_dir)
train_data_file = record_file_name_format.format(dataset_dir, Modes.TRAIN)
eval_data_file = record_file_name_format.format(dataset_dir, Modes.EVAL)
meta_data_filename = meta_data_file_name_format.format(dataset_dir)
train_input_fn = create_input_data_fn(
mode=Modes.TRAIN,
pipeline_config=TFRecordSequencePipelineConfig(
dynamic_pad=True,
bucket_boundaries=bucket_boundaries,
data_files=train_data_file,
meta_data_file=meta_data_filename))
eval_input_fn = create_input_data_fn(
mode=Modes.EVAL,
pipeline_config=TFRecordSequencePipelineConfig(
dynamic_pad=True,
bucket_boundaries=bucket_boundaries,
data_files=eval_data_file,
meta_data_file=meta_data_filename))
return train_input_fn, eval_input_fn
def test_train_config(self):
config_dict = {
'data_pipeline': TFRecordSequencePipelineConfig(
data_files=['~/data_file'],
meta_data_file='~/meta_data_file',
shuffle=True,
num_epochs=10,
batch_size=64).to_schema(),
'steps': 100,
'hooks': [
StepLoggingTensorHookConfig(['Dense_1', 'Conv2D_4'], every_n_iter=100).to_schema()
]
}
config = TrainConfig.from_dict(config_dict)
assert_equal_dict(config.to_dict(), config_dict)
def test_tf_record_sequence_pipeline_config(self):
config_dict = {
'batch_size': 64,
'num_epochs': 10,
'shuffle': True,
'data_files': ['train_data_file'],
'meta_data_file': 'meta_data_file',
}
config = TFRecordSequencePipelineConfig.from_dict(config_dict)
config_to_dict = config.to_dict()
assert config_to_dict['num_epochs'] == config_dict['num_epochs']
assert config_to_dict['shuffle'] == config_dict['shuffle']
assert config_to_dict['data_files'] == config_dict['data_files']
assert config_to_dict['meta_data_file'] == config_dict[
'meta_data_file']
def test_eval_config(self):
config_dict = {
'data_pipeline': TFRecordSequencePipelineConfig(
data_files=['~/data_file'],
meta_data_file='~/meta_data_file',
shuffle=True,
num_epochs=10,
batch_size=64).to_schema(),
'steps': 10,
'hooks': [
StepLoggingTensorHookConfig(['Dense_1', 'Conv2D_4'], every_n_iter=100).to_schema()
],
'delay_secs': 0,
'continuous_eval_throttle_secs': 60,
}
config = EvalConfig.from_dict(config_dict)
assert_equal_dict(config.to_dict(), config_dict)
def create_sequence_dataset_predict_input_fn(dataset_dir, prepare_fn,
record_file_name_format,
meta_data_file_name_format,
bucket_boundaries):
prepare_fn(dataset_dir)
test_data_file = record_file_name_format.format(dataset_dir, Modes.PREDICT)
meta_data_filename = meta_data_file_name_format.format(dataset_dir)
test_input_fn = create_input_data_fn(
mode=Modes.PREDICT,
pipeline_config=TFRecordSequencePipelineConfig(
dynamic_pad=True,
num_epochs=1,
batch_size=4,
min_after_dequeue=0,
bucket_boundaries=bucket_boundaries,
data_files=test_data_file,
meta_data_file=meta_data_filename))
return test_input_fn