def generate_tfrecords(self):
"""
Generate tensorflow records file
:return:
"""
def _read_training_example_index_file(_index_file_path):
assert ops.exists(_index_file_path)
_example_gt_path_info = []
_example_gt_binary_path_info = []
_example_gt_instance_path_info = []
with open(_index_file_path, 'r') as _file:
for _line in _file:
_example_info = _line.rstrip('\r').rstrip('\n').split(' ')
_example_gt_path_info.append(_example_info[0])
_example_gt_binary_path_info.append(_example_info[1])
_example_gt_instance_path_info.append(_example_info[2])
ret = {
'gt_path_info': _example_gt_path_info,
'gt_binary_path_info': _example_gt_binary_path_info,
'gt_instance_path_info': _example_gt_instance_path_info
}
return ret
# make save dirs
os.makedirs(self._tfrecords_save_dir, exist_ok=True)
# start generating training example tfrecords
LOG.info('Start generating training example tfrecords')
# collecting train images paths info
train_image_paths_info = _read_training_example_index_file(
self._train_example_index_file_path)
train_gt_images_paths = train_image_paths_info['gt_path_info']
train_gt_binary_images_paths = train_image_paths_info[
'gt_binary_path_info']
train_gt_instance_images_paths = train_image_paths_info[
'gt_instance_path_info']
train_tfrecords_paths = ops.join(self._tfrecords_save_dir,
'tusimple_train.tfrecords')
tf_io_pipline_tools.write_example_tfrecords(
train_gt_images_paths, train_gt_binary_images_paths,
train_gt_instance_images_paths, train_tfrecords_paths)
LOG.info('Generating training example tfrecords complete')
# start generating validation example tfrecords
LOG.info('Start generating validation example tfrecords')
# collecting validation images paths info
val_image_paths_info = _read_training_example_index_file(
self._val_example_index_file_path)
val_gt_images_paths = val_image_paths_info['gt_path_info']
val_gt_binary_images_paths = val_image_paths_info[
'gt_binary_path_info']
val_gt_instance_images_paths = val_image_paths_info[
'gt_instance_path_info']
val_tfrecords_paths = ops.join(self._tfrecords_save_dir,
'tusimple_val.tfrecords')
tf_io_pipline_tools.write_example_tfrecords(
val_gt_images_paths, val_gt_binary_images_paths,
val_gt_instance_images_paths, val_tfrecords_paths)
LOG.info('Generating validation example tfrecords complete')
# generate test example tfrecords
LOG.info('Start generating testing example tfrecords')
# collecting test images paths info
test_image_paths_info = _read_training_example_index_file(
self._test_example_index_file_path)
test_gt_images_paths = test_image_paths_info['gt_path_info']
test_gt_binary_images_paths = test_image_paths_info[
'gt_binary_path_info']
test_gt_instance_images_paths = test_image_paths_info[
'gt_instance_path_info']
test_tfrecords_paths = ops.join(self._tfrecords_save_dir,
'tusimple_test.tfrecords')
tf_io_pipline_tools.write_example_tfrecords(
test_gt_images_paths, test_gt_binary_images_paths,
test_gt_instance_images_paths, test_tfrecords_paths)
LOG.info('Generating testing example tfrecords complete')
return
def generate_tfrecords(self, save_dir, step_size=10000):
"""
Generate tensorflow records file
:param save_dir:
:param step_size: generate a tfrecord every step_size examples
:return:
"""
def _read_training_example_index_file(_index_file_path):
assert ops.exists(_index_file_path)
_example_gt_path_info = []
_example_gt_binary_path_info = []
_example_gt_instance_path_info = []
with open(_index_file_path, 'r') as _file:
for _line in _file:
_example_info = _line.rstrip('\r').rstrip('\n').split(' ')
_example_gt_path_info.append(_example_info[0])
_example_gt_binary_path_info.append(_example_info[1])
_example_gt_instance_path_info.append(_example_info[2])
ret = {
'gt_path_info': _example_gt_path_info,
'gt_binary_path_info': _example_gt_binary_path_info,
'gt_instance_path_info': _example_gt_instance_path_info
}
return ret
def _split_writing_tfrecords_task(
_example_gt_paths, _example_gt_binary_paths, _example_gt_instance_paths, _flags='train'):
_split_example_gt_paths = []
_split_example_gt_binary_paths = []
_split_example_gt_instance_paths = []
_split_tfrecords_save_paths = []
for i in range(0, len(_example_gt_paths), step_size):
_split_example_gt_paths.append(_example_gt_paths[i:i + step_size])
_split_example_gt_binary_paths.append(_example_gt_binary_paths[i:i + step_size])
_split_example_gt_instance_paths.append(_example_gt_instance_paths[i:i + step_size])
if i + step_size > len(_example_gt_paths):
_split_tfrecords_save_paths.append(
ops.join(save_dir, '{:s}_{:d}_{:d}.tfrecords'.format(_flags, i, len(_example_gt_paths))))
else:
_split_tfrecords_save_paths.append(
ops.join(save_dir, '{:s}_{:d}_{:d}.tfrecords'.format(_flags, i, i + step_size)))
ret = {
'gt_paths': _split_example_gt_paths,
'gt_binary_paths': _split_example_gt_binary_paths,
'gt_instance_paths': _split_example_gt_instance_paths,
'tfrecords_paths': _split_tfrecords_save_paths
}
return ret
# make save dirs
os.makedirs(save_dir, exist_ok=True)
# start generating training example tfrecords
log.info('Start generating training example tfrecords')
# collecting train images paths info
train_image_paths_info = _read_training_example_index_file(self._train_example_index_file_path)
train_gt_images_paths = train_image_paths_info['gt_path_info']
train_gt_binary_images_paths = train_image_paths_info['gt_binary_path_info']
train_gt_instance_images_paths = train_image_paths_info['gt_instance_path_info']
# split training images according step size
train_split_result = _split_writing_tfrecords_task(
train_gt_images_paths, train_gt_binary_images_paths, train_gt_instance_images_paths, _flags='train')
train_example_gt_paths = train_split_result['gt_paths']
train_example_gt_binary_paths = train_split_result['gt_binary_paths']
train_example_gt_instance_paths = train_split_result['gt_instance_paths']
train_example_tfrecords_paths = train_split_result['tfrecords_paths']
for index, example_gt_paths in enumerate(train_example_gt_paths):
tf_io_pipline_tools.write_example_tfrecords(
example_gt_paths,
train_example_gt_binary_paths[index],
train_example_gt_instance_paths[index],
train_example_tfrecords_paths[index]
)
log.info('Generating training example tfrecords complete')
# start generating validation example tfrecords
log.info('Start generating validation example tfrecords')
# collecting validation images paths info
val_image_paths_info = _read_training_example_index_file(self._val_example_index_file_path)
val_gt_images_paths = val_image_paths_info['gt_path_info']
val_gt_binary_images_paths = val_image_paths_info['gt_binary_path_info']
val_gt_instance_images_paths = val_image_paths_info['gt_instance_path_info']
# split validation images according step size
val_split_result = _split_writing_tfrecords_task(
val_gt_images_paths, val_gt_binary_images_paths, val_gt_instance_images_paths, _flags='val')
val_example_gt_paths = val_split_result['gt_paths']
val_example_gt_binary_paths = val_split_result['gt_binary_paths']
val_example_gt_instance_paths = val_split_result['gt_instance_paths']
val_example_tfrecords_paths = val_split_result['tfrecords_paths']
for index, example_gt_paths in enumerate(val_example_gt_paths):
tf_io_pipline_tools.write_example_tfrecords(
example_gt_paths,
val_example_gt_binary_paths[index],
val_example_gt_instance_paths[index],
val_example_tfrecords_paths[index]
)
log.info('Generating validation example tfrecords complete')
# generate test example tfrecords
log.info('Start generating testing example tfrecords')
# collecting test images paths info
test_image_paths_info = _read_training_example_index_file(self._test_example_index_file_path)
test_gt_images_paths = test_image_paths_info['gt_path_info']
test_gt_binary_images_paths = test_image_paths_info['gt_binary_path_info']
test_gt_instance_images_paths = test_image_paths_info['gt_instance_path_info']
# split validating images according step size
test_split_result = _split_writing_tfrecords_task(
test_gt_images_paths, test_gt_binary_images_paths, test_gt_instance_images_paths, _flags='test')
test_example_gt_paths = test_split_result['gt_paths']
test_example_gt_binary_paths = test_split_result['gt_binary_paths']
test_example_gt_instance_paths = test_split_result['gt_instance_paths']
test_example_tfrecords_paths = test_split_result['tfrecords_paths']
for index, example_gt_paths in enumerate(test_example_gt_paths):
tf_io_pipline_tools.write_example_tfrecords(
example_gt_paths,
test_example_gt_binary_paths[index],
test_example_gt_instance_paths[index],
test_example_tfrecords_paths[index]
)
log.info('Generating testing example tfrecords complete')
return
def generate_tfrecords(self, save_dir, step_size=10000):
"""
Generate tensorflow records file
:param save_dir:
:param step_size: generate a tfrecord every step_size examples
:return:
"""
def _read_training_example_index_file(_index_file_path):
assert ops.exists(_index_file_path)
_example_rain_path_info = []
_example_clean_path_info = []
with open(_index_file_path, 'r') as _file:
for _line in _file:
_example_info = _line.rstrip('\r').rstrip('\n').split(' ')
_example_rain_path_info.append(_example_info[0])
_example_clean_path_info.append(_example_info[1])
return _example_rain_path_info, _example_clean_path_info
def _split_writing_tfrecords_task(_example_paths,
_example_labels,
_flags='train'):
_split_example_paths = []
_split_example_labels = []
_split_tfrecords_save_paths = []
for i in range(0, len(_example_paths), step_size):
_split_example_paths.append(_example_paths[i:i + step_size])
_split_example_labels.append(_example_labels[i:i + step_size])
if i + step_size > len(_example_paths):
_split_tfrecords_save_paths.append(
ops.join(
save_dir, '{:s}_{:d}_{:d}.tfrecords'.format(
_flags, i, len(_example_paths))))
else:
_split_tfrecords_save_paths.append(
ops.join(
save_dir, '{:s}_{:d}_{:d}.tfrecords'.format(
_flags, i, i + step_size)))
return _split_example_paths, _split_example_labels, _split_tfrecords_save_paths
# make save dirs
os.makedirs(save_dir, exist_ok=True)
# generate training example tfrecords
log.info('Generating training example tfrecords')
train_rain_images_paths, train_clean_images_paths = _read_training_example_index_file(
self._train_example_index_file_path)
train_rain_images_paths_split, train_clean_images_paths_split, train_tfrecords_save_paths = \
_split_writing_tfrecords_task(train_rain_images_paths, train_clean_images_paths, _flags='train')
for index, example_paths in enumerate(train_rain_images_paths_split):
tf_io_pipline_tools.write_example_tfrecords(
example_paths, train_clean_images_paths_split[index],
train_tfrecords_save_paths[index])
log.info('Generate training example tfrecords complete')
# generate val example tfrecords
log.info('Generating validation example tfrecords')
val_rain_images_paths, val_clean_images_paths = _read_training_example_index_file(
self._val_example_index_file_path)
val_rain_images_paths_split, val_clean_images_paths_split, val_tfrecords_save_paths = \
_split_writing_tfrecords_task(val_rain_images_paths, val_clean_images_paths, _flags='val')
for index, example_paths in enumerate(val_rain_images_paths_split):
tf_io_pipline_tools.write_example_tfrecords(
example_paths, val_clean_images_paths_split[index],
val_tfrecords_save_paths[index])
log.info('Generate validation example tfrecords complete')
# generate test example tfrecords
log.info('Generating testing example tfrecords')
test_rain_images_paths, test_clean_images_paths = _read_training_example_index_file(
self._test_example_index_file_path)
test_rain_images_paths_split, test_clean_images_paths_split, test_tfrecords_save_paths = \
_split_writing_tfrecords_task(test_rain_images_paths, test_clean_images_paths, _flags='test')
for index, example_paths in enumerate(test_rain_images_paths_split):
tf_io_pipline_tools.write_example_tfrecords(
example_paths, test_clean_images_paths_split[index],
test_tfrecords_save_paths[index])
log.info('Generate testing example tfrecords complete')
return
def generate_tfrecords(self, save_dir, step_size=10000):
"""
Generate tensorflow records file
:param save_dir:
:param step_size: generate a tfrecord every step_size examples
:return:
"""
def _read_training_example_index_file(_index_file_path):
"""
从train.txt val.txt test.txt中读取数据,将其中的数据保存在dict中
:param _index_file_path: train.txt或val.txt或test.txt
:return: ret:一个字典，包含‘gt_path_info’, 'gt_binary_paht_info'的字典
"""
assert ops.exists(_index_file_path)
_example_gt_path_info = []
_example_gt_binary_path_info = []
with open(_index_file_path, 'r') as _file:
for _line in _file:
_example_info = _line.rstrip('\r').rstrip('\n').split(' ')
_example_gt_path_info.append(_example_info[0])
_example_gt_binary_path_info.append(_example_info[1])
ret = {
'gt_path_info': _example_gt_path_info,
'gt_binary_path_info': _example_gt_binary_path_info,
}
return ret
def _split_writing_tfrecords_task(_example_gt_paths,
_example_gt_binary_paths,
_flags='train'):
"""
为了防止tf.record文件过大，内存溢出所做的操作；将各部分数据信息分成若干个相同大小的部分
:param _example_gt_paths: list，images下图片的地址
:param _example_gt_binary_paths: list, gt_image_binary下标签图片的地址
:param _flags: 'train'、'val'、'test'
:return:
"""
_split_example_gt_paths = []
_split_example_gt_binary_paths = []
_split_tfrecords_save_paths = []
for i in range(0, len(_example_gt_paths), step_size):
_split_example_gt_paths.append(_example_gt_paths[i:i +
step_size])
_split_example_gt_binary_paths.append(
_example_gt_binary_paths[i:i + step_size])
if i + step_size > len(_example_gt_paths):
_split_tfrecords_save_paths.append(
ops.join(
save_dir, '{:s}_{:d}_{:d}.tfrecords'.format(
_flags, i, len(_example_gt_paths))))
else:
_split_tfrecords_save_paths.append(
ops.join(
save_dir, '{:s}_{:d}_{:d}.tfrecords'.format(
_flags, i, i + step_size)))
ret = {
'gt_paths': _split_example_gt_paths,
'gt_binary_paths': _split_example_gt_binary_paths,
'tfrecords_paths': _split_tfrecords_save_paths
}
return ret
os.makedirs(save_dir, exist_ok=True)
log.info('Start generating training example tfrecords')
train_image_paths_info = _read_training_example_index_file(
self._train_example_index_file_path)
train_gt_images_paths = train_image_paths_info['gt_path_info']
train_gt_binary_images_paths = train_image_paths_info[
'gt_binary_path_info']
train_split_result = _split_writing_tfrecords_task(
train_gt_images_paths,
train_gt_binary_images_paths,
_flags='train')
train_example_gt_paths = train_split_result['gt_paths']
train_example_gt_binary_paths = train_split_result['gt_binary_paths']
train_example_tfrecords_paths = train_split_result['tfrecords_paths']
for index, example_gt_paths in enumerate(train_example_gt_paths):
tf_io_pipline_tools.write_example_tfrecords(
example_gt_paths, train_example_gt_binary_paths[index],
train_example_tfrecords_paths[index])
log.info('Generating training example tfrecords complete')
log.info('Start generating validation example tfrecords')
val_image_paths_info = _read_training_example_index_file(
self._val_example_index_file_path)
val_gt_images_paths = val_image_paths_info['gt_path_info']
val_gt_binary_images_paths = val_image_paths_info[
'gt_binary_path_info']
val_split_result = _split_writing_tfrecords_task(
val_gt_images_paths, val_gt_binary_images_paths, _flags='val')
val_example_gt_paths = val_split_result['gt_paths']
val_example_gt_binary_paths = val_split_result['gt_binary_paths']
val_example_tfrecords_paths = val_split_result['tfrecords_paths']
for index, example_gt_paths in enumerate(val_example_gt_paths):
tf_io_pipline_tools.write_example_tfrecords(
example_gt_paths, val_example_gt_binary_paths[index],
val_example_tfrecords_paths[index])
log.info('Generating validation example tfrecords complete')
log.info('Start generating testing example tfrecords')
test_image_paths_info = _read_training_example_index_file(
self._test_example_index_file_path)
test_gt_images_paths = test_image_paths_info['gt_path_info']
test_gt_binary_images_paths = test_image_paths_info[
'gt_binary_path_info']
test_split_result = _split_writing_tfrecords_task(
test_gt_images_paths, test_gt_binary_images_paths, _flags='test')
test_example_gt_paths = test_split_result['gt_paths']
test_example_gt_binary_paths = test_split_result['gt_binary_paths']
test_example_tfrecords_paths = test_split_result['tfrecords_paths']
for index, example_gt_paths in enumerate(test_example_gt_paths):
tf_io_pipline_tools.write_example_tfrecords(
example_gt_paths, test_example_gt_binary_paths[index],
test_example_tfrecords_paths[index])
log.info('Generating testing example tfrecords complete')
return