def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir,
output_suffix):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id, output_suffix)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d, %s' %
(i + 1, len(filenames), shard_id, filenames[i]))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i],
'r').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
if sys.version_info[0] == 3:
example = dataset_utils.image_to_tfexample(
image_data, 'jpg'.encode(), height, width,
class_id)
else:
example = dataset_utils.image_to_tfexample(
image_data, 'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset. 图像转成tfr数据
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images. 存放所有图片的路径 list
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(
len(filenames) /
float(_NUM_SHARDS))) # 将总的文件分成_NUM_SHARDS份，每一份是num_per_shard个图片
with tf.Graph().as_default(): # 新建图表
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames)) # 每num_per_shard个图片生成一个tfr文件
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id)) # 输出进度条
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(
filenames[i],
'rb').read() # 数据类型也是bytes ，如果不是 使用tobytes转换
height, width = image_reader.read_image_dims(
sess, image_data)
# 这里可以使用PIL，scipy，skimage，opencv等对图像操作，image_data需要用tobytes转成bytes格式
# 如：
# from PIL import Image
# image_data=Image(filenames[i]).convert('RGB')
# height, width=image_data.size
# image_data=image_data.tobytes()
class_name = os.path.basename(
os.path.dirname(filenames[i])) # 类名（文件夹名）
class_id = class_names_to_ids[
class_name] # 换成对应的类的id 如 0,1等
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(filename,
tfrecord_writer,
labels_to_class_names,
offset=0):
"""Loads pic data from the filename and writes files to a TFRecord.
Args:
filename: The filename of one picture .
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
image = tf.gfile.FastGFile(filename, 'r').read()
label = labels_to_class_names[filename.split('/')[-2]]
with tf.Graph().as_default():
with tf.Session('') as sess:
example = dataset_utils.image_to_tfexample(image, b'jpg',
_IMAGE_SIZE_HEIGHT,
_IMAGE_SIZE_WIDTH,
label)
tfrecord_writer.write(example.SerializeToString())
return offset + 1
def _add_to_tfrecord(data_filename, labels_filename, num_images,
tfrecord_writer):
"""Loads data from the binary MNIST files and writes files to a TFRecord.
Args:
data_filename: The filename of the MNIST images.
labels_filename: The filename of the MNIST labels.
num_images: The number of images in the dataset.
tfrecord_writer: The TFRecord writer to use for writing.
"""
images = _extract_images(data_filename, num_images)
labels = _extract_labels(labels_filename, num_images)
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Converting image %d/%d' % (j + 1, num_images))
sys.stdout.flush()
png_string = sess.run(encoded_png, feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE, labels[j])
tfrecord_writer.write(example.SerializeToString())
def get_feature(data_set='train', data_type='bytes'):
if not os.path.exists(output_directory) or os.path.isfile(
output_directory):
os.makedirs(output_directory)
_examples, _labels, examples_num = load_file(train_file)
filename = output_directory + "/" + name + '.tfrecords'
writer = tf.python_io.TFRecordWriter(filename)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for i, [example, label] in enumerate(zip(_examples, _labels)):
print('No.%d' % (i))
img = read_image(data_set + '/' + example + '.' + data_type)
#print('shape: %d, %d, %d, label: %d' % (image.shape[0], image.shape[1], 1, label))
#image_raw = image.tostring()
png_string = sess.run(encoded_png, feed_dict={image: img})
example = dataset_utils.image_to_tfexample(
png_string, 'png', resize_height, resize_width, label - 1)
writer.write(example.SerializeToString())
writer.close()
def _to_tfrecord(image_ids_file, tfrecord_writer, source_dir):
with open(image_ids_file) as f:
img_ids = f.readlines()
num_images = len(img_ids)
with tf.Graph().as_default():
coder = dataset_utils.ImageCoder()
with tf.Session('') as sess:
for j in range(num_images):
xml_path = os.path.join(
source_dir, 'VOC2007/Annotations',
'{}.xml'.format(img_ids[j].strip('\n')))
img_path, label = _parse_xml(xml_path, source_dir)
sys.stdout.write('\r>> Reading file [%s] image %d/%d' %
(img_path, j + 1, num_images))
sys.stdout.flush()
# Get image, edge-map and cartooned image
img = misc.imread(img_path)
# Resize the image
h = np.size(img, 0)
w = np.size(img, 1)
# Encode the images
image_str = coder.encode_jpeg(img)
# Buil example
example = dataset_utils.image_to_tfexample(
image_str, 'jpg', h, w, label.tolist())
tfrecord_writer.write(example.SerializeToString())
def _add_to_tfrecord(img_filenames, type, tfrecord_writer):
"""Loads data from the binary MNIST files and writes files to a TFRecord.
Args:
img_filenames: The filename of the UC Merced images.
type: The category of those images.
tfrecord_writer: The TFRecord writer to use for writing.
"""
images, n = _extract_images(img_filenames)
labels = np.zeros((n, ), np.int64)
labels.fill(type)
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(n):
sys.stdout.write('\r>> Converting image %d/%d\n' % (j + 1, n))
sys.stdout.flush()
png_string = sess.run(encoded_png,
feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE,
labels[j])
tfrecord_writer.write(example.SerializeToString())
def _add_to_tfrecord(data_filename, labels_filename, num_images,
tfrecord_writer):
"""Loads data from the binary MNIST files and writes files to a TFRecord.
Args:
data_filename: The filename of the MNIST images.
labels_filename: The filename of the MNIST labels.
num_images: The number of images in the dataset.
tfrecord_writer: The TFRecord writer to use for writing.
"""
images = _extract_images(data_filename, num_images)
labels = _extract_labels(labels_filename, num_images)
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Converting image %d/%d' %
(j + 1, num_images))
sys.stdout.flush()
png_string = sess.run(encoded_png,
feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE,
labels[j])
tfrecord_writer.write(example.SerializeToString())
def convert_dataset(list_dir, split_name, img_dir, output_dir, num_tfrecord=3):
list_name = "dataset_" + split_name + ".txt"
list_path = os.path.join(list_dir, list_name)
fd = open(list_path)
lines = [line.split() for line in fd]
fd.close()
num_per_shard = int(math.ceil(len(lines) / float(num_tfrecord)))
with tf.Graph().as_default():
decode_jpeg_data = tf.placeholder(dtype=tf.string)
decode_jpeg = tf.image.decode_jpeg(decode_jpeg_data, channels=3)
with tf.Session('') as sess:
for shard_id in range(num_tfrecord):
output_path = os.path.join(
output_dir, 'cifar10_{:03}-of-{:03}-{}.tfrecord'.format(
shard_id + 1, num_tfrecord, split_name))
tfrecord_writer = tf.python_io.TFRecordWriter(output_path)
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard, len(lines))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image {}/{} to the shard {}'.format(
i + 1, len(lines), shard_id + 1))
sys.stdout.flush()
image_data = tf.gfile.FastGFile(
os.path.join(img_dir, lines[i][0]), 'rb').read()
image = sess.run(decode_jpeg,
feed_dict={decode_jpeg_data: image_data})
height, width = image.shape[0], image.shape[1]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width,
int(lines[i][1])) #btype object(b'jpg') btype.feature
tfrecord_writer.write(example.SerializeToString())
tfrecord_writer.close()
sys.stdout.write('\n') #输出不会换行
sys.stdout.flush()
def convert_dataset(list_path, data_dir, output_dir, _NUM_SHARDS=5):
fd = open(list_path)
lines = [line.split() for line in fd]
fd.close()
num_per_shard = int(math.ceil(len(lines) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
decode_jpeg_data = tf.placeholder(dtype=tf.string)
decode_jpeg = tf.image.decode_jpeg(decode_jpeg_data, channels=3)
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_path = os.path.join(
output_dir, 'data_{:05}-of-{:05}.tfrecord'.format(
shard_id, _NUM_SHARDS))
tfrecord_writer = tf.python_io.TFRecordWriter(output_path)
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard, len(lines))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image {}/{} shard {}'.format(
i + 1, len(lines), shard_id))
sys.stdout.flush()
image_data = tf.gfile.FastGFile(
os.path.join(data_dir, lines[i][0]), 'rb').read()
image = sess.run(decode_jpeg,
feed_dict={decode_jpeg_data: image_data})
height, width = image.shape[0], image.shape[1]
example = dataset_utils.image_to_tfexample(
image_data, b'png', height, width, int(lines[i][1]))
tfrecord_writer.write(example.SerializeToString())
tfrecord_writer.close()
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(data_filename, tfrecord_writer, augment=False):
"""Loads data from the Pickle files and writes files to a TFRecord.
Args:
data_filename: The filename of the images and labels.
tfrecord_writer: The TFRecord writer to use for writing.
"""
images, labels = _extract_images_labels(data_filename)
num_images = images.shape[0]
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
# Just copy!
for j in range(num_images):
sys.stdout.write('\r>> Converting image %d/%d' %
(j + 1, num_images))
sys.stdout.flush()
png_string = sess.run(encoded_png,
feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE,
int(labels[j]))
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(split_name, image_filenames, dataset_dir):
assert split_name in ['train', 'validation', 'test']
num_per_shard = int(math.ceil(len(image_filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = dataset_utils.ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(image_filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(image_filenames), shard_id))
sys.stdout.flush()
# read the image
img_filename = image_filenames[i]
img_data = tf.gfile.FastGFile(img_filename, 'r').read()
img_shape = image_reader.read_image_dims(
sess, img_data)
example = dataset_utils.image_to_tfexample(
img_data, img_filename[-3:], img_shape,
img_filename)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.compat.v1.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
print("start_ndx: %d" % start_ndx)
print("end_ndx: %d" % end_ndx)
for i in range(start_ndx, end_ndx):
#sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
# i+1, len(filenames), shard_id))
#sys.stdout.flush()
if i % 100 == 0:
print("Convert dataset %d, shard %d" %
(i, shard_id))
# Read the filename:
image_data = tf.io.gfile.GFile(filenames[i],
'rb').read()
if filenames[i].endswith(
".jpg") | filenames[i].endswith(".jpeg"):
height, width = image_reader.read_jpeg_dims(
sess, image_data)
elif filenames[i].endswith(".png"):
height, width = image_reader.read_png_dims(
sess, image_data)
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(data_filepath, label_filepath, num_images,
tfrecord_writer):
images = _extract_images(data_filepath, num_images)
labels = _extract_labels(label_filepath, num_images)
label_cn = {}
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Converting image %d/%d' %
(j + 1, num_images))
sys.stdout.flush()
png_string = sess.run(encoded_png,
feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE,
labels[j])
tfrecord_writer.write(example.SerializeToString())
label_cn[labels[j]] = label_cn.get(labels[j], 0) + 1
print('\nds=%s' % (path.basename(data_filepath)))
for i in range(len(_CLASS_NAMES)):
print('\t#%s=%d' % (_CLASS_NAMES[i], label_cn[i]))
def run(datadir, dstdir):
kaggle_test = os.path.join(dstdir, "kaggle_test.tfrecord")
if not tf.gfile.Exists(dstdir):
tf.gfile.MakeDirs(dstdir)
if tf.gfile.Exists(kaggle_test):
print('Dataset files already exist. Exiting without re-creating them.')
return
image_filename = tf.placeholder(dtype=tf.string)
image_file = tf.read_file(image_filename)
# image_placeholder = tf.placeholder(dtype=tf.uint8)
# encoded_image = tf.image.encode_png(image_placeholder)
tfrecord_writer = tf.python_io.TFRecordWriter(kaggle_test)
with tf.Session() as sess:
tf.global_variables_initializer().run()
print("begin")
for j in range(1, 300001):
filename = os.path.join(datadir, ("%d.png" % j))
sys.stdout.write('\r>> Reading %s' % filename)
sys.stdout.flush()
# statinfo = os.stat(filename)
# print(statinfo)
png_string = sess.run(image_file,
feed_dict={image_filename: filename})
example = dataset_utils.image_to_tfexample(png_string,
'png'.encode(), 32, 32,
0)
tfrecord_writer.write(example.SerializeToString())
tfrecord_writer.close()
print("over create %s\n", kaggle_test)
def _convert_to_tfrecord(dataset_dir, split_name, data_filename,
label_filename):
"""Loads data from the image list files and writes images to a TFRecord.
Args:
dataset_dir: The root directory.
split_name: The name of the train/test split.
data_filename: The filename of the MNIST images.
label_filename: The filename of the MNIST labels.
"""
imageList = _extract_imagelist(data_filename)
labels = _extract_labels(label_filename)
num_images = len(imageList)
assert (len(labels) == num_images)
num_shards = int(math.floor(float(num_images) / _NUM_PER_SHARD))
for i in range(num_shards + 1):
st_idx = i * _NUM_PER_SHARD
ed_idx = min(st_idx + _NUM_PER_SHARD, num_images)
output_filename = _get_tfrecord_filename(dataset_dir, split_name, i,
num_shards)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for j in range(st_idx, ed_idx):
sys.stdout.write('\r>> Converting image %d/%d' %
(j + 1, num_images))
sys.stdout.flush()
image_path = os.path.join(_IMAGE_DIRECTORY, imageList[j])
image_string = tf.gfile.FastGFile(image_path, 'rb').read()
image = Image.open(image_path)
example = dataset_utils.image_to_tfexample(
image_string, image.format.encode(), image.size[0],
image.size[1], labels[j], image_path, True)
tfrecord_writer.write(example.SerializeToString())
def convert_dataset(filenames, class_names_to_ids, dataset_dir):
num_per_shard = int(math.ceil(len(filenames) / float(NUM_CLASS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for class_id in range(NUM_CLASS):
output_filename = get_dataset_filename(dataset_dir, class_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = class_id * num_per_shard
end_ndx = min((class_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d class %d' %
(i + 1, len(filenames), class_id))
sys.stdout.flush()
image_data = tf.gfile.FastGFile(filenames[i],
'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(tfrecord_writer):
"""Loads data from the binary MNIST files and writes files to a TFRecord.
Args:
data_filename: The filename of the MNIST images.
labels_filename: The filename of the MNIST labels.
num_images: The number of images in the dataset.
tfrecord_writer: The TFRecord writer to use for writing.
"""
data_ori = pd.read_csv(image_file)
labels_ori = pd.read_csv(train_file)
data = pd.merge(data_ori, labels_ori, on='Id')
labels = data.Class
data.drop(["Class", "Id"], axis=1, inplace=True)
train_data = data.values[:, :]
images = train_data.reshape(_IMAGE_NUM, _IMAGE_SIZE, _IMAGE_SIZE,
_IMAGE_CHANEL)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(_IMAGE_NUM):
sys.stdout.write('\r>> Converting image %d/%d' %
(j + 1, _IMAGE_NUM))
sys.stdout.flush()
png_string = sess.run(encoded_png,
feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png', _IMAGE_SIZE, _IMAGE_SIZE, labels[j])
tfrecord_writer.write(example.SerializeToString())
def _rotate_flip_core(sess, patch_image, rot90_k, tfrecord_writer, height,
width, class_id):
k = 0
rotated_image = tf.image.rot90(patch_image, k=rot90_k)
rotated_image_data = tf.image.encode_jpeg(rotated_image)
example = dataset_utils.image_to_tfexample(sess.run(rotated_image_data),
'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
k += 1
flipped_image = tf.image.flip_left_right(patch_image)
flipped_image_data = tf.image.encode_jpeg(flipped_image)
example = dataset_utils.image_to_tfexample(sess.run(flipped_image_data),
'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
k += 1
return k
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(FLAGS.num_shards)))
sys.stdout.write('\nWriting {} records in each of {} shards\n'.format(
num_per_shard, FLAGS.num_shards))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(FLAGS.num_shards):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
sys.stdout.write(
'\nUsing {} as outname\n'.format(output_filename))
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
try:
sys.stdout.write(
'\r>> Converting image %d/%d on shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(
filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
#print('Size is {} by {}'.format(height, width))
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
except:
print('\nUnable to parse image {}. Skipping...\n'.
format(i))
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(filename, tfrecord_writer, offset=0):
"""Loads data from the cifar10 pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar10 pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
absolute(adj. 纯粹的; 绝对的; 完全的; 专制的) previously(adv. 事先; 仓促地; 以前; 不成熟地)
Returns:
The new offset.
"""
with tf.gfile.Open(filename, 'rb') as f:
if sys.version_info < (3, ):
data = cPickle.load(f)
else:
data = cPickle.load(f, encoding='bytes')
# 总的图片数
images = data[b'data']
num_images = images.shape[0]
# 将图片转为三通道的32x32的彩图
images = images.reshape((num_images, 3, 32, 32))
labels = data[b'labels']
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
# 读取每一张图片做处理
for j in range(num_images):
sys.stdout.write(
'\r>> Reading file [%s] image %d/%d' %
(filename, offset + j + 1, offset + num_images))
sys.stdout.flush()
# 把原有的(1,3,32,32)转成(32,32,3)
image = np.squeeze(images[j]).transpose((1, 2, 0))
label = labels[j]
# 把(32,32,3)转成str格式
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label)
# def image_to_tfexample(image_data, image_format, height, width, class_id):
# return tf.train.Example(features=tf.train.Features(feature={
# 'image/encoded': bytes_feature(image_data),
# 'image/format': bytes_feature(image_format),
# 'image/class/label': int64_feature(class_id),
# 'image/height': int64_feature(height),
# 'image/width': int64_feature(width),
# }))
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir, dataset_name, image_height, image_width):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation', 'test']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
# print('##############', filenames[0])
num_samples_per_class = dict()
with tf.Graph().as_default():
image_reader = ImageReaderPNG()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id, dataset_name)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id+1) * num_per_shard, len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i+1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.GFile(filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(sess, image_data)
# print('################# before', height, width)
if image_height and image_width:
image_data = image_reader.resize_image(sess, image_data, image_height, image_width)
height, width = image_reader.read_image_dims(sess, image_data)
# print('################# after', height, width)
class_name = os.path.basename(os.path.dirname(filenames[i]))
if class_name not in num_samples_per_class:
num_samples_per_class[class_name] = 0
num_samples_per_class[class_name] += 1
class_id = class_names_to_ids[class_name]
image_name = bytes(filenames[i], 'utf-8')
# print('####',image_name,height, width, class_id )
example = dataset_utils.image_to_tfexample(
image_data, image_name, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
return num_samples_per_class
def _add_to_tfrecord(filename, tfrecord_writer, dataset, offset=0):
"""Loads data from the cifar pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
with open(filename, 'rb') as f:
if six.PY3:
data = cPickle.load(f, encoding='bytes')
else:
data = cPickle.load(f)
images = data[six.b('data')]
num_images = images.shape[0]
images = images.reshape((num_images, 3, 32, 32))
labels = data[six.b('labels' if dataset == 'cifar10' else 'fine_labels')]
if dataset == 'cifar100':
coarse_labels = data[six.b('coarse_labels')]
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write(
'\r>> Reading file [%s] image %d/%d' %
(filename, offset + j + 1, offset + num_images))
sys.stdout.flush()
image = np.squeeze(images[j]).transpose((1, 2, 0))
label = labels[j]
coarse_label = coarse_labels[
j] if dataset == 'cifar100' else None
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string,
six.b('png'),
_IMAGE_SIZE,
_IMAGE_SIZE,
label,
coarse_class_id=coarse_label)
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
def _add_to_tfrecord(filename,
class_names_to_labels,
tfrecord_writer,
offset=0):
"""Loads data from the image_path and writes files to a TFRecord.
Args:
filename: The filename of the cifar10 pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
# with tf.gfile.Open(filename, 'rb') as f:
# if sys.version_info < (3,):
# data = cPickle.load(f)
# else:
# data = cPickle.load(f, encoding='bytes')
# images = data[b'data']
# num_images = images.shape[0]
# images = images.reshape((num_images, 3, 32, 32))
# labels = data[b'labels']
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_jpeg(image_placeholder)
with tf.Session('') as sess:
num_images = len(filename)
for j, images in enumerate(filename):
# sys.stdout.write('\r>> Reading file [%s] image %d/%d' % (
# filename, offset + j + 1, offset + num_images))
# sys.stdout.flush()
label = class_names_to_labels[images.split('/')[-2]]
images = cv2.imread(images)
try:
image = cv2.resize(images,
dsize=(_IMAGE_SIZE, _IMAGE_SIZE),
interpolation=cv2.INTER_CUBIC)
# image = np.squeeze(images).transpose((1, 2, 0))
jpg_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
jpg_string, b'jpg', _IMAGE_SIZE, _IMAGE_SIZE, label)
tfrecord_writer.write(example.SerializeToString())
except:
pass
return offset + num_images
def _convert_dataset(split_name, dataset, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'testing'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'test']
num_per_shard = int(math.ceil(len(dataset) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 1.0
with tf.Session(config=config) as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard, len(dataset))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting %s image %d/%d shard %d' %
(split_name, i + 1, len(dataset), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(
dataset[i]['filename'], 'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
label = dataset[i]['label']
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, label)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def make_TFRecord(list_file_path, tfrecord_file_path=None):
from datasets import dataset_utils
from PIL import Image
import tensorflow as tf
import time
# 默认情况下保存为列表名_tfrecord.tfrecords
if tfrecord_file_path is None:
tfrecord_file_path = file_tools.getFileName(list_file_path).split('.')[0] + '_tfrecord.tfrecord'
# 图片列表文件
list_file = open(list_file_path)
# 获取每行，并且分隔
lines = [line.split() for line in list_file]
list_file.close()
tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_file_path)
last_time = time.time()
with tf.Graph().as_default():
# 读取格式信息
decode_jpeg_data = tf.placeholder(dtype=tf.string)
decode_jpeg = tf.image.decode_jpeg(decode_jpeg_data, channels=3)
with tf.Session('') as sess:
num = 0
max_num = len(lines)
for line in lines:
# 处理路径名，去除空格的影响
if len(line) != 2:
for index, i in enumerate(line):
if index != 0 and index < len(line) - 1:
line[0] = line[0] + ' ' + i
line[1] = line[len(line) - 1]
# 判断是否为数字
if line[1].isdigit():
image_data = tf.gfile.FastGFile(line[0], 'rb').read()
image = sess.run(decode_jpeg, feed_dict={decode_jpeg_data: image_data})
height, width = image.shape[0], image.shape[1]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, int(line[1]))
tfrecord_writer.write(example.SerializeToString())
num = num + 1
if num % 1000 == 0:
spend_time = float(1000) / (time.time() - last_time)
print('Finish %d/%d(%.2f/sec)........%.2f' % (num, max_num, spend_time, max_num / spend_time))
last_time = time.time()
# 防止每个tfrecord太大，进行分割
tfrecord_writer.close()
tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_file_path.replace('.', '_%d.' % int(num / 1000)))
tfrecord_writer.close()
def _convert_dataset_train(dataset_dir, output_filename, split_name, cat2idx, _image_count):
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
print("Processing {}".format(split_name))
data = bson.decode_file_iter(open(split_name, 'rb'))
for c, d in enumerate(data):
product_id = d['_id']
category_id = d['category_id']
class_id = cat2idx[category_id]
for e, pic in enumerate(d['imgs']):
example = dataset_utils.image_to_tfexample(pic['picture'], b'png', _IMAGE_HEIGHT, _IMAGE_WIDTH,
class_id, product_id)
tfrecord_writer.write(example.SerializeToString())
_image_count[0] += 1
def _add_to_tfrecord_my(filename,
tfrecord_writer,
rand_idx,
split_num,
offset=0):
"""Loads data from the cifar10 pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar10 pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
with tf.gfile.Open(filename, 'rb') as f:
if sys.version_info < (3, ):
data = cPickle.load(f)
else:
data = cPickle.load(f, encoding='bytes')
images = data[b'data']
num_images = images.shape[0]
images = images.reshape((num_images, 3, 32, 32))
labels = data[b'labels']
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
for j in range(split_num):
sys.stdout.write('\r>> Reading file [%s] image %d/%d' %
(filename, offset + j + 1, num_images))
sys.stdout.flush()
image = np.squeeze(images[rand_idx[j + offset]]).transpose(
(1, 2, 0))
label = labels[rand_idx[j + offset]]
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label)
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
def _add_to_tfrecord(filename, tfrecord_writer, dataset, offset=0):
"""Loads data from the cifar pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
with open(filename, 'rb') as f:
if six.PY3:
data = cPickle.load(f, encoding='bytes')
else:
data = cPickle.load(f)
images = data[six.b('data')]
num_images = images.shape[0]
images = images.reshape((num_images, 3, 32, 32))
labels = data[six.b('labels' if dataset == 'cifar10' else 'fine_labels')]
if dataset == 'cifar100':
coarse_labels = data[six.b('coarse_labels')]
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Reading file [%s] image %d/%d' % (
filename, offset + j + 1, offset + num_images))
sys.stdout.flush()
image = np.squeeze(images[j]).transpose((1, 2, 0))
label = labels[j]
coarse_label = coarse_labels[j] if dataset == 'cifar100' else None
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, six.b('png'),
_IMAGE_SIZE, _IMAGE_SIZE,
label, coarse_class_id = coarse_label)
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
#assert 断言 assert expression 相当于 if not expression raise AssertionError
assert split_name in ['train', 'test']
#计算每个数据块有多少个数据
num_per_shard = int(len(filenames) / _NUM_SHARDS)
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session() as sess:
for shard_id in range(_NUM_SHARDS):
#定义tfrecord文件的路径+名字
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
#每一个数据块开始的位置
start_ndx = shard_id * num_per_shard
#每一个数据块最后的位置
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
try:
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.GFile(filenames[i],
'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
except IOError as e:
print("Could not read: ", filenames[i])
print("Error: ", e)
print("Skip it \n")
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
try:
# Read the filename:
image_data = tf.gfile.FastGFile(
filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
except:
continue
sys.stdout.write('\n')
sys.stdout.flush()
def _image_random_X(sess, patch_image, tfrecord_writer, height, width,
class_id):
k = 0
## random_brightness
patch_image_data = tf.image.encode_jpeg(
tf.image.random_brightness(patch_image, max_delta=64 / 255))
example = dataset_utils.image_to_tfexample(sess.run(patch_image_data),
'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
k += 1
## random_saturation
patch_image_data = tf.image.encode_jpeg(
tf.image.random_saturation(patch_image, lower=0, upper=0.25))
example = dataset_utils.image_to_tfexample(sess.run(patch_image_data),
'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
k += 1
## random_hue
atch_image_data = tf.image.encode_jpeg(
tf.image.random_hue(patch_image, max_delta=0.04))
example = dataset_utils.image_to_tfexample(sess.run(patch_image_data),
'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
k += 1
## random_constrast
patch_image_data = tf.image.encode_jpeg(
tf.image.random_contrast(patch_image, lower=0, upper=0.75))
example = dataset_utils.image_to_tfexample(sess.run(patch_image_data),
'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
k += 1
return k
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id+1) * num_per_shard, len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i+1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(sess, image_data)
class_name = os.path.basename(os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(filename, tfrecord_writer, offset=0):
"""Loads data from the cifar10 pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar10 pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
with tf.gfile.Open(filename, 'rb') as f:
data = cPickle.load(f, encoding="bytes")
# print(data.keys())
images = data[b'data']
num_images = images.shape[0]
images = images.reshape((num_images, 3, 32, 32))
labels = data[b'labels']
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Reading file [%s] image %d/%d' % (
filename, offset + j + 1, offset + num_images))
sys.stdout.flush()
image = np.squeeze(images[j]).transpose((1, 2, 0))
label = labels[j]
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label)
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
