def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
split_name, shard_id)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id+1) * num_per_shard, len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i+1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(sess, image_data)
class_name = os.path.basename(os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _create_tfrecord_train(dataset_dir, tfrecord_writer, classes_train):
count = 0
all_image_paths, class_labels = [], []
for cls_train in classes_train:
images_dir = glob.glob(join(dataset_dir, 'train', cls_train, '*.JPG')) +\
glob.glob(join(dataset_dir, 'train', cls_train, '*.jpg'))
all_image_paths.extend(images_dir)
class_labels.extend(
[classes_train[cls_train] for _ in range(len(images_dir))])
combined = list(zip(all_image_paths, class_labels))
shuffle(combined)
all_image_paths, class_labels = zip(*combined)
for i, im_path in enumerate(all_image_paths):
with tf.gfile.Open(im_path, 'rb') as f:
img = cv2.imread(im_path)
img_shape = img.shape
if img_shape != (256, 256, 3):
print('Resizing!!!!!!!!!!!!!!')
img = cv2.resize(img, (256, 256))
_, encoded_image = cv2.imencode('.jpg', img)
label = class_labels[i]
encoded_image = encoded_image.tobytes()
example = dataset_utils.image_to_tfexample(encoded_image, b'jpg',
256, 256, label)
output_shard_index = count % _NUM_TRAIN_FILES
tfrecord_writer[output_shard_index].write(
example.SerializeToString())
count += 1
print('Processed {} images'.format(count))
def _add_to_tfrecord(data_filename, labels_filename, num_images,
tfrecord_writer):
"""Loads data from the binary MNIST files and writes files to a TFRecord.
Args:
data_filename: The filename of the MNIST images.
labels_filename: The filename of the MNIST labels.
num_images: The number of images in the dataset.
tfrecord_writer: The TFRecord writer to use for writing.
"""
images = _extract_images(data_filename, num_images)
labels = _extract_labels(labels_filename, num_images)
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Converting image %d/%d' % (j + 1, num_images))
sys.stdout.flush()
png_string = sess.run(encoded_png, feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE, labels[j])
tfrecord_writer.write(example.SerializeToString())
def tf_writer(phase):
print 'phase: {}'.format(phase)
pairs_npy_path = os.path.join(cfg.pairs_dir, 'leaders_{}_pairs.npy'.format(phase))
pairs_npy = np.load(pairs_npy_path)
nofpairs = len(pairs_npy)
print 'will write {} pairs'.format(nofpairs)
tfrecords_name = os.path.join(cfg.tfrecord_dir, phase, '{}_0.tfrecords'.format(phase))
print 'creating tfrecords writer: {}'.format(tfrecords_name)
writer = tf.python_io.TFRecordWriter(tfrecords_name)
for ind in range(nofpairs):
pair = pairs_npy[ind]
if ind > 0 and ind % 1000 == 0:
writer.close()
tfrecords_name = os.path.join(cfg.tfrecord_dir, phase,'{}_{}.tfrecords'.format(phase, ind))
print 'creating tfrecords writer: {}'.format(tfrecords_name)
writer = tf.python_io.TFRecordWriter(tfrecords_name)
fn = pair[0]
label = pair[1]
try:
im = Image.open(fn)
except IOError:
print 'cannot load file: {}'.format(fn)
continue
im = np.array(im)
im = im.astype(np.uint8)
# read file name
image_data = tf.gfile.FastGFile(fn, 'rb').read()
height , width = im.shape[0], im.shape[1]
example = dataset_utils.image_to_tfexample(image_data, b'jpg', height, width, int(label))
writer.write(example.SerializeToString())
def tf_writer(phase):
print 'phase: {}'.format(phase)
pairs_npy_path = os.path.join(cfg.pairs_dir,
'leaders_{}_pairs.npy'.format(phase))
pairs_npy = np.load(pairs_npy_path)
nofpairs = len(pairs_npy)
print 'will write {} pairs'.format(nofpairs)
tfrecords_name = os.path.join(cfg.tfrecord_dir, phase,
'{}_0.tfrecords'.format(phase))
print 'creating tfrecords writer: {}'.format(tfrecords_name)
writer = tf.python_io.TFRecordWriter(tfrecords_name)
for ind in range(nofpairs):
pair = pairs_npy[ind]
if ind > 0 and ind % 1000 == 0:
writer.close()
tfrecords_name = os.path.join(cfg.tfrecord_dir, phase,
'{}_{}.tfrecords'.format(phase, ind))
print 'creating tfrecords writer: {}'.format(tfrecords_name)
writer = tf.python_io.TFRecordWriter(tfrecords_name)
fn = pair[0]
label = pair[1]
try:
im = Image.open(fn)
except IOError:
print 'cannot load file: {}'.format(fn)
continue
im = np.array(im)
im = im.astype(np.uint8)
# read file name
image_data = tf.gfile.FastGFile(fn, 'rb').read()
height, width = im.shape[0], im.shape[1]
example = dataset_utils.image_to_tfexample(image_data, b'jpg', height,
width, int(label))
writer.write(example.SerializeToString())
def _convert_dataset(split_name, filenames, dataset_dir, class_id):
record_file_num = math.ceil(len(filenames) / _NUM_PER_RECORD)
f = open('./log.txt', 'a')
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for record_id in range(record_file_num):
output_filename = _get_dataset_filename(
dataset_dir, split_name, class_id, record_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
for i in range(_NUM_PER_RECORD):
if record_id * _NUM_PER_RECORD + i >= len(filenames):
break
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i],
'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
print('write ' + filenames[i] + ' to ' +
output_filename)
f.write('write ' + filenames[i] + ' to ' +
output_filename + '\n')
f.close()
def create_tf(split_name, filenames, class_names_to_ids, dataset_dir,
num_shards, shard_id, num_per_shard, image_reader, sess):
output_filename = _get_dataset_filename(
OUTPUT_PATH, split_name, shard_id, num_shards)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard, len(filenames))
for i in range(start_ndx, end_ndx):
# sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
# i + 1, len(filenames), shard_id))
# sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(sess, image_data)
class_name = os.path.basename(os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
g_process.add_finish(output_filename)
sys.stdout.write('\r>> Converting image %d/%d of files: %d' % (
len(g_process.finished_file), g_process.all_step, len(filenames)))
sys.stdout.flush()
def _add_to_tfrecord(data_filename, labels_filename, num_images,
tfrecord_writer):
"""Loads data from the binary MNIST files and writes files to a TFRecord.
Args:
data_filename: The filename of the MNIST images.
labels_filename: The filename of the MNIST labels.
num_images: The number of images in the dataset.
tfrecord_writer: The TFRecord writer to use for writing.
"""
images = _extract_images(data_filename, num_images)
labels = _extract_labels(labels_filename, num_images)
shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS)
with tf.Graph().as_default():
image = tf.placeholder(dtype=tf.uint8, shape=shape)
encoded_png = tf.image.encode_png(image)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Converting image %d/%d' %
(j + 1, num_images))
sys.stdout.flush()
png_string = sess.run(encoded_png,
feed_dict={image: images[j]})
example = dataset_utils.image_to_tfexample(
png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE,
labels[j])
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(dataset_name, split_name, filenames, class_names_to_ids, dataset_dir, num_shards, num_channels=3):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation', 'test']
num_per_shard = int(math.ceil(len(filenames) / float(num_shards)))
with tf.Graph().as_default():
image_reader = ImageReader(num_channels)
with tf.Session('') as sess:
for shard_id in range(num_shards):
output_filename = _get_dataset_filename(dataset_name, dataset_dir, split_name, shard_id, num_shards)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard, len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write('\r>> Converting image %d/%d shard %d, %s' % (
i + 1, len(filenames), shard_id, filenames[i]))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(sess, image_data)
class_name = os.path.basename(os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
if sys.version_info[0] == 3:
example = dataset_utils.image_to_tfexample(
image_data, 'jpg'.encode(), height, width, class_id)
else:
example = dataset_utils.image_to_tfexample(
image_data, 'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _build_shards(filenames, shard_name):
"""Loads data from the raw CIFAR10 files and writes to TFRecord files.
Parameters
----------
filenames : list of str
The name of the CIFAR10 pickle files.
shard_name : str
A string template for the shard name.
"""
images = []
labels = []
for filename in filenames:
with open(filename, 'rb') as f:
if six.PY3:
data = cPickle.load(f, encoding='latin1')
else:
data = cPickle.load(f)
images.append(data['data'].reshape((-1, 3, 32, 32)))
labels.append(data['labels'])
images = np.concatenate(images, 0)
labels = np.concatenate(labels, 0)
# num_images = images.shape[0]
# num_shards = -(-num_images // _SHARD_SIZE) # ceiling function
# sort images by class such that every shard only contains one class
for c in range(len(_CLASS_NAMES)):
c_labels = labels[labels == c]
c_images = images[labels == c]
c_num_images = c_images.shape[0]
c_num_shards = -(-c_num_images // _SHARD_SIZE) # ceiling function
with tf.Graph().as_default(), tf.Session('') as session:
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
for s in range(c_num_shards):
shard = shard_name.format(c, s, c_num_shards)
with tf.python_io.TFRecordWriter(shard) as tfrecord_writer:
i_min, i_max = s * _SHARD_SIZE, (s + 1) * _SHARD_SIZE
for i in range(i_min, min(c_num_images, i_max)):
sys.stdout.write(
'\r>> Building class {}/{}, shard {}/{}, image {}/{}'
.format(c, len(_CLASS_NAMES), s, c_num_shards, i,
c_num_images))
sys.stdout.flush()
image = np.squeeze(c_images[i]).transpose((1, 2, 0))
label = c_labels[i]
png_string = session.run(
encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE,
label)
tfrecord_writer.write(example.SerializeToString())
def _add_to_tfrecord(filename, tfrecord_writer, offset=0):
"""Loads data from the cifar10 pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar10 pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
# file_list = glob.glob(filename + '/*.jpg')
file_list = glob.glob(filename + '/*.JPG')
images = []
labels = []
count = 0
for file_name in file_list:
count += 1
image = misc.imread(file_name)
height, width, _ = image.shape
# if height<200 or width<200:
# os.remove(file_name)
# continue
# image_resize = misc.imresize(image, (299,299,3))
input_img = np.array(image, dtype='uint8')
# label_name = int(os.path.basename(file_name).split('_')[0]) - 1 #start at 0
label_name = int(os.path.basename(file_name).split('.')[0])
images.append(input_img)
labels.append(label_name)
print(file_name)
# if count>100:
# break
num_images = len(images)
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write('\r>> Reading file [%s] image %d/%d' % (
filename, offset + j + 1, offset + num_images))
sys.stdout.flush()
image = np.squeeze(images[j])
label = labels[j]
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, b'png', height, width, label)
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
img = Image.open(
filenames[i]) ###########################
if img.mode != 'RGB': #test by penglong
print(filenames[i] + ', ' +
img.mode) ############################
img.close()
os.remove(filenames[i])
continue
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i],
'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_name = os.path.basename(
os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(split_name, filenames, filename_to_class_ids,
dataset_dir):
""" Convert the given list of files to a TFRecord dataset.
Args:
split_name: The name of the dataset split, either 'train' or 'test'.
filenames: A list of paths to image files
filename_to_class_ids: A dictionary from file name to its class id.
dataset_dir: The path to the dataset directory.
"""
assert split_name in ['train', 'test']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
# shuffle file names
np.random.shuffle(filenames)
with tf.Graph().as_default():
image_reader = ImageReader()
# just run on CPU
config = tf.ConfigProto(device_count={'GPU': 0})
with tf.Session(config=config) as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_idx, end_idx):
# write in a progress style
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# read file
image_data = tf.gfile.FastGFile(filenames[i],
'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
basename = os.path.basename(filenames[i])
class_id = filename_to_class_ids[basename] - 1
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def get_data_images(ROOT, set_id):
save_path = os.path.join(DATA_DIR, set_id)
if os.path.isdir(save_path):
pass
#sys.exit('{} is already exist'.format(save_path))
else:
os.makedirs(save_path)
log_f = open(os.path.join(DATA_DIR, set_id+'_info'), 'a')
pids = glob.glob(os.path.join(ROOT, '*'))
pids = sorted(pids, key=lambda x: int(x.split('/')[-1]))
#pids = pids[131:]
for i, pname in enumerate(pids):
tfrecord_filename = os.path.join(save_path, PRE+str(i)+'.tfrecord')
tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_filename)
# read images
print(pname)
filenames = glob.glob(os.path.join(pname, '*'))
#filenames = sorted(filenames, key=lambda x: int(x.split('/')[-1].split('_')[0]))
for j, imgname in enumerate(filenames):
#print(imgname)
if imgname.endswith('.png'):
img = cv2.imread(imgname)
img = img[...,0]
oriimg = img[:,0:256]
label = img[:,256*1:256*2]
label_wall = (label==255*np.ones(label.shape))
label_endo = (label==127*np.ones(label.shape))
pred = img[:,256*2:256*3]
pred_wall = (pred==255*np.ones(pred.shape))
pred_endo = (pred==127*np.ones(pred.shape))
masks_wall = [label_wall, pred_wall]
masks_endo = [label_endo, pred_endo]
masks = [label, pred]
for ind in range(2):
#Input = np.stack([oriimg*masks_wall[ind], oriimg*masks_endo[ind]], -1)
#Input = np.stack([oriimg, masks[ind]], -1)
Input = np.stack([oriimg, masks_wall[ind]*255, masks_endo[ind]*255], -1)
Input = Input.astype(np.int8)
label = ind
#label = random.randint(0,1)
data_point = Input.tostring()
example = dataset_utils.image_to_tfexample(data_point, label, subject_id=i, index=j)
tfrecord_writer.write(example.SerializeToString())
print('Finish writing data from {}'.format(i))
log_f.write('{}: {}\n'.format(i, len(imgname)))
def _add_to_tfrecord(data_filename,
tfrecord_writer,
label_filename=None,
subset=None,
test_fold=False):
"""Loads data from the stl10 files and writes files to a TFRecord.
Args:
data_filename: The filename of the stl10 file.
tfrecord_writer: The TFRecord writer to use for writing.
Returns:
The new offset.
"""
images = read_all_images(data_filename)
labels = None
if label_filename:
labels = read_labels(label_filename)
if subset:
if test_fold:
images = images[subset]
labels = labels[subset]
else:
images = np.delete(images, subset, axis=0)
labels = np.delete(labels, subset, axis=0)
num_images = images.shape[0]
with tf.Graph().as_default():
coder = dataset_utils.ImageCoder()
with tf.Session(''):
for j in range(num_images):
sys.stdout.write('\r>> Reading file [%s] image %d/%d' %
(data_filename, j + 1, num_images))
sys.stdout.flush()
# Get image, edge-map and cartooned image
image = np.squeeze(images[j])
if label_filename:
label = labels[j] - 1 # labels should be 0 indexed!
else:
label = -1
# Encode the images
image_str = coder.encode_jpeg(image)
# Build example
example = dataset_utils.image_to_tfexample(
image_str, 'jpg', HEIGHT, WIDTH, int(label))
tfrecord_writer.write(example.SerializeToString())
return num_images
def _convert_dataset(split_name, filenames, tps, Qs, classnames, classids,
output_dir, NUM_SHARDS):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(NUM_SHARDS):
output_filename = _get_dataset_filename(
output_dir, split_name, shard_id, NUM_SHARDS)
print(output_filename)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
print("reading file" + filenames[i])
image_data = tf.gfile.FastGFile(filenames[i],
'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_id = classids[i]
tp = tps[i]
filename = filenames[i]
Q = Qs[i]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, int(class_id),
int(tp), filename, Q)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(filename, tfrecord_writer, offset=0):
"""Loads data from the cifar10 pickle files and writes files to a TFRecord.
Args:
filename: The filename of the cifar10 pickle file.
tfrecord_writer: The TFRecord writer to use for writing.
offset: An offset into the absolute number of images previously written.
Returns:
The new offset.
"""
with tf.gfile.Open(filename, 'rb') as f:
if sys.version_info < (3, ):
data = cPickle.load(f)
else:
data = cPickle.load(f, encoding='bytes')
images = data[b'data']
num_images = images.shape[0]
images = images.reshape((num_images, 3, 32, 32))
labels = data[b'labels']
with tf.Graph().as_default():
image_placeholder = tf.placeholder(dtype=tf.uint8)
encoded_image = tf.image.encode_png(image_placeholder)
with tf.Session('') as sess:
for j in range(num_images):
sys.stdout.write(
'\r>> Reading file [%s] image %d/%d' %
(filename, offset + j + 1, offset + num_images))
sys.stdout.flush()
image = np.squeeze(images[j]).transpose((1, 2, 0))
label = labels[j]
png_string = sess.run(encoded_image,
feed_dict={image_placeholder: image})
example = dataset_utils.image_to_tfexample(
png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label)
tfrecord_writer.write(example.SerializeToString())
return offset + num_images
def _convert_dataset(split_name, filenames, features, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
features: A lisr of absolute paths to feature binary file
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = load_image(filenames[i])
#read the feature binary files
feature_data = np.fromfile(features[i],
dtype=np.float32)
example = dataset_utils.image_to_tfexample(
image_data, feature_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id+1) * num_per_shard, len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i+1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
height, width = image_reader.read_image_dims(sess, image_data)
class_name = os.path.basename(os.path.dirname(filenames[i]))
class_id = class_names_to_ids[class_name]
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(split_name, filenames, labels, bbox_dict, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation', 'test']
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
output_filename = _get_dataset_filename(dataset_dir, split_name)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
for i in range(len(filenames)):
sys.stdout.write('\r>> Converting image %d/%d' %
(i + 1, len(filenames)))
sys.stdout.flush()
# Read the filename:
image_data = tf.gfile.FastGFile(filenames[i], 'r').read()
height, width = image_reader.read_image_dims(
sess, image_data)
x_min = bbox_dict[filenames[i]][0] / width
y_min = bbox_dict[filenames[i]][1] / height
x_max = bbox_dict[filenames[i]][2] / width
y_max = bbox_dict[filenames[i]][3] / height
example = dataset_utils.image_to_tfexample(
image_data, 'jpeg', height, width, labels[i],
[[x_min], [y_min], [x_max], [y_max]])
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _create_tfrecord_test(dataset_dir, tfrecord_writer, classes_train):
count = 0
image_paths = sorted(glob.glob(join(dataset_dir, 'test_imgs', '*.JPG'))) +\
glob.glob(join(dataset_dir, 'test_imgs', '*.jpg'))
num_images = len(image_paths)
for i, im_path in enumerate(image_paths):
with tf.gfile.Open(im_path, 'rb') as f:
img = cv2.imread(im_path)
img_shape = img.shape
if img_shape != (256, 256, 3):
print('Resizing!!!!!!!!!!!!!!!!')
img = cv2.resize(img, (256, 256))
_, encoded_image = cv2.imencode('.jpg', img)
encoded_image = encoded_image.tobytes()
example = dataset_utils.image_to_tfexample(encoded_image, b'jpg',
256, 256, 0)
output_shard_index = count % _NUM_TRAIN_FILES
tfrecord_writer[output_shard_index].write(
example.SerializeToString())
count += 1
print('Processed {} images'.format(count))
def _convert_dataset(specs_name, filenames, class_ids):
with tf.Graph().as_default():
image_reader = ImageReader()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
output_filename = specs_name + '.tfrecord'
if os.path.exists(output_filename):
os.remove(output_filename)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
for i in range(len(filenames)):
# Read the filename:
print str(i) + ' ' + filenames[i]
img_path = filenames[i]
image_data = tf.gfile.FastGFile(img_path, 'rb').read()
height, width = image_reader.read_image_dims(
sess, image_data)
class_id = int(class_ids[i])
example = dataset_utils.image_to_tfexample(
image_data, b'jpg', height, width, class_id)
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(split_name, filenames, dataset_dir):
assert split_name in ['train', 'test']
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
for i in range(2):
image_data = tf.gfile.FastGFile(
dataset_dir + '/data/' + filenames[i] + '_' +
str(i + 1) + '.png', 'r').read()
height, width = image_reader.read_image_dims(
sess, image_data)
example = dataset_utils.image_to_tfexample(
image_data, 'png', height, width, -1)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def get_data_pickle(raw_data_path, set_id, seq_length, stride_frame, stride_seq, subject_index=[]):
""" Convert data in pickle to tfrecord
Args:
raw_data_path: location of the raw data
set_id: unique name for the created tfrecord dataset
seq_length: the length of frames used for one input
stride_frame: the step size of frames
stride_seq: the step size of sequences
subject_index: the subject id
"""
save_path = os.path.join(DATA_DIR, set_id+'_{}_{}_{}'.format(seq_length, stride_frame, stride_seq))
if os.path.isdir(save_path):
pass
#sys.exit('{} is already exist'.format(save_path))
else:
os.makedirs(save_path)
log_f = open(os.path.join(DATA_DIR, set_id+'_{}_{}_{}'.format(seq_length, stride_frame, stride_seq)+'_info'), 'a')
dataset_stats = {}
for each_class in CLASS:
dataset_stats.setdefault(each_class,0)
for pf in subject_index:
tfrecord_filename = os.path.join(save_path, PRE+pf+'.tfrecord')
tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_filename)
subject_stats = {}
for each_class in CLASS:
subject_stats.setdefault(each_class,0)
total = 0
files = glob.glob(os.path.join(raw_data_path, '*'+pf+'*'))
for each_file in files:
fname = each_file.split('/')[-1]
with open(each_file, 'rb') as rf:
[imgs, annots, eye, head, mouth]= pickle.load(rf)
for index in range (0,len(imgs)-seq_length*stride_frame,stride_seq):
# IF 2D input
if seq_length == 1:
data_point = imgs[index,:,:,:]
label = int(annots[index])
subject_stats[label] += 1
dataset_stats[label] += 1
total += 1
# IF 3D input
else:
data_point = imgs[index:index+seq_length*stride_frame:stride_frame,:,:]
# For drowsiness
annots = annots.astype(np.int32)
label_array = annots[index:index+seq_length*stride_frame:stride_frame]
#if int(label[-1])==1:
if sum(label_array)==seq_length:
label = 1
#elif int(label[-1])==0:
elif sum(label_array)==0:
label = 0
else:
continue
subject_stats[label] += 1
dataset_stats[label] += 1
total += 1
data_point = data_point.tostring()
example = dataset_utils.image_to_tfexample(data_point, label)
tfrecord_writer.write(example.SerializeToString())
rf.close()
print('Finish extracting data from %s'%(each_file))
print('Finish writing data from {}'.format(pf))
log_f.write('{}: {}\t {}\n'.format(pf, total, json.dumps(subject_stats)))
log_f.write('{}'.format(json.dumps(dataset_stats)))
def _convert_dataset(split_name, filenames, dataset_dir):
"""Converts the given filenames to a TFRecord dataset.
Args:
split_name: The name of the dataset, either 'train' or 'validation'.
filenames: A list of absolute paths to png or jpg images.
class_names_to_ids: A dictionary from class names (strings) to ids
(integers).
dataset_dir: The directory where the converted datasets are stored.
"""
assert split_name in ['train', 'validation']
img_files_a = []
img_files_b = []
class_names = []
for filename in filenames:
img_files_a.append(
os.path.join(dataset_dir, parts[0],
filename.strip().split()[0]))
img_files_b.append(
os.path.join(dataset_dir, parts[1],
filename.strip().split()[0]))
class_names.append(filename.strip().split()[1])
num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS)))
with tf.Graph().as_default():
image_reader = ImageReader()
with tf.Session('') as sess:
for shard_id in range(_NUM_SHARDS):
output_filename = _get_dataset_filename(
dataset_dir, split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the filename:
image_a = tf.gfile.FastGFile(img_files_a[i],
'rb').read()
image_b = tf.gfile.FastGFile(img_files_b[i],
'rb').read()
class_id = int(class_names[i])
example = dataset_utils.image_to_tfexample(
image_a, image_b, b'jpg', class_id)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
