def convert_lips(dataset_dir,
tfrecord_dir,
sep='user',
num_shards=5,
num_val_samples=None,
num_frames=12):
"""Runs the conversion operation.
Args:
dataset_dir: Where the data (i.e. .mat videos) is stored.
tfrecord_dir: Where to store the generated data (i.e. TFRecords).
sep: The way to separate train and validation data.
'user'- uses the given separation (`train` and `validation`
directories).
'mixed'- put all the data samples toghether and
conducts a random split.
num_shards: The number of shards per dataset split.
num_val_samples: Used only when sep=='mixed', the number of
samples in validation set.
num_frames: The number of frames of the stored videos.
"""
if not tf.gfile.Exists(tfrecord_dir):
tf.gfile.MakeDirs(tfrecord_dir)
train_dir = os.path.join(dataset_dir, 'train')
training_filenames = [os.path.join(train_dir, filename)
for filename in os.listdir(train_dir)]
validation_dir = os.path.join(dataset_dir, 'validation')
validation_filenames = [os.path.join(validation_dir, filename)
for filename in os.listdir(validation_dir)]
alphabets = [chr(i) for i in range(ord('A'), ord('Z')+1)]
class_names_to_ids = dict(zip(alphabets, range(26)))
assert sep in ['user', 'mixed']
if sep == 'user':
random.shuffle(training_filenames)
random.shuffle(validation_filenames)
elif sep == 'mixed':
if num_val_samples is None:
num_val_samples = len(validation_filenames)
all_filenames = training_filenames + validation_filenames
random.shuffle(all_filenames)
training_filenames = all_filenames[:-num_val_samples]
validation_filenames = all_filenames[-num_val_samples:]
convert_dataset('train', training_filenames,
class_names_to_ids, tfrecord_dir,
num_shards=num_shards, num_frames=num_frames)
convert_dataset('validation', validation_filenames,
class_names_to_ids, tfrecord_dir,
num_shards=num_shards, num_frames=num_frames)
labels_to_class_names = dict(zip(range(26), alphabets))
dataset_utils.write_label_file(labels_to_class_names, tfrecord_dir)
print('\nFinished converting dataset!')
def run(dataset_dir):
"""Runs the download and conversion operation.
Args:
dataset_dir: The dataset directory where the dataset is stored.
"""
if not tf.gfile.Exists(dataset_dir):
tf.gfile.MakeDirs(dataset_dir)
if _dataset_exists(dataset_dir):
print('Dataset files already exist. Exiting without re-creating them.')
return
dataset_utils.download_and_uncompress_tarball(_DATA_URL, dataset_dir)
photo_filenames, class_names = _get_filenames_and_classes(dataset_dir)
class_names_to_ids = dict(zip(class_names, range(len(class_names))))
# Divide into train and test:
random.seed(_RANDOM_SEED)
random.shuffle(photo_filenames)
training_filenames = photo_filenames[_NUM_VALIDATION:]
validation_filenames = photo_filenames[:_NUM_VALIDATION]
# First, convert the training and validation sets.
_convert_dataset('train', training_filenames, class_names_to_ids,
dataset_dir)
_convert_dataset('validation', validation_filenames, class_names_to_ids,
dataset_dir)
# Finally, write the labels file:
labels_to_class_names = dict(zip(range(len(class_names)), class_names))
dataset_utils.write_label_file(labels_to_class_names, dataset_dir)
_clean_up_temporary_files(dataset_dir)
print('\nFinished converting the Flowers dataset!')
def convert_montalbano(dataset_dir_train,
dataset_dir_validation,
tfrecord_dir,
num_shards=10,
width=100,
height=100,
num_frames=40,
grayscale=True):
if not tf.gfile.Exists(tfrecord_dir):
tf.gfile.MakeDirs(tfrecord_dir)
training_data = []
for i in range(1, 404):
basename = os.path.join(dataset_dir_train,
'Sample' + '{:5d}'.format(i).replace(' ', '0'))
sys.stdout.write('\r >> Processing training session %d/403' % i)
sys.stdout.flush()
processed_data = process_session(basename, num_frames, width, height,
grayscale)
if processed_data is not None:
training_data.extend(processed_data)
sys.stdout.write('\n>> Shuffling training session data\n')
sys.stdout.flush()
random.shuffle(training_data)
convert_dataset('train', training_data, tfrecord_dir, num_shards)
validation_data = []
for i in range(410, 711):
basename = os.path.join(dataset_dir_validation,
'Sample' + '{:5d}'.format(i).replace(' ', '0'))
sys.stdout.write('\r >> Processing validation session %d/301' % i -
410)
sys.stdout.flush()
processed_data = process_session(basename, num_frames, width, height)
if processed_data is not None:
validation_data.extend(processed_data)
sys.stdout.write('\n>> Shuffling validation_data session data\n')
sys.stdout.flush()
random.shuffle(validation_data)
convert_dataset('validation', validation_data, tfrecord_dir, num_shards)
dataset_utils.write_label_file(labels_to_class_names, tfrecord_dir)
print('\nFinished converting dataset!')
def run(dataset_dir, dataset_name):
"""Runs the download and conversion operation.
Args:
dataset_dir: The dataset directory where the dataset is stored.
dataset_name: The dataset name (cifar10 or cifar100)
"""
dataset = datasets[dataset_name]
if not tf.gfile.Exists(dataset_dir):
tf.gfile.MakeDirs(dataset_dir)
training_filename = _get_output_filename(dataset_dir, 'train',
dataset_name)
testing_filename = _get_output_filename(dataset_dir, 'test', dataset_name)
if tf.gfile.Exists(training_filename) and tf.gfile.Exists(
testing_filename):
print('Dataset files already exist. Exiting without re-creating them.')
return
dataset_utils.download_and_uncompress_tarball(dataset['url'], dataset_dir)
# First, process the training data:
with tf.python_io.TFRecordWriter(training_filename) as tfrecord_writer:
offset = 0
for i in range(dataset['num_train_files']):
filename = os.path.join(dataset_dir, dataset['folder'],
dataset['train_filename_fn'](i))
offset = _add_to_tfrecord(filename, tfrecord_writer, offset)
# Next, process the testing data:
with tf.python_io.TFRecordWriter(testing_filename) as tfrecord_writer:
filename = os.path.join(dataset_dir, dataset['folder'],
dataset['test_filename'])
_add_to_tfrecord(filename, tfrecord_writer)
# Finally, write the labels file:
labels_to_class_names = dict(
zip(range(len(dataset['labels'])), dataset['labels']))
dataset_utils.write_label_file(labels_to_class_names, dataset_dir)
_clean_up_temporary_files(dataset_dir, dataset)
print('\nFinished converting the %s dataset!' % dataset_name)
def convert_to_numpy(dataset_dir, dataset_name):
"""Runs the download and conversion operation.
Args:
dataset_dir: The dataset directory where the dataset is stored.
dataset_name: The dataset name (cifar10 or cifar100)
"""
dataset = datasets[dataset_name]
if not tf.gfile.Exists(dataset_dir):
tf.gfile.MakeDirs(dataset_dir)
dataset_utils.download_and_uncompress_tarball(dataset['url'], dataset_dir)
images = []
labels = []
for i in range(dataset['num_train_files']):
filename = os.path.join(dataset_dir, dataset['folder'],
dataset['train_filename_fn'](i))
im, lbl = _extract(filename)
images.append(im)
labels.append(lbl)
images = np.concatenate(images, 0)
labels = np.concatenate(labels, 0)
np.save(os.path.join(dataset_dir, "X_train.npy"), images)
np.save(os.path.join(dataset_dir, "Y_train.npy"), labels)
filename = os.path.join(dataset_dir, dataset['folder'],
dataset['test_filename'])
images, labels = _extract(filename)
np.save(os.path.join(dataset_dir, "X_test.npy"), images)
np.save(os.path.join(dataset_dir, "Y_test.npy"), labels)
# Finally, write the labels file:
labels_to_class_names = dict(
zip(range(len(dataset['labels'])), dataset['labels']))
dataset_utils.write_label_file(labels_to_class_names, dataset_dir)
_clean_up_temporary_files(dataset_dir, dataset)
print('\nFinished converting the %s dataset!' % dataset_name)
def convert_avicar(dataset_dir,
tfrecord_dir,
num_shards=5,
num_val_samples=2000,
num_frames=20):
if not tf.gfile.Exists(tfrecord_dir):
tf.gfile.MakeDirs(tfrecord_dir)
filenames = [
os.path.join(dataset_dir, filename)
for filename in os.listdir(dataset_dir)
]
random.shuffle(filenames)
alphabets = [chr(i) for i in range(ord('A'), ord('Z') + 1)]
class_names_to_ids = dict(zip(alphabets, range(26)))
training_filenames = filenames[:-num_val_samples]
validation_filenames = filenames[-num_val_samples:]
convert_dataset('train',
training_filenames,
class_names_to_ids,
tfrecord_dir,
num_shards=num_shards,
num_frames=num_frames)
convert_dataset('validation',
validation_filenames,
class_names_to_ids,
tfrecord_dir,
num_shards=num_shards,
num_frames=num_frames)
labels_to_class_names = dict(zip(range(26), alphabets))
dataset_utils.write_label_file(labels_to_class_names, tfrecord_dir)
print('\nFinished converting dataset!')
def run(dataset_dir):
"""Runs the download and conversion operation.
Args:
dataset_dir: The dataset directory where the dataset is stored.
"""
if not tf.gfile.Exists(dataset_dir):
tf.gfile.MakeDirs(dataset_dir)
training_filename = _get_output_filename(dataset_dir, 'train')
testing_filename = _get_output_filename(dataset_dir, 'test')
if tf.gfile.Exists(training_filename) and tf.gfile.Exists(
testing_filename):
print('Dataset files already exist. Exiting without re-creating them.')
return
_download_dataset(dataset_dir)
# First, process the training data:
with tf.python_io.TFRecordWriter(training_filename) as tfrecord_writer:
data_filename = os.path.join(dataset_dir, _TRAIN_DATA_FILENAME)
labels_filename = os.path.join(dataset_dir, _TRAIN_LABELS_FILENAME)
_add_to_tfrecord(data_filename, labels_filename, 60000,
tfrecord_writer)
# Next, process the testing data:
with tf.python_io.TFRecordWriter(testing_filename) as tfrecord_writer:
data_filename = os.path.join(dataset_dir, _TEST_DATA_FILENAME)
labels_filename = os.path.join(dataset_dir, _TEST_LABELS_FILENAME)
_add_to_tfrecord(data_filename, labels_filename, 10000,
tfrecord_writer)
# Finally, write the labels file:
labels_to_class_names = dict(zip(range(len(_CLASS_NAMES)), _CLASS_NAMES))
dataset_utils.write_label_file(labels_to_class_names, dataset_dir)
_clean_up_temporary_files(dataset_dir)
print('\nFinished converting the MNIST dataset!')
def __init__(self, config_dict, is_train=True):
#self.polices = str2polices(polices_str)
self.img_root_dir = config_dict['DATASET']['DATASET_ROOT_DIR']
self.image_size= config_dict['DATASET']['IMAGE_SIZE']
self.batch_size= config_dict['TRAIN']['BATCH_SIZE']
self.train_data_proportion = config_dict['DATASET']['TRAIN_DATA_PROPORTION']
self.dataset_random_seed = config_dict['DATASET']['DATASET_RANDOM_SEED']
self.max_sample_per_class = config_dict['DATASET']['MAX_SAMPLE_PER_CLASS']
self.augment_method = config_dict['DATASET']['AUGMENT_METHOD']
self.is_train= is_train
output_paras = config_dict['OUTPUT']
experiment_base_dir = os.path.join(output_paras['OUTPUT_SAVE_DIR'], output_paras['EXPERIMENT_NAME'])
model_save_dir = os.path.join(experiment_base_dir, 'weights')
# get label file path
label_file = os.path.join(model_save_dir, 'label.txt')
# show the basic infomation of dataset
label_idx = 0
label_names = []
for label_name in os.listdir(self.img_root_dir):
print("label index: {}, label name: {}".format(label_idx, label_name))
label_names.append(label_name)
label_idx += 1
self.classes= len(label_names)
labels_to_class_names = dict(zip(range(len(label_names)), label_names))
dataset_utils.write_label_file(labels_to_class_names, model_save_dir)
# get dataset mean std info
mean_std_file = os.path.join(model_save_dir, 'dataset_mean_var.txt')
self.dataset_rgb_mean, self.dataset_rgb_std = dataset_utils.load_dataset_mean_std_file(mean_std_file)
# set back color
self.back_color = tuple([int(x) for x in self.dataset_rgb_mean])
self.on_epoch_end()
def convert_color_depth(dataset_dir,
tfrecord_dir,
subjects=True,
sep='user',
num_val_clss=2,
num_shards=5):
"""Runs the conversion operation.
Args:
dataset_dir: Where the data (i.e. images) is stored.
tfrecord_dir: Where to store the generated data (i.e. TFRecords).
keywords: Filenames must contain these keywords.
subjects: Determine directory structure, please refer to
`get_filenames_and_classes`.
sep: The way to separate train and validation data.
'user'- uses the given separation (`train` and `validation`
directories).
'mixed'- put all the data samples toghether and
conducts a random split.
'class'- puts different classes in training and validation set
(used for some particular purpose).
num_val_clss: Used only when sep=='class', the number of classes
in validation set.
num_shards: The number of shards per dataset split.
"""
if not tf.gfile.Exists(tfrecord_dir):
tf.gfile.MakeDirs(tfrecord_dir)
# get filenames and classnames
training_filename_pairs, validation_filename_pairs, class_names = \
train_validate_filename_pairs_classes(dataset_dir, subjects=subjects)
class_names_to_ids = dict(zip(class_names, range(len(class_names))))
assert sep in ['user', 'mixed', 'class']
if sep == 'user':
random.shuffle(training_filename_pairs)
random.shuffle(validation_filename_pairs)
elif sep == 'mixed':
num_train_ex = len(training_filename_pairs)
all_pairs = training_filename_pairs + validation_filename_pairs
random.shuffle(all_pairs)
training_filename_pairs = all_pairs[:num_train_ex]
validation_filename_pairs = all_pairs[num_train_ex:]
elif sep == 'class':
all_pairs = training_filename_pairs + validation_filename_pairs
random.shuffle(all_pairs)
training_filename_pairs = []
validation_filename_pairs = []
for fpair in all_pairs:
cls = os.path.basename(os.path.dirname(fpair[0]))
if cls in class_names[:-num_val_clss]:
training_filename_pairs.append(fpair)
else:
assert cls in class_names[-num_val_clss:]
validation_filename_pairs.append(fpair)
# convert datasets
convert_dataset('train', training_filename_pairs,
class_names_to_ids, tfrecord_dir, num_shards=num_shards)
convert_dataset('validation', validation_filename_pairs,
class_names_to_ids, tfrecord_dir, num_shards=num_shards)
# write the label file
labels_to_class_names = dict(zip(range(len(class_names)), class_names))
dataset_utils.write_label_file(labels_to_class_names, tfrecord_dir)
print('\nFinished converting dataset!')
def convert_mfcc_lips(dataset_dir_audio,
dataset_dir_video,
tfrecord_dir,
num_shards=5,
num_val_samples=100,
num_frames_audio=24,
num_frames_video=12):
"""Runs the conversion operation.
Args:
dataset_dir_audio: Where the audio data (mfcc ascii) is stored.
dataset_dir_video: Where the video data (.mat) is stored.
tfrecord_dir: Where to store the generated data (i.e. TFRecords).
num_shards: The number of shards per dataset split.
num_val_samples: The number of samples in 'validation' part.
num_frames_audio: The number of frames of the stored audios.
num_frames_video: The number of frames of the stored videos.
"""
if not tf.gfile.Exists(tfrecord_dir):
tf.gfile.MakeDirs(tfrecord_dir)
filename_pairs = []
for split_name in ['train', 'validation']:
dirname = os.path.join(dataset_dir_audio, split_name)
audio_filenames = sorted([
os.path.join(dirname, filename) for filename in os.listdir(dirname)
])
dirname = os.path.join(dataset_dir_video, split_name)
video_filenames = sorted([
os.path.join(dirname, filename) for filename in os.listdir(dirname)
])
filename_pairs.extend(list(zip(audio_filenames, video_filenames)))
alphabets = [chr(i) for i in range(ord('A'), ord('Z') + 1)]
class_names_to_ids = dict(zip(alphabets, range(26)))
random.shuffle(filename_pairs)
training_pairs = filename_pairs[:-num_val_samples]
validation_pairs = filename_pairs[-num_val_samples:]
train_AT_pairs = []
train_UZ_pairs = []
for p in training_pairs:
if 'U' <= os.path.basename(p[0])[0] <= 'Z':
train_UZ_pairs.append(p)
else:
train_AT_pairs.append(p)
validation_AT_pairs = []
validation_UZ_pairs = []
for p in validation_pairs:
if 'U' <= os.path.basename(p[0])[0] <= 'Z':
validation_UZ_pairs.append(p)
else:
validation_AT_pairs.append(p)
split_names = [
'train_all', 'trainAT', 'trainUZ', 'validation', 'validationAT',
'validationUZ'
]
pairs = [
training_pairs, train_AT_pairs, train_UZ_pairs, validation_pairs,
validation_AT_pairs, validation_UZ_pairs
]
for i in range(6):
convert_dataset(split_names[i],
pairs[i],
class_names_to_ids,
tfrecord_dir,
num_shards=num_shards,
num_frames_audio=num_frames_audio,
num_frames_video=num_frames_video)
labels_to_class_names = dict(zip(range(26), alphabets))
dataset_utils.write_label_file(labels_to_class_names, tfrecord_dir)
print('\nFinished converting dataset!')
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""Gets a dataset tuple with instructions for reading ImageNet.
Args:
split_name: A train/test split name.
dataset_dir: The base directory of the dataset sources.
file_pattern: The file pattern to use when matching the dataset sources.
It is assumed that the pattern contains a '%s' string so that the split
name can be inserted.
reader: The TensorFlow reader type.
Returns:
A `Dataset` namedtuple.
Raises:
ValueError: if `split_name` is not a valid train/test split.
"""
if split_name not in _SPLITS_TO_SIZES:
raise ValueError('split name %s was not recognized.' % split_name)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/class/label':
tf.FixedLenFeature([], dtype=tf.int64, default_value=-1),
'image/class/text':
tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/object/bbox/xmin':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax':
tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax':
tf.VarLenFeature(dtype=tf.float32),
'image/object/class/label':
tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image':
slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'label':
slim.tfexample_decoder.Tensor('image/class/label'),
'label_text':
slim.tfexample_decoder.Tensor('image/class/text'),
'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
else:
labels_to_names = create_readable_names_for_imagenet_labels()
dataset_utils.write_label_file(labels_to_names, dataset_dir)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=_SPLITS_TO_SIZES[split_name],
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=_NUM_CLASSES,
labels_to_names=labels_to_names)