def main():
dataset_count = 10
def create_dataset(i):
return Dataset.range(4 * i, 4 * (i + 1))
dataset = Dataset.range(dataset_count).map(create_dataset)
for d in dataset:
show_dataset(d)
d = dataset.flat_map(lambda x: x)
show_dataset(d)
d = dataset.interleave(lambda x: x, cycle_length=2, block_length=3)
show_dataset(d)
# Repeat two datasets of different lengths and interleave them.
a = Dataset.from_tensor_slices(np.arange(10)).repeat()
b = Dataset.from_tensor_slices(100 + np.arange(17)).repeat()
datasets = [a, b]
n = len(datasets)
c = Dataset.from_tensor_slices(datasets)
d = c.interleave(lambda x: x, cycle_length=n).take(50)
show_dataset(d)
def prepare_train_generator(self):
image_names = glob.glob(self.dir_name +
"/training_data/images/images/*.jpg")
image_names.extend(
glob.glob(self.dir_name + "/training_data/images/images/*.png"))
image_names.extend(
glob.glob(self.dir_name + "/training_data/images/images/*.bmp"))
image_names.extend(
glob.glob(self.dir_name + "/training_data/images/images/*.tif"))
sample_img = cv2.imread(image_names[0])
target_shape = (sample_img.shape[0], sample_img.shape[1])
crop_generator = CropGenerator(self.dir_name, target_shape)
#image_dataset = tf.data.Dataset.list_files(self.dir_name + '/training_data/images/images/*')
total_dataset = Dataset.range(1, 8).interleave(
lambda x: Dataset.from_generator(
CropGenerator(self.dir_name, target_shape),
output_types=(tf.float32, tf.float32)),
cycle_length=8)
total_dataset = total_dataset.shuffle(buffer_size=20)
#total_dataset = total_dataset.cache("./data_cache.")
total_dataset = total_dataset.repeat()
total_dataset = total_dataset.prefetch(buffer_size=20)
data_tf = total_dataset.make_one_shot_iterator().get_next()
return data_tf, crop_generator()
def create_dataset(i):
return Dataset.range(4 * i, 4 * (i + 1))
def compose_datasets(datasets):
n = len(datasets)
return Dataset.range(n).map(lambda i: datasets[i])
import tensorflow as tf
from tensorflow.data import Dataset, Iterator
dataset_train = Dataset.range(10)
dataset_val = Dataset.range(90, 100)
iter_train_handle = dataset_train.make_one_shot_iterator().string_handle()
iter_val_handle = dataset_val.make_one_shot_iterator().string_handle()
handle = tf.placeholder(tf.string, shape=[])
iterator = Iterator.from_string_handle(handle, dataset_train.output_types,
dataset_train.output_shapes)
next_batch = iterator.get_next()
with tf.train.MonitoredTrainingSession() as sess:
handle_train, handle_val = sess.run([iter_train_handle, iter_val_handle])
for step in range(10):
print('train', sess.run(next_batch, feed_dict={handle: handle_train}))
if step % 3 == 0:
print('val', sess.run(next_batch, feed_dict={handle: handle_val}))
from tensorflow.data import Dataset
from numpy import array
num_words = 10000
words_per_example = 20
examples_per_batch = 100
words_per_batch = examples_per_batch * words_per_example
num_batches = num_words // words_per_batch
num_words = num_batches * words_per_batch
num_examples = num_words // words_per_example
a = Dataset.range(num_words).batch(words_per_example)
batch_stride = num_batches
a = a.batch(batch_stride)
a = a.interleave(lambda a: Dataset.from_tensor_slices(a),
cycle_length=examples_per_batch)
a = a.batch(examples_per_batch)
print(a.output_shapes)
n = a.make_one_shot_iterator().get_next()
print("-"*80)
with tf.Session() as sess:
try:
def extract_examples(audio, length, stride):
indices = Dataset.range(
0, tf.cast(tf.shape(audio)[0] - length + 1, dtype='int64'), stride)
return indices.map(lambda i: audio[i:i + length, ...])
