def imgaug_file_batch_manager_example():
num_examples = 100
is_label_augmented = False
images, labels = generate_dataset(num_examples, is_label_augmented)
batch_size = 12
num_epochs = 7
shuffle = True
is_time_major = False
augmenter = iaa.Sequential(
[iaa.Fliplr(0.5),
iaa.CoarseDropout(p=0.1, size_percent=0.1)])
batch_dir_path_prefix = './batch_dir'
num_batch_dirs = 5
dirMgr = WorkingDirectoryManager(batch_dir_path_prefix, num_batch_dirs)
#--------------------
for epoch in range(num_epochs):
print('>>>>> Epoch #{}.'.format(epoch))
dir_path = dirMgr.requestAvailableDirectory()
if dir_path is None:
break
print('\t>>>>> Directory: {}.'.format(dir_path))
batchMgr = ImgaugFileBatchManager(augmenter, images, labels,
batch_size, shuffle,
is_label_augmented, is_time_major)
batchMgr.putBatches(
dir_path) # Generates, augments, and saves batches.
batches = batchMgr.getBatches(dir_path) # Loads batches.
for idx, batch in enumerate(batches):
# Train with each batch (images & labels).
#print('\t{}: {}, {}'.format(idx, batch[0].shape, batch[1].shape))
print('{}: {}-{}, {}-{}'.format(
idx, batch[0].shape,
np.max(np.reshape(batch[0], (batch[0].shape[0], -1)), axis=-1),
batch[1].shape,
np.max(np.reshape(batch[1], (batch[1].shape[0], -1)),
axis=-1)))
dirMgr.returnDirectory(dir_path)
def simple_file_batch_manager_example():
num_examples = 100
images, labels = generate_dataset(num_examples)
batch_size = 12
num_epochs = 7
shuffle = True
is_time_major = False
batch_dir_path_prefix = './batch_dir'
num_batch_dirs = 5
dirMgr = WorkingDirectoryManager(batch_dir_path_prefix, num_batch_dirs)
#--------------------
for epoch in range(num_epochs):
print('>>>>> Epoch #{}.'.format(epoch))
while True:
dir_path = dirMgr.requestAvailableDirectory()
if dir_path is not None:
break
else:
time.sleep(0.1)
print('\t>>>>> Directory: {}.'.format(dir_path))
batchMgr = SimpleFileBatchManager(images, labels, batch_size, shuffle,
is_time_major)
batchMgr.putBatches(dir_path) # Generates and saves batches.
batches = batchMgr.getBatches(dir_path) # Loads batches.
for idx, batch in enumerate(batches):
# Can run in an individual thread or process.
# Augment each batch (images & labels).
# Train with each batch (images & labels).
#print('\t{}: {}, {}'.format(idx, batch[0].shape, batch[1].shape))
print('\t{}: {}-{}, {}-{}'.format(
idx, batch[0].shape,
np.max(np.reshape(batch[0], (batch[0].shape[0], -1)), axis=-1),
batch[1].shape,
np.max(np.reshape(batch[1], (batch[1].shape[0], -1)),
axis=-1)))
dirMgr.returnDirectory(dir_path)
def imgaug_file_batch_manager_with_file_input_example():
num_examples = 300
is_label_augmented = False
num_files = generate_file_dataset('./batches', num_examples,
is_label_augmented)
npy_filepath_pairs = list()
for idx in range(num_files):
npy_filepath_pairs.append(('./batches/images_{}.npy'.format(idx),
'./batches/labels_{}.npy'.format(idx)))
npy_filepath_pairs = np.array(npy_filepath_pairs)
num_file_pairs = 3
num_file_pair_steps = ((num_files - 1) // num_file_pairs +
1) if num_files > 0 else 0
batch_size = 12
num_epochs = 7
shuffle = True
is_time_major = False
augmenter = iaa.Sequential(
[iaa.Fliplr(0.5),
iaa.CoarseDropout(p=0.1, size_percent=0.1)])
batch_dir_path_prefix = './batch_dir'
num_batch_dirs = 5
dirMgr = WorkingDirectoryManager(batch_dir_path_prefix, num_batch_dirs)
#--------------------
for epoch in range(num_epochs):
print('>>>>> Epoch #{}.'.format(epoch))
dir_path = dirMgr.requestAvailableDirectory()
if dir_path is None:
break
print('\t>>>>> Directory: {}.'.format(dir_path))
indices = np.arange(num_files)
if shuffle:
np.random.shuffle(indices)
for step in range(num_file_pair_steps):
print('\t\t>>>>> File pairs #{}.'.format(step))
start = step * num_file_pairs
end = start + num_file_pairs
file_pair_indices = indices[start:end]
if file_pair_indices.size > 0: # If file_pair_indices is non-empty.
sub_filepath_pairs = npy_filepath_pairs[file_pair_indices]
if sub_filepath_pairs.size > 0: # If sub_filepath_pairs is non-empty.
# Can run in an individual thread or process.
batchMgr = ImgaugFileBatchManagerWithFileInput(
augmenter, sub_filepath_pairs, batch_size, shuffle,
is_label_augmented, is_time_major)
batchMgr.putBatches(
dir_path) # Generates, augments, and saves batches.
batches = batchMgr.getBatches(dir_path) # Loads batches.
for idx, batch in enumerate(batches):
# Train with each batch (images & labels).
#print('\t\t{}: {}, {}'.format(idx, batch[0].shape, batch[1].shape))
print('{}: {}-{}, {}-{}'.format(
idx, batch[0].shape,
np.max(np.reshape(batch[0],
(batch[0].shape[0], -1)),
axis=-1), batch[1].shape,
np.max(np.reshape(batch[1],
(batch[1].shape[0], -1)),
axis=-1)))
dirMgr.returnDirectory(dir_path)