def imgaug_file_batch_manager_example(): num_examples = 100 is_label_augmented = False images, labels = generate_dataset(num_examples, is_label_augmented) batch_size = 12 num_epochs = 7 shuffle = True is_time_major = False augmenter = iaa.Sequential( [iaa.Fliplr(0.5), iaa.CoarseDropout(p=0.1, size_percent=0.1)]) batch_dir_path_prefix = './batch_dir' num_batch_dirs = 5 dirMgr = WorkingDirectoryManager(batch_dir_path_prefix, num_batch_dirs) #-------------------- for epoch in range(num_epochs): print('>>>>> Epoch #{}.'.format(epoch)) dir_path = dirMgr.requestAvailableDirectory() if dir_path is None: break print('\t>>>>> Directory: {}.'.format(dir_path)) batchMgr = ImgaugFileBatchManager(augmenter, images, labels, batch_size, shuffle, is_label_augmented, is_time_major) batchMgr.putBatches( dir_path) # Generates, augments, and saves batches. batches = batchMgr.getBatches(dir_path) # Loads batches. for idx, batch in enumerate(batches): # Train with each batch (images & labels). #print('\t{}: {}, {}'.format(idx, batch[0].shape, batch[1].shape)) print('{}: {}-{}, {}-{}'.format( idx, batch[0].shape, np.max(np.reshape(batch[0], (batch[0].shape[0], -1)), axis=-1), batch[1].shape, np.max(np.reshape(batch[1], (batch[1].shape[0], -1)), axis=-1))) dirMgr.returnDirectory(dir_path)
def simple_file_batch_manager_example(): num_examples = 100 images, labels = generate_dataset(num_examples) batch_size = 12 num_epochs = 7 shuffle = True is_time_major = False batch_dir_path_prefix = './batch_dir' num_batch_dirs = 5 dirMgr = WorkingDirectoryManager(batch_dir_path_prefix, num_batch_dirs) #-------------------- for epoch in range(num_epochs): print('>>>>> Epoch #{}.'.format(epoch)) while True: dir_path = dirMgr.requestAvailableDirectory() if dir_path is not None: break else: time.sleep(0.1) print('\t>>>>> Directory: {}.'.format(dir_path)) batchMgr = SimpleFileBatchManager(images, labels, batch_size, shuffle, is_time_major) batchMgr.putBatches(dir_path) # Generates and saves batches. batches = batchMgr.getBatches(dir_path) # Loads batches. for idx, batch in enumerate(batches): # Can run in an individual thread or process. # Augment each batch (images & labels). # Train with each batch (images & labels). #print('\t{}: {}, {}'.format(idx, batch[0].shape, batch[1].shape)) print('\t{}: {}-{}, {}-{}'.format( idx, batch[0].shape, np.max(np.reshape(batch[0], (batch[0].shape[0], -1)), axis=-1), batch[1].shape, np.max(np.reshape(batch[1], (batch[1].shape[0], -1)), axis=-1))) dirMgr.returnDirectory(dir_path)
def imgaug_file_batch_manager_with_file_input_example(): num_examples = 300 is_label_augmented = False num_files = generate_file_dataset('./batches', num_examples, is_label_augmented) npy_filepath_pairs = list() for idx in range(num_files): npy_filepath_pairs.append(('./batches/images_{}.npy'.format(idx), './batches/labels_{}.npy'.format(idx))) npy_filepath_pairs = np.array(npy_filepath_pairs) num_file_pairs = 3 num_file_pair_steps = ((num_files - 1) // num_file_pairs + 1) if num_files > 0 else 0 batch_size = 12 num_epochs = 7 shuffle = True is_time_major = False augmenter = iaa.Sequential( [iaa.Fliplr(0.5), iaa.CoarseDropout(p=0.1, size_percent=0.1)]) batch_dir_path_prefix = './batch_dir' num_batch_dirs = 5 dirMgr = WorkingDirectoryManager(batch_dir_path_prefix, num_batch_dirs) #-------------------- for epoch in range(num_epochs): print('>>>>> Epoch #{}.'.format(epoch)) dir_path = dirMgr.requestAvailableDirectory() if dir_path is None: break print('\t>>>>> Directory: {}.'.format(dir_path)) indices = np.arange(num_files) if shuffle: np.random.shuffle(indices) for step in range(num_file_pair_steps): print('\t\t>>>>> File pairs #{}.'.format(step)) start = step * num_file_pairs end = start + num_file_pairs file_pair_indices = indices[start:end] if file_pair_indices.size > 0: # If file_pair_indices is non-empty. sub_filepath_pairs = npy_filepath_pairs[file_pair_indices] if sub_filepath_pairs.size > 0: # If sub_filepath_pairs is non-empty. # Can run in an individual thread or process. batchMgr = ImgaugFileBatchManagerWithFileInput( augmenter, sub_filepath_pairs, batch_size, shuffle, is_label_augmented, is_time_major) batchMgr.putBatches( dir_path) # Generates, augments, and saves batches. batches = batchMgr.getBatches(dir_path) # Loads batches. for idx, batch in enumerate(batches): # Train with each batch (images & labels). #print('\t\t{}: {}, {}'.format(idx, batch[0].shape, batch[1].shape)) print('{}: {}-{}, {}-{}'.format( idx, batch[0].shape, np.max(np.reshape(batch[0], (batch[0].shape[0], -1)), axis=-1), batch[1].shape, np.max(np.reshape(batch[1], (batch[1].shape[0], -1)), axis=-1))) dirMgr.returnDirectory(dir_path)