def test_load_data(dataset_finder_class, file_pattern, maxfiles):
file_finder = FileFinder(file_pattern, maxfiles)
print("Constructing dataset...")
start = time.time()
datasets = dataset_finder_class(file_finder.get_files())
print("... done in {:.3f}.".format(time.time() - start))
print("Loading views...")
start = time.time()
views = datasets.get_datasets()
print("... done in {:.3f}. Total views {}.".format(time.time() - start,
len(views)))
print("Processing images in batches...")
start = time.time()
batch_size = 128
n_batches = len(views) // batch_size
for batch_id in range(n_batches):
start_index = batch_id * batch_size
end_index = start_index + batch_size
print("batch_id {}/{}, start_index {}, end_index {}".format(
batch_id, n_batches, start_index, end_index))
images, labels = load_h5view_data(views[start_index:end_index])
print(len(images), images[0].dtype, type(images[0]))
print("... done in {:.3f}.".format(time.time() - start))
def test_finder(dataset_finder_class, file_pattern, maxfiles):
file_finder = FileFinder(file_pattern, maxfiles)
start = time.time()
print("Constructing dataset...")
dataset_finder = dataset_finder_class(file_finder.get_files())
print("Done. Time to construct dataset {:.3f}".format(time.time() - start))
start = time.time()
print("Loading views...")
dataset_finder.load_dataset_views()
print("Done. Time to load views {:.3f}".format(time.time() - start))
start = time.time()
views = dataset_finder.get_datasets()
print("Time to get datasets {:.3f}, size {}".format(
time.time() - start, len(views)))
