def test_load_data(dataset_finder_class, file_pattern, maxfiles): file_finder = FileFinder(file_pattern, maxfiles) print("Constructing dataset...") start = time.time() datasets = dataset_finder_class(file_finder.get_files()) print("... done in {:.3f}.".format(time.time() - start)) print("Loading views...") start = time.time() views = datasets.get_datasets() print("... done in {:.3f}. Total views {}.".format(time.time() - start, len(views))) print("Processing images in batches...") start = time.time() batch_size = 128 n_batches = len(views) // batch_size for batch_id in range(n_batches): start_index = batch_id * batch_size end_index = start_index + batch_size print("batch_id {}/{}, start_index {}, end_index {}".format( batch_id, n_batches, start_index, end_index)) images, labels = load_h5view_data(views[start_index:end_index]) print(len(images), images[0].dtype, type(images[0])) print("... done in {:.3f}.".format(time.time() - start))
def test_finder(dataset_finder_class, file_pattern, maxfiles): file_finder = FileFinder(file_pattern, maxfiles) start = time.time() print("Constructing dataset...") dataset_finder = dataset_finder_class(file_finder.get_files()) print("Done. Time to construct dataset {:.3f}".format(time.time() - start)) start = time.time() print("Loading views...") dataset_finder.load_dataset_views() print("Done. Time to load views {:.3f}".format(time.time() - start)) start = time.time() views = dataset_finder.get_datasets() print("Time to get datasets {:.3f}, size {}".format( time.time() - start, len(views)))