def test_read_cache_file(self): # Expect None from non-existent file self.assertIsNone(cache.read_cache_file('asdf')) # Expect correct data without image ID check np.testing.assert_equal(self.test_data['Y'], cache.read_cache_file(self.test_file))
def test_read_cache_file_imageIDs(cachefile, example_cache): image_IDs = example_cache['image_IDs'] # Expect None since image IDs won't match assert None is cache.read_cache_file(cachefile, image_IDs[::-1]) np.testing.assert_equal(example_cache['Y'], cache.read_cache_file(cachefile, image_IDs))
def test_read_cache_file_imageIDs(self): # Expect None because image IDs won't match image_IDs = self.test_data['image_IDs'][:-1] self.assertIsNone(cache.read_cache_file(self.test_file, image_IDs)) # Expect correct data with image ID check np.testing.assert_equal( self.test_data['Y'], cache.read_cache_file(self.test_file, self.test_data['image_IDs']))
def test_read_cache_file(cachefile, example_cache): assert None is cache.read_cache_file('asdf') np.testing.assert_equal(example_cache['Y'], cache.read_cache_file(cachefile))
def read_line(line, images, image_IDs, dataset_config, ncol, nband, dtype, read_cache=False, write_cache=False, validate_cache=False): """ Reads in dataset from cache or images if required Args: line (int): line to read in from images images (list): list of image filenames to read from image_IDs (iterable): list image identifying strings dataset_config (dict): dictionary of dataset configuration options ncol (int): number of columns nband (int): number of bands dtype (type): NumPy datatype read_cache (bool, optional): try to read from cache directory (default: False) write_cache (bool, optional): try to to write to cache directory (default: False) validate_cache (bool, optional): validate that cache data come from same images specified in `images` (default: False) Returns: Y (np.ndarray): 3D array of image data (nband, n_image, n_cols) """ start_time = time.time() read_from_disk = True cache_filename = get_line_cache_name( dataset_config, len(images), line, nband) Y_shape = (nband, len(images), ncol) if read_cache: Y = read_cache_file(cache_filename, image_IDs if validate_cache else None) if Y is not None and Y.shape == Y_shape: logger.debug('Read in Y from cache file') read_from_disk = False elif Y is not None and Y.shape != Y_shape: logger.warning( 'Data from cache file does not meet size requested ' '({y} versus {r})'.format(y=Y.shape, r=Y_shape)) if read_from_disk: # Read in Y if dataset_config['use_bip_reader']: # Use BIP reader logger.debug('Reading in data from disk using BIP reader') Y = read_row_BIP(images, line, (ncol, nband), dtype) else: # Read in data just using GDAL logger.debug('Reading in data from disk using GDAL') Y = read_row_GDAL(images, line) logger.debug('Took {s}s to read in the data'.format( s=round(time.time() - start_time, 2))) if write_cache and read_from_disk: logger.debug('Writing Y data to cache file {f}'.format( f=cache_filename)) write_cache_file(cache_filename, Y, image_IDs) return Y
def read_line(line, images, image_IDs, dataset_config, ncol, nband, dtype, read_cache=False, write_cache=False, validate_cache=False): """ Reads in dataset from cache or images if required Args: line (int): line to read in from images images (list): list of image filenames to read from image_IDs (iterable): list image identifying strings dataset_config (dict): dictionary of dataset configuration options ncol (int): number of columns nband (int): number of bands dtype (type): NumPy datatype read_cache (bool, optional): try to read from cache directory (default: False) write_cache (bool, optional): try to to write to cache directory (default: False) validate_cache (bool, optional): validate that cache data come from same images specified in `images` (default: False) Returns: Y (np.ndarray): 3D array of image data (nband, n_image, n_cols) """ start_time = time.time() read_from_disk = True cache_filename = get_line_cache_name(dataset_config, len(images), line, nband) Y_shape = (nband, len(images), ncol) if read_cache: Y = read_cache_file(cache_filename, image_IDs if validate_cache else None) if Y is not None and Y.shape == Y_shape: logger.debug('Read in Y from cache file') read_from_disk = False elif Y is not None and Y.shape != Y_shape: logger.warning('Data from cache file does not meet size requested ' '({y} versus {r})'.format(y=Y.shape, r=Y_shape)) if read_from_disk: # Read in Y if dataset_config['use_bip_reader']: # Use BIP reader logger.debug('Reading in data from disk using BIP reader') Y = read_row_BIP(images, line, (ncol, nband), dtype) else: # Read in data just using GDAL logger.debug('Reading in data from disk using GDAL') Y = read_row_GDAL(images, line) logger.debug('Took {s}s to read in the data'.format( s=round(time.time() - start_time, 2))) if write_cache and read_from_disk: logger.debug( 'Writing Y data to cache file {f}'.format(f=cache_filename)) write_cache_file(cache_filename, Y, image_IDs) return Y