def read_blocks(band: gdal.Band) -> BandData: # Reading the 'natural' blocks of the dataset is faster and should # help with not consuming too much memory, compared to # reading the whole band at once or reading parts not aligned to the blocks. dtype = gdal_array.GDALTypeCodeToNumericTypeCode(band.DataType) no_data = band.GetNoDataValue() # type: Union[float,None] block_size_x, block_size_y = band.GetBlockSize() # See GDALRasterBand::ReadBlock documentation. block_count_x = (band.XSize + block_size_x - 1) // block_size_x block_count_y = (band.YSize + block_size_y - 1) // block_size_y for y in range(block_count_y): for x in range(block_count_x): block = band.ReadBlock(x, y) data = np.frombuffer(block, dtype).reshape(block_size_y, block_size_x) w, h = band.GetActualBlockSize(x, y) data = data[:h,:w] if no_data is not None: data = ma.masked_equal(data, no_data) yield data
def convert_to_wps_binary(input_path: str, output_folder: str, is_categorical: bool, units: Optional[str]=None, description: Optional[str]=None, strict_datum: bool=True) -> GeogridBinaryDataset: ''' Losslessly convert common geo formats to WPS binary format. If the given input file has a CRS or data type unsupported by WRF then an error is raised. :param input_path: Path to GDAL-supported raster file. :param output_folder: Path to output folder, will be created if not existing :param is_categorical: Whether the data is categorical, otherwise continuous :param units: units for continuous data :param description: single-line dataset description :param strict_datum: if True, fail if the input datum is not supported by WRF, otherwise ignore mismatch ''' os.makedirs(output_folder, exist_ok=True) if os.listdir(output_folder): raise ValueError('Output folder must be empty') # FIXME if there is no nodata value, ask the user if it really has no nodata or ask for the value src_ds = gdal.Open(input_path) # type: gdal.Dataset xsize, ysize = src_ds.RasterXSize, src_ds.RasterYSize if xsize > MAX_SIZE or ysize > MAX_SIZE: raise UserError(f'Dataset has more than {MAX_SIZE} rows or columns: {ysize} x {xsize}, consider downsampling') filename_digits = 6 if xsize > 99999 or ysize > 99999 else 5 if src_ds.GetLayerCount() > 1: raise UnsupportedError('Dataset has more than one layer which is unsupported') band = src_ds.GetRasterBand(1) # type: gdal.Band src_no_data_value = band.GetNoDataValue() has_no_data_value = src_no_data_value is not None tilesize_x = find_tile_size(xsize, try_hard=not has_no_data_value) tilesize_y = find_tile_size(ysize, try_hard=not has_no_data_value) is_perfect_tiling = xsize % tilesize_x == 0 and ysize % tilesize_y == 0 if is_categorical or (tilesize_x == xsize and tilesize_y == ysize): tile_bdr = 0 else: # TODO write unit test that checks whether halo areas have correct values tile_bdr = 3 if tile_bdr > 0 and not has_no_data_value: raise UserError('No-data value required as dataset is continuous and halo is non-zero') if not is_perfect_tiling and not has_no_data_value: raise UserError('No-data value required as no perfect tile size could be found') tilesize_bdr_x = tilesize_x + 2*tile_bdr tilesize_bdr_y = tilesize_y + 2*tile_bdr tiles_x = list(range(0, xsize, tilesize_x)) tiles_y = list(range(0, ysize, tilesize_y)) ysize_pad = tilesize_y * len(tiles_y) # ysize including padding caused by imperfect tiling # write 'index' file with metadata index_path = os.path.join(output_folder, 'index') index_dict, datum_mismatch, inv_scale_factor, dst_dtype, dst_no_data_value = create_index_dict( src_ds, tilesize_x, tilesize_y, ysize_pad, tile_bdr, filename_digits, is_categorical, units, description, strict_datum) write_index_file(index_path, index_dict) np_dst_dtype = gdal_array.GDALTypeCodeToNumericTypeCode(dst_dtype) needs_scaling = inv_scale_factor is not None # As we have no control over the auxiliarly files that are created as well during conversion # we do everything in a temporary folder and move the binary file out after the conversion. # This keeps everything clean and tidy. tmp_dir = tempfile.mkdtemp() tmp_bin_path = os.path.join(tmp_dir, 'data.bin') driver = gdal.GetDriverByName('ENVI') # type: gdal.Driver# dy = src_ds.GetGeoTransform()[5] try: for start_x in tiles_x: for start_y in tiles_y: end_x = start_x + tilesize_x - 1 end_y = start_y + tilesize_y - 1 start_bdr_x = start_x - tile_bdr start_bdr_y = start_y - tile_bdr end_bdr_x = end_x + tile_bdr end_bdr_y = end_y + tile_bdr # read source data offset_x = max(0, start_bdr_x) offset_y = max(0, start_bdr_y) if end_bdr_x >= xsize: datasize_x = xsize - offset_x else: datasize_x = end_bdr_x - offset_x + 1 if end_bdr_y >= ysize: datasize_y = ysize - offset_y else: datasize_y = end_bdr_y - offset_y + 1 src_data = band.ReadAsArray(offset_x, offset_y, datasize_x, datasize_y) if dy > 0: src_data = src_data[::-1] # scale if necessary (float data only) if needs_scaling: # TODO test if scaling with no-data works if has_no_data_value: src_data = ma.masked_equal(src_data, src_no_data_value) src_data *= inv_scale_factor np.round(src_data, out=src_data) if has_no_data_value: src_data = ma.filled(src_data, dst_no_data_value) # pad incomplete tile with nodata value if datasize_x == tilesize_bdr_x and datasize_y == tilesize_bdr_y: dst_data = src_data else: assert has_no_data_value dst_data = np.empty((tilesize_bdr_y, tilesize_bdr_x), np_dst_dtype) data_start_x = offset_x - start_bdr_x data_start_y = offset_y - start_bdr_y dst_data[data_start_y:data_start_y+datasize_y,data_start_x:data_start_x+datasize_x] = src_data if start_bdr_x < 0: dst_data[:,:data_start_x] = dst_no_data_value if start_bdr_y < 0: dst_data[:data_start_y,:] = dst_no_data_value if end_bdr_x >= xsize: dst_data[:,data_start_x+datasize_x:] = dst_no_data_value if end_bdr_y >= ysize: dst_data[data_start_y+datasize_y:,:] = dst_no_data_value # create tile file dst_ds = driver.Create(tmp_bin_path, tilesize_bdr_x, tilesize_bdr_y, 1, dst_dtype) # type: gdal.Dataset dst_band = dst_ds.GetRasterBand(1) # type: gdal.Band dst_band.WriteArray(dst_data) # write to disk dst_ds.FlushCache() del dst_ds # move to final location with WPS-specific filename convention fmt_int = '{:0' + str(filename_digits) + 'd}' fmt_filename = '{fmt}-{fmt}.{fmt}-{fmt}'.format(fmt=fmt_int) if dy < 0: end_y = ysize_pad - start_y - 1 start_y = end_y - tilesize_y + 1 final_path = os.path.join(output_folder, fmt_filename.format( start_x + 1, end_x + 1, start_y + 1, end_y + 1)) shutil.move(tmp_bin_path, final_path) return GeogridBinaryDataset(index_path, datum_mismatch) finally: shutil.rmtree(tmp_dir)
def get_gdal_dtype_range(dtype: int) -> Tuple[int,int]: np_dtype = gdal_array.GDALTypeCodeToNumericTypeCode(dtype) return (np.iinfo(np_dtype).min, np.iinfo(np_dtype).max)