def test_utils_py_1(): """ test get_ovr_idx, create_flat_raster """ filename = 'tmp/raster.tif' temp_files.append(filename) overview_list = [2, 4] raster_creation.create_flat_raster(filename, overview_list=overview_list) ds = util.open_ds(filename) compression = util.get_image_structure_metadata(filename, 'COMPRESSION') assert compression == 'DEFLATE' ovr_count = util.get_ovr_count(ds) + 1 assert ovr_count == 3 pixel_size = util.get_pixel_size(ds) assert pixel_size == (10, -10) for i in range(-ovr_count, ovr_count): assert util.get_ovr_idx(filename, ovr_idx=i) == (i if i >= 0 else ovr_count + i) ovr_factor = [1] + overview_list ras_size = ds.RasterXSize, ds.RasterYSize for res, ovr_idx in [(5, 0), (10, 0), (11, 0), (19.99, 0), (20, 1), (20.1, 1), (39, 1), (40, 2), (41, 2), (400, 2)]: assert util.get_ovr_idx(ds, ovr_res=res) == ovr_idx assert util.get_ovr_idx( ds, float(res)) == ovr_idx # noqa secret functionality bands = util.get_bands(ds, ovr_idx=ovr_idx) assert len(bands) == 1 f = ovr_factor[ovr_idx] assert (bands[0].XSize, bands[0].XSize) == (ras_size[0] // f, ras_size[1] // f) # test open_ds with multiple different inputs filename2 = 'tmp/raster2.tif' temp_files.append(filename2) raster_creation.create_flat_raster(filename2) ds_list = util.open_ds([ds, filename2]) assert tuple(util.get_ovr_count(ds) for ds in ds_list) == (2, 0) ds_list = None
def gdallocationinfo( filename_or_ds: PathOrDS, x: ArrayOrScalarLike, y: ArrayOrScalarLike, srs: CoordinateTransformationOrSRS = None, axis_order: Optional[OAMS_AXIS_ORDER] = None, open_options: Optional[dict] = None, ovr_idx: Optional[int] = None, band_nums: Optional[Sequence[int]] = None, inline_xy_replacement: bool = False, return_ovr_pixel_line: bool = False, transform_round_digits: Optional[float] = None, allow_xy_outside_extent: bool = True, pixel_offset: Real = -0.5, line_offset: Real = -0.5, resample_alg=gdalconst.GRIORA_NearestNeighbour, quiet_mode: bool = True, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: ds = open_ds(filename_or_ds, open_options=open_options) filename = filename_or_ds if is_path_like(filename_or_ds) else '' if ds is None: raise Exception(f'Could not open {filename}.') if not isinstance(x, ArrayLike.__args__): x = [x] if not isinstance(y, ArrayLike.__args__): y = [y] if len(x) != len(y): raise Exception( f'len(x)={len(x)} should be the same as len(y)={len(y)}') point_count = len(x) dtype = np.float64 if not isinstance(x, np.ndarray) or not isinstance(y, np.ndarray): x = np.array(x, dtype=dtype) y = np.array(y, dtype=dtype) inline_xy_replacement = True if srs is None: srs = LocationInfoSRS.PixelLine # Build Spatial Reference object based on coordinate system, fetched from the opened dataset if srs != LocationInfoSRS.PixelLine: if srs != LocationInfoSRS.SameAsDS_SRS: ds_srs = ds.GetSpatialRef() ct = None if isinstance(srs, osr.CoordinateTransformation): ct = srs else: if srs == LocationInfoSRS.SameAsDS_SRS_GeogCS: points_srs = ds_srs.CloneGeogCS() else: points_srs = get_srs(srs, axis_order=axis_order) ct = get_transform(points_srs, ds_srs) if ct is not None: if not inline_xy_replacement: x = x.copy() y = y.copy() inline_xy_replacement = True transform_points(ct, x, y) if transform_round_digits is not None: x.round(transform_round_digits, out=x) y.round(transform_round_digits, out=y) # Read geotransform matrix and calculate corresponding pixel coordinates geotransform = ds.GetGeoTransform() inv_geotransform = gdal.InvGeoTransform(geotransform) if inv_geotransform is None: raise Exception("Failed InvGeoTransform()") # can we inline this transformation ? x, y = \ (inv_geotransform[0] + inv_geotransform[1] * x + inv_geotransform[2] * y), \ (inv_geotransform[3] + inv_geotransform[4] * x + inv_geotransform[5] * y) inline_xy_replacement = True xsize, ysize = ds.RasterXSize, ds.RasterYSize bands = get_bands(ds, band_nums, ovr_idx=ovr_idx) ovr_xsize, ovr_ysize = bands[0].XSize, bands[0].YSize pixel_fact, line_fact = (ovr_xsize / xsize, ovr_ysize / ysize) if ovr_idx else (1, 1) bnd_count = len(bands) shape = (bnd_count, point_count) np_dtype, np_dtype = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds) results = np.empty(shape=shape, dtype=np_dtype) check_outside = not quiet_mode or not allow_xy_outside_extent if check_outside and (np.any(x < 0) or np.any(x >= xsize) or np.any(y < 0) or np.any(y >= ysize)): msg = 'Passed coordinates are not in dataset extent!' if not allow_xy_outside_extent: raise Exception(msg) elif not quiet_mode: print(msg) if pixel_fact == 1: pixels_q = x elif return_ovr_pixel_line: x *= pixel_fact pixels_q = x else: pixels_q = x * pixel_fact if line_fact == 1: lines_q = y elif return_ovr_pixel_line: y *= line_fact lines_q = y else: lines_q = y * line_fact buf_xsize = buf_ysize = 1 buf_type, typecode = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds) buf_obj = np.empty([buf_ysize, buf_xsize], dtype=typecode) for idx, (pixel, line) in enumerate(zip(pixels_q, lines_q)): for bnd_idx, band in enumerate(bands): if BandRasterIONumPy(band, 0, pixel - 0.5, line - 0.5, 1, 1, buf_obj, buf_type, resample_alg, None, None) == 0: results[bnd_idx][idx] = buf_obj[0][0] is_scaled, scales, offsets = get_scales_and_offsets(bands) if is_scaled: for bnd_idx, (scale, offset) in enumerate(zip(scales, offsets)): results[bnd_idx] = results[bnd_idx] * scale + offset return x, y, results
def gdal2xyz( srcfile: PathOrDS, dstfile: PathLikeOrStr = None, srcwin: Optional[Sequence[int]] = None, skip: Union[int, Sequence[int]] = 1, band_nums: Optional[Sequence[int]] = None, delim: str = ' ', skip_nodata: bool = False, src_nodata: Optional[Union[Sequence, Number]] = None, dst_nodata: Optional[Union[Sequence, Number]] = None, return_np_arrays: bool = False, pre_allocate_np_arrays: bool = True, progress_callback: OptionalProgressCallback = ...) -> Optional[Tuple]: """ translates a raster file (or dataset) into xyz format skip - how many rows/cols to skip each iteration srcwin (xoff, yoff, xsize, ysize) - Selects a subwindow from the source image for copying based on pixel/line location. band_nums - selected input bands to process, None to process all. delim - the delimiter to use between values in a line skip_nodata - Exclude the output lines with nodata value (as determined by srcnodata) src_nodata - The nodata value of the dataset (for skipping or replacing) default (`None`) - Use the dataset NoDataValue; `Sequence`/`Number` - use the given nodata value (per band or per dataset). dst_nodata - Replace source nodata with a given nodata. Has an effect only if not setting `-skipnodata` default(`None`) - use srcnodata, no replacement; `Sequence`/`Number` - replace the `srcnodata` with the given nodata value (per band or per dataset). srcfile - The source dataset filename or dataset object dstfile - The output dataset filename; for dstfile=None - if return_np_arrays=False then output will be printed to stdout return_np_arrays - return numpy arrays of the result, otherwise returns None pre_allocate_np_arrays - pre-allocated result arrays. Should be faster unless skip_nodata and the input is very sparse thus most data points will be skipped. progress_callback - progress callback function. use None for quiet or Ellipsis for using the default callback """ result = None progress_callback = get_progress_callback(progress_callback) # Open source file. ds = open_ds(srcfile) if ds is None: raise Exception(f'Could not open {srcfile}.') bands = get_bands(ds, band_nums) band_count = len(bands) gt = ds.GetGeoTransform() # Collect information on all the source files. if srcwin is None: srcwin = (0, 0, ds.RasterXSize, ds.RasterYSize) dt, np_dt = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds) # Open the output file. if dstfile is not None: dst_fh = open(dstfile, 'wt') elif return_np_arrays: dst_fh = None else: dst_fh = sys.stdout if dst_fh: if dt == gdal.GDT_Int32 or dt == gdal.GDT_UInt32: band_format = (("%d" + delim) * len(bands)).rstrip(delim) + '\n' else: band_format = (("%g" + delim) * len(bands)).rstrip(delim) + '\n' # Setup an appropriate print format. if abs(gt[0]) < 180 and abs(gt[3]) < 180 \ and abs(ds.RasterXSize * gt[1]) < 180 \ and abs(ds.RasterYSize * gt[5]) < 180: frmt = '%.10g' + delim + '%.10g' + delim + '%s' else: frmt = '%.3f' + delim + '%.3f' + delim + '%s' if isinstance(src_nodata, Number): src_nodata = [src_nodata] * band_count elif src_nodata is None: src_nodata = list(band.GetNoDataValue() for band in bands) if None in src_nodata: src_nodata = None if src_nodata is not None: src_nodata = np.asarray(src_nodata, dtype=np_dt) if isinstance(dst_nodata, Number): dst_nodata = [dst_nodata] * band_count if (dst_nodata is None) or (None in dst_nodata) or (src_nodata is None): dst_nodata = None if dst_nodata is not None: dst_nodata = np.asarray(dst_nodata, dtype=np_dt) skip_nodata = skip_nodata and (src_nodata is not None) replace_nodata = (not skip_nodata) and (dst_nodata is not None) process_nodata = skip_nodata or replace_nodata if isinstance(skip, Sequence): x_skip, y_skip = skip else: x_skip = y_skip = skip x_off, y_off, x_size, y_size = srcwin bands_count = len(bands) nXBlocks = (x_size - x_off) // x_skip nYBlocks = (y_size - y_off) // y_skip progress_end = nXBlocks * nYBlocks progress_curr = 0 progress_prev = -1 progress_parts = 100 if return_np_arrays: size = progress_end if pre_allocate_np_arrays else 0 all_geo_x = np.empty(size) all_geo_y = np.empty(size) all_data = np.empty((size, band_count), dtype=np_dt) # Loop emitting data. idx = 0 for y in range(y_off, y_off + y_size, y_skip): size = bands_count if pre_allocate_np_arrays else 0 data = np.empty((size, x_size), dtype=np_dt) # dims: (bands_count, x_size) for i_bnd, band in enumerate(bands): band_data = band.ReadAsArray(x_off, y, x_size, 1) # read one band line if pre_allocate_np_arrays: data[i_bnd] = band_data[0] else: data = np.append(data, band_data, axis=0) for x_i in range(0, x_size, x_skip): progress_curr += 1 if progress_callback: progress_frac = progress_curr / progress_end progress = int(progress_frac * progress_parts) if progress > progress_prev: progress_prev = progress progress_callback(progress_frac) x_i_data = data[:, x_i] # single pixel, dims: (bands) if process_nodata and np.array_equal(src_nodata, x_i_data): if skip_nodata: continue elif replace_nodata: x_i_data = dst_nodata x = x_i + x_off geo_x = gt[0] + (x + 0.5) * gt[1] + (y + 0.5) * gt[2] geo_y = gt[3] + (x + 0.5) * gt[4] + (y + 0.5) * gt[5] if dst_fh: band_str = band_format % tuple(x_i_data) line = frmt % (float(geo_x), float(geo_y), band_str) dst_fh.write(line) if return_np_arrays: if pre_allocate_np_arrays: all_geo_x[idx] = geo_x all_geo_y[idx] = geo_y all_data[idx] = x_i_data else: all_geo_x = np.append(all_geo_x, geo_x) all_geo_y = np.append(all_geo_y, geo_y) all_data = np.append(all_data, [x_i_data], axis=0) idx += 1 if return_np_arrays: nodata = None if skip_nodata else dst_nodata if replace_nodata else src_nodata if idx != progress_curr: all_geo_x = all_geo_x[:idx] all_geo_y = all_geo_y[:idx] all_data = all_data[:idx, :] result = all_geo_x, all_geo_y, all_data.transpose(), nodata return result
def gdallocationinfo( filename_or_ds: PathOrDS, x: NumpyCompatibleArrayOrReal, y: NumpyCompatibleArrayOrReal, gis_order: bool = False, open_options: Optional[dict] = None, ovr_idx: Optional[int] = None, band_nums: Optional[Sequence[int]] = None, srs: CoordinateTransformationOrSRS = None, inline_xy_replacement: bool = True, quiet_mode: bool = True, allow_xy_outside_extent: bool = True, pixel_offset: Real = -0.5, line_offset: Real = -0.5, resample_alg=gdalconst.GRIORA_NearestNeighbour ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: ds = open_ds(filename_or_ds, open_options=open_options) filename = filename_or_ds if is_path_like(filename_or_ds) else '' if ds is None: raise Exception(f'Could not open {filename}.') if not isinstance(x, NumpyCompatibleArray.__args__): x = [x] if not isinstance(y, NumpyCompatibleArray.__args__): y = [y] if len(x) != len(y): raise Exception( f'len(x)={len(x)} should be the same as len(y)={len(y)}') point_count = len(x) if not isinstance(x, np.ndarray): x = np.ndarray(x) if not isinstance(y, np.ndarray): y = np.ndarray(y) if srs is None: srs = LocationInfoSRS.PixelLine # Build Spatial Reference object based on coordinate system, fetched from the opened dataset if srs != LocationInfoSRS.PixelLine: if srs != LocationInfoSRS.SameAsDS_SRS: ds_srs = ds.GetSpatialRef() ct = None if isinstance(srs, osr.CoordinateTransformation): ct = srs else: if srs == LocationInfoSRS.SameAsDS_SRS_GeogCS: points_srs = ds_srs.CloneGeogCS() else: points_srs = get_srs(srs, gis_order=gis_order) ct = get_transform(points_srs, ds_srs) x, y, _z = transform_points(ct, x, y) # Read geotransform matrix and calculate corresponding pixel coordinates geomatrix = ds.GetGeoTransform() inv_geometrix = gdal.InvGeoTransform(geomatrix) if inv_geometrix is None: raise Exception("Failed InvGeoTransform()") x, y = \ (inv_geometrix[0] + inv_geometrix[1] * x + inv_geometrix[2] * y), \ (inv_geometrix[3] + inv_geometrix[4] * x + inv_geometrix[5] * y) xsize, ysize = ds.RasterXSize, ds.RasterYSize bands = get_bands(ds, band_nums, ovr_idx=ovr_idx) ovr_xsize, ovr_ysize = bands[0].XSize, bands[0].YSize pixel_fact, line_fact = (ovr_xsize / xsize, ovr_ysize / ysize) if ovr_idx else (1, 1) bnd_count = len(bands) shape = (bnd_count, point_count) np_dtype, np_dtype = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds) results = np.empty(shape=shape, dtype=np_dtype) check_outside = not quiet_mode or not allow_xy_outside_extent if check_outside and (np.any(x < 0) or np.any(x >= xsize) or np.any(y < 0) or np.any(y >= ysize)): msg = 'Passed coordinates are not in dataset extent!' if not allow_xy_outside_extent: raise Exception(msg) elif not quiet_mode: print(msg) pixels = np.clip(x * pixel_fact + pixel_offset, 0, ovr_xsize - 1, out=x if inline_xy_replacement else None) lines = np.clip(y * line_fact + line_offset, 0, ovr_ysize - 1, out=y if inline_xy_replacement else None) for idx, (pixel, line) in enumerate(zip(pixels, lines)): for bnd_idx, band in enumerate(bands): val = band.ReadAsArray(pixel, line, 1, 1, resample_alg=resample_alg) val = val[0][0] results[bnd_idx][idx] = val is_scaled, scales, offsets = get_scales_and_offsets(bands) if is_scaled: for bnd_idx, scale, offset in enumerate(zip(scales, offsets)): results[bnd_idx] = results[bnd_idx] * scale + offset return pixels, lines, results