def test_utils_py_1():
""" test get_ovr_idx, create_flat_raster """
filename = 'tmp/raster.tif'
temp_files.append(filename)
overview_list = [2, 4]
raster_creation.create_flat_raster(filename, overview_list=overview_list)
ds = util.open_ds(filename)
compression = util.get_image_structure_metadata(filename, 'COMPRESSION')
assert compression == 'DEFLATE'
ovr_count = util.get_ovr_count(ds) + 1
assert ovr_count == 3
pixel_size = util.get_pixel_size(ds)
assert pixel_size == (10, -10)
for i in range(-ovr_count, ovr_count):
assert util.get_ovr_idx(filename,
ovr_idx=i) == (i if i >= 0 else ovr_count + i)
ovr_factor = [1] + overview_list
ras_size = ds.RasterXSize, ds.RasterYSize
for res, ovr_idx in [(5, 0), (10, 0), (11, 0), (19.99, 0), (20, 1),
(20.1, 1), (39, 1), (40, 2), (41, 2), (400, 2)]:
assert util.get_ovr_idx(ds, ovr_res=res) == ovr_idx
assert util.get_ovr_idx(
ds, float(res)) == ovr_idx # noqa secret functionality
bands = util.get_bands(ds, ovr_idx=ovr_idx)
assert len(bands) == 1
f = ovr_factor[ovr_idx]
assert (bands[0].XSize, bands[0].XSize) == (ras_size[0] // f,
ras_size[1] // f)
# test open_ds with multiple different inputs
filename2 = 'tmp/raster2.tif'
temp_files.append(filename2)
raster_creation.create_flat_raster(filename2)
ds_list = util.open_ds([ds, filename2])
assert tuple(util.get_ovr_count(ds) for ds in ds_list) == (2, 0)
ds_list = None
def gdallocationinfo(
filename_or_ds: PathOrDS,
x: ArrayOrScalarLike,
y: ArrayOrScalarLike,
srs: CoordinateTransformationOrSRS = None,
axis_order: Optional[OAMS_AXIS_ORDER] = None,
open_options: Optional[dict] = None,
ovr_idx: Optional[int] = None,
band_nums: Optional[Sequence[int]] = None,
inline_xy_replacement: bool = False,
return_ovr_pixel_line: bool = False,
transform_round_digits: Optional[float] = None,
allow_xy_outside_extent: bool = True,
pixel_offset: Real = -0.5,
line_offset: Real = -0.5,
resample_alg=gdalconst.GRIORA_NearestNeighbour,
quiet_mode: bool = True,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
ds = open_ds(filename_or_ds, open_options=open_options)
filename = filename_or_ds if is_path_like(filename_or_ds) else ''
if ds is None:
raise Exception(f'Could not open {filename}.')
if not isinstance(x, ArrayLike.__args__):
x = [x]
if not isinstance(y, ArrayLike.__args__):
y = [y]
if len(x) != len(y):
raise Exception(
f'len(x)={len(x)} should be the same as len(y)={len(y)}')
point_count = len(x)
dtype = np.float64
if not isinstance(x, np.ndarray) or not isinstance(y, np.ndarray):
x = np.array(x, dtype=dtype)
y = np.array(y, dtype=dtype)
inline_xy_replacement = True
if srs is None:
srs = LocationInfoSRS.PixelLine
# Build Spatial Reference object based on coordinate system, fetched from the opened dataset
if srs != LocationInfoSRS.PixelLine:
if srs != LocationInfoSRS.SameAsDS_SRS:
ds_srs = ds.GetSpatialRef()
ct = None
if isinstance(srs, osr.CoordinateTransformation):
ct = srs
else:
if srs == LocationInfoSRS.SameAsDS_SRS_GeogCS:
points_srs = ds_srs.CloneGeogCS()
else:
points_srs = get_srs(srs, axis_order=axis_order)
ct = get_transform(points_srs, ds_srs)
if ct is not None:
if not inline_xy_replacement:
x = x.copy()
y = y.copy()
inline_xy_replacement = True
transform_points(ct, x, y)
if transform_round_digits is not None:
x.round(transform_round_digits, out=x)
y.round(transform_round_digits, out=y)
# Read geotransform matrix and calculate corresponding pixel coordinates
geotransform = ds.GetGeoTransform()
inv_geotransform = gdal.InvGeoTransform(geotransform)
if inv_geotransform is None:
raise Exception("Failed InvGeoTransform()")
# can we inline this transformation ?
x, y = \
(inv_geotransform[0] + inv_geotransform[1] * x + inv_geotransform[2] * y), \
(inv_geotransform[3] + inv_geotransform[4] * x + inv_geotransform[5] * y)
inline_xy_replacement = True
xsize, ysize = ds.RasterXSize, ds.RasterYSize
bands = get_bands(ds, band_nums, ovr_idx=ovr_idx)
ovr_xsize, ovr_ysize = bands[0].XSize, bands[0].YSize
pixel_fact, line_fact = (ovr_xsize / xsize,
ovr_ysize / ysize) if ovr_idx else (1, 1)
bnd_count = len(bands)
shape = (bnd_count, point_count)
np_dtype, np_dtype = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
results = np.empty(shape=shape, dtype=np_dtype)
check_outside = not quiet_mode or not allow_xy_outside_extent
if check_outside and (np.any(x < 0) or np.any(x >= xsize) or np.any(y < 0)
or np.any(y >= ysize)):
msg = 'Passed coordinates are not in dataset extent!'
if not allow_xy_outside_extent:
raise Exception(msg)
elif not quiet_mode:
print(msg)
if pixel_fact == 1:
pixels_q = x
elif return_ovr_pixel_line:
x *= pixel_fact
pixels_q = x
else:
pixels_q = x * pixel_fact
if line_fact == 1:
lines_q = y
elif return_ovr_pixel_line:
y *= line_fact
lines_q = y
else:
lines_q = y * line_fact
buf_xsize = buf_ysize = 1
buf_type, typecode = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
buf_obj = np.empty([buf_ysize, buf_xsize], dtype=typecode)
for idx, (pixel, line) in enumerate(zip(pixels_q, lines_q)):
for bnd_idx, band in enumerate(bands):
if BandRasterIONumPy(band, 0, pixel - 0.5, line - 0.5, 1, 1,
buf_obj, buf_type, resample_alg, None,
None) == 0:
results[bnd_idx][idx] = buf_obj[0][0]
is_scaled, scales, offsets = get_scales_and_offsets(bands)
if is_scaled:
for bnd_idx, (scale, offset) in enumerate(zip(scales, offsets)):
results[bnd_idx] = results[bnd_idx] * scale + offset
return x, y, results
def gdal2xyz(
srcfile: PathOrDS,
dstfile: PathLikeOrStr = None,
srcwin: Optional[Sequence[int]] = None,
skip: Union[int, Sequence[int]] = 1,
band_nums: Optional[Sequence[int]] = None,
delim: str = ' ',
skip_nodata: bool = False,
src_nodata: Optional[Union[Sequence, Number]] = None,
dst_nodata: Optional[Union[Sequence, Number]] = None,
return_np_arrays: bool = False,
pre_allocate_np_arrays: bool = True,
progress_callback: OptionalProgressCallback = ...) -> Optional[Tuple]:
"""
translates a raster file (or dataset) into xyz format
skip - how many rows/cols to skip each iteration
srcwin (xoff, yoff, xsize, ysize) - Selects a subwindow from the source image for copying based on pixel/line location.
band_nums - selected input bands to process, None to process all.
delim - the delimiter to use between values in a line
skip_nodata - Exclude the output lines with nodata value (as determined by srcnodata)
src_nodata - The nodata value of the dataset (for skipping or replacing)
default (`None`) - Use the dataset NoDataValue;
`Sequence`/`Number` - use the given nodata value (per band or per dataset).
dst_nodata - Replace source nodata with a given nodata. Has an effect only if not setting `-skipnodata`
default(`None`) - use srcnodata, no replacement;
`Sequence`/`Number` - replace the `srcnodata` with the given nodata value (per band or per dataset).
srcfile - The source dataset filename or dataset object
dstfile - The output dataset filename; for dstfile=None - if return_np_arrays=False then output will be printed to stdout
return_np_arrays - return numpy arrays of the result, otherwise returns None
pre_allocate_np_arrays - pre-allocated result arrays.
Should be faster unless skip_nodata and the input is very sparse thus most data points will be skipped.
progress_callback - progress callback function. use None for quiet or Ellipsis for using the default callback
"""
result = None
progress_callback = get_progress_callback(progress_callback)
# Open source file.
ds = open_ds(srcfile)
if ds is None:
raise Exception(f'Could not open {srcfile}.')
bands = get_bands(ds, band_nums)
band_count = len(bands)
gt = ds.GetGeoTransform()
# Collect information on all the source files.
if srcwin is None:
srcwin = (0, 0, ds.RasterXSize, ds.RasterYSize)
dt, np_dt = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
# Open the output file.
if dstfile is not None:
dst_fh = open(dstfile, 'wt')
elif return_np_arrays:
dst_fh = None
else:
dst_fh = sys.stdout
if dst_fh:
if dt == gdal.GDT_Int32 or dt == gdal.GDT_UInt32:
band_format = (("%d" + delim) * len(bands)).rstrip(delim) + '\n'
else:
band_format = (("%g" + delim) * len(bands)).rstrip(delim) + '\n'
# Setup an appropriate print format.
if abs(gt[0]) < 180 and abs(gt[3]) < 180 \
and abs(ds.RasterXSize * gt[1]) < 180 \
and abs(ds.RasterYSize * gt[5]) < 180:
frmt = '%.10g' + delim + '%.10g' + delim + '%s'
else:
frmt = '%.3f' + delim + '%.3f' + delim + '%s'
if isinstance(src_nodata, Number):
src_nodata = [src_nodata] * band_count
elif src_nodata is None:
src_nodata = list(band.GetNoDataValue() for band in bands)
if None in src_nodata:
src_nodata = None
if src_nodata is not None:
src_nodata = np.asarray(src_nodata, dtype=np_dt)
if isinstance(dst_nodata, Number):
dst_nodata = [dst_nodata] * band_count
if (dst_nodata is None) or (None in dst_nodata) or (src_nodata is None):
dst_nodata = None
if dst_nodata is not None:
dst_nodata = np.asarray(dst_nodata, dtype=np_dt)
skip_nodata = skip_nodata and (src_nodata is not None)
replace_nodata = (not skip_nodata) and (dst_nodata is not None)
process_nodata = skip_nodata or replace_nodata
if isinstance(skip, Sequence):
x_skip, y_skip = skip
else:
x_skip = y_skip = skip
x_off, y_off, x_size, y_size = srcwin
bands_count = len(bands)
nXBlocks = (x_size - x_off) // x_skip
nYBlocks = (y_size - y_off) // y_skip
progress_end = nXBlocks * nYBlocks
progress_curr = 0
progress_prev = -1
progress_parts = 100
if return_np_arrays:
size = progress_end if pre_allocate_np_arrays else 0
all_geo_x = np.empty(size)
all_geo_y = np.empty(size)
all_data = np.empty((size, band_count), dtype=np_dt)
# Loop emitting data.
idx = 0
for y in range(y_off, y_off + y_size, y_skip):
size = bands_count if pre_allocate_np_arrays else 0
data = np.empty((size, x_size),
dtype=np_dt) # dims: (bands_count, x_size)
for i_bnd, band in enumerate(bands):
band_data = band.ReadAsArray(x_off, y, x_size,
1) # read one band line
if pre_allocate_np_arrays:
data[i_bnd] = band_data[0]
else:
data = np.append(data, band_data, axis=0)
for x_i in range(0, x_size, x_skip):
progress_curr += 1
if progress_callback:
progress_frac = progress_curr / progress_end
progress = int(progress_frac * progress_parts)
if progress > progress_prev:
progress_prev = progress
progress_callback(progress_frac)
x_i_data = data[:, x_i] # single pixel, dims: (bands)
if process_nodata and np.array_equal(src_nodata, x_i_data):
if skip_nodata:
continue
elif replace_nodata:
x_i_data = dst_nodata
x = x_i + x_off
geo_x = gt[0] + (x + 0.5) * gt[1] + (y + 0.5) * gt[2]
geo_y = gt[3] + (x + 0.5) * gt[4] + (y + 0.5) * gt[5]
if dst_fh:
band_str = band_format % tuple(x_i_data)
line = frmt % (float(geo_x), float(geo_y), band_str)
dst_fh.write(line)
if return_np_arrays:
if pre_allocate_np_arrays:
all_geo_x[idx] = geo_x
all_geo_y[idx] = geo_y
all_data[idx] = x_i_data
else:
all_geo_x = np.append(all_geo_x, geo_x)
all_geo_y = np.append(all_geo_y, geo_y)
all_data = np.append(all_data, [x_i_data], axis=0)
idx += 1
if return_np_arrays:
nodata = None if skip_nodata else dst_nodata if replace_nodata else src_nodata
if idx != progress_curr:
all_geo_x = all_geo_x[:idx]
all_geo_y = all_geo_y[:idx]
all_data = all_data[:idx, :]
result = all_geo_x, all_geo_y, all_data.transpose(), nodata
return result
def gdallocationinfo(
filename_or_ds: PathOrDS,
x: NumpyCompatibleArrayOrReal,
y: NumpyCompatibleArrayOrReal,
gis_order: bool = False,
open_options: Optional[dict] = None,
ovr_idx: Optional[int] = None,
band_nums: Optional[Sequence[int]] = None,
srs: CoordinateTransformationOrSRS = None,
inline_xy_replacement: bool = True,
quiet_mode: bool = True,
allow_xy_outside_extent: bool = True,
pixel_offset: Real = -0.5,
line_offset: Real = -0.5,
resample_alg=gdalconst.GRIORA_NearestNeighbour
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
ds = open_ds(filename_or_ds, open_options=open_options)
filename = filename_or_ds if is_path_like(filename_or_ds) else ''
if ds is None:
raise Exception(f'Could not open {filename}.')
if not isinstance(x, NumpyCompatibleArray.__args__):
x = [x]
if not isinstance(y, NumpyCompatibleArray.__args__):
y = [y]
if len(x) != len(y):
raise Exception(
f'len(x)={len(x)} should be the same as len(y)={len(y)}')
point_count = len(x)
if not isinstance(x, np.ndarray):
x = np.ndarray(x)
if not isinstance(y, np.ndarray):
y = np.ndarray(y)
if srs is None:
srs = LocationInfoSRS.PixelLine
# Build Spatial Reference object based on coordinate system, fetched from the opened dataset
if srs != LocationInfoSRS.PixelLine:
if srs != LocationInfoSRS.SameAsDS_SRS:
ds_srs = ds.GetSpatialRef()
ct = None
if isinstance(srs, osr.CoordinateTransformation):
ct = srs
else:
if srs == LocationInfoSRS.SameAsDS_SRS_GeogCS:
points_srs = ds_srs.CloneGeogCS()
else:
points_srs = get_srs(srs, gis_order=gis_order)
ct = get_transform(points_srs, ds_srs)
x, y, _z = transform_points(ct, x, y)
# Read geotransform matrix and calculate corresponding pixel coordinates
geomatrix = ds.GetGeoTransform()
inv_geometrix = gdal.InvGeoTransform(geomatrix)
if inv_geometrix is None:
raise Exception("Failed InvGeoTransform()")
x, y = \
(inv_geometrix[0] + inv_geometrix[1] * x + inv_geometrix[2] * y), \
(inv_geometrix[3] + inv_geometrix[4] * x + inv_geometrix[5] * y)
xsize, ysize = ds.RasterXSize, ds.RasterYSize
bands = get_bands(ds, band_nums, ovr_idx=ovr_idx)
ovr_xsize, ovr_ysize = bands[0].XSize, bands[0].YSize
pixel_fact, line_fact = (ovr_xsize / xsize,
ovr_ysize / ysize) if ovr_idx else (1, 1)
bnd_count = len(bands)
shape = (bnd_count, point_count)
np_dtype, np_dtype = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
results = np.empty(shape=shape, dtype=np_dtype)
check_outside = not quiet_mode or not allow_xy_outside_extent
if check_outside and (np.any(x < 0) or np.any(x >= xsize) or np.any(y < 0)
or np.any(y >= ysize)):
msg = 'Passed coordinates are not in dataset extent!'
if not allow_xy_outside_extent:
raise Exception(msg)
elif not quiet_mode:
print(msg)
pixels = np.clip(x * pixel_fact + pixel_offset,
0,
ovr_xsize - 1,
out=x if inline_xy_replacement else None)
lines = np.clip(y * line_fact + line_offset,
0,
ovr_ysize - 1,
out=y if inline_xy_replacement else None)
for idx, (pixel, line) in enumerate(zip(pixels, lines)):
for bnd_idx, band in enumerate(bands):
val = band.ReadAsArray(pixel,
line,
1,
1,
resample_alg=resample_alg)
val = val[0][0]
results[bnd_idx][idx] = val
is_scaled, scales, offsets = get_scales_and_offsets(bands)
if is_scaled:
for bnd_idx, scale, offset in enumerate(zip(scales, offsets)):
results[bnd_idx] = results[bnd_idx] * scale + offset
return pixels, lines, results