def copy_raster_and_add_overviews(filename_src: PathLikeOrStr,
output_filename_template: str,
overview_list: Sequence[int],
overview_alg='bilinear',
create_file_per_ovr: bool = True,
driver_name: str = 'GTiff'):
files_list = []
ds_with_ovrs = output_filename_template.format('')
shutil.copy(filename_src, ds_with_ovrs)
files_list.append(ds_with_ovrs)
ds_base = output_filename_template.format(0)
shutil.copy(filename_src, ds_base)
files_list.append(ds_base)
ds = open_ds(ds_with_ovrs, gdal.GA_Update)
size = (ds.RasterXSize, ds.RasterYSize)
ds.BuildOverviews(overview_alg, overviewlist=overview_list)
driver = gdal.GetDriverByName(driver_name)
all_ovrs = [1]
all_ovrs.extend(overview_list)
for ovr_idx, f in enumerate(all_ovrs):
filename_i = output_filename_template.format(ovr_idx)
if ovr_idx == 0:
ds1 = open_ds(filename_i)
else:
ds1 = open_ds(ds_with_ovrs, ovr_idx=ovr_idx, ovr_only=True)
if create_file_per_ovr:
driver.CreateCopy(filename_i, ds1)
files_list.append(filename_i)
assert ds1.RasterXSize == int(size[0] / f) and ds1.RasterYSize == int(
size[1] / f) and ds1.RasterCount == 1
return all_ovrs, files_list
def test_utils_py_1():
""" test get_ovr_idx, create_flat_raster """
filename = 'tmp/raster.tif'
temp_files.append(filename)
raster_creation.create_flat_raster(filename, overview_list=[2, 4])
ds = util.open_ds(filename)
compression = util.get_image_structure_metadata(filename, 'COMPRESSION')
assert compression == 'DEFLATE'
ovr_count = util.get_ovr_count(ds) + 1
assert ovr_count == 3
pixel_size = util.get_pixel_size(ds)
assert pixel_size == (10, -10)
for i in range(-ovr_count, ovr_count):
assert util.get_ovr_idx(filename,
ovr_idx=i) == (i if i >= 0 else ovr_count + i)
for res, ovr in [(5, 0), (10, 0), (11, 0), (19.99, 0), (20, 1), (20.1, 1),
(39, 1), (40, 2), (41, 2), (400, 2)]:
assert util.get_ovr_idx(filename, ovr_res=res) == ovr
assert util.get_ovr_idx(filename,
float(res)) == ovr # noqa secret functionality
# test open_ds with multiple different inputs
filename2 = 'tmp/raster2.tif'
temp_files.append(filename2)
raster_creation.create_flat_raster(filename2)
ds_list = util.open_ds([ds, filename2])
assert tuple(util.get_ovr_count(ds) for ds in ds_list) == (2, 0)
ds_list = None
def get_color_table_from_raster(
path_or_ds: PathOrDS) -> Optional[gdal.ColorTable]:
ds = open_ds(path_or_ds, silent_fail=True)
if ds is not None:
ct = ds.GetRasterBand(1).GetRasterColorTable()
return ct.Clone()
return None
def calc_fwd(self, filename_or_ds, ovr_idx):
self.ox = np.array(self.ox, dtype=np.float32)
self.oy = np.array(self.oy, dtype=np.float32)
abs_oz = np.array(self.oz, dtype=np.float32)
ds = open_ds(filename_or_ds)
az = np.array(self.get_grid_azimuth(), dtype=np.float32)
el = np.array(self.elevation, dtype=np.float32)
r = np.array(self.max_r, dtype=np.float32) if len(
self.max_r) == len(el) else np.full_like(el, self.max_r[0])
a = (90 - az) * M_PI_180
e = el * M_PI_180
ground_r = r * np.cos(e)
self.tmsl = True
if not self.omsl:
_pixels, _lines, alts = gdallocationinfo(
ds,
band_nums=1,
x=self.ox,
y=self.oy,
srs=LocationInfoSRS.SameAsDS_SRS,
inline_xy_replacement=False,
ovr_idx=ovr_idx,
axis_order=osr.OAMS_TRADITIONAL_GIS_ORDER)
abs_oz = abs_oz + alts
earth_d = 6378137.0 * 2
earth_curvature = (1 - self.refraction_coeff) / earth_d
self.tz = abs_oz + r * np.sin(
e) + ground_r * ground_r * earth_curvature
self.tx = self.ox + np.cos(a) * ground_r
self.ty = self.oy + np.sin(a) * ground_r
def gdallocationinfo_util(filename_or_ds: PathOrDS,
x: ArrayOrScalarLike,
y: ArrayOrScalarLike,
open_options: Optional[dict] = None,
band_nums: Optional[Sequence[int]] = None,
resample_alg=gdalconst.GRIORA_NearestNeighbour,
output_mode: Optional[LocationInfoOutput] = None,
**kwargs):
if output_mode is None:
output_mode = LocationInfoOutput.Quiet
if output_mode in [LocationInfoOutput.XML, LocationInfoOutput.LifOnly]:
raise Exception(
f'Sorry, output mode {output_mode} is not implemented yet. you may use the c++ version.'
)
quiet_mode = output_mode == LocationInfoOutput.Quiet
print_mode = output_mode in \
[LocationInfoOutput.ValOnly, LocationInfoOutput.PixelLineVal, LocationInfoOutput.PixelLineValVerbose]
inline_xy_replacement = not print_mode
ds = open_ds(filename_or_ds, open_options=open_options)
band_nums = get_band_nums(ds, band_nums)
x, y, results = gdallocationinfo(
filename_or_ds=ds,
x=x,
y=y,
band_nums=band_nums,
resample_alg=resample_alg,
inline_xy_replacement=inline_xy_replacement,
quiet_mode=quiet_mode,
**kwargs)
xsize, ysize = ds.RasterXSize, ds.RasterYSize
is_nearest_neighbour = resample_alg == gdal.GRIORA_NearestNeighbour
if print_mode:
if output_mode == LocationInfoOutput.PixelLineValVerbose:
print('Report:')
for idx, (pixel, line) in enumerate(zip(x, y)):
if not quiet_mode and pixel < 0 or pixel >= xsize or line < 0 or line >= ysize:
print(f'Location {pixel} {line} is off this file!')
else:
if is_nearest_neighbour:
pixel, line = int(pixel), int(line)
if output_mode == LocationInfoOutput.PixelLineValVerbose:
print(f' Location: ({pixel}P,{line}L)')
for bnd_idx, band_num in enumerate(band_nums):
val = results[bnd_idx][idx]
if output_mode == LocationInfoOutput.ValOnly:
print(val)
elif output_mode == LocationInfoOutput.PixelLineVal:
print(f'{pixel} {line} {val}')
elif output_mode == LocationInfoOutput.PixelLineValVerbose:
print(f' Band {band_num}:')
print(f' Value: {val}')
return results
def doit(src_filename,
pct_filename: Optional[ColorTableLike],
dst_filename: Optional[PathLikeOrStr] = None,
driver_name: Optional[str] = None):
# =============================================================================
# Get the PCT.
# =============================================================================
ct = get_color_table(pct_filename)
if pct_filename is not None and ct is None:
print('No color table on file ', pct_filename)
return None, 1
# =============================================================================
# Create a MEM clone of the source file.
# =============================================================================
src_ds = open_ds(src_filename)
mem_ds = gdal.GetDriverByName('MEM').CreateCopy('mem', src_ds)
# =============================================================================
# Assign the color table in memory.
# =============================================================================
mem_ds.GetRasterBand(1).SetRasterColorTable(ct)
mem_ds.GetRasterBand(1).SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
# =============================================================================
# Write the dataset to the output file.
# =============================================================================
if not driver_name:
driver_name = GetOutputDriverFor(dst_filename)
dst_driver = gdal.GetDriverByName(driver_name)
if dst_driver is None:
print('"%s" driver not registered.' % driver_name)
return None, 1
if driver_name.upper() == 'MEM':
out_ds = mem_ds
else:
out_ds = dst_driver.CreateCopy(dst_filename or '', mem_ds)
mem_ds = None
src_ds = None
return out_ds, 0
def test_utils_py_1():
""" test get_ovr_idx, create_flat_raster """
filename = 'tmp/raster.tif'
temp_files.append(filename)
raster_creation.create_flat_raster(filename, overview_list=[2, 4])
ds = util.open_ds(filename)
compression = util.get_image_structure_metadata(filename, 'COMPRESSION')
assert compression == 'DEFLATE'
ras_count = util.get_ovr_count(ds) + 1
assert ras_count == 3
pixel_size = util.get_pixel_size(ds)
assert pixel_size == (10, -10)
for i in range(-ras_count, ras_count):
assert util.get_ovr_idx(filename,
ovr_idx=i) == (i if i >= 0 else ras_count + i)
for res, ovr in [(5, 0), (10, 0), (11, 0), (19.99, 0), (20, 1), (20.1, 1),
(39, 1), (40, 2), (41, 2), (400, 2)]:
assert util.get_ovr_idx(filename, ovr_res=res) == ovr
assert util.get_ovr_idx(
filename, ovr_idx=float(res)) == ovr # noqa secret functionality
def gdallocationinfo(
filename_or_ds: PathOrDS,
x: ArrayOrScalarLike,
y: ArrayOrScalarLike,
srs: CoordinateTransformationOrSRS = None,
axis_order: Optional[OAMS_AXIS_ORDER] = None,
open_options: Optional[dict] = None,
ovr_idx: Optional[int] = None,
band_nums: Optional[Sequence[int]] = None,
inline_xy_replacement: bool = False,
return_ovr_pixel_line: bool = False,
transform_round_digits: Optional[float] = None,
allow_xy_outside_extent: bool = True,
pixel_offset: Real = -0.5,
line_offset: Real = -0.5,
resample_alg=gdalconst.GRIORA_NearestNeighbour,
quiet_mode: bool = True,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
ds = open_ds(filename_or_ds, open_options=open_options)
filename = filename_or_ds if is_path_like(filename_or_ds) else ''
if ds is None:
raise Exception(f'Could not open {filename}.')
if not isinstance(x, ArrayLike.__args__):
x = [x]
if not isinstance(y, ArrayLike.__args__):
y = [y]
if len(x) != len(y):
raise Exception(
f'len(x)={len(x)} should be the same as len(y)={len(y)}')
point_count = len(x)
dtype = np.float64
if not isinstance(x, np.ndarray) or not isinstance(y, np.ndarray):
x = np.array(x, dtype=dtype)
y = np.array(y, dtype=dtype)
inline_xy_replacement = True
if srs is None:
srs = LocationInfoSRS.PixelLine
# Build Spatial Reference object based on coordinate system, fetched from the opened dataset
if srs != LocationInfoSRS.PixelLine:
if srs != LocationInfoSRS.SameAsDS_SRS:
ds_srs = ds.GetSpatialRef()
ct = None
if isinstance(srs, osr.CoordinateTransformation):
ct = srs
else:
if srs == LocationInfoSRS.SameAsDS_SRS_GeogCS:
points_srs = ds_srs.CloneGeogCS()
else:
points_srs = get_srs(srs, axis_order=axis_order)
ct = get_transform(points_srs, ds_srs)
if ct is not None:
if not inline_xy_replacement:
x = x.copy()
y = y.copy()
inline_xy_replacement = True
transform_points(ct, x, y)
if transform_round_digits is not None:
x.round(transform_round_digits, out=x)
y.round(transform_round_digits, out=y)
# Read geotransform matrix and calculate corresponding pixel coordinates
geotransform = ds.GetGeoTransform()
inv_geotransform = gdal.InvGeoTransform(geotransform)
if inv_geotransform is None:
raise Exception("Failed InvGeoTransform()")
# can we inline this transformation ?
x, y = \
(inv_geotransform[0] + inv_geotransform[1] * x + inv_geotransform[2] * y), \
(inv_geotransform[3] + inv_geotransform[4] * x + inv_geotransform[5] * y)
inline_xy_replacement = True
xsize, ysize = ds.RasterXSize, ds.RasterYSize
bands = get_bands(ds, band_nums, ovr_idx=ovr_idx)
ovr_xsize, ovr_ysize = bands[0].XSize, bands[0].YSize
pixel_fact, line_fact = (ovr_xsize / xsize,
ovr_ysize / ysize) if ovr_idx else (1, 1)
bnd_count = len(bands)
shape = (bnd_count, point_count)
np_dtype, np_dtype = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
results = np.empty(shape=shape, dtype=np_dtype)
check_outside = not quiet_mode or not allow_xy_outside_extent
if check_outside and (np.any(x < 0) or np.any(x >= xsize) or np.any(y < 0)
or np.any(y >= ysize)):
msg = 'Passed coordinates are not in dataset extent!'
if not allow_xy_outside_extent:
raise Exception(msg)
elif not quiet_mode:
print(msg)
if pixel_fact == 1:
pixels_q = x
elif return_ovr_pixel_line:
x *= pixel_fact
pixels_q = x
else:
pixels_q = x * pixel_fact
if line_fact == 1:
lines_q = y
elif return_ovr_pixel_line:
y *= line_fact
lines_q = y
else:
lines_q = y * line_fact
buf_xsize = buf_ysize = 1
buf_type, typecode = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
buf_obj = np.empty([buf_ysize, buf_xsize], dtype=typecode)
for idx, (pixel, line) in enumerate(zip(pixels_q, lines_q)):
for bnd_idx, band in enumerate(bands):
if BandRasterIONumPy(band, 0, pixel - 0.5, line - 0.5, 1, 1,
buf_obj, buf_type, resample_alg, None,
None) == 0:
results[bnd_idx][idx] = buf_obj[0][0]
is_scaled, scales, offsets = get_scales_and_offsets(bands)
if is_scaled:
for bnd_idx, (scale, offset) in enumerate(zip(scales, offsets)):
results[bnd_idx] = results[bnd_idx] * scale + offset
return x, y, results
def Calc(calc: MaybeSequence[str], outfile: Optional[PathLikeOrStr] = None, NoDataValue: Optional[Number] = None,
type: Optional[Union[GDALDataType, str]] = None, format: Optional[str] = None,
creation_options: Optional[Sequence[str]] = None, allBands: str = '', overwrite: bool = False,
hideNoData: bool = False, projectionCheck: bool = False,
color_table: Optional[ColorTableLike] = None,
extent: Optional[Extent] = None, projwin: Optional[Union[Tuple, GeoRectangle]] = None,
user_namespace: Optional[Dict]=None,
debug: bool = False, quiet: bool = False, **input_files):
if debug:
print(f"gdal_calc.py starting calculation {calc}")
# Single calc value compatibility
if isinstance(calc, (list, tuple)):
calc = calc
else:
calc = [calc]
calc = [c.strip('"') for c in calc]
creation_options = creation_options or []
# set up global namespace for eval with all functions of gdal_array, numpy
global_namespace = {key: getattr(module, key)
for module in [gdal_array, numpy] for key in dir(module) if not key.startswith('__')}
if user_namespace:
global_namespace.update(user_namespace)
if not calc:
raise Exception("No calculation provided.")
elif not outfile and format.upper() != 'MEM':
raise Exception("No output file provided.")
if format is None:
format = GetOutputDriverFor(outfile)
if isinstance(extent, GeoRectangle):
pass
elif projwin:
if isinstance(projwin, GeoRectangle):
extent = projwin
else:
extent = GeoRectangle.from_lurd(*projwin)
elif not extent:
extent = Extent.IGNORE
else:
extent = extent_util.parse_extent(extent)
compatible_gt_eps = 0.000001
gt_diff_support = {
GT.INCOMPATIBLE_OFFSET: extent != Extent.FAIL,
GT.INCOMPATIBLE_PIXEL_SIZE: False,
GT.INCOMPATIBLE_ROTATION: False,
GT.NON_ZERO_ROTATION: False,
}
gt_diff_error = {
GT.INCOMPATIBLE_OFFSET: 'different offset',
GT.INCOMPATIBLE_PIXEL_SIZE: 'different pixel size',
GT.INCOMPATIBLE_ROTATION: 'different rotation',
GT.NON_ZERO_ROTATION: 'non zero rotation',
}
################################################################
# fetch details of input layers
################################################################
# set up some lists to store data for each band
myFileNames = [] # input filenames
myFiles = [] # input DataSets
myBands = [] # input bands
myAlphaList = [] # input alpha letter that represents each input file
myDataType = [] # string representation of the datatype of each input file
myDataTypeNum = [] # datatype of each input file
myNDV = [] # nodatavalue for each input file
DimensionsCheck = None # dimensions of the output
Dimensions = [] # Dimensions of input files
ProjectionCheck = None # projection of the output
GeoTransformCheck = None # GeoTransform of the output
GeoTransforms = [] # GeoTransform of each input file
GeoTransformDiffer = False # True if we have inputs with different GeoTransforms
myTempFileNames = [] # vrt filename from each input file
myAlphaFileLists = [] # list of the Alphas which holds a list of inputs
# loop through input files - checking dimensions
for alphas, filenames in input_files.items():
if isinstance(filenames, (list, tuple)):
# alpha is a list of files
myAlphaFileLists.append(alphas)
elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset):
# alpha is a single filename or a Dataset
filenames = [filenames]
alphas = [alphas]
else:
# I guess this alphas should be in the global_namespace,
# It would have been better to pass it as user_namepsace, but I'll accept it anyway
global_namespace[alphas] = filenames
continue
for alpha, filename in zip(alphas * len(filenames), filenames):
if not alpha.endswith("_band"):
# check if we have asked for a specific band...
alpha_band = f"{alpha}_band"
if alpha_band in input_files:
myBand = input_files[alpha_band]
else:
myBand = 1
myF_is_ds = not is_path_like(filename)
if myF_is_ds:
myFile = filename
filename = None
else:
myFile = open_ds(filename, gdal.GA_ReadOnly)
if not myFile:
raise IOError(f"No such file or directory: '{filename}'")
myFileNames.append(filename)
myFiles.append(myFile)
myBands.append(myBand)
myAlphaList.append(alpha)
dt = myFile.GetRasterBand(myBand).DataType
myDataType.append(gdal.GetDataTypeName(dt))
myDataTypeNum.append(dt)
myNDV.append(None if hideNoData else myFile.GetRasterBand(myBand).GetNoDataValue())
# check that the dimensions of each layer are the same
myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize]
if DimensionsCheck:
if DimensionsCheck != myFileDimensions:
GeoTransformDiffer = True
if extent in [Extent.IGNORE, Extent.FAIL]:
raise Exception(
f"Error! Dimensions of file {filename} ({myFileDimensions[0]:d}, "
f"{myFileDimensions[1]:d}) are different from other files "
f"({DimensionsCheck[0]:d}, {DimensionsCheck[1]:d}). Cannot proceed")
else:
DimensionsCheck = myFileDimensions
# check that the Projection of each layer are the same
myProjection = myFile.GetProjection()
if ProjectionCheck:
if projectionCheck and ProjectionCheck != myProjection:
raise Exception(
f"Error! Projection of file {filename} {myProjection} "
f"are different from other files {ProjectionCheck}. Cannot proceed")
else:
ProjectionCheck = myProjection
# check that the GeoTransforms of each layer are the same
myFileGeoTransform = myFile.GetGeoTransform(can_return_null=True)
if extent == Extent.IGNORE:
GeoTransformCheck = myFileGeoTransform
else:
Dimensions.append(myFileDimensions)
GeoTransforms.append(myFileGeoTransform)
if not GeoTransformCheck:
GeoTransformCheck = myFileGeoTransform
else:
my_gt_diff = extent_util.gt_diff(GeoTransformCheck, myFileGeoTransform, eps=compatible_gt_eps,
diff_support=gt_diff_support)
if my_gt_diff not in [GT.SAME, GT.ALMOST_SAME]:
GeoTransformDiffer = True
if my_gt_diff != GT.COMPATIBLE_DIFF:
raise Exception(
f"Error! GeoTransform of file {filename} {myFileGeoTransform} is incompatible "
f"({gt_diff_error[my_gt_diff]}), first file GeoTransform is {GeoTransformCheck}. "
f"Cannot proceed")
if debug:
print(
f"file {alpha}: {filename}, dimensions: "
f"{DimensionsCheck[0]}, {DimensionsCheck[1]}, type: {myDataType[-1]}")
# process allBands option
allBandsIndex = None
allBandsCount = 1
if allBands:
if len(calc) > 1:
raise Exception("Error! --allBands implies a single --calc")
try:
allBandsIndex = myAlphaList.index(allBands)
except ValueError:
raise Exception(f"Error! allBands option was given but Band {allBands} not found. Cannot proceed")
allBandsCount = myFiles[allBandsIndex].RasterCount
if allBandsCount <= 1:
allBandsIndex = None
else:
allBandsCount = len(calc)
if extent not in [Extent.IGNORE, Extent.FAIL] and (
GeoTransformDiffer or isinstance(extent, GeoRectangle)):
# mixing different GeoTransforms/Extents
GeoTransformCheck, DimensionsCheck, ExtentCheck = extent_util.calc_geotransform_and_dimensions(
GeoTransforms, Dimensions, extent)
if GeoTransformCheck is None:
raise Exception("Error! The requested extent is empty. Cannot proceed")
for i in range(len(myFileNames)):
temp_vrt_filename, temp_vrt_ds = extent_util.make_temp_vrt(myFiles[i], ExtentCheck)
myTempFileNames.append(temp_vrt_filename)
myFiles[i] = None # close original ds
myFiles[i] = temp_vrt_ds # replace original ds with vrt_ds
# update the new precise dimensions and gt from the new ds
GeoTransformCheck = temp_vrt_ds.GetGeoTransform()
DimensionsCheck = [temp_vrt_ds.RasterXSize, temp_vrt_ds.RasterYSize]
temp_vrt_ds = None
################################################################
# set up output file
################################################################
# open output file exists
if outfile and os.path.isfile(outfile) and not overwrite:
if allBandsIndex is not None:
raise Exception("Error! allBands option was given but Output file exists, must use --overwrite option!")
if len(calc) > 1:
raise Exception(
"Error! multiple calc options were given but Output file exists, must use --overwrite option!")
if debug:
print(f"Output file {outfile} exists - filling in results into file")
myOut = open_ds(outfile, gdal.GA_Update)
if myOut is None:
error = 'but cannot be opened for update'
elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck:
error = 'but is the wrong size'
elif ProjectionCheck and ProjectionCheck != myOut.GetProjection():
error = 'but is the wrong projection'
elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform(can_return_null=True):
error = 'but is the wrong geotransform'
else:
error = None
if error:
raise Exception(
f"Error! Output exists, {error}. Use the --overwrite option "
f"to automatically overwrite the existing file")
myOutB = myOut.GetRasterBand(1)
myOutNDV = myOutB.GetNoDataValue()
myOutType = myOutB.DataType
else:
if outfile:
# remove existing file and regenerate
if os.path.isfile(outfile):
os.remove(outfile)
# create a new file
if debug:
print(f"Generating output file {outfile}")
else:
outfile = ''
# find data type to use
if not type:
# use the largest type of the input files
myOutType = max(myDataTypeNum)
else:
myOutType = type
if isinstance(myOutType, str):
myOutType = gdal.GetDataTypeByName(myOutType)
# create file
myOutDrv = gdal.GetDriverByName(format)
myOut = myOutDrv.Create(
os.fspath(outfile), DimensionsCheck[0], DimensionsCheck[1], allBandsCount,
myOutType, creation_options)
# set output geo info based on first input layer
if not GeoTransformCheck:
GeoTransformCheck = myFiles[0].GetGeoTransform(can_return_null=True)
if GeoTransformCheck:
myOut.SetGeoTransform(GeoTransformCheck)
if not ProjectionCheck:
ProjectionCheck = myFiles[0].GetProjection()
if ProjectionCheck:
myOut.SetProjection(ProjectionCheck)
if NoDataValue is None:
myOutNDV = None if hideNoData else DefaultNDVLookup[
myOutType] # use the default noDataValue for this datatype
elif isinstance(NoDataValue, str) and NoDataValue.lower() == 'none':
myOutNDV = None # not to set any noDataValue
else:
myOutNDV = NoDataValue # use the given noDataValue
for i in range(1, allBandsCount + 1):
myOutB = myOut.GetRasterBand(i)
if myOutNDV is not None:
myOutB.SetNoDataValue(myOutNDV)
if color_table:
# set color table and color interpretation
if is_path_like(color_table):
color_table = get_color_table(color_table)
myOutB.SetRasterColorTable(color_table)
myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
myOutB = None # write to band
myOutTypeName = gdal.GetDataTypeName(myOutType)
if debug:
print(f"output file: {outfile}, dimensions: {myOut.RasterXSize}, {myOut.RasterYSize}, type: {myOutTypeName}")
################################################################
# find block size to chop grids into bite-sized chunks
################################################################
# use the block size of the first layer to read efficiently
myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize()
# find total x and y blocks to be read
nXBlocks = (int)((DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0])
nYBlocks = (int)((DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1])
myBufSize = myBlockSize[0] * myBlockSize[1]
if debug:
print(f"using blocksize {myBlockSize[0]} x {myBlockSize[1]}")
# variables for displaying progress
ProgressCt = -1
ProgressMk = -1
ProgressEnd = nXBlocks * nYBlocks * allBandsCount
################################################################
# start looping through each band in allBandsCount
################################################################
for bandNo in range(1, allBandsCount + 1):
################################################################
# start looping through blocks of data
################################################################
# store these numbers in variables that may change later
nXValid = myBlockSize[0]
nYValid = myBlockSize[1]
# loop through X-lines
for X in range(0, nXBlocks):
# in case the blocks don't fit perfectly
# change the block size of the final piece
if X == nXBlocks - 1:
nXValid = DimensionsCheck[0] - X * myBlockSize[0]
# find X offset
myX = X * myBlockSize[0]
# reset buffer size for start of Y loop
nYValid = myBlockSize[1]
myBufSize = nXValid * nYValid
# loop through Y lines
for Y in range(0, nYBlocks):
ProgressCt += 1
if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not quiet:
ProgressMk = 10 * ProgressCt / ProgressEnd % 10
from sys import version_info
if version_info >= (3, 0, 0):
exec('print("%d.." % (10*ProgressMk), end=" ")')
else:
exec('print 10*ProgressMk, "..",')
# change the block size of the final piece
if Y == nYBlocks - 1:
nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
myBufSize = nXValid * nYValid
# find Y offset
myY = Y * myBlockSize[1]
# create empty buffer to mark where nodata occurs
myNDVs = None
# make local namespace for calculation
local_namespace = {}
val_lists = defaultdict(list)
# fetch data for each input layer
for i, Alpha in enumerate(myAlphaList):
# populate lettered arrays with values
if allBandsIndex is not None and allBandsIndex == i:
myBandNo = bandNo
else:
myBandNo = myBands[i]
myval = gdal_array.BandReadAsArray(myFiles[i].GetRasterBand(myBandNo),
xoff=myX, yoff=myY,
win_xsize=nXValid, win_ysize=nYValid)
if myval is None:
raise Exception(f'Input block reading failed from filename {filename[i]}')
# fill in nodata values
if myNDV[i] is not None:
# myNDVs is a boolean buffer.
# a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands.
if myNDVs is None:
# this is the first band that has NDV set. we initializes myNDVs to a zero buffer
# as we didn't see any NDV value yet.
myNDVs = numpy.zeros(myBufSize)
myNDVs.shape = (nYValid, nXValid)
myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i])
# add an array of values for this block to the eval namespace
if Alpha in myAlphaFileLists:
val_lists[Alpha].append(myval)
else:
local_namespace[Alpha] = myval
myval = None
for lst in myAlphaFileLists:
local_namespace[lst] = val_lists[lst]
# try the calculation on the array blocks
this_calc = calc[bandNo - 1 if len(calc) > 1 else 0]
try:
myResult = eval(this_calc, global_namespace, local_namespace)
except:
print(f"evaluation of calculation {this_calc} failed")
raise
# Propagate nodata values (set nodata cells to zero
# then add nodata value to these cells).
if myNDVs is not None and myOutNDV is not None:
myResult = ((1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs)
elif not isinstance(myResult, numpy.ndarray):
myResult = numpy.ones((nYValid, nXValid)) * myResult
# write data block to the output file
myOutB = myOut.GetRasterBand(bandNo)
if gdal_array.BandWriteArray(myOutB, myResult, xoff=myX, yoff=myY) != 0:
raise Exception('Block writing failed')
myOutB = None # write to band
# remove temp files
for idx, tempFile in enumerate(myTempFileNames):
myFiles[idx] = None
os.remove(tempFile)
gdal.ErrorReset()
myOut.FlushCache()
if gdal.GetLastErrorMsg() != '':
raise Exception('Dataset writing failed')
if not quiet:
print("100 - Done")
return myOut
def gdal2xyz(
srcfile: PathOrDS,
dstfile: PathLikeOrStr = None,
srcwin: Optional[Sequence[int]] = None,
skip: Union[int, Sequence[int]] = 1,
band_nums: Optional[Sequence[int]] = None,
delim: str = ' ',
skip_nodata: bool = False,
src_nodata: Optional[Union[Sequence, Number]] = None,
dst_nodata: Optional[Union[Sequence, Number]] = None,
return_np_arrays: bool = False,
pre_allocate_np_arrays: bool = True,
progress_callback: OptionalProgressCallback = ...) -> Optional[Tuple]:
"""
translates a raster file (or dataset) into xyz format
skip - how many rows/cols to skip each iteration
srcwin (xoff, yoff, xsize, ysize) - Selects a subwindow from the source image for copying based on pixel/line location.
band_nums - selected input bands to process, None to process all.
delim - the delimiter to use between values in a line
skip_nodata - Exclude the output lines with nodata value (as determined by srcnodata)
src_nodata - The nodata value of the dataset (for skipping or replacing)
default (`None`) - Use the dataset NoDataValue;
`Sequence`/`Number` - use the given nodata value (per band or per dataset).
dst_nodata - Replace source nodata with a given nodata. Has an effect only if not setting `-skipnodata`
default(`None`) - use srcnodata, no replacement;
`Sequence`/`Number` - replace the `srcnodata` with the given nodata value (per band or per dataset).
srcfile - The source dataset filename or dataset object
dstfile - The output dataset filename; for dstfile=None - if return_np_arrays=False then output will be printed to stdout
return_np_arrays - return numpy arrays of the result, otherwise returns None
pre_allocate_np_arrays - pre-allocated result arrays.
Should be faster unless skip_nodata and the input is very sparse thus most data points will be skipped.
progress_callback - progress callback function. use None for quiet or Ellipsis for using the default callback
"""
result = None
progress_callback = get_progress_callback(progress_callback)
# Open source file.
ds = open_ds(srcfile)
if ds is None:
raise Exception(f'Could not open {srcfile}.')
bands = get_bands(ds, band_nums)
band_count = len(bands)
gt = ds.GetGeoTransform()
# Collect information on all the source files.
if srcwin is None:
srcwin = (0, 0, ds.RasterXSize, ds.RasterYSize)
dt, np_dt = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
# Open the output file.
if dstfile is not None:
dst_fh = open(dstfile, 'wt')
elif return_np_arrays:
dst_fh = None
else:
dst_fh = sys.stdout
if dst_fh:
if dt == gdal.GDT_Int32 or dt == gdal.GDT_UInt32:
band_format = (("%d" + delim) * len(bands)).rstrip(delim) + '\n'
else:
band_format = (("%g" + delim) * len(bands)).rstrip(delim) + '\n'
# Setup an appropriate print format.
if abs(gt[0]) < 180 and abs(gt[3]) < 180 \
and abs(ds.RasterXSize * gt[1]) < 180 \
and abs(ds.RasterYSize * gt[5]) < 180:
frmt = '%.10g' + delim + '%.10g' + delim + '%s'
else:
frmt = '%.3f' + delim + '%.3f' + delim + '%s'
if isinstance(src_nodata, Number):
src_nodata = [src_nodata] * band_count
elif src_nodata is None:
src_nodata = list(band.GetNoDataValue() for band in bands)
if None in src_nodata:
src_nodata = None
if src_nodata is not None:
src_nodata = np.asarray(src_nodata, dtype=np_dt)
if isinstance(dst_nodata, Number):
dst_nodata = [dst_nodata] * band_count
if (dst_nodata is None) or (None in dst_nodata) or (src_nodata is None):
dst_nodata = None
if dst_nodata is not None:
dst_nodata = np.asarray(dst_nodata, dtype=np_dt)
skip_nodata = skip_nodata and (src_nodata is not None)
replace_nodata = (not skip_nodata) and (dst_nodata is not None)
process_nodata = skip_nodata or replace_nodata
if isinstance(skip, Sequence):
x_skip, y_skip = skip
else:
x_skip = y_skip = skip
x_off, y_off, x_size, y_size = srcwin
bands_count = len(bands)
nXBlocks = (x_size - x_off) // x_skip
nYBlocks = (y_size - y_off) // y_skip
progress_end = nXBlocks * nYBlocks
progress_curr = 0
progress_prev = -1
progress_parts = 100
if return_np_arrays:
size = progress_end if pre_allocate_np_arrays else 0
all_geo_x = np.empty(size)
all_geo_y = np.empty(size)
all_data = np.empty((size, band_count), dtype=np_dt)
# Loop emitting data.
idx = 0
for y in range(y_off, y_off + y_size, y_skip):
size = bands_count if pre_allocate_np_arrays else 0
data = np.empty((size, x_size),
dtype=np_dt) # dims: (bands_count, x_size)
for i_bnd, band in enumerate(bands):
band_data = band.ReadAsArray(x_off, y, x_size,
1) # read one band line
if pre_allocate_np_arrays:
data[i_bnd] = band_data[0]
else:
data = np.append(data, band_data, axis=0)
for x_i in range(0, x_size, x_skip):
progress_curr += 1
if progress_callback:
progress_frac = progress_curr / progress_end
progress = int(progress_frac * progress_parts)
if progress > progress_prev:
progress_prev = progress
progress_callback(progress_frac)
x_i_data = data[:, x_i] # single pixel, dims: (bands)
if process_nodata and np.array_equal(src_nodata, x_i_data):
if skip_nodata:
continue
elif replace_nodata:
x_i_data = dst_nodata
x = x_i + x_off
geo_x = gt[0] + (x + 0.5) * gt[1] + (y + 0.5) * gt[2]
geo_y = gt[3] + (x + 0.5) * gt[4] + (y + 0.5) * gt[5]
if dst_fh:
band_str = band_format % tuple(x_i_data)
line = frmt % (float(geo_x), float(geo_y), band_str)
dst_fh.write(line)
if return_np_arrays:
if pre_allocate_np_arrays:
all_geo_x[idx] = geo_x
all_geo_y[idx] = geo_y
all_data[idx] = x_i_data
else:
all_geo_x = np.append(all_geo_x, geo_x)
all_geo_y = np.append(all_geo_y, geo_y)
all_data = np.append(all_data, [x_i_data], axis=0)
idx += 1
if return_np_arrays:
nodata = None if skip_nodata else dst_nodata if replace_nodata else src_nodata
if idx != progress_curr:
all_geo_x = all_geo_x[:idx]
all_geo_y = all_geo_y[:idx]
all_data = all_data[:idx, :]
result = all_geo_x, all_geo_y, all_data.transpose(), nodata
return result
def pct2rgb(src_filename: PathLikeOrStr, pct_filename: Optional[PathLikeOrStr], dst_filename: PathLikeOrStr,
band_number: int = 1, out_bands: int = 3, driver_name: Optional[str] = None):
# Open source file
src_ds = open_ds(src_filename)
if src_ds is None:
raise Exception(f'Unable to open {src_filename} ')
src_band = src_ds.GetRasterBand(band_number)
# ----------------------------------------------------------------------------
# Ensure we recognise the driver.
if driver_name is None:
driver_name = GetOutputDriverFor(dst_filename)
dst_driver = gdal.GetDriverByName(driver_name)
if dst_driver is None:
raise Exception(f'"{driver_name}" driver not registered.')
# ----------------------------------------------------------------------------
# Build color table.
if pct_filename is not None:
pal = get_color_palette(pct_filename)
if pal.has_percents():
min_val = src_band.GetMinimum()
max_val = src_band.GetMinimum()
pal.apply_percent(min_val, max_val)
ct = get_color_table(pal)
else:
ct = src_band.GetRasterColorTable()
ct_size = ct.GetCount()
lookup = [np.arange(ct_size),
np.arange(ct_size),
np.arange(ct_size),
np.ones(ct_size) * 255]
if ct is not None:
for i in range(ct_size):
entry = ct.GetColorEntry(i)
for c in range(4):
lookup[c][i] = entry[c]
# ----------------------------------------------------------------------------
# Create the working file.
if driver_name.lower() == 'gtiff':
tif_filename = dst_filename
else:
tif_filename = 'temp.tif'
gtiff_driver = gdal.GetDriverByName('GTiff')
tif_ds = gtiff_driver.Create(tif_filename, src_ds.RasterXSize, src_ds.RasterYSize, out_bands)
# ----------------------------------------------------------------------------
# We should copy projection information and so forth at this point.
tif_ds.SetProjection(src_ds.GetProjection())
tif_ds.SetGeoTransform(src_ds.GetGeoTransform())
if src_ds.GetGCPCount() > 0:
tif_ds.SetGCPs(src_ds.GetGCPs(), src_ds.GetGCPProjection())
# ----------------------------------------------------------------------------
# Do the processing one scanline at a time.
progress(0.0)
for iY in range(src_ds.RasterYSize):
src_data = src_band.ReadAsArray(0, iY, src_ds.RasterXSize, 1)
for iBand in range(out_bands):
band_lookup = lookup[iBand]
dst_data = np.take(band_lookup, src_data)
tif_ds.GetRasterBand(iBand + 1).WriteArray(dst_data, 0, iY)
progress((iY + 1.0) / src_ds.RasterYSize)
# ----------------------------------------------------------------------------
# Translate intermediate file to output format if desired format is not TIFF.
if tif_filename == dst_filename:
dst_ds = tif_ds
else:
dst_ds = dst_driver.CreateCopy(dst_filename or '', tif_ds)
tif_ds = None
gtiff_driver.Delete(tif_filename)
return dst_ds
def test_gdallocationinfo_py_7():
filename_template = 'tmp/byte{}.tif'
overview_list = [2]
overview_list, file_list = copy_raster_and_add_overviews(
filename_src='../gcore/data/byte.tif',
output_filename_template=filename_template,
overview_list=overview_list)
temp_files.extend(file_list)
ds = open_ds(filename_template.format(''))
ds_srs = ds.GetSpatialRef()
gt = ds.GetGeoTransform()
ovr_fact = 1
pix_offset = 0
p0 = list(i * ovr_fact + pix_offset for i in range(0, 3))
l0 = list(i * ovr_fact + pix_offset for i in range(0, 3))
p0, l0 = zip(*product(p0, l0))
x0 = list(gt[0] + pixel * gt[1] for pixel in p0)
y0 = list(gt[3] + line * gt[5] for line in l0)
dtype = np.float64
p0 = np.array(p0, dtype=dtype)
l0 = np.array(l0, dtype=dtype)
x0 = np.array(x0, dtype=dtype)
y0 = np.array(y0, dtype=dtype)
expected_results = [[(0.0, 0.0, 107), (0.0, 1.0, 115), (0.0, 2.0, 115),
(1.0, 0.0, 123), (1.0, 1.0, 132), (1.0, 2.0, 132),
(2.0, 0.0, 132), (2.0, 1.0, 107), (2.0, 2.0, 140)],
[(0.0, 0.0, 120), (0.0, 0.5, 120), (0.0, 1.0, 132),
(0.5, 0.0, 120), (0.5, 0.5, 120), (0.5, 1.0, 132),
(1.0, 0.0, 124), (1.0, 0.5, 124), (1.0, 1.0, 129)]]
expected_results = np.array(expected_results, dtype=dtype)
do_print = False
for ovr_idx, f in enumerate(overview_list):
for srs in [
LocationInfoSRS.PixelLine, LocationInfoSRS.SameAsDS_SRS, 4326
]:
if srs == LocationInfoSRS.PixelLine:
x = p0
y = l0
elif srs == LocationInfoSRS.SameAsDS_SRS:
x = x0
y = y0
else:
points_srs = get_srs(srs,
axis_order=osr.OAMS_TRADITIONAL_GIS_ORDER)
ct = get_transform(ds_srs, points_srs)
x = x0.copy()
y = y0.copy()
transform_points(ct, x, y)
pixels, lines, results = gdallocationinfo.gdallocationinfo(
filename_or_ds=ds,
ovr_idx=ovr_idx,
x=x,
y=y,
transform_round_digits=2,
return_ovr_pixel_line=True,
srs=srs,
axis_order=osr.OAMS_TRADITIONAL_GIS_ORDER)
actual = list(zip(pixels, lines, *results))
actual = np.array(actual, dtype=dtype)
expected = expected_results[ovr_idx]
if do_print:
print(actual)
print(expected)
outputs = list(zip(x, y, pixels, lines, *results))
print(
f'ovr: {ovr_idx}, srs: {srs}, x/y/pixel/line/result: {outputs}'
)
assert_allclose(expected, actual, rtol=1e-4, atol=1e-3)
def rgb2pct(src_filename: PathLikeOrStr,
pct_filename: Optional[PathLikeOrStr] = None,
dst_filename: Optional[PathLikeOrStr] = None,
color_count: int = 256,
driver_name: Optional[str] = None):
# Open source file
src_ds = open_ds(src_filename)
if src_ds is None:
raise Exception(f'Unable to open {src_filename}')
if src_ds.RasterCount < 3:
raise Exception(
f'{src_filename} has {src_ds.RasterCount} band(s), need 3 for inputs red, green and blue.'
)
# Ensure we recognise the driver.
if not driver_name:
driver_name = GetOutputDriverFor(dst_filename)
dst_driver = gdal.GetDriverByName(driver_name)
if dst_driver is None:
raise Exception(f'"{driver_name}" driver not registered.')
# Generate palette
if pct_filename is None:
ct = gdal.ColorTable()
err = gdal.ComputeMedianCutPCT(src_ds.GetRasterBand(1),
src_ds.GetRasterBand(2),
src_ds.GetRasterBand(3),
color_count,
ct,
callback=gdal.TermProgress_nocb)
else:
ct = get_color_table(pct_filename)
# Create the working file. We have to use TIFF since there are few formats
# that allow setting the color table after creation.
if driver_name.lower() == 'gtiff':
tif_filename = dst_filename
else:
import tempfile
tif_filedesc, tif_filename = tempfile.mkstemp(suffix='.tif')
gtiff_driver = gdal.GetDriverByName('GTiff')
tif_ds = gtiff_driver.Create(tif_filename, src_ds.RasterXSize,
src_ds.RasterYSize, 1)
tif_ds.GetRasterBand(1).SetRasterColorTable(ct)
# ----------------------------------------------------------------------------
# We should copy projection information and so forth at this point.
tif_ds.SetProjection(src_ds.GetProjection())
tif_ds.SetGeoTransform(src_ds.GetGeoTransform())
if src_ds.GetGCPCount() > 0:
tif_ds.SetGCPs(src_ds.GetGCPs(), src_ds.GetGCPProjection())
# ----------------------------------------------------------------------------
# Actually transfer and dither the data.
err = gdal.DitherRGB2PCT(src_ds.GetRasterBand(1),
src_ds.GetRasterBand(2),
src_ds.GetRasterBand(3),
tif_ds.GetRasterBand(1),
ct,
callback=gdal.TermProgress_nocb)
if err != gdal.CE_None:
raise Exception('DitherRGB2PCT failed')
if tif_filename == dst_filename:
dst_ds = tif_ds
else:
dst_ds = dst_driver.CreateCopy(dst_filename or '', tif_ds)
tif_ds = None
os.close(tif_filedesc)
gtiff_driver.Delete(tif_filename)
return dst_ds
def gdallocationinfo(
filename_or_ds: PathOrDS,
x: NumpyCompatibleArrayOrReal,
y: NumpyCompatibleArrayOrReal,
gis_order: bool = False,
open_options: Optional[dict] = None,
ovr_idx: Optional[int] = None,
band_nums: Optional[Sequence[int]] = None,
srs: CoordinateTransformationOrSRS = None,
inline_xy_replacement: bool = True,
quiet_mode: bool = True,
allow_xy_outside_extent: bool = True,
pixel_offset: Real = -0.5,
line_offset: Real = -0.5,
resample_alg=gdalconst.GRIORA_NearestNeighbour
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
ds = open_ds(filename_or_ds, open_options=open_options)
filename = filename_or_ds if is_path_like(filename_or_ds) else ''
if ds is None:
raise Exception(f'Could not open {filename}.')
if not isinstance(x, NumpyCompatibleArray.__args__):
x = [x]
if not isinstance(y, NumpyCompatibleArray.__args__):
y = [y]
if len(x) != len(y):
raise Exception(
f'len(x)={len(x)} should be the same as len(y)={len(y)}')
point_count = len(x)
if not isinstance(x, np.ndarray):
x = np.ndarray(x)
if not isinstance(y, np.ndarray):
y = np.ndarray(y)
if srs is None:
srs = LocationInfoSRS.PixelLine
# Build Spatial Reference object based on coordinate system, fetched from the opened dataset
if srs != LocationInfoSRS.PixelLine:
if srs != LocationInfoSRS.SameAsDS_SRS:
ds_srs = ds.GetSpatialRef()
ct = None
if isinstance(srs, osr.CoordinateTransformation):
ct = srs
else:
if srs == LocationInfoSRS.SameAsDS_SRS_GeogCS:
points_srs = ds_srs.CloneGeogCS()
else:
points_srs = get_srs(srs, gis_order=gis_order)
ct = get_transform(points_srs, ds_srs)
x, y, _z = transform_points(ct, x, y)
# Read geotransform matrix and calculate corresponding pixel coordinates
geomatrix = ds.GetGeoTransform()
inv_geometrix = gdal.InvGeoTransform(geomatrix)
if inv_geometrix is None:
raise Exception("Failed InvGeoTransform()")
x, y = \
(inv_geometrix[0] + inv_geometrix[1] * x + inv_geometrix[2] * y), \
(inv_geometrix[3] + inv_geometrix[4] * x + inv_geometrix[5] * y)
xsize, ysize = ds.RasterXSize, ds.RasterYSize
bands = get_bands(ds, band_nums, ovr_idx=ovr_idx)
ovr_xsize, ovr_ysize = bands[0].XSize, bands[0].YSize
pixel_fact, line_fact = (ovr_xsize / xsize,
ovr_ysize / ysize) if ovr_idx else (1, 1)
bnd_count = len(bands)
shape = (bnd_count, point_count)
np_dtype, np_dtype = GDALTypeCodeAndNumericTypeCodeFromDataSet(ds)
results = np.empty(shape=shape, dtype=np_dtype)
check_outside = not quiet_mode or not allow_xy_outside_extent
if check_outside and (np.any(x < 0) or np.any(x >= xsize) or np.any(y < 0)
or np.any(y >= ysize)):
msg = 'Passed coordinates are not in dataset extent!'
if not allow_xy_outside_extent:
raise Exception(msg)
elif not quiet_mode:
print(msg)
pixels = np.clip(x * pixel_fact + pixel_offset,
0,
ovr_xsize - 1,
out=x if inline_xy_replacement else None)
lines = np.clip(y * line_fact + line_offset,
0,
ovr_ysize - 1,
out=y if inline_xy_replacement else None)
for idx, (pixel, line) in enumerate(zip(pixels, lines)):
for bnd_idx, band in enumerate(bands):
val = band.ReadAsArray(pixel,
line,
1,
1,
resample_alg=resample_alg)
val = val[0][0]
results[bnd_idx][idx] = val
is_scaled, scales, offsets = get_scales_and_offsets(bands)
if is_scaled:
for bnd_idx, scale, offset in enumerate(zip(scales, offsets)):
results[bnd_idx] = results[bnd_idx] * scale + offset
return pixels, lines, results
def test_min_max(data, name, min, max, approx_ok):
ds = util.open_ds(test_py_scripts.get_data_path(data) + name)
min_max = util.get_raster_min_max(ds, approx_ok=approx_ok)
assert min_max == (min, max)