def test_srtm():
logging.basicConfig(level=logging.DEBUG)
my_extent = GeoRectangle.from_min_max(5, 85, 30, 40)
srtm_path = r'd:\maps\srtm.tif'
warp_srs = [None, 32, 34]
gdalos_trans(filename=srtm_path,
extent=my_extent,
warp_srs=warp_srs,
dst_nodatavalue=0,
hide_nodatavalue=True,
ovr_type=OvrType.create_external_auto)
def gdalos_qt_main():
from fidget.backend.QtWidgets import QApplication
from fidget.backend.QtWidgets import QVBoxLayout
from fidget.core import Fidget
from fidget.widgets import FidgetMatrix, FidgetMinimal, FidgetQuestion
from fidget.widgets.__util__ import CountBounds
from gdalos.gdalos_trans import gdalos_trans
from gdalos_qt.gdalos_widget import GdalosWidget
app = QApplication(
[]
) # this app variable is needed, because otherwise python will delete the QApplication
q = FidgetQuestion(
FidgetMatrix(
FidgetMinimal(
GdalosWidget(make_title=True,
make_plaintext=True,
make_indicator=True),
make_title=False,
make_plaintext=False,
make_indicator=False,
initial_value={},
),
rows=CountBounds(1, 1, None),
columns=1,
make_plaintext=True,
make_title=True,
make_indicator=False,
layout_cls=QVBoxLayout,
scrollable=True,
),
flags=Fidget.FLAGS,
)
q.show()
result = q.exec()
print(result)
if result.is_ok() and result.value is not None:
for v in result.value:
d = dict(v[0])
d2 = dict()
for k, v in d.items():
new_k = k.replace(" ", "_")
d2[new_k] = d[k]
print(gdalos_trans(**d2))
np_dtype = np.uint8
if color_palette is ...:
color_palette = ColorPalette.get_xkcd_palette()
color_table = gdalos_color.get_color_table(color_palette)
for i in range(levels):
size = 1 << levels - 1 - i
shape = (size, size)
filename_i = filename + '.ovr' * i
ds = driver.Create(filename_i,
xsize=shape[0],
ysize=shape[1],
bands=1,
eType=gdal_dtype)
pixel_size = 1 << i
gt = [0, pixel_size, 0, 0, 0, -pixel_size]
ds.SetGeoTransform(gt)
bnd = ds.GetRasterBand(1)
bnd.SetRasterColorTable(color_table)
bnd.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
arr = np.full(shape, i, np_dtype)
bnd.WriteArray(arr, 0, 0)
ds.FlushCache()
if __name__ == '__main__':
filename = r'd:\Maps.temp\test.tif'
test_data_generator(filename=filename)
gdalos_trans(filename, overwrite=True)
def viewshed_calc_to_ds(
vp_array,
input_filename: Union[gdal.Dataset, PathLikeOrStr, DataSetSelector],
extent=Extent.UNION, cutline=None, calc_cutline: bool = True,
operation: Optional[CalcOperation] = CalcOperation.count, operation_hidendv=True,
in_coords_srs=None, out_crs=None,
color_palette: Optional[ColorPaletteOrPathOrStrings] = None,
discrete_mode: DiscreteMode = DiscreteMode.interp,
bi=1, ovr_idx=0, co=None, threads=0,
vp_slice=None,
backend: ViewshedBackend = None,
output_ras: Optional[list] = None,
temp_files=None,
files=None):
input_selector = None
input_ds = None
calc_cutline = None if not calc_cutline else cutline if isinstance(calc_cutline, bool) else calc_cutline
if isinstance(input_filename, PathLikeOrStr.__args__):
input_ds = gdalos_util.open_ds(input_filename, ovr_idx=ovr_idx)
elif isinstance(input_filename, DataSetSelector):
input_selector = input_filename
else:
input_ds = input_filename
if isinstance(extent, int):
extent = Extent(extent)
elif isinstance(extent, str):
extent = Extent[extent]
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False)
color_palette = gdalos_color.get_color_palette(color_palette)
color_table = None
if operation == CalcOperation.viewshed:
operation = None
do_post_color = False
temp_files = temp_files if temp_files is not None else []
vp_slice = make_slice(vp_slice)
if isinstance(discrete_mode, str):
discrete_mode = DiscreteMode[discrete_mode]
if not files:
files = []
else:
files = files.copy()[vp_slice]
if not files:
if isinstance(vp_array, ViewshedParams):
vp_array = [vp_array]
else:
if isinstance(vp_array, dict):
vp_array = ViewshedParams.get_list_from_lists_dict(vp_array)
vp_array = vp_array[vp_slice]
if operation:
# restore viewshed consts default values
my_viewshed_defaults = viewshed_params.viewshed_defaults
for a in vp_array:
a.update(my_viewshed_defaults)
else:
vp_array = vp_array[0:1]
max_rasters_count = 1 if operation is None else 254 if operation == CalcOperation.unique else 1000
if len(vp_array) > max_rasters_count:
vp_array = vp_array[0:max_rasters_count]
srs_4326 = projdef.get_srs(4326)
pjstr_4326 = srs_4326.ExportToProj4()
first_vs = True
if in_coords_srs is not None:
in_coords_srs = projdef.get_proj_string(in_coords_srs)
for vp in vp_array:
# vp might get changed, so make a copy
vp = copy.copy(vp)
if first_vs or input_selector is not None:
first_vs = False
# figure out in_coords_crs_pj for getting the geo_ox, geo_oy
if input_selector is None:
if in_coords_srs is None:
if input_ds is None:
input_ds = gdalos_util.open_ds(input_filename, ovr_idx=ovr_idx)
in_coords_srs = projdef.get_srs_from_ds(input_ds)
else:
if in_coords_srs is None:
in_coords_srs = pjstr_4326
transform_coords_to_4326 = projdef.get_transform(in_coords_srs, pjstr_4326)
if transform_coords_to_4326:
geo_ox, geo_oy, _ = transform_coords_to_4326.TransformPoint(vp.ox, vp.oy)
else:
geo_ox, geo_oy = vp.ox, vp.oy
# select the ds
if input_selector is not None:
input_filename, input_ds = input_selector.get_item_projected(geo_ox, geo_oy)
input_filename = Path(input_filename).resolve()
if input_ds is None:
input_ds = gdalos_util.open_ds(input_filename, ovr_idx=ovr_idx)
if input_ds is None:
raise Exception(f'cannot open input file: {input_filename}')
# figure out the input, output and intermediate srs
# the intermediate srs will be used for combining the output rasters, if needed
pjstr_input_srs = projdef.get_srs_pj(input_ds)
pjstr_output_srs = projdef.get_proj_string(out_crs) if out_crs is not None else \
pjstr_input_srs if input_selector is None else pjstr_4326
if input_selector is None:
pjstr_inter_srs = pjstr_input_srs
else:
pjstr_inter_srs = pjstr_output_srs
input_srs = projdef.get_srs_from_ds(input_ds)
input_raster_is_projected = input_srs.IsProjected()
if input_raster_is_projected:
transform_coords_to_raster = projdef.get_transform(in_coords_srs, pjstr_input_srs)
else:
raise Exception(f'input raster has to be projected')
zone_lon0 = input_srs.GetProjParm('central_meridian')
vp.convergence = utm_convergence(vp.ox, vp.oy, zone_lon0)
if input_raster_is_projected:
projected_filename = input_filename
if transform_coords_to_raster:
vp.ox, vp.oy, _ = transform_coords_to_raster.TransformPoint(vp.ox, vp.oy)
else:
projected_pj = get_projected_pj(geo_ox, geo_oy)
transform_coords_to_raster = projdef.get_transform(in_coords_srs, projected_pj)
vp.ox, vp.oy, _ = transform_coords_to_raster.TransformPoint(vp.ox, vp.oy)
d = gdalos_extent.transform_resolution_p(transform_coords_to_raster, 10, 10, vp.ox, vp.oy)
extent = GeoRectangle.from_center_and_radius(vp.ox, vp.oy, vp.max_r + d, vp.max_r + d)
projected_filename = tempfile.mktemp('.tif')
projected_ds = gdalos_trans(
input_ds, out_filename=projected_filename, warp_srs=projected_pj,
extent=extent, return_ds=True, write_info=False, write_spec=False)
if not projected_ds:
raise Exception('input raster projection failed')
input_ds = projected_ds
if backend is None:
backend = default_ViewshedBackend
elif isinstance(backend, str):
backend = ViewshedBackend[backend]
if backend == ViewshedBackend.radio:
backend = ViewshedBackend.talos
is_base_calc = True
if backend == ViewshedBackend.gdal:
# TypeError: '>' not supported between instances of 'NoneType' and 'int'
bnd_type = gdal.GDT_Byte
# todo: why dosn't it work without it?
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(True)
inputs = vp.get_as_gdal_params()
print(inputs)
input_band: Optional[gdal.Band] = input_ds.GetRasterBand(bi)
if input_band is None:
raise Exception('band number out of range')
ds = gdal.ViewshedGenerate(input_band, gdal_out_format, str(d_path), co, **inputs)
input_band = None # close band
if not ds:
raise Exception('Viewshed calculation failed')
elif backend == ViewshedBackend.talos:
# is_temp_file = True # output is file, not ds
if not projected_filename:
raise Exception('to use talos backend you need to provide an input filename')
from talosgis import talos
talosgis_version = talos_module_init()
dtm_open_err = talos.GS_DtmOpenDTM(str(projected_filename))
talos.GS_SetProjectCRSFromActiveDTM()
ovr_idx = get_ovr_idx(projected_filename, ovr_idx)
talos.GS_DtmSelectOvle(ovr_idx)
talos.GS_DtmSetCalcThreadsCount(threads or 0)
if dtm_open_err != 0:
raise Exception('talos could not open input file {}'.format(projected_filename))
talos.GS_SetRefractionCoeff(vp.refraction_coeff)
inputs = vp.get_as_talos_params()
bnd_type = inputs['result_dt']
is_base_calc = bnd_type in [gdal.GDT_Byte]
inputs['low_nodata'] = is_base_calc or operation == CalcOperation.max
if hasattr(talos, 'GS_SetCalcModule'):
module = vp.get_calc_module()
talos.GS_SetCalcModule(module)
if vp.is_radio():
talos_radio_init()
radio_params = vp.get_radio_as_talos_params(0)
talos.GS_SetRadioParameters(**radio_params)
talos.GS_SetInterestAreaCalcMethod(
CalcOnlyInInterestArea=bool(calc_cutline), ClearOutsideInterestArea=False)
X0Pixel = Y0Pixel = ras = h_ras = e_ras = a_ras = r_ras = None
if talosgis_version >= (3, 6):
if calc_cutline:
vertex_count, xys = polygon_to_np(calc_cutline)
talos.GS_SetInterestArea1(vertex_count, xys, False)
result = talos.GS_Viewshed_Calc(**inputs, CheckNonVoid=False)
_unexpected, X0Pixel, Y0Pixel, ras, h_ras, e_ras, a_ras, r_ras = result
elif 'GS_Viewshed_Calc2' in dir(talos):
ras = talos.GS_Viewshed_Calc2(**inputs)
else:
del inputs['out_res']
ras = talos.GS_Viewshed_Calc1(**inputs)
if ras is None:
raise Exception(f'fail to calc viewshed: {inputs}')
do_post_color = color_palette and (bnd_type not in [gdal.GDT_Byte, gdal.GDT_UInt16])
ras_map = {
'v': ras, # visibility
'h': h_ras, # heights / dtm
'r': r_ras, # ranges
'a': a_ras, # azimuths
'e': e_ras, # elevations
}
output_ras = output_ras or ['v']
output_ras = [s[0].lower() for s in output_ras]
my_rasters = [v for k, v in ras_map.items() if k in output_ras and v is not None]
my_ds = []
for r in my_rasters:
# talos supports only file output (not ds)
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(True)
temp_files.append(d_path)
talos.GS_SaveRaster(r, str(d_path))
# I will reopen the ds to change the color table and ndv
# ds = gdalos_util.open_ds(d_path, access_mode=gdal.OF_UPDATE)
ds: gdal.Dataset = gdal.OpenEx(str(d_path), gdal.OF_RASTER | gdal.OF_UPDATE)
my_ds.append(ds)
if len(my_ds) > 1:
d_path = str(d_path) + '_vrt.vrt'
# let's stack all these bands into a single vrt
ds = gdal.BuildVRT(d_path, my_ds, separate=True)
temp_files.append(d_path)
my_ds = None
else:
raise Exception('unknown backend {}'.format(backend))
input_ds = None
set_nodata = backend == ViewshedBackend.gdal
# set_nodata = is_base_calc
bnd = ds.GetRasterBand(1)
if set_nodata:
base_calc_ndv = vp.ndv
bnd.SetNoDataValue(base_calc_ndv)
else:
base_calc_ndv = bnd.GetNoDataValue()
if color_palette and not do_post_color:
if bnd_type != bnd.DataType:
raise Exception('Unexpected band type, expected: {}, got {}'.format(bnd_type, bnd.DataType))
if not color_palette.is_numeric():
min_max = bnd.ComputeRasterMinMax()
color_palette.apply_percent(*min_max)
color_table = gdalos_color.get_color_table(color_palette)
if color_table is None:
raise Exception('Could not create color table')
bnd.SetRasterColorTable(color_table)
bnd.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
bnd = None
# if is_temp_file:
# # close original ds and reopen
# ds = None
# ds = gdalos_util.open_ds(d_path)
cut_sector = (backend == ViewshedBackend.gdal) and not vp.is_omni_h()
# warp_result = False
warp_result = (input_selector is not None)
if warp_result or cut_sector:
if cut_sector:
ring = PolygonizeSector(vp.ox, vp.oy, vp.max_r, vp.max_r, vp.get_grid_azimuth(), vp.h_aperture)
post_calc_cutline = tempfile.mktemp(suffix='.gpkg')
temp_files.append(post_calc_cutline)
create_layer_from_geometries([ring], post_calc_cutline)
else:
post_calc_cutline = None
# todo: check why without temp file it crashes on operation
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(True)
scale = ds.GetRasterBand(1).GetScale()
ds = gdalos_trans(ds, out_filename=d_path, warp_srs=pjstr_inter_srs,
cutline=post_calc_cutline, of=gdal_out_format, return_ds=return_ds,
ovr_type=OvrType.no_overviews)
if is_temp_file:
# close original ds and reopen
ds = None
ds = gdalos_util.open_ds(d_path)
if scale and workaround_warp_scale_bug:
ds.GetRasterBand(1).SetScale(scale)
temp_files.append(d_path)
if not ds:
raise Exception('Viewshed calculation failed to cut')
if operation:
files.append(ds)
if operation:
# alpha_pattern = '1*({{}}>{})'.format(viewshed_thresh)
# alpha_pattern = 'np.multiply({{}}>{}, dtype=np.uint8)'.format(viewshed_thresh)
no_data_value = base_calc_ndv
if operation == CalcOperation.viewshed:
# no_data_value = viewshed_params.viewshed_ndv
f = gdalos_combine.get_by_index
# calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), sum
elif operation == CalcOperation.max:
# no_data_value = viewshed_params.viewshed_ndv
f = gdalos_combine.vs_max
# calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), sum
elif operation == CalcOperation.min:
f = gdalos_combine.vs_min
elif operation == CalcOperation.count:
no_data_value = 0
f = gdalos_combine.vs_count
# calc_expr, calc_kwargs = gdal_calc.make_calc_with_operand(files, alpha_pattern, '+')
# calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern), sum
elif operation == CalcOperation.count_z:
no_data_value = viewshed_params.viewshed_comb_ndv
f = partial(gdalos_combine.vs_count_z, in_ndv=base_calc_ndv)
# calc_expr, calc_kwargs f, = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), sum
elif operation == CalcOperation.unique:
no_data_value = viewshed_params.viewshed_comb_ndv
f = gdalos_combine.vs_unique
# calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), unique
else:
raise Exception('Unknown operation: {}'.format(operation))
calc_expr = 'f(x)'
calc_kwargs = dict(x=files)
user_namespace = dict(f=f)
debug_time = 1
t = time.time()
for i in range(debug_time):
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False)
ds = gdal_calc.Calc(
calc_expr, outfile=str(d_path), extent=extent, format=gdal_out_format,
color_table=color_table, overwrite=True,
NoDataValue=no_data_value, hideNoData=operation_hidendv,
user_namespace=user_namespace, **calc_kwargs)
t = time.time() - t
print('time for calc: {:.3f} seconds'.format(t))
if not ds:
raise Exception('error occurred')
for i in range(len(files)):
files[i] = None # close calc input ds(s)
combined_post_process_needed = cutline or not projdef.are_srs_equivalent(pjstr_inter_srs, pjstr_output_srs)
if combined_post_process_needed:
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False)
ds = gdalos_trans(ds, out_filename=d_path, warp_srs=pjstr_output_srs,
cutline=cutline, of=gdal_out_format, return_ds=return_ds, ovr_type=OvrType.no_overviews)
if return_ds:
if not ds:
raise Exception('error occurred')
if do_post_color:
is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False)
ds = gdalos_raster_color(ds, out_filename=d_path, color_palette=color_palette, discrete_mode=discrete_mode)
if not ds:
raise Exception('Viewshed calculation failed to color result')
removed = []
if temp_files:
for f in temp_files:
try:
os.remove(f)
removed.append(f)
except:
pass
# probably this is a file that backs the ds that we'll return
# print('failed to remove temp file:{}'.format(f))
for f in removed:
temp_files.remove(f)
return ds
raise Exception('error occurred')
del dest
if __name__ == "__main__":
input_filename = Path(
r'd:\Maps\w84u36\dtm\SRTM1_hgt.x[27.97,37.98]_y[27.43,37.59].cog.tif')
output_path = Path(r'd:\dev\gis\maps')
# dir_path = Path('/home/idan/maps')
# input_filename = dir_path / Path('srtm1_x35_y32.tif')
# output_path = dir_path / Path('comb')
srtm_filename = Path(output_path) / Path('srtm1_36_sample.tif')
vp = ViewshedGridParams()
make_map = True
if make_map:
frame = 500
full_extent = calc_extent(vp.oxy, vp.grid_range, vp.interval, vp.max_r,
frame)
gdalos_trans(input_filename,
extent=full_extent,
warp_srs=36,
extent_in_4326=False,
out_filename=srtm_filename)
viewshed_run(vp, output_path, srtm_filename)
def gdalos_rasterize(in_filename: str,
shp_filename_or_ds: str,
out_filename: str = None,
shp_layer_name: str = None,
shp_z_attribute: str = 'Height',
add: bool = True,
extent: GeoRectangle = ...,
**kwargs):
"""rasterize a vector layer into a given raster layer, overriding or adding the values
Parameters
----------
in_filename:str
input raster filename
out_filename:str
ouput raster filename, if None - input raster would be updated inplace
shp_filename_or_ds: str
input vector filename or dataset
shp_layer_name: str
name of the layer from the vector dataset to use
shp_z_attribute: str='Height'
name of the attribute to extract the z value from
add: bool=True
True to add to dtm values to shape values into the raster, False to burn shape values into the raster
extent: GeoRectangle = ...
None - use the extent of the input raster
... - use the extent of the input vector layer
GeoRectangle - custom extent
out_res: Real
output resolution (if None then auto select)
warp_srs: str=None
output srs
overwrite: bool=True
what to do if the output exists (fail of overwrite)
Returns
-------
output raster dataset
"""
# shp = gdalos_util.open_ds(shp_filename, gdal.gdalconst.OF_VECTOR)
if shp_filename_or_ds is None:
shp = None
elif isinstance(shp_filename_or_ds, (str, Path)):
shp = gdal.OpenEx(str(shp_filename_or_ds), gdal.gdalconst.OF_VECTOR)
else:
shp = shp_filename_or_ds
if out_filename is None:
dstDs = gdalos_util.open_ds(in_filename, gdal.GA_Update)
else:
pj4326 = projdef.get_srs_pj(4326)
if extent is ...:
cov_pj_srs = projdef.get_srs_pj(shp)
if not cov_pj_srs:
cov_pj_srs = pj4326
cov_extent = ogr_get_layer_extent(shp.GetLayer())
transform = projdef.get_transform(cov_pj_srs, pj4326)
extent = gdalos_extent.transform_extent(cov_extent, transform)
dstDs = gdalos_trans(in_filename,
out_filename,
extent=extent,
cog=False,
ovr_type=OvrType.no_overviews,
return_ds=True,
**kwargs)
if shp is not None:
rasteize_options = gdal.RasterizeOptions(layers=shp_layer_name,
add=add,
attribute=shp_z_attribute)
ret = gdal.Rasterize(dstDs, shp, options=rasteize_options)
if ret != 1:
raise Exception('Rasterize failed')
return dstDs
else:
shp_filename_or_ds = None
out_extent = GeoRectangle.from_min_max(34.0, 34.2, 32.0, 32.2)
if do_inplace:
input_raster = out_dir / 'input.tif'
out_filename = None
else:
input_raster = Path(
r'd:\Maps\w84geo\dtm\SRTM1_hgt.tif.x20-80_y20-40.cog.tif')
out_filename = my_out_filename
dstDs = gdalos_rasterize(input_raster,
shp_filename_or_ds=shp_filename_or_ds,
out_filename=out_filename,
add=do_add,
extent=out_extent,
warp_srs=my_out_srs,
out_res=out_res,
overwrite=do_overwrite)
if do_cog:
if out_filename is None:
out_filename = my_out_filename
gdalos_trans(dstDs,
out_filename=str(out_filename) + '.cog.tif',
overwrite=do_overwrite)
del dstDs
print('done!')