def test_srtm(): logging.basicConfig(level=logging.DEBUG) my_extent = GeoRectangle.from_min_max(5, 85, 30, 40) srtm_path = r'd:\maps\srtm.tif' warp_srs = [None, 32, 34] gdalos_trans(filename=srtm_path, extent=my_extent, warp_srs=warp_srs, dst_nodatavalue=0, hide_nodatavalue=True, ovr_type=OvrType.create_external_auto)
def gdalos_qt_main(): from fidget.backend.QtWidgets import QApplication from fidget.backend.QtWidgets import QVBoxLayout from fidget.core import Fidget from fidget.widgets import FidgetMatrix, FidgetMinimal, FidgetQuestion from fidget.widgets.__util__ import CountBounds from gdalos.gdalos_trans import gdalos_trans from gdalos_qt.gdalos_widget import GdalosWidget app = QApplication( [] ) # this app variable is needed, because otherwise python will delete the QApplication q = FidgetQuestion( FidgetMatrix( FidgetMinimal( GdalosWidget(make_title=True, make_plaintext=True, make_indicator=True), make_title=False, make_plaintext=False, make_indicator=False, initial_value={}, ), rows=CountBounds(1, 1, None), columns=1, make_plaintext=True, make_title=True, make_indicator=False, layout_cls=QVBoxLayout, scrollable=True, ), flags=Fidget.FLAGS, ) q.show() result = q.exec() print(result) if result.is_ok() and result.value is not None: for v in result.value: d = dict(v[0]) d2 = dict() for k, v in d.items(): new_k = k.replace(" ", "_") d2[new_k] = d[k] print(gdalos_trans(**d2))
np_dtype = np.uint8 if color_palette is ...: color_palette = ColorPalette.get_xkcd_palette() color_table = gdalos_color.get_color_table(color_palette) for i in range(levels): size = 1 << levels - 1 - i shape = (size, size) filename_i = filename + '.ovr' * i ds = driver.Create(filename_i, xsize=shape[0], ysize=shape[1], bands=1, eType=gdal_dtype) pixel_size = 1 << i gt = [0, pixel_size, 0, 0, 0, -pixel_size] ds.SetGeoTransform(gt) bnd = ds.GetRasterBand(1) bnd.SetRasterColorTable(color_table) bnd.SetRasterColorInterpretation(gdal.GCI_PaletteIndex) arr = np.full(shape, i, np_dtype) bnd.WriteArray(arr, 0, 0) ds.FlushCache() if __name__ == '__main__': filename = r'd:\Maps.temp\test.tif' test_data_generator(filename=filename) gdalos_trans(filename, overwrite=True)
def viewshed_calc_to_ds( vp_array, input_filename: Union[gdal.Dataset, PathLikeOrStr, DataSetSelector], extent=Extent.UNION, cutline=None, calc_cutline: bool = True, operation: Optional[CalcOperation] = CalcOperation.count, operation_hidendv=True, in_coords_srs=None, out_crs=None, color_palette: Optional[ColorPaletteOrPathOrStrings] = None, discrete_mode: DiscreteMode = DiscreteMode.interp, bi=1, ovr_idx=0, co=None, threads=0, vp_slice=None, backend: ViewshedBackend = None, output_ras: Optional[list] = None, temp_files=None, files=None): input_selector = None input_ds = None calc_cutline = None if not calc_cutline else cutline if isinstance(calc_cutline, bool) else calc_cutline if isinstance(input_filename, PathLikeOrStr.__args__): input_ds = gdalos_util.open_ds(input_filename, ovr_idx=ovr_idx) elif isinstance(input_filename, DataSetSelector): input_selector = input_filename else: input_ds = input_filename if isinstance(extent, int): extent = Extent(extent) elif isinstance(extent, str): extent = Extent[extent] is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False) color_palette = gdalos_color.get_color_palette(color_palette) color_table = None if operation == CalcOperation.viewshed: operation = None do_post_color = False temp_files = temp_files if temp_files is not None else [] vp_slice = make_slice(vp_slice) if isinstance(discrete_mode, str): discrete_mode = DiscreteMode[discrete_mode] if not files: files = [] else: files = files.copy()[vp_slice] if not files: if isinstance(vp_array, ViewshedParams): vp_array = [vp_array] else: if isinstance(vp_array, dict): vp_array = ViewshedParams.get_list_from_lists_dict(vp_array) vp_array = vp_array[vp_slice] if operation: # restore viewshed consts default values my_viewshed_defaults = viewshed_params.viewshed_defaults for a in vp_array: a.update(my_viewshed_defaults) else: vp_array = vp_array[0:1] max_rasters_count = 1 if operation is None else 254 if operation == CalcOperation.unique else 1000 if len(vp_array) > max_rasters_count: vp_array = vp_array[0:max_rasters_count] srs_4326 = projdef.get_srs(4326) pjstr_4326 = srs_4326.ExportToProj4() first_vs = True if in_coords_srs is not None: in_coords_srs = projdef.get_proj_string(in_coords_srs) for vp in vp_array: # vp might get changed, so make a copy vp = copy.copy(vp) if first_vs or input_selector is not None: first_vs = False # figure out in_coords_crs_pj for getting the geo_ox, geo_oy if input_selector is None: if in_coords_srs is None: if input_ds is None: input_ds = gdalos_util.open_ds(input_filename, ovr_idx=ovr_idx) in_coords_srs = projdef.get_srs_from_ds(input_ds) else: if in_coords_srs is None: in_coords_srs = pjstr_4326 transform_coords_to_4326 = projdef.get_transform(in_coords_srs, pjstr_4326) if transform_coords_to_4326: geo_ox, geo_oy, _ = transform_coords_to_4326.TransformPoint(vp.ox, vp.oy) else: geo_ox, geo_oy = vp.ox, vp.oy # select the ds if input_selector is not None: input_filename, input_ds = input_selector.get_item_projected(geo_ox, geo_oy) input_filename = Path(input_filename).resolve() if input_ds is None: input_ds = gdalos_util.open_ds(input_filename, ovr_idx=ovr_idx) if input_ds is None: raise Exception(f'cannot open input file: {input_filename}') # figure out the input, output and intermediate srs # the intermediate srs will be used for combining the output rasters, if needed pjstr_input_srs = projdef.get_srs_pj(input_ds) pjstr_output_srs = projdef.get_proj_string(out_crs) if out_crs is not None else \ pjstr_input_srs if input_selector is None else pjstr_4326 if input_selector is None: pjstr_inter_srs = pjstr_input_srs else: pjstr_inter_srs = pjstr_output_srs input_srs = projdef.get_srs_from_ds(input_ds) input_raster_is_projected = input_srs.IsProjected() if input_raster_is_projected: transform_coords_to_raster = projdef.get_transform(in_coords_srs, pjstr_input_srs) else: raise Exception(f'input raster has to be projected') zone_lon0 = input_srs.GetProjParm('central_meridian') vp.convergence = utm_convergence(vp.ox, vp.oy, zone_lon0) if input_raster_is_projected: projected_filename = input_filename if transform_coords_to_raster: vp.ox, vp.oy, _ = transform_coords_to_raster.TransformPoint(vp.ox, vp.oy) else: projected_pj = get_projected_pj(geo_ox, geo_oy) transform_coords_to_raster = projdef.get_transform(in_coords_srs, projected_pj) vp.ox, vp.oy, _ = transform_coords_to_raster.TransformPoint(vp.ox, vp.oy) d = gdalos_extent.transform_resolution_p(transform_coords_to_raster, 10, 10, vp.ox, vp.oy) extent = GeoRectangle.from_center_and_radius(vp.ox, vp.oy, vp.max_r + d, vp.max_r + d) projected_filename = tempfile.mktemp('.tif') projected_ds = gdalos_trans( input_ds, out_filename=projected_filename, warp_srs=projected_pj, extent=extent, return_ds=True, write_info=False, write_spec=False) if not projected_ds: raise Exception('input raster projection failed') input_ds = projected_ds if backend is None: backend = default_ViewshedBackend elif isinstance(backend, str): backend = ViewshedBackend[backend] if backend == ViewshedBackend.radio: backend = ViewshedBackend.talos is_base_calc = True if backend == ViewshedBackend.gdal: # TypeError: '>' not supported between instances of 'NoneType' and 'int' bnd_type = gdal.GDT_Byte # todo: why dosn't it work without it? is_temp_file, gdal_out_format, d_path, return_ds = temp_params(True) inputs = vp.get_as_gdal_params() print(inputs) input_band: Optional[gdal.Band] = input_ds.GetRasterBand(bi) if input_band is None: raise Exception('band number out of range') ds = gdal.ViewshedGenerate(input_band, gdal_out_format, str(d_path), co, **inputs) input_band = None # close band if not ds: raise Exception('Viewshed calculation failed') elif backend == ViewshedBackend.talos: # is_temp_file = True # output is file, not ds if not projected_filename: raise Exception('to use talos backend you need to provide an input filename') from talosgis import talos talosgis_version = talos_module_init() dtm_open_err = talos.GS_DtmOpenDTM(str(projected_filename)) talos.GS_SetProjectCRSFromActiveDTM() ovr_idx = get_ovr_idx(projected_filename, ovr_idx) talos.GS_DtmSelectOvle(ovr_idx) talos.GS_DtmSetCalcThreadsCount(threads or 0) if dtm_open_err != 0: raise Exception('talos could not open input file {}'.format(projected_filename)) talos.GS_SetRefractionCoeff(vp.refraction_coeff) inputs = vp.get_as_talos_params() bnd_type = inputs['result_dt'] is_base_calc = bnd_type in [gdal.GDT_Byte] inputs['low_nodata'] = is_base_calc or operation == CalcOperation.max if hasattr(talos, 'GS_SetCalcModule'): module = vp.get_calc_module() talos.GS_SetCalcModule(module) if vp.is_radio(): talos_radio_init() radio_params = vp.get_radio_as_talos_params(0) talos.GS_SetRadioParameters(**radio_params) talos.GS_SetInterestAreaCalcMethod( CalcOnlyInInterestArea=bool(calc_cutline), ClearOutsideInterestArea=False) X0Pixel = Y0Pixel = ras = h_ras = e_ras = a_ras = r_ras = None if talosgis_version >= (3, 6): if calc_cutline: vertex_count, xys = polygon_to_np(calc_cutline) talos.GS_SetInterestArea1(vertex_count, xys, False) result = talos.GS_Viewshed_Calc(**inputs, CheckNonVoid=False) _unexpected, X0Pixel, Y0Pixel, ras, h_ras, e_ras, a_ras, r_ras = result elif 'GS_Viewshed_Calc2' in dir(talos): ras = talos.GS_Viewshed_Calc2(**inputs) else: del inputs['out_res'] ras = talos.GS_Viewshed_Calc1(**inputs) if ras is None: raise Exception(f'fail to calc viewshed: {inputs}') do_post_color = color_palette and (bnd_type not in [gdal.GDT_Byte, gdal.GDT_UInt16]) ras_map = { 'v': ras, # visibility 'h': h_ras, # heights / dtm 'r': r_ras, # ranges 'a': a_ras, # azimuths 'e': e_ras, # elevations } output_ras = output_ras or ['v'] output_ras = [s[0].lower() for s in output_ras] my_rasters = [v for k, v in ras_map.items() if k in output_ras and v is not None] my_ds = [] for r in my_rasters: # talos supports only file output (not ds) is_temp_file, gdal_out_format, d_path, return_ds = temp_params(True) temp_files.append(d_path) talos.GS_SaveRaster(r, str(d_path)) # I will reopen the ds to change the color table and ndv # ds = gdalos_util.open_ds(d_path, access_mode=gdal.OF_UPDATE) ds: gdal.Dataset = gdal.OpenEx(str(d_path), gdal.OF_RASTER | gdal.OF_UPDATE) my_ds.append(ds) if len(my_ds) > 1: d_path = str(d_path) + '_vrt.vrt' # let's stack all these bands into a single vrt ds = gdal.BuildVRT(d_path, my_ds, separate=True) temp_files.append(d_path) my_ds = None else: raise Exception('unknown backend {}'.format(backend)) input_ds = None set_nodata = backend == ViewshedBackend.gdal # set_nodata = is_base_calc bnd = ds.GetRasterBand(1) if set_nodata: base_calc_ndv = vp.ndv bnd.SetNoDataValue(base_calc_ndv) else: base_calc_ndv = bnd.GetNoDataValue() if color_palette and not do_post_color: if bnd_type != bnd.DataType: raise Exception('Unexpected band type, expected: {}, got {}'.format(bnd_type, bnd.DataType)) if not color_palette.is_numeric(): min_max = bnd.ComputeRasterMinMax() color_palette.apply_percent(*min_max) color_table = gdalos_color.get_color_table(color_palette) if color_table is None: raise Exception('Could not create color table') bnd.SetRasterColorTable(color_table) bnd.SetRasterColorInterpretation(gdal.GCI_PaletteIndex) bnd = None # if is_temp_file: # # close original ds and reopen # ds = None # ds = gdalos_util.open_ds(d_path) cut_sector = (backend == ViewshedBackend.gdal) and not vp.is_omni_h() # warp_result = False warp_result = (input_selector is not None) if warp_result or cut_sector: if cut_sector: ring = PolygonizeSector(vp.ox, vp.oy, vp.max_r, vp.max_r, vp.get_grid_azimuth(), vp.h_aperture) post_calc_cutline = tempfile.mktemp(suffix='.gpkg') temp_files.append(post_calc_cutline) create_layer_from_geometries([ring], post_calc_cutline) else: post_calc_cutline = None # todo: check why without temp file it crashes on operation is_temp_file, gdal_out_format, d_path, return_ds = temp_params(True) scale = ds.GetRasterBand(1).GetScale() ds = gdalos_trans(ds, out_filename=d_path, warp_srs=pjstr_inter_srs, cutline=post_calc_cutline, of=gdal_out_format, return_ds=return_ds, ovr_type=OvrType.no_overviews) if is_temp_file: # close original ds and reopen ds = None ds = gdalos_util.open_ds(d_path) if scale and workaround_warp_scale_bug: ds.GetRasterBand(1).SetScale(scale) temp_files.append(d_path) if not ds: raise Exception('Viewshed calculation failed to cut') if operation: files.append(ds) if operation: # alpha_pattern = '1*({{}}>{})'.format(viewshed_thresh) # alpha_pattern = 'np.multiply({{}}>{}, dtype=np.uint8)'.format(viewshed_thresh) no_data_value = base_calc_ndv if operation == CalcOperation.viewshed: # no_data_value = viewshed_params.viewshed_ndv f = gdalos_combine.get_by_index # calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), sum elif operation == CalcOperation.max: # no_data_value = viewshed_params.viewshed_ndv f = gdalos_combine.vs_max # calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), sum elif operation == CalcOperation.min: f = gdalos_combine.vs_min elif operation == CalcOperation.count: no_data_value = 0 f = gdalos_combine.vs_count # calc_expr, calc_kwargs = gdal_calc.make_calc_with_operand(files, alpha_pattern, '+') # calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern), sum elif operation == CalcOperation.count_z: no_data_value = viewshed_params.viewshed_comb_ndv f = partial(gdalos_combine.vs_count_z, in_ndv=base_calc_ndv) # calc_expr, calc_kwargs f, = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), sum elif operation == CalcOperation.unique: no_data_value = viewshed_params.viewshed_comb_ndv f = gdalos_combine.vs_unique # calc_expr, calc_kwargs, f = gdal_calc.make_calc_with_func(files, alpha_pattern, 'f'), unique else: raise Exception('Unknown operation: {}'.format(operation)) calc_expr = 'f(x)' calc_kwargs = dict(x=files) user_namespace = dict(f=f) debug_time = 1 t = time.time() for i in range(debug_time): is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False) ds = gdal_calc.Calc( calc_expr, outfile=str(d_path), extent=extent, format=gdal_out_format, color_table=color_table, overwrite=True, NoDataValue=no_data_value, hideNoData=operation_hidendv, user_namespace=user_namespace, **calc_kwargs) t = time.time() - t print('time for calc: {:.3f} seconds'.format(t)) if not ds: raise Exception('error occurred') for i in range(len(files)): files[i] = None # close calc input ds(s) combined_post_process_needed = cutline or not projdef.are_srs_equivalent(pjstr_inter_srs, pjstr_output_srs) if combined_post_process_needed: is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False) ds = gdalos_trans(ds, out_filename=d_path, warp_srs=pjstr_output_srs, cutline=cutline, of=gdal_out_format, return_ds=return_ds, ovr_type=OvrType.no_overviews) if return_ds: if not ds: raise Exception('error occurred') if do_post_color: is_temp_file, gdal_out_format, d_path, return_ds = temp_params(False) ds = gdalos_raster_color(ds, out_filename=d_path, color_palette=color_palette, discrete_mode=discrete_mode) if not ds: raise Exception('Viewshed calculation failed to color result') removed = [] if temp_files: for f in temp_files: try: os.remove(f) removed.append(f) except: pass # probably this is a file that backs the ds that we'll return # print('failed to remove temp file:{}'.format(f)) for f in removed: temp_files.remove(f) return ds
raise Exception('error occurred') del dest if __name__ == "__main__": input_filename = Path( r'd:\Maps\w84u36\dtm\SRTM1_hgt.x[27.97,37.98]_y[27.43,37.59].cog.tif') output_path = Path(r'd:\dev\gis\maps') # dir_path = Path('/home/idan/maps') # input_filename = dir_path / Path('srtm1_x35_y32.tif') # output_path = dir_path / Path('comb') srtm_filename = Path(output_path) / Path('srtm1_36_sample.tif') vp = ViewshedGridParams() make_map = True if make_map: frame = 500 full_extent = calc_extent(vp.oxy, vp.grid_range, vp.interval, vp.max_r, frame) gdalos_trans(input_filename, extent=full_extent, warp_srs=36, extent_in_4326=False, out_filename=srtm_filename) viewshed_run(vp, output_path, srtm_filename)
def gdalos_rasterize(in_filename: str, shp_filename_or_ds: str, out_filename: str = None, shp_layer_name: str = None, shp_z_attribute: str = 'Height', add: bool = True, extent: GeoRectangle = ..., **kwargs): """rasterize a vector layer into a given raster layer, overriding or adding the values Parameters ---------- in_filename:str input raster filename out_filename:str ouput raster filename, if None - input raster would be updated inplace shp_filename_or_ds: str input vector filename or dataset shp_layer_name: str name of the layer from the vector dataset to use shp_z_attribute: str='Height' name of the attribute to extract the z value from add: bool=True True to add to dtm values to shape values into the raster, False to burn shape values into the raster extent: GeoRectangle = ... None - use the extent of the input raster ... - use the extent of the input vector layer GeoRectangle - custom extent out_res: Real output resolution (if None then auto select) warp_srs: str=None output srs overwrite: bool=True what to do if the output exists (fail of overwrite) Returns ------- output raster dataset """ # shp = gdalos_util.open_ds(shp_filename, gdal.gdalconst.OF_VECTOR) if shp_filename_or_ds is None: shp = None elif isinstance(shp_filename_or_ds, (str, Path)): shp = gdal.OpenEx(str(shp_filename_or_ds), gdal.gdalconst.OF_VECTOR) else: shp = shp_filename_or_ds if out_filename is None: dstDs = gdalos_util.open_ds(in_filename, gdal.GA_Update) else: pj4326 = projdef.get_srs_pj(4326) if extent is ...: cov_pj_srs = projdef.get_srs_pj(shp) if not cov_pj_srs: cov_pj_srs = pj4326 cov_extent = ogr_get_layer_extent(shp.GetLayer()) transform = projdef.get_transform(cov_pj_srs, pj4326) extent = gdalos_extent.transform_extent(cov_extent, transform) dstDs = gdalos_trans(in_filename, out_filename, extent=extent, cog=False, ovr_type=OvrType.no_overviews, return_ds=True, **kwargs) if shp is not None: rasteize_options = gdal.RasterizeOptions(layers=shp_layer_name, add=add, attribute=shp_z_attribute) ret = gdal.Rasterize(dstDs, shp, options=rasteize_options) if ret != 1: raise Exception('Rasterize failed') return dstDs
else: shp_filename_or_ds = None out_extent = GeoRectangle.from_min_max(34.0, 34.2, 32.0, 32.2) if do_inplace: input_raster = out_dir / 'input.tif' out_filename = None else: input_raster = Path( r'd:\Maps\w84geo\dtm\SRTM1_hgt.tif.x20-80_y20-40.cog.tif') out_filename = my_out_filename dstDs = gdalos_rasterize(input_raster, shp_filename_or_ds=shp_filename_or_ds, out_filename=out_filename, add=do_add, extent=out_extent, warp_srs=my_out_srs, out_res=out_res, overwrite=do_overwrite) if do_cog: if out_filename is None: out_filename = my_out_filename gdalos_trans(dstDs, out_filename=str(out_filename) + '.cog.tif', overwrite=do_overwrite) del dstDs print('done!')