def test_misc_14(): import collections import logging class MockLoggingHandler(logging.Handler): def __init__(self, *args, **kwargs): super(MockLoggingHandler, self).__init__(*args, **kwargs) self.messages = collections.defaultdict(list) def emit(self, record): self.messages[record.levelname].append(record.getMessage()) logger = logging.getLogger('gdal_logging_test') logger.setLevel(logging.DEBUG) logger.propagate = False handler = MockLoggingHandler(level=logging.DEBUG) logger.addHandler(handler) prev_debug = gdal.GetConfigOption("CPL_DEBUG") try: gdal.ConfigurePythonLogging(logger_name='gdal_logging_test', enable_debug=True) assert gdal.GetConfigOption("CPL_DEBUG") == "ON", "should have enabled debug" gdal.Debug("test1", "debug1") gdal.Error(gdal.CE_Debug, gdal.CPLE_FileIO, "debug2") gdal.Error(gdal.CE_None, gdal.CPLE_AppDefined, "info1") gdal.Error(gdal.CE_Warning, gdal.CPLE_AssertionFailed, "warning1") gdal.Error(gdal.CE_Failure, 99999, "error1") expected = { 'DEBUG': ["test1: debug1", "FileIO: debug2"], 'INFO': ["AppDefined: info1"], 'WARNING': ["AssertionFailed: warning1"], 'ERROR': ["99999: error1"], } assert handler.messages == expected, "missing log messages" gdal.SetErrorHandler('CPLDefaultErrorHandler') handler.messages.clear() gdal.SetConfigOption('CPL_DEBUG', "OFF") gdal.ConfigurePythonLogging(logger_name='gdal_logging_test') assert gdal.GetConfigOption("CPL_DEBUG") == "OFF", \ "shouldn't have enabled debug" # these get suppressed by CPL_DEBUG gdal.Debug("test1", "debug3") # these don't gdal.Error(gdal.CE_Debug, gdal.CPLE_None, "debug4") assert handler.messages['DEBUG'] == ['debug4'], "unexpected log messages" finally: gdal.SetErrorHandler('CPLDefaultErrorHandler') gdal.SetConfigOption('CPL_DEBUG', prev_debug) logger.removeHandler(handler)
def vector_translate( input_path: Union[Path, str], output_path: Path, translate_description: Optional[str] = None, input_layers: Union[List[str], str, None] = None, output_layer: Optional[str] = None, spatial_filter: Optional[Tuple[float, float, float, float]] = None, clip_bounds: Optional[Tuple[float, float, float, float]] = None, sql_stmt: Optional[str] = None, sql_dialect: Optional[str] = None, transaction_size: int = 65536, append: bool = False, update: bool = False, create_spatial_index: Optional[bool] = None, explodecollections: bool = False, force_output_geometrytype: Optional[GeometryType] = None, sqlite_journal_mode: Optional[str] = None, verbose: bool = False) -> bool: # Remark: when executing a select statement, I keep getting error that # there are two columns named "geom" as he doesnt see the "geom" column # in the select as a geometry column. Probably a version issue. Maybe # try again later. args = [] # Cleanup the input_layers variable. if isinstance(input_path, Path) and input_path.suffix.lower() == '.shp': # For shapefiles, having input_layers not None gives issues input_layers = None elif sql_stmt is not None: # If a sql statement is passed, the input layers are not relevant, # and ogr2ogr will give a warning, so clear it. input_layers = None # Sql'ing, Filtering, clipping if spatial_filter is not None: args.extend(['-spat', str(spatial_filter[0]), str(spatial_filter[1]), str(spatial_filter[2]), str(spatial_filter[3])]) if clip_bounds is not None: args.extend(['-clipsrc', str(clip_bounds[0]), str(clip_bounds[1]), str(clip_bounds[2]), str(clip_bounds[3])]) ''' if sqlite_stmt is not None: args.extend(['-sql', sqlite_stmt, '-dialect', 'sqlite']) ''' # Output file options if output_path.exists() is True: if append is True: args.append('-append') if update is True: args.append('-update') # Files #args.append(output_path) #args.append(input_path) # Output layer options if explodecollections is True: args.append('-explodecollections') if output_layer is not None: args.extend(['-nln', output_layer]) if force_output_geometrytype is not None: args.extend(['-nlt', force_output_geometrytype.name]) args.extend(['-nlt', 'PROMOTE_TO_MULTI']) if transaction_size is not None: args.extend(['-gt', str(transaction_size)]) # Output layer creation options layerCreationOptions = [] # TODO: should check if the layer exists instead of the file if not output_path.exists(): if create_spatial_index is not None: if create_spatial_index is True: layerCreationOptions.extend(['SPATIAL_INDEX=YES']) else: layerCreationOptions.extend(['SPATIAL_INDEX=NO']) # Get output format from the filename output_filetype = GeofileType(output_path) # Sqlite specific options datasetCreationOptions = [] if output_filetype == GeofileType.SQLite: # Use the spatialite type of sqlite #datasetCreationOptions.extend(['-dsco', 'SPATIALITE=YES']) datasetCreationOptions.append('SPATIALITE=YES') ''' # Try if the busy_timeout isn't giving problems rather than solving them... if sqlite_journal_mode is not None: datasetCreationOptions.extend(['--config', 'OGR_SQLITE_PRAGMA', f"journal_mode={sqlite_journal_mode},busy_timeout=5000"]) else: datasetCreationOptions.extend(['--config OGR_SQLITE_PRAGMA busy_timeout=5000']) ''' if sqlite_journal_mode is not None: gdal.SetConfigOption('OGR_SQLITE_PRAGMA', f"journal_mode={sqlite_journal_mode}") #if append is False: # args.extend(['--config', 'OGR_SQLITE_PRAGMA', '"journal_mode=WAL"']) # args.extend(['-dsco', 'ADD_GPKG_OGR_CONTENTS=NO']) #else: # args.extend(['--config', 'OGR_SQLITE_PRAGMA', 'busy_timeout=-1']) #args.extend(['--config', 'OGR_SQLITE_SYNCHRONOUS', 'OFF']) gdal.SetConfigOption('OGR_SQLITE_CACHE', '128') options = gdal.VectorTranslateOptions( options=args, format=output_filetype.ogrdriver, accessMode=None, srcSRS=None, dstSRS=None, reproject=False, SQLStatement=sql_stmt, SQLDialect=sql_dialect, where=None, #"geom IS NOT NULL", selectFields=None, addFields=False, forceNullable=False, spatFilter=spatial_filter, spatSRS=None, datasetCreationOptions=datasetCreationOptions, layerCreationOptions=layerCreationOptions, layers=input_layers, layerName=output_layer, geometryType=None, dim=None, segmentizeMaxDist=None, zField=None, skipFailures=False, limit=None, callback=None, callback_data=None) input_ds = None try: # In some cases gdal only raises the last exception instead of the stack in VectorTranslate, # so you lose necessary details! -> uncomment gdal.DontUseExceptions() when debugging! #gdal.DontUseExceptions() gdal.UseExceptions() gdal.ConfigurePythonLogging(logger_name='gdal', enable_debug=False) logger.debug(f"Execute {sql_stmt} on {input_path}") input_ds = gdal.OpenEx(str(input_path)) # TODO: memory output support might be interesting to support result_ds = gdal.VectorTranslate( destNameOrDestDS=str(output_path), srcDS=input_ds, #SQLStatement=sql_stmt, #SQLDialect=sql_dialect, #layerName=output_layer options=options) if result_ds is None: raise Exception("BOEM") else: if result_ds.GetLayerCount() == 0: del result_ds if output_path.exists(): gfo.remove(output_path) except Exception as ex: message = f"Error executing {sql_stmt}" logger.exception(message) raise Exception(message) from ex finally: if input_ds is not None: del input_ds return True
from io import StringIO import logging import os from pathlib import Path import pprint import re import subprocess import tempfile from threading import Lock import time from typing import List, Optional, Tuple, Union import geopandas as gpd from osgeo import gdal gdal.UseExceptions() gdal.ConfigurePythonLogging(logger_name='gdal', enable_debug=False) import geofileops as gfo from geofileops.util.geofiletype import GeofileType from geofileops.util.geometry_util import GeometryType #------------------------------------------------------------- # First define/init some general variables/constants #------------------------------------------------------------- # Make sure only one instance per process is running lock = Lock() # Get a logger... logger = logging.getLogger(__name__)