def init(self): self.ogr = ogr # http://trac.osgeo.org/gdal/wiki/PythonGotchas self.gdal = gdal self.gdal.UseExceptions() log.info("Using GDAL/OGR version: %d" % int(gdal.VersionInfo('VERSION_NUM'))) # GDAL error handler function # http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html def gdal_error_handler(err_class, err_num, err_msg): err_type = { gdal.CE_None: 'None', gdal.CE_Debug: 'Debug', gdal.CE_Warning: 'Warning', gdal.CE_Failure: 'Failure', gdal.CE_Fatal: 'Fatal' } err_msg = err_msg.replace('\n', ' ') err_class = err_type.get(err_class, 'None') log.error('Error Number: %s, Type: %s, Msg: %s' % (err_num, err_class, err_msg)) # install error handler self.gdal.PushErrorHandler(gdal_error_handler) # Raise a dummy error for testing # self.gdal.Error(1, 2, 'test error') if self.source_options: for k in self.source_options: self.gdal.SetConfigOption(k, self.source_options[k]) # Open OGR data source in read-only mode. if self.source_format: self.data_source_p = ogr.GetDriverByName(self.source_format).Open(self.data_source, 0) else: self.data_source_p = self.ogr.Open(self.data_source, 0) # Report failure if failed if self.data_source_p is None: log.error("Cannot open OGR datasource: %s with the following drivers." % Util.safe_string_value(self.data_source)) for iDriver in range(self.ogr.GetDriverCount()): log.info(" -> " + self.ogr.GetDriver(iDriver).GetName()) raise Exception() else: # Open ok: initialize self.layer = None if self.sql: self.layer_count = 1 self.layer_idx = -1 else: self.layer_count = self.data_source_p.GetLayerCount() self.layer_idx = 0 log.info("Opened OGR source ok: %s layer count=%d" % (Util.safe_string_value(self.data_source), self.layer_count))
def read(self, packet): if not self.data_source_p: log.info("End reading from: %s" % Util.safe_string_value(self.data_source)) return packet if self.layer is None: if self.sql and self.layer_idx == -1: # PostgreSQL: Layer is gotten via Query # http://trac.osgeo.org/postgis/wiki/UsersWikiOGR self.layer = self.data_source_p.ExecuteSQL(self.sql) self.layer_idx = 0 elif self.layer_idx < self.layer_count: self.layer = self.data_source_p.GetLayer(self.layer_idx) self.layer_idx += 1 if self.layer is None: log.error("Could not fetch layer %d" % 0) raise Exception() log.info("Start reading from OGR Source: %s, Layer: %s" % (Util.safe_string_value( self.data_source), self.layer.GetName())) else: # No more Layers left: cleanup packet.set_end_of_stream() log.info("Closing OGR source: %s" % Util.safe_string_value(self.data_source)) # Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas # self.data_source_p.Destroy() self.data_source_p = None return packet # Return all features from Layer (ogr_feature_array) or next feature (ogr_feature) if self.output_format == FORMAT.ogr_feature_array: # Assemble all features features = list() for feature in self.layer: features.append(feature) packet.data = features log.info("End reading all features from Layer: %s count=%d" % (self.layer.GetName(), len(features))) packet.set_end_of_doc() self.layer = None else: # Next feature feature = self.layer.GetNextFeature() if feature: packet.data = feature else: log.info("End reading from Layer: %s" % self.layer.GetName()) packet.set_end_of_doc() self.layer = None return packet
def read(self, packet): if not self.data_source_p: log.info("End reading from: %s" % Util.safe_string_value(self.data_source)) return packet if self.layer is None: if self.sql and self.layer_idx == -1: # PostgreSQL: Layer is gotten via Query # http://trac.osgeo.org/postgis/wiki/UsersWikiOGR self.layer = self.data_source_p.ExecuteSQL(self.sql) self.layer_idx = 0 elif self.layer_idx < self.layer_count: self.layer = self.data_source_p.GetLayer(self.layer_idx) self.layer_idx += 1 if self.layer is None: log.error("Could not fetch layer %d" % 0) raise Exception() log.info("Start reading from OGR Source: %s, Layer: %s" % (Util.safe_string_value(self.data_source), self.layer.GetName())) else: # No more Layers left: cleanup packet.set_end_of_stream() log.info("Closing OGR source: %s" % Util.safe_string_value(self.data_source)) # Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas # self.data_source_p.Destroy() self.data_source_p = None return packet # Return all features from Layer (ogr_feature_array) or next feature (ogr_feature) if self.output_format == FORMAT.ogr_feature_array: # Assemble all features features = list() for feature in self.layer: features.append(feature) packet.data = features log.info("End reading all features from Layer: %s count=%d" % (self.layer.GetName(), len(features))) packet.set_end_of_doc() self.layer = None else: # Next feature feature = self.layer.GetNextFeature() if feature: packet.data = feature else: log.info("End reading from Layer: %s" % self.layer.GetName()) packet.set_end_of_doc() self.layer = None return packet
def write(self, packet): # Are we all done? if packet.data is None or self.dest_fd is None: self.write_end(packet) return packet if self.layer is None: log.info("No Layer, end writing to: %s" % Util.safe_string_value(self.dest_data_source)) return packet # Assume ogr_feature_array input, otherwise convert ogr_feature to list if type(packet.data) is list: # Write feature collection to OGR Layer output for feature in packet.data: self.write_feature(feature) self.write_end(packet) else: # Write single feature to OGR Layer output if packet.end_of_stream or packet.end_of_doc: self.write_end(packet) return packet self.write_feature(packet.data) return packet
def write_end(self, packet): # Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas # self.dest_fd.Destroy() log.info("End writing to: %s" % Util.safe_string_value(self.dest_data_source)) self.dest_fd = None self.layer = None return packet
def exec_cmd(self): log.info("start ogr2ogr cmd = %s" % Util.safe_string_value(repr(self.cmd))) self.ogr_process = subprocess.Popen(self.cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) err_line = self.readline_err() if err_line: log.warning('ogr2ogr: %s ' % err_line)
def execute_cmd(self, cmd): env_vars = Util.string_to_dict(self.env_args, self.env_separator) old_environ = os.environ.copy() try: os.environ.update(env_vars) log.info("executing cmd=%s" % Util.safe_string_value(cmd)) subprocess.call(cmd, shell=True) log.info("execute done") finally: os.environ = old_environ
def init(self): self.ogr = ogr # http://trac.osgeo.org/gdal/wiki/PythonGotchas self.gdal = gdal self.gdal.UseExceptions() log.info("Using GDAL/OGR version: %d" % int(gdal.VersionInfo('VERSION_NUM'))) # GDAL error handler function # http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html def gdal_error_handler(err_class, err_num, err_msg): err_type = { gdal.CE_None: 'None', gdal.CE_Debug: 'Debug', gdal.CE_Warning: 'Warning', gdal.CE_Failure: 'Failure', gdal.CE_Fatal: 'Fatal' } err_msg = err_msg.replace('\n', ' ') err_class = err_type.get(err_class, 'None') log.error('Error Number: %s, Type: %s, Msg: %s' % (err_num, err_class, err_msg)) # install error handler self.gdal.PushErrorHandler(gdal_error_handler) # Raise a dummy error for testing # self.gdal.Error(1, 2, 'test error') self.update = self.overwrite or self.append if self.dest_options: for k in self.dest_options: self.gdal.SetConfigOption(k, self.dest_options[k]) self.dest_driver = None self.dest_fd = None # Loosely based on https://github.com/OSGeo/gdal/blob/trunk/gdal/swig/python/samples/ogr2ogr.py # /* -------------------------------------------------------------------- */ # /* Try opening the output data source as an existing, writable */ # /* -------------------------------------------------------------------- */ if self.update: # Try opening in update mode self.dest_fd = ogr.Open(self.dest_data_source, True) if self.dest_fd is not None: if len(self.dest_create_options) > 0: log.warn( "Datasource creation options ignored since an existing datasource being updated." ) self.dest_driver = self.dest_fd.GetDriver() if self.overwrite: self.dest_driver.DeleteDataSource(self.dest_data_source) self.dest_fd = None self.dest_driver = None self.update = False # /* -------------------------------------------------------------------- */ # /* Find the output driver. */ # /* -------------------------------------------------------------------- */ if self.dest_driver is None: # Open OGR data dest in write-only mode. self.dest_driver = ogr.GetDriverByName(self.dest_format) # Report failure if failed if self.dest_driver is None: log.error( "Cannot open OGR data destination: %s with the following drivers." % self.dest_data_source) for iDriver in range(self.ogr.GetDriverCount()): log.info(" -> " + self.ogr.GetDriver(iDriver).GetName()) raise Exception() if self.dest_driver.TestCapability( ogr.ODrCCreateDataSource) is False: log.error("%s driver does not support data source creation." % self.dest_format) raise Exception() # /* -------------------------------------------------------------------- */ # /* Create the output data source. */ # /* -------------------------------------------------------------------- */ if self.dest_fd is None: self.dest_fd = self.dest_driver.CreateDataSource( self.dest_data_source, options=self.dest_create_options) if self.dest_fd is None: log.error("%s driver failed to create %s" % (self.dest_format, Util.safe_string_value(self.dest_data_source))) raise Exception() # /* -------------------------------------------------------------------- */ # /* Parse the output SRS definition if possible. */ # /* -------------------------------------------------------------------- */ output_srs_ref = None if self.target_srs is not None: output_srs_ref = osr.SpatialReference() if output_srs_ref.SetFromUserInput(self.target_srs) != 0: log.error("Failed to process SRS definition: %s" % self.target_srs) raise Exception() self.layer = self.dest_fd.CreateLayer(self.new_layer_name, output_srs_ref, ogr.wkbUnknown, self.layer_create_options) self.feature_def = None log.info("Opened OGR dest ok: %s " % Util.safe_string_value(self.dest_data_source))
def init(self): self.ogr = ogr # http://trac.osgeo.org/gdal/wiki/PythonGotchas self.gdal = gdal self.gdal.UseExceptions() log.info("Using GDAL/OGR version: %d" % int(gdal.VersionInfo('VERSION_NUM'))) # GDAL error handler function # http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html def gdal_error_handler(err_class, err_num, err_msg): err_type = { gdal.CE_None: 'None', gdal.CE_Debug: 'Debug', gdal.CE_Warning: 'Warning', gdal.CE_Failure: 'Failure', gdal.CE_Fatal: 'Fatal' } err_msg = err_msg.replace('\n', ' ') err_class = err_type.get(err_class, 'None') log.error('Error Number: %s, Type: %s, Msg: %s' % (err_num, err_class, err_msg)) # install error handler self.gdal.PushErrorHandler(gdal_error_handler) # Raise a dummy error for testing # self.gdal.Error(1, 2, 'test error') self.update = self.overwrite or self.append if self.dest_options: for k in self.dest_options: self.gdal.SetConfigOption(k, self.dest_options[k]) self.dest_driver = None self.dest_fd = None # Loosely based on https://github.com/OSGeo/gdal/blob/trunk/gdal/swig/python/samples/ogr2ogr.py # /* -------------------------------------------------------------------- */ # /* Try opening the output data source as an existing, writable */ # /* -------------------------------------------------------------------- */ if self.update: # Try opening in update mode self.dest_fd = ogr.Open(self.dest_data_source, True) if self.dest_fd is not None: if len(self.dest_create_options) > 0: log.warn("Datasource creation options ignored since an existing datasource being updated.") self.dest_driver = self.dest_fd.GetDriver() if self.overwrite: self.dest_driver.DeleteDataSource(self.dest_data_source) self.dest_fd = None self.dest_driver = None self.update = False # /* -------------------------------------------------------------------- */ # /* Find the output driver. */ # /* -------------------------------------------------------------------- */ if self.dest_driver is None: # Open OGR data dest in write-only mode. self.dest_driver = ogr.GetDriverByName(self.dest_format) # Report failure if failed if self.dest_driver is None: log.error("Cannot open OGR data destination: %s with the following drivers." % self.dest_data_source) for iDriver in range(self.ogr.GetDriverCount()): log.info(" -> " + self.ogr.GetDriver(iDriver).GetName()) raise Exception() if self.dest_driver.TestCapability(ogr.ODrCCreateDataSource) is False: log.error("%s driver does not support data source creation." % self.dest_format) raise Exception() # /* -------------------------------------------------------------------- */ # /* Create the output data source. */ # /* -------------------------------------------------------------------- */ if self.dest_fd is None: self.dest_fd = self.dest_driver.CreateDataSource(self.dest_data_source, options=self.dest_create_options) if self.dest_fd is None: log.error("%s driver failed to create %s" % (self.dest_format, Util.safe_string_value(self.dest_data_source))) raise Exception() # /* -------------------------------------------------------------------- */ # /* Parse the output SRS definition if possible. */ # /* -------------------------------------------------------------------- */ output_srs_ref = None if self.target_srs is not None: output_srs_ref = osr.SpatialReference() if output_srs_ref.SetFromUserInput(self.target_srs) != 0: log.error("Failed to process SRS definition: %s" % self.target_srs) raise Exception() self.layer = self.dest_fd.CreateLayer(self.new_layer_name, output_srs_ref, ogr.wkbUnknown, self.layer_create_options) self.feature_def = None log.info("Opened OGR dest ok: %s " % Util.safe_string_value(self.dest_data_source))
def init(self): self.ogr = ogr # http://trac.osgeo.org/gdal/wiki/PythonGotchas self.gdal = gdal self.gdal.UseExceptions() log.info("Using GDAL/OGR version: %d" % int(gdal.VersionInfo('VERSION_NUM'))) # GDAL error handler function # http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html def gdal_error_handler(err_class, err_num, err_msg): err_type = { gdal.CE_None: 'None', gdal.CE_Debug: 'Debug', gdal.CE_Warning: 'Warning', gdal.CE_Failure: 'Failure', gdal.CE_Fatal: 'Fatal' } err_msg = err_msg.replace('\n', ' ') err_class = err_type.get(err_class, 'None') log.error('Error Number: %s, Type: %s, Msg: %s' % (err_num, err_class, err_msg)) # install error handler self.gdal.PushErrorHandler(gdal_error_handler) # Raise a dummy error for testing # self.gdal.Error(1, 2, 'test error') if self.source_options: for k in self.source_options: self.gdal.SetConfigOption(k, self.source_options[k]) # Open OGR data source in read-only mode. if self.source_format: self.data_source_p = ogr.GetDriverByName(self.source_format).Open( self.data_source, 0) else: self.data_source_p = self.ogr.Open(self.data_source, 0) # Report failure if failed if self.data_source_p is None: log.error( "Cannot open OGR datasource: %s with the following drivers." % Util.safe_string_value(self.data_source)) for iDriver in range(self.ogr.GetDriverCount()): log.info(" -> " + self.ogr.GetDriver(iDriver).GetName()) raise Exception() else: # Open ok: initialize self.layer = None if self.sql: self.layer_count = 1 self.layer_idx = -1 else: self.layer_count = self.data_source_p.GetLayerCount() self.layer_idx = 0 log.info( "Opened OGR source ok: %s layer count=%d" % (Util.safe_string_value(self.data_source), self.layer_count))