def init(self):
self.ogr = ogr
# http://trac.osgeo.org/gdal/wiki/PythonGotchas
self.gdal = gdal
self.gdal.UseExceptions()
log.info("Using GDAL/OGR version: %d" % int(gdal.VersionInfo('VERSION_NUM')))
# GDAL error handler function
# http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html
def gdal_error_handler(err_class, err_num, err_msg):
err_type = {
gdal.CE_None: 'None',
gdal.CE_Debug: 'Debug',
gdal.CE_Warning: 'Warning',
gdal.CE_Failure: 'Failure',
gdal.CE_Fatal: 'Fatal'
}
err_msg = err_msg.replace('\n', ' ')
err_class = err_type.get(err_class, 'None')
log.error('Error Number: %s, Type: %s, Msg: %s' % (err_num, err_class, err_msg))
# install error handler
self.gdal.PushErrorHandler(gdal_error_handler)
# Raise a dummy error for testing
# self.gdal.Error(1, 2, 'test error')
if self.source_options:
for k in self.source_options:
self.gdal.SetConfigOption(k, self.source_options[k])
# Open OGR data source in read-only mode.
if self.source_format:
self.data_source_p = ogr.GetDriverByName(self.source_format).Open(self.data_source, 0)
else:
self.data_source_p = self.ogr.Open(self.data_source, 0)
# Report failure if failed
if self.data_source_p is None:
log.error("Cannot open OGR datasource: %s with the following drivers." % Util.safe_string_value(self.data_source))
for iDriver in range(self.ogr.GetDriverCount()):
log.info(" -> " + self.ogr.GetDriver(iDriver).GetName())
raise Exception()
else:
# Open ok: initialize
self.layer = None
if self.sql:
self.layer_count = 1
self.layer_idx = -1
else:
self.layer_count = self.data_source_p.GetLayerCount()
self.layer_idx = 0
log.info("Opened OGR source ok: %s layer count=%d" % (Util.safe_string_value(self.data_source), self.layer_count))
def read(self, packet):
if not self.data_source_p:
log.info("End reading from: %s" %
Util.safe_string_value(self.data_source))
return packet
if self.layer is None:
if self.sql and self.layer_idx == -1:
# PostgreSQL: Layer is gotten via Query
# http://trac.osgeo.org/postgis/wiki/UsersWikiOGR
self.layer = self.data_source_p.ExecuteSQL(self.sql)
self.layer_idx = 0
elif self.layer_idx < self.layer_count:
self.layer = self.data_source_p.GetLayer(self.layer_idx)
self.layer_idx += 1
if self.layer is None:
log.error("Could not fetch layer %d" % 0)
raise Exception()
log.info("Start reading from OGR Source: %s, Layer: %s" %
(Util.safe_string_value(
self.data_source), self.layer.GetName()))
else:
# No more Layers left: cleanup
packet.set_end_of_stream()
log.info("Closing OGR source: %s" %
Util.safe_string_value(self.data_source))
# Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas
# self.data_source_p.Destroy()
self.data_source_p = None
return packet
# Return all features from Layer (ogr_feature_array) or next feature (ogr_feature)
if self.output_format == FORMAT.ogr_feature_array:
# Assemble all features
features = list()
for feature in self.layer:
features.append(feature)
packet.data = features
log.info("End reading all features from Layer: %s count=%d" %
(self.layer.GetName(), len(features)))
packet.set_end_of_doc()
self.layer = None
else:
# Next feature
feature = self.layer.GetNextFeature()
if feature:
packet.data = feature
else:
log.info("End reading from Layer: %s" % self.layer.GetName())
packet.set_end_of_doc()
self.layer = None
return packet
def read(self, packet):
if not self.data_source_p:
log.info("End reading from: %s" % Util.safe_string_value(self.data_source))
return packet
if self.layer is None:
if self.sql and self.layer_idx == -1:
# PostgreSQL: Layer is gotten via Query
# http://trac.osgeo.org/postgis/wiki/UsersWikiOGR
self.layer = self.data_source_p.ExecuteSQL(self.sql)
self.layer_idx = 0
elif self.layer_idx < self.layer_count:
self.layer = self.data_source_p.GetLayer(self.layer_idx)
self.layer_idx += 1
if self.layer is None:
log.error("Could not fetch layer %d" % 0)
raise Exception()
log.info("Start reading from OGR Source: %s, Layer: %s" % (Util.safe_string_value(self.data_source), self.layer.GetName()))
else:
# No more Layers left: cleanup
packet.set_end_of_stream()
log.info("Closing OGR source: %s" % Util.safe_string_value(self.data_source))
# Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas
# self.data_source_p.Destroy()
self.data_source_p = None
return packet
# Return all features from Layer (ogr_feature_array) or next feature (ogr_feature)
if self.output_format == FORMAT.ogr_feature_array:
# Assemble all features
features = list()
for feature in self.layer:
features.append(feature)
packet.data = features
log.info("End reading all features from Layer: %s count=%d" % (self.layer.GetName(), len(features)))
packet.set_end_of_doc()
self.layer = None
else:
# Next feature
feature = self.layer.GetNextFeature()
if feature:
packet.data = feature
else:
log.info("End reading from Layer: %s" % self.layer.GetName())
packet.set_end_of_doc()
self.layer = None
return packet
def write(self, packet):
# Are we all done?
if packet.data is None or self.dest_fd is None:
self.write_end(packet)
return packet
if self.layer is None:
log.info("No Layer, end writing to: %s" %
Util.safe_string_value(self.dest_data_source))
return packet
# Assume ogr_feature_array input, otherwise convert ogr_feature to list
if type(packet.data) is list:
# Write feature collection to OGR Layer output
for feature in packet.data:
self.write_feature(feature)
self.write_end(packet)
else:
# Write single feature to OGR Layer output
if packet.end_of_stream or packet.end_of_doc:
self.write_end(packet)
return packet
self.write_feature(packet.data)
return packet
def write_end(self, packet):
# Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas
# self.dest_fd.Destroy()
log.info("End writing to: %s" % Util.safe_string_value(self.dest_data_source))
self.dest_fd = None
self.layer = None
return packet
def write(self, packet):
# Are we all done?
if packet.data is None or self.dest_fd is None:
self.write_end(packet)
return packet
if self.layer is None:
log.info("No Layer, end writing to: %s" % Util.safe_string_value(self.dest_data_source))
return packet
# Assume ogr_feature_array input, otherwise convert ogr_feature to list
if type(packet.data) is list:
# Write feature collection to OGR Layer output
for feature in packet.data:
self.write_feature(feature)
self.write_end(packet)
else:
# Write single feature to OGR Layer output
if packet.end_of_stream or packet.end_of_doc:
self.write_end(packet)
return packet
self.write_feature(packet.data)
return packet
def write_end(self, packet):
# Destroy not required anymore: http://trac.osgeo.org/gdal/wiki/PythonGotchas
# self.dest_fd.Destroy()
log.info("End writing to: %s" %
Util.safe_string_value(self.dest_data_source))
self.dest_fd = None
self.layer = None
return packet
def exec_cmd(self):
log.info("start ogr2ogr cmd = %s" % Util.safe_string_value(repr(self.cmd)))
self.ogr_process = subprocess.Popen(self.cmd,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
err_line = self.readline_err()
if err_line:
log.warning('ogr2ogr: %s ' % err_line)
def execute_cmd(self, cmd):
env_vars = Util.string_to_dict(self.env_args, self.env_separator)
old_environ = os.environ.copy()
try:
os.environ.update(env_vars)
log.info("executing cmd=%s" % Util.safe_string_value(cmd))
subprocess.call(cmd, shell=True)
log.info("execute done")
finally:
os.environ = old_environ
def execute_cmd(self, cmd):
env_vars = Util.string_to_dict(self.env_args, self.env_separator)
old_environ = os.environ.copy()
try:
os.environ.update(env_vars)
log.info("executing cmd=%s" % Util.safe_string_value(cmd))
subprocess.call(cmd, shell=True)
log.info("execute done")
finally:
os.environ = old_environ
def exec_cmd(self):
log.info("start ogr2ogr cmd = %s" %
Util.safe_string_value(repr(self.cmd)))
self.ogr_process = subprocess.Popen(self.cmd,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
err_line = self.readline_err()
if err_line:
log.warning('ogr2ogr: %s ' % err_line)
def init(self):
self.ogr = ogr
# http://trac.osgeo.org/gdal/wiki/PythonGotchas
self.gdal = gdal
self.gdal.UseExceptions()
log.info("Using GDAL/OGR version: %d" %
int(gdal.VersionInfo('VERSION_NUM')))
# GDAL error handler function
# http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html
def gdal_error_handler(err_class, err_num, err_msg):
err_type = {
gdal.CE_None: 'None',
gdal.CE_Debug: 'Debug',
gdal.CE_Warning: 'Warning',
gdal.CE_Failure: 'Failure',
gdal.CE_Fatal: 'Fatal'
}
err_msg = err_msg.replace('\n', ' ')
err_class = err_type.get(err_class, 'None')
log.error('Error Number: %s, Type: %s, Msg: %s' %
(err_num, err_class, err_msg))
# install error handler
self.gdal.PushErrorHandler(gdal_error_handler)
# Raise a dummy error for testing
# self.gdal.Error(1, 2, 'test error')
self.update = self.overwrite or self.append
if self.dest_options:
for k in self.dest_options:
self.gdal.SetConfigOption(k, self.dest_options[k])
self.dest_driver = None
self.dest_fd = None
# Loosely based on https://github.com/OSGeo/gdal/blob/trunk/gdal/swig/python/samples/ogr2ogr.py
# /* -------------------------------------------------------------------- */
# /* Try opening the output data source as an existing, writable */
# /* -------------------------------------------------------------------- */
if self.update:
# Try opening in update mode
self.dest_fd = ogr.Open(self.dest_data_source, True)
if self.dest_fd is not None:
if len(self.dest_create_options) > 0:
log.warn(
"Datasource creation options ignored since an existing datasource being updated."
)
self.dest_driver = self.dest_fd.GetDriver()
if self.overwrite:
self.dest_driver.DeleteDataSource(self.dest_data_source)
self.dest_fd = None
self.dest_driver = None
self.update = False
# /* -------------------------------------------------------------------- */
# /* Find the output driver. */
# /* -------------------------------------------------------------------- */
if self.dest_driver is None:
# Open OGR data dest in write-only mode.
self.dest_driver = ogr.GetDriverByName(self.dest_format)
# Report failure if failed
if self.dest_driver is None:
log.error(
"Cannot open OGR data destination: %s with the following drivers."
% self.dest_data_source)
for iDriver in range(self.ogr.GetDriverCount()):
log.info(" -> " + self.ogr.GetDriver(iDriver).GetName())
raise Exception()
if self.dest_driver.TestCapability(
ogr.ODrCCreateDataSource) is False:
log.error("%s driver does not support data source creation." %
self.dest_format)
raise Exception()
# /* -------------------------------------------------------------------- */
# /* Create the output data source. */
# /* -------------------------------------------------------------------- */
if self.dest_fd is None:
self.dest_fd = self.dest_driver.CreateDataSource(
self.dest_data_source, options=self.dest_create_options)
if self.dest_fd is None:
log.error("%s driver failed to create %s" %
(self.dest_format,
Util.safe_string_value(self.dest_data_source)))
raise Exception()
# /* -------------------------------------------------------------------- */
# /* Parse the output SRS definition if possible. */
# /* -------------------------------------------------------------------- */
output_srs_ref = None
if self.target_srs is not None:
output_srs_ref = osr.SpatialReference()
if output_srs_ref.SetFromUserInput(self.target_srs) != 0:
log.error("Failed to process SRS definition: %s" %
self.target_srs)
raise Exception()
self.layer = self.dest_fd.CreateLayer(self.new_layer_name,
output_srs_ref, ogr.wkbUnknown,
self.layer_create_options)
self.feature_def = None
log.info("Opened OGR dest ok: %s " %
Util.safe_string_value(self.dest_data_source))
def init(self):
self.ogr = ogr
# http://trac.osgeo.org/gdal/wiki/PythonGotchas
self.gdal = gdal
self.gdal.UseExceptions()
log.info("Using GDAL/OGR version: %d" % int(gdal.VersionInfo('VERSION_NUM')))
# GDAL error handler function
# http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html
def gdal_error_handler(err_class, err_num, err_msg):
err_type = {
gdal.CE_None: 'None',
gdal.CE_Debug: 'Debug',
gdal.CE_Warning: 'Warning',
gdal.CE_Failure: 'Failure',
gdal.CE_Fatal: 'Fatal'
}
err_msg = err_msg.replace('\n', ' ')
err_class = err_type.get(err_class, 'None')
log.error('Error Number: %s, Type: %s, Msg: %s' % (err_num, err_class, err_msg))
# install error handler
self.gdal.PushErrorHandler(gdal_error_handler)
# Raise a dummy error for testing
# self.gdal.Error(1, 2, 'test error')
self.update = self.overwrite or self.append
if self.dest_options:
for k in self.dest_options:
self.gdal.SetConfigOption(k, self.dest_options[k])
self.dest_driver = None
self.dest_fd = None
# Loosely based on https://github.com/OSGeo/gdal/blob/trunk/gdal/swig/python/samples/ogr2ogr.py
# /* -------------------------------------------------------------------- */
# /* Try opening the output data source as an existing, writable */
# /* -------------------------------------------------------------------- */
if self.update:
# Try opening in update mode
self.dest_fd = ogr.Open(self.dest_data_source, True)
if self.dest_fd is not None:
if len(self.dest_create_options) > 0:
log.warn("Datasource creation options ignored since an existing datasource being updated.")
self.dest_driver = self.dest_fd.GetDriver()
if self.overwrite:
self.dest_driver.DeleteDataSource(self.dest_data_source)
self.dest_fd = None
self.dest_driver = None
self.update = False
# /* -------------------------------------------------------------------- */
# /* Find the output driver. */
# /* -------------------------------------------------------------------- */
if self.dest_driver is None:
# Open OGR data dest in write-only mode.
self.dest_driver = ogr.GetDriverByName(self.dest_format)
# Report failure if failed
if self.dest_driver is None:
log.error("Cannot open OGR data destination: %s with the following drivers." % self.dest_data_source)
for iDriver in range(self.ogr.GetDriverCount()):
log.info(" -> " + self.ogr.GetDriver(iDriver).GetName())
raise Exception()
if self.dest_driver.TestCapability(ogr.ODrCCreateDataSource) is False:
log.error("%s driver does not support data source creation." % self.dest_format)
raise Exception()
# /* -------------------------------------------------------------------- */
# /* Create the output data source. */
# /* -------------------------------------------------------------------- */
if self.dest_fd is None:
self.dest_fd = self.dest_driver.CreateDataSource(self.dest_data_source, options=self.dest_create_options)
if self.dest_fd is None:
log.error("%s driver failed to create %s" % (self.dest_format, Util.safe_string_value(self.dest_data_source)))
raise Exception()
# /* -------------------------------------------------------------------- */
# /* Parse the output SRS definition if possible. */
# /* -------------------------------------------------------------------- */
output_srs_ref = None
if self.target_srs is not None:
output_srs_ref = osr.SpatialReference()
if output_srs_ref.SetFromUserInput(self.target_srs) != 0:
log.error("Failed to process SRS definition: %s" % self.target_srs)
raise Exception()
self.layer = self.dest_fd.CreateLayer(self.new_layer_name, output_srs_ref, ogr.wkbUnknown,
self.layer_create_options)
self.feature_def = None
log.info("Opened OGR dest ok: %s " % Util.safe_string_value(self.dest_data_source))
def init(self):
self.ogr = ogr
# http://trac.osgeo.org/gdal/wiki/PythonGotchas
self.gdal = gdal
self.gdal.UseExceptions()
log.info("Using GDAL/OGR version: %d" %
int(gdal.VersionInfo('VERSION_NUM')))
# GDAL error handler function
# http://pcjericks.github.io/py-gdalogr-cookbook/gdal_general.html
def gdal_error_handler(err_class, err_num, err_msg):
err_type = {
gdal.CE_None: 'None',
gdal.CE_Debug: 'Debug',
gdal.CE_Warning: 'Warning',
gdal.CE_Failure: 'Failure',
gdal.CE_Fatal: 'Fatal'
}
err_msg = err_msg.replace('\n', ' ')
err_class = err_type.get(err_class, 'None')
log.error('Error Number: %s, Type: %s, Msg: %s' %
(err_num, err_class, err_msg))
# install error handler
self.gdal.PushErrorHandler(gdal_error_handler)
# Raise a dummy error for testing
# self.gdal.Error(1, 2, 'test error')
if self.source_options:
for k in self.source_options:
self.gdal.SetConfigOption(k, self.source_options[k])
# Open OGR data source in read-only mode.
if self.source_format:
self.data_source_p = ogr.GetDriverByName(self.source_format).Open(
self.data_source, 0)
else:
self.data_source_p = self.ogr.Open(self.data_source, 0)
# Report failure if failed
if self.data_source_p is None:
log.error(
"Cannot open OGR datasource: %s with the following drivers." %
Util.safe_string_value(self.data_source))
for iDriver in range(self.ogr.GetDriverCount()):
log.info(" -> " + self.ogr.GetDriver(iDriver).GetName())
raise Exception()
else:
# Open ok: initialize
self.layer = None
if self.sql:
self.layer_count = 1
self.layer_idx = -1
else:
self.layer_count = self.data_source_p.GetLayerCount()
self.layer_idx = 0
log.info(
"Opened OGR source ok: %s layer count=%d" %
(Util.safe_string_value(self.data_source), self.layer_count))
