def save_as_gml(layer, path):
transform_context = QgsProject.instance().transformContext()
save_options = QgsVectorFileWriter.SaveVectorOptions()
save_options.driverName = QgsVectorFileWriter.driverForExtension("gml")
save_options.fileEncoding = "UTF-8"
QgsVectorFileWriter.writeAsVectorFormatV2(
layer,
path,
transform_context,
save_options,
)
output = QgsVectorLayer(path, "output", "ogr")
assert output.isValid()
return output
def save_as_geojson(self, output_path: Path) -> Path:
output_file = Path(output_path, f"{self.resource_name}.geojson")
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = "GeoJSON"
options.fileEncoding = "utf-8"
src_crs = self.layer.crs()
dst_crs = QgsCoordinateReferenceSystem("EPSG:4326")
options.ct = QgsCoordinateTransform(src_crs, dst_crs,
QgsProject.instance())
if hasattr(QgsVectorFileWriter, "writeAsVectorFormatV3"):
writer_, msg, _, _ = QgsVectorFileWriter.writeAsVectorFormatV3(
self.layer,
str(output_file),
QgsProject.instance().transformContext(),
options,
)
else:
writer_, msg = QgsVectorFileWriter.writeAsVectorFormatV2(
self.layer,
str(output_file),
QgsProject.instance().transformContext(),
options,
)
if msg != "":
raise DataPackageException(
tr("Could not write layer {} to disk", output_file),
bar_msg(tr("Check the log for more details")),
)
return output_file
def save_vector_layer_as_gpkg(layer, target_dir, update_datasource=False):
"""Save layer as a single table GeoPackage in the target_dir. Update the original layer datasource if needed.
If the original layer has already a fid column with non-unique values, it will be renamed to first free fid_x.
"""
layer_name = remove_forbidden_chars("_".join(layer.name().split()))
layer_filename = get_unique_filename(
os.path.join(target_dir, f"{layer_name}.gpkg"))
transform_context = QgsProject.instance().transformContext()
writer_opts = QgsVectorFileWriter.SaveVectorOptions()
writer_opts.fileEncoding = "UTF-8"
writer_opts.layerName = layer_name
writer_opts.driverName = "GPKG"
if layer.fields().lookupField("fid") >= 0:
converter = FieldConverter(layer)
writer_opts.fieldValueConverter = converter
res, err = QgsVectorFileWriter.writeAsVectorFormatV2(
layer, layer_filename, transform_context, writer_opts)
if res != QgsVectorFileWriter.NoError:
return layer_filename, f"Couldn't properly save layer: {layer_filename}. \n{err}"
if update_datasource:
provider_opts = QgsDataProvider.ProviderOptions()
provider_opts.fileEncoding = "UTF-8"
provider_opts.layerName = layer_name
provider_opts.driverName = "GPKG"
datasource = f"{layer_filename}|layername={layer_name}"
layer.setDataSource(datasource, layer_name, "ogr", provider_opts)
return layer_filename, None
def response_gpx_mode(self, request):
"""
Download GPX of data
:param request: Http Django request object
:return: http response with attached file
"""
if not self.layer.download_gpx:
return HttpResponseForbidden()
# check for vector type
if self.metadata_layer.qgis_layer.geometryType(
) == QgsWkbTypes.PolygonGeometry:
return HttpResponseForbidden()
tmp_dir = tempfile.TemporaryDirectory()
filename = self.metadata_layer.qgis_layer.name()
# Apply filter backends, store original subset string
qgs_request = QgsFeatureRequest()
original_subset_string = self.metadata_layer.qgis_layer.subsetString()
if hasattr(self, 'filter_backends'):
for backend in self.filter_backends:
backend().apply_filter(request, self.metadata_layer.qgis_layer,
qgs_request, self)
save_options = QgsVectorFileWriter.SaveVectorOptions()
save_options.driverName = 'GPX'
save_options.fileEncoding = 'utf-8'
save_options.datasourceOptions = [
"GPX_USE_EXTENSIONS=1",
"GPX_EXTENSIONS_NS_URL=http://osgeo.org/gdal",
"GPX_EXTENSIONS_NS=ogr"
]
filename = self.metadata_layer.qgis_layer.name() + '.gpx'
# Make a selection based on the request
self._selection_responde_download_mode(qgs_request, save_options)
gpx_tmp_path = os.path.join(tmp_dir.name, filename)
error_code, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
self.metadata_layer.qgis_layer, gpx_tmp_path,
self.metadata_layer.qgis_layer.transformContext(), save_options)
# Restore the original subset string and select no features
self.metadata_layer.qgis_layer.selectByIds([])
self.metadata_layer.qgis_layer.setSubsetString(original_subset_string)
if error_code != QgsVectorFileWriter.NoError:
tmp_dir.cleanup()
return HttpResponse(status=500, reason=error_message)
response = HttpResponse(open(gpx_tmp_path, 'rb').read(),
content_type='application/octet-stream')
tmp_dir.cleanup()
response['Content-Disposition'] = f'attachment; filename={filename}'
response.set_cookie('fileDownload', 'true')
return response
def _save_layer_to_file(layer: QgsVectorLayer, output_path: Path) -> Path:
""" Save layer to file"""
output_file = output_path / f'{layer.name().replace(" ", "")}.csv'
LOGGER.debug(f'Saving layer to a file {output_file.name}')
converter = CsvFieldValueConverter(layer)
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = "csv"
options.fileEncoding = "utf-8"
options.layerOptions = ["SEPARATOR=COMMA"]
options.fieldValueConverter = converter
if hasattr(QgsVectorFileWriter, "writeAsVectorFormatV3"):
# noinspection PyCallByClass
writer_, msg, _, _ = QgsVectorFileWriter.writeAsVectorFormatV3(layer, str(output_file),
QgsProject.instance().transformContext(),
options)
else:
writer_, msg = QgsVectorFileWriter.writeAsVectorFormatV2(layer, str(output_file),
QgsProject.instance().transformContext(), options)
if msg:
raise ProcessInterruptedException(tr('Process ended'),
bar_msg=bar_msg(tr('Exception occurred during data extraction: {}', msg)))
return output_file
def testShapefilesWithNoAttributes(self):
"""Test issue GH #38834"""
ml = QgsVectorLayer('Point?crs=epsg:4326', 'test', 'memory')
self.assertTrue(ml.isValid())
d = QTemporaryDir()
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = 'ESRI Shapefile'
options.layerName = 'writetest'
err, _ = QgsVectorFileWriter.writeAsVectorFormatV2(
ml, os.path.join(d.path(), 'writetest.shp'),
QgsCoordinateTransformContext(), options)
self.assertEqual(err, QgsVectorFileWriter.NoError)
self.assertTrue(os.path.isfile(os.path.join(d.path(),
'writetest.shp')))
vl = QgsVectorLayer(os.path.join(d.path(), 'writetest.shp'))
self.assertTrue(
bool(vl.dataProvider().capabilities()
& QgsVectorDataProvider.AddFeatures))
# Let's try if we can really add features
feature = QgsFeature(vl.fields())
geom = QgsGeometry.fromWkt('POINT(9 45)')
feature.setGeometry(geom)
self.assertTrue(vl.startEditing())
self.assertTrue(vl.addFeatures([feature]))
self.assertTrue(vl.commitChanges())
del (vl)
vl = QgsVectorLayer(os.path.join(d.path(), 'writetest.shp'))
self.assertEqual(vl.featureCount(), 1)
def export_layer_to_shape(self, context, sql_layer, dpt, table, dirname, feedback):
# Construction filename
prefixe = "N_3V_"
suffixe = "_" + dpt
tablename = table.upper()
if table == 'element':
tablename = 'R_' + tablename + '_PORTION'
if 'poi' in table and 'portion' not in table:
tablename = tablename[4:]
if "val" in table:
prefixe = "3V_"
suffixe = ""
if "avancement" in table:
tablename = "AVANCEMENT_VAL"
if "statut" in table:
tablename = "STATUT_VAL"
if "portion" in table or "repere" in table:
tablename = "TYPE" + tablename
geomtype = {
QgsWkbTypes.LineGeometry: '_L',
QgsWkbTypes.PointGeometry: '_P',
QgsWkbTypes.NullGeometry: ''
}
geomcode = geomtype[sql_layer.geometryType()]
filename = prefixe + tablename + geomcode + suffixe
transform_context = context.project().transformContext()
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = "ESRI ShapeFile"
options.fileEncoding = "utf-8"
# construction du répertoire
if not os.path.exists(dirname):
os.mkdir(dirname)
ext = '.shp'
if geomcode == '':
ext = '.dbf'
file_path = os.path.join(dirname, filename + ext)
# Enregistrement du fichier shape
error = QgsVectorFileWriter.writeAsVectorFormatV2(
layer=sql_layer,
fileName=file_path,
transformContext=transform_context,
options=options
)
# Vérification du retour de l'enregistrement et des erreurs potentiels
if error[0] == QgsVectorFileWriter.ErrCreateLayer:
raise QgsProcessingException(
"Erreur lors de l'écriture du fichier :\n"
"Vérifiez que le chemin de destination est valide")
if error[0] != QgsVectorFileWriter.NoError:
raise QgsProcessingException("Erreur lors de l'écriture du fichier :" + error[1])
return [file_path, filename]
def _write_to_ogr(destination_path, new_layer_name, driverName=None):
"""Writes features to new or existing OGR layer"""
tmp_dir = QTemporaryDir()
tmp_path = os.path.join(tmp_dir.path(), 'isochrone.json')
with open(tmp_path, 'w+') as f:
f.write(geojson)
tmp_layer = QgsVectorLayer(tmp_path, 'tmp_isochrone', 'ogr')
if not tmp_layer.isValid():
raise Exception(
_('Cannot create temporary layer for isochrone result.'))
# Note: shp attribute names are max 10 chars long
save_options = QgsVectorFileWriter.SaveVectorOptions()
if driverName is not None:
save_options.driverName = driverName
save_options.layerName = new_layer_name
save_options.fileEncoding = 'utf-8'
# This is nonsense to me: if the file does not exist the actionOnExistingFile
# should be ignored instead of raising an error, probable QGIS bug
if os.path.exists(destination_path):
# Check if the layer already exists
layer_exists = QgsVectorFileWriter.targetLayerExists(
destination_path, new_layer_name)
if layer_exists:
raise Exception(
_('Cannot save isochrone result to destination layer: layer already exists (use "qgis_layer_id" instead)!'
))
save_options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
error_code, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
tmp_layer, destination_path,
project.qgis_project.transformContext(), save_options)
if error_code != QgsVectorFileWriter.NoError:
raise Exception(
_('Cannot save isochrone result to destination layer: ') +
error_message)
layer_uri = destination_path
if driverName != 'ESRI Shapefile':
layer_uri += '|layername=' + new_layer_name
provider = 'ogr'
return layer_uri, provider
def writeAsVectorFormat(self, name, driver_name, target_crs=None):
transform_context = QgsProject.instance().transformContext()
save_options = QgsVectorFileWriter.SaveVectorOptions()
save_options.driverName = driver_name
save_options.fileEncoding = "UTF-8"
save_options.onlySelectedFeatures = self.selectedFeatureCount() != 0
if target_crs is not None or target_crs != self.crs():
save_options.ct = QgsCoordinateTransform(
self.crs(),
QgsCoordinateReferenceSystem(target_crs),
transform_context,
)
return QgsVectorFileWriter.writeAsVectorFormatV2(
self, name, transform_context, save_options)
def response_csv_mode(self, request):
"""
Download csv of data
:param request: Http Django request object
:return: http response with attached file
"""
if not self.layer.download_csv:
return HttpResponseForbidden()
# Apply filter backends, store original subset string
qgs_request = QgsFeatureRequest()
original_subset_string = self.metadata_layer.qgis_layer.subsetString()
if hasattr(self, 'filter_backends'):
for backend in self.filter_backends:
backend().apply_filter(request, self.metadata_layer, qgs_request, self)
save_options = QgsVectorFileWriter.SaveVectorOptions()
save_options.driverName = 'csv'
save_options.fileEncoding = 'utf-8'
tmp_dir = tempfile.TemporaryDirectory()
filename = self._build_download_filename(request) + '.csv'
# Make a selection based on the request
self._selection_responde_download_mode(qgs_request, save_options)
xls_tmp_path = os.path.join(tmp_dir.name, filename)
error_code, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
self.metadata_layer.qgis_layer,
xls_tmp_path,
self.metadata_layer.qgis_layer.transformContext(),
save_options
)
# Restore the original subset string and select no features
self.metadata_layer.qgis_layer.selectByIds([])
self.metadata_layer.qgis_layer.setSubsetString(original_subset_string)
if error_code != QgsVectorFileWriter.NoError:
tmp_dir.cleanup()
return HttpResponse(status=500, reason=error_message)
response = HttpResponse(
open(xls_tmp_path, 'rb').read(), content_type='text/csv')
tmp_dir.cleanup()
response['Content-Disposition'] = f'attachment; filename={filename}'
response.set_cookie('fileDownload', 'true')
return response
def export_as_xlsx(context, file_path, input_layer) -> None:
""" Export the layer to XLSX to the given path. """
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = QgsVectorFileWriter.driverForExtension('xlsx')
options.fileEncoding = 'UTF-8'
options.layerName = input_layer.name()
options.layerOptions = ['OGR_XLSX_FIELD_TYPES=AUTO']
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
input_layer,
file_path,
context.project().transformContext(),
options,
)
if write_result != QgsVectorFileWriter.NoError:
raise QgsProcessingException(error_message)
def write_layer_to_gpkg2(layer, gpkgfile, layername):
options = QgsVectorFileWriter.SaveVectorOptions()
from pathlib import Path
if Path(gpkgfile).exists():
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
else:
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteFile
# to get rid of spaces in the layer name
options.layerName = layername
context = QgsProject.instance().transformContext()
if hasattr(QgsVectorFileWriter, "writeAsVectorFormatV3"):
return QgsVectorFileWriter.writeAsVectorFormatV3(layer, gpkgfile, context, options)
else:
return QgsVectorFileWriter.writeAsVectorFormatV2(layer, gpkgfile, context, options)
def save_layer_as(orig_layer, dest_path, save_format, crs=None):
if crs is None:
crs = orig_layer.crs()
old_lc_numeric = locale.getlocale(locale.LC_NUMERIC)
locale.setlocale(locale.LC_NUMERIC, 'C')
if Qgis.QGIS_VERSION_INT < 31003:
writer_error = QgsVectorFileWriter.writeAsVectorFormat(
orig_layer, dest_path, 'utf-8', crs, save_format)
else:
options = QgsVectorFileWriter.SaveVectorOptions()
options.fileEncoding = 'utf8'
options.driverName = save_format
ctc = QgsCoordinateTransformContext()
if crs is not None:
ctc.addCoordinateOperation(orig_layer.crs(), crs, 'proj')
writer_error = QgsVectorFileWriter.writeAsVectorFormatV2(
orig_layer, dest_path, ctc, options)
locale.setlocale(locale.LC_NUMERIC, old_lc_numeric)
return writer_error
def save_as_geojson(self, output_path: Path) -> Path:
output_file = Path(output_path, f"{self.resource_name}.geojson")
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = "GeoJSON"
options.fileEncoding = "utf-8"
src_crs = self.layer.crs()
dst_crs = QgsCoordinateReferenceSystem("EPSG:4326")
options.ct = QgsCoordinateTransform(src_crs, dst_crs,
QgsProject.instance())
writer_, msg = QgsVectorFileWriter.writeAsVectorFormatV2(
self.layer,
str(output_file),
QgsProject.instance().transformContext(),
options,
)
return output_file
def send_output_file(self, handler):
format_dict = WFSFormats[self.format]
# read the GML
gml_path = join(self.temp_dir, '{}.gml'.format(self.filename))
output_layer = QgsVectorLayer(gml_path, 'qgis_server_wfs_features',
'ogr')
if not output_layer.isValid():
handler.appendBody(b'')
self.logger.critical(
'Output layer {} is not valid.'.format(gml_path))
return False
# Temporary file where to write the output
temporary = QTemporaryFile(
join(
self.temp_dir,
'to-{}-XXXXXX.{}'.format(self.format,
format_dict['filenameExt'])))
temporary.open()
output_file = temporary.fileName()
temporary.remove() # Fix issue #18
try:
# create save options
options = QgsVectorFileWriter.SaveVectorOptions()
# driver name
options.driverName = format_dict['ogrProvider']
# file encoding
options.fileEncoding = 'utf-8'
# coordinate transformation
if format_dict['forceCRS']:
options.ct = QgsCoordinateTransform(
output_layer.crs(),
QgsCoordinateReferenceSystem(format_dict['forceCRS']),
QgsProject.instance())
# datasource options
if format_dict['ogrDatasourceOptions']:
options.datasourceOptions = format_dict['ogrDatasourceOptions']
# write file
# noinspection PyUnresolvedReferences
if Qgis.QGIS_VERSION_INT >= 31003:
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
output_layer, output_file,
QgsProject.instance().transformContext(), options)
else:
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat(
output_layer, output_file, options)
if write_result != QgsVectorFileWriter.NoError:
handler.appendBody(b'')
self.logger.critical(error_message)
return False
except Exception as e:
handler.appendBody(b'')
exc_type, _, exc_tb = exc_info()
self.logger.critical(str(e))
self.logger.critical(exc_type)
self.logger.critical('\n'.join(traceback.format_tb(exc_tb)))
return False
if format_dict['zip']:
# compress files
import zipfile
try:
import zlib # NOQA
compression = zipfile.ZIP_DEFLATED
except ImportError:
compression = zipfile.ZIP_STORED
# create the zip file
base_filename = splitext(output_file)[0]
zip_file_path = join(self.temp_dir, '{}.zip'.format(base_filename))
with zipfile.ZipFile(zip_file_path, 'w') as zf:
# Add the main file
arc_filename = '{}.{}'.format(self.typename,
format_dict['filenameExt'])
zf.write(output_file,
compress_type=compression,
arcname=arc_filename)
for extension in format_dict['extToZip']:
file_path = join(self.temp_dir,
'{}.{}'.format(base_filename, extension))
if exists(file_path):
arc_filename = '{}.{}'.format(self.typename, extension)
zf.write(file_path,
compress_type=compression,
arcname=arc_filename)
zf.close()
f = QFile(zip_file_path)
if f.open(QFile.ReadOnly):
ba = f.readAll()
handler.appendBody(ba)
return True
else:
# return the file created without zip
f = QFile(output_file)
if f.open(QFile.ReadOnly):
ba = f.readAll()
handler.appendBody(ba)
return True
handler.appendBody(b'')
self.logger.critical('Error no output file')
return False
def run(self):
"""Run method that performs all the real work"""
# Create the dialog with elements (after translation) and keep reference
# Only create GUI ONCE in callback, so that it will only load when the plugin is started
if self.first_start == True:
self.first_start = False
self.dlg = HarmonyQGISDialog()
# get stored settings
settings = QgsSettings()
# Fetch the currently loaded layers
layers = QgsProject.instance().layerTreeRoot().children()
layerNames = [layer.name() for layer in layers]
# Clear the contents of the comboBox from previous runs
self.dlg.comboBox.clear()
# Populate the comboBox with names of all the loaded layers
self.dlg.comboBox.addItems(layerNames)
# use the previous layer as the default if it is in the existing layers
# layerName = settings.value("harmony_qgis/layer")
# if layerName and layerName in layerNames:
# self.dlg.comboBox.setCurrentIndex(layerNames.index(layerName))
layer = self.iface.activeLayer()
if layer:
self.dlg.comboBox.setCurrentIndex(layerNames.index(layer.name()))
# fill the harmnoy url input with the saved setting if available
harmonyUrl = settings.value("harmony_qgis/harmony_url")
if harmonyUrl:
self.dlg.harmonyUrlLineEdit.setText(harmonyUrl)
collectionId = settings.value("harmony_qgis/collection_id")
if collectionId:
self.dlg.collectionField.setText(collectionId)
version = settings.value("harmony_qgis/version") or "1.0.0"
self.dlg.versionField.setText(version)
variable = settings.value("harmony_qgis/variable")
if variable:
self.dlg.variableField.setText(variable)
# clear the table
self.dlg.tableWidget.setRowCount(0)
# set the table header
self.dlg.tableWidget.setHorizontalHeaderLabels('Parameter;Value'.split(';'))
# add a parameter/value when the 'Add' button is clicked
self.dlg.addButton.clicked.connect(self.addSearchParameter)
# show the dialog
self.dlg.show()
# Run the dialog event loop
result = self.dlg.exec_()
# See if OK was pressed
if result:
collectionId = str(self.dlg.collectionField.text())
version = str(self.dlg.versionField.text())
variable = str(self.dlg.variableField.text())
layerName = str(self.dlg.comboBox.currentText())
# TODO handle the case where there is more than one layer by this name
layer = QgsProject.instance().mapLayersByName(layerName)[0]
opts = QgsVectorFileWriter.SaveVectorOptions()
opts.driverName = 'GeoJson'
tempFile = '/tmp/qgis.json'
QgsVectorFileWriter.writeAsVectorFormatV2(layer, tempFile, QgsCoordinateTransformContext(), opts)
harmonyUrl = self.dlg.harmonyUrlLineEdit.text()
path = collectionId + "/" + "ogc-api-coverages/" + version + "/collections/" + variable + "/coverage/rangeset"
url = harmonyUrl + "/" + path
print(url)
tempFileHandle = open(tempFile, 'r')
contents = tempFileHandle.read()
tempFileHandle.close()
geoJson = rewind(contents)
tempFileHandle = open(tempFile, 'w')
tempFileHandle.write(geoJson)
tempFileHandle.close()
tempFileHandle = open(tempFile, 'rb')
multipart_form_data = {
'shapefile': (layerName + '.geojson', tempFileHandle, 'application/geo+json')
}
rowCount = self.dlg.tableWidget.rowCount()
for row in range(rowCount):
parameter = self.dlg.tableWidget.item(row, 0).text()
value = self.dlg.tableWidget.item(row, 1).text()
multipart_form_data[parameter] = (None, value)
resp = requests.post(url, files=multipart_form_data, stream=True)
tempFileHandle.close()
# print(resp)
# print(resp.text)
with open('/tmp/harmony_output_image.tif', 'wb') as fd:
for chunk in resp.iter_content(chunk_size=128):
fd.write(chunk)
os.remove(tempFile)
self.iface.addRasterLayer('/tmp/harmony_output_image.tif', layerName + '-' + variable)
# QgsRasterLayer('/tmp/harmony_output_image.tif', layerName)
# save settings
if collectionId != "":
settings.setValue("harmony_qgis/collection_id", collectionId)
if version != "":
settings.setValue("harmony_qgis/version", version)
if variable != "":
settings.setValue("harmony_qgis/variable", variable)
if harmonyUrl != "":
settings.setValue("harmony_qgis/harmony_url", harmonyUrl)
settings.setValue("harmony_qgis/layer", layerName)
def create_geopackage(project_type: ProjectType, file_path, crs,
transform_context) -> None:
""" Create the geopackage for the given path. """
encoding = 'UTF-8'
driver_name = QgsVectorFileWriter.driverForExtension('gpkg')
for table in project_type.layers:
layer_path = str(tables[table])
if layer_path != 'None':
layer_path += "?crs={}".format(crs.authid())
vector_layer = QgsVectorLayer(layer_path, table, "memory")
data_provider = vector_layer.dataProvider()
fields = QgsFields()
path = resources_path('data_models', '{}.csv'.format(table))
csv = load_csv(table, path)
for csv_feature in csv.getFeatures():
field = QgsField(name=csv_feature['name'],
type=int(csv_feature['type']))
field.setComment(csv_feature['comment'])
field.setAlias(csv_feature['alias'])
fields.append(field)
del csv
# add fields
data_provider.addAttributes(fields)
vector_layer.updateFields()
# set create file layer options
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = driver_name
options.fileEncoding = encoding
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteFile
if os.path.exists(file_path):
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
options.layerName = vector_layer.name()
options.layerOptions = ['FID=id']
# write file
if Qgis.QGIS_VERSION_INT >= 31900:
write_result, error_message, _, _ = QgsVectorFileWriter.writeAsVectorFormatV3(
vector_layer, file_path, transform_context, options)
else:
# 3.10 <= QGIS <3.18
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
vector_layer, file_path, transform_context, options)
# result
if write_result != QgsVectorFileWriter.NoError:
raise QgsProcessingException(
'* ERROR: {}'.format(error_message))
del fields
del data_provider
del vector_layer
def _test(autoTransaction):
"""Test buffer methods within and without transactions
- create a feature
- save
- retrieve the feature
- change geom and attrs
- test changes are seen in the buffer
"""
def _check_feature(wkt):
f = next(layer_a.getFeatures())
self.assertEqual(f.geometry().asWkt().upper(), wkt)
f = list(buffer.addedFeatures().values())[0]
self.assertEqual(f.geometry().asWkt().upper(), wkt)
ml = QgsVectorLayer('Point?crs=epsg:4326&field=int:integer', 'test', 'memory')
self.assertTrue(ml.isValid())
d = QTemporaryDir()
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = 'GPKG'
options.layerName = 'layer_a'
err, _ = QgsVectorFileWriter.writeAsVectorFormatV2(ml, os.path.join(d.path(), 'transaction_test.gpkg'), QgsCoordinateTransformContext(), options)
self.assertEqual(err, QgsVectorFileWriter.NoError)
self.assertTrue(os.path.isfile(os.path.join(d.path(), 'transaction_test.gpkg')))
options.layerName = 'layer_b'
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
err, _ = QgsVectorFileWriter.writeAsVectorFormatV2(ml, os.path.join(d.path(), 'transaction_test.gpkg'), QgsCoordinateTransformContext(), options)
layer_a = QgsVectorLayer(os.path.join(d.path(), 'transaction_test.gpkg|layername=layer_a'))
self.assertTrue(layer_a.isValid())
project = QgsProject()
project.setAutoTransaction(autoTransaction)
project.addMapLayers([layer_a])
###########################################
# Tests with a new feature
self.assertTrue(layer_a.startEditing())
buffer = layer_a.editBuffer()
f = QgsFeature(layer_a.fields())
f.setAttribute('int', 123)
f.setGeometry(QgsGeometry.fromWkt('point(7 45)'))
self.assertTrue(layer_a.addFeatures([f]))
_check_feature('POINT (7 45)')
# Need to fetch the feature because its ID is NULL (-9223372036854775808)
f = next(layer_a.getFeatures())
self.assertEqual(len(buffer.addedFeatures()), 1)
layer_a.undoStack().undo()
self.assertEqual(len(buffer.addedFeatures()), 0)
layer_a.undoStack().redo()
self.assertEqual(len(buffer.addedFeatures()), 1)
f = list(buffer.addedFeatures().values())[0]
self.assertEqual(f.attribute('int'), 123)
# Now change attribute
self.assertEqual(buffer.changedAttributeValues(), {})
layer_a.changeAttributeValue(f.id(), 1, 321)
self.assertEqual(len(buffer.addedFeatures()), 1)
# This is surprising: because it was a new feature it has been changed directly
self.assertEqual(buffer.changedAttributeValues(), {})
f = list(buffer.addedFeatures().values())[0]
self.assertEqual(f.attribute('int'), 321)
layer_a.undoStack().undo()
self.assertEqual(buffer.changedAttributeValues(), {})
f = list(buffer.addedFeatures().values())[0]
self.assertEqual(f.attribute('int'), 123)
f = next(layer_a.getFeatures())
self.assertEqual(f.attribute('int'), 123)
# Change geometry
f = next(layer_a.getFeatures())
self.assertTrue(layer_a.changeGeometry(f.id(), QgsGeometry.fromWkt('point(9 43)')))
_check_feature('POINT (9 43)')
self.assertEqual(buffer.changedGeometries(), {})
layer_a.undoStack().undo()
_check_feature('POINT (7 45)')
self.assertEqual(buffer.changedGeometries(), {})
self.assertTrue(layer_a.changeGeometry(f.id(), QgsGeometry.fromWkt('point(9 43)')))
_check_feature('POINT (9 43)')
self.assertTrue(layer_a.changeGeometry(f.id(), QgsGeometry.fromWkt('point(10 44)')))
_check_feature('POINT (10 44)')
# This is anothr surprise: geometry edits get collapsed into a single
# one because they have the same hardcoded id
layer_a.undoStack().undo()
_check_feature('POINT (7 45)')
self.assertTrue(layer_a.commitChanges())
###########################################
# Tests with the existing feature
# Get the feature
f = next(layer_a.getFeatures())
self.assertTrue(f.isValid())
self.assertEqual(f.attribute('int'), 123)
self.assertEqual(f.geometry().asWkt().upper(), 'POINT (7 45)')
self.assertTrue(layer_a.startEditing())
layer_a.changeAttributeValue(f.id(), 1, 321)
buffer = layer_a.editBuffer()
self.assertEqual(buffer.changedAttributeValues(), {1: {1: 321}})
layer_a.undoStack().undo()
self.assertEqual(buffer.changedAttributeValues(), {})
# Change geometry
self.assertTrue(layer_a.changeGeometry(f.id(), QgsGeometry.fromWkt('point(9 43)')))
f = next(layer_a.getFeatures())
self.assertEqual(f.geometry().asWkt().upper(), 'POINT (9 43)')
self.assertEqual(buffer.changedGeometries()[1].asWkt().upper(), 'POINT (9 43)')
layer_a.undoStack().undo()
self.assertEqual(buffer.changedGeometries(), {})
f = next(layer_a.getFeatures())
self.assertEqual(f.geometry().asWkt().upper(), 'POINT (7 45)')
self.assertEqual(buffer.changedGeometries(), {})
# Delete an existing feature
self.assertTrue(layer_a.deleteFeature(f.id()))
with self.assertRaises(StopIteration):
next(layer_a.getFeatures())
self.assertEqual(buffer.deletedFeatureIds(), [f.id()])
layer_a.undoStack().undo()
self.assertTrue(layer_a.getFeature(f.id()).isValid())
self.assertEqual(buffer.deletedFeatureIds(), [])
###########################################
# Test delete
# Delete a new feature
f = QgsFeature(layer_a.fields())
f.setAttribute('int', 555)
f.setGeometry(QgsGeometry.fromWkt('point(8 46)'))
self.assertTrue(layer_a.addFeatures([f]))
f = [f for f in layer_a.getFeatures() if f.attribute('int') == 555][0]
self.assertTrue(f.id() in buffer.addedFeatures())
self.assertTrue(layer_a.deleteFeature(f.id()))
self.assertFalse(f.id() in buffer.addedFeatures())
self.assertFalse(f.id() in buffer.deletedFeatureIds())
layer_a.undoStack().undo()
self.assertTrue(f.id() in buffer.addedFeatures())
###########################################
# Add attribute
field = QgsField('attr1', QVariant.String)
self.assertTrue(layer_a.addAttribute(field))
self.assertNotEqual(layer_a.fields().lookupField(field.name()), -1)
self.assertEqual(buffer.addedAttributes(), [field])
layer_a.undoStack().undo()
self.assertEqual(layer_a.fields().lookupField(field.name()), -1)
self.assertEqual(buffer.addedAttributes(), [])
layer_a.undoStack().redo()
self.assertNotEqual(layer_a.fields().lookupField(field.name()), -1)
self.assertEqual(buffer.addedAttributes(), [field])
self.assertTrue(layer_a.commitChanges())
###########################################
# Remove attribute
self.assertTrue(layer_a.startEditing())
buffer = layer_a.editBuffer()
attr_idx = layer_a.fields().lookupField(field.name())
self.assertNotEqual(attr_idx, -1)
self.assertTrue(layer_a.deleteAttribute(attr_idx))
self.assertEqual(buffer.deletedAttributeIds(), [2])
self.assertEqual(layer_a.fields().lookupField(field.name()), -1)
layer_a.undoStack().undo()
self.assertEqual(buffer.deletedAttributeIds(), [])
self.assertEqual(layer_a.fields().lookupField(field.name()), attr_idx)
layer_a.undoStack().redo()
self.assertEqual(buffer.deletedAttributeIds(), [2])
self.assertEqual(layer_a.fields().lookupField(field.name()), -1)
self.assertTrue(layer_a.rollBack())
###########################################
# Rename attribute
self.assertTrue(layer_a.startEditing())
attr_idx = layer_a.fields().lookupField(field.name())
self.assertNotEqual(attr_idx, -1)
self.assertEqual(layer_a.fields().lookupField('new_name'), -1)
self.assertTrue(layer_a.renameAttribute(attr_idx, 'new_name'))
self.assertEqual(layer_a.fields().lookupField('new_name'), attr_idx)
layer_a.undoStack().undo()
self.assertEqual(layer_a.fields().lookupField(field.name()), attr_idx)
self.assertEqual(layer_a.fields().lookupField('new_name'), -1)
layer_a.undoStack().redo()
self.assertEqual(layer_a.fields().lookupField('new_name'), attr_idx)
self.assertEqual(layer_a.fields().lookupField(field.name()), -1)
def response_shp_mode(self, request):
"""
Download Shapefile of data
:param request: Http Django request object
:return: http response with attached file
"""
if not self.layer.download:
return HttpResponseForbidden()
tmp_dir = tempfile.TemporaryDirectory()
filename = self._build_download_filename(request)
# Apply filter backends, store original subset string
qgs_request = QgsFeatureRequest()
original_subset_string = self.metadata_layer.qgis_layer.subsetString()
if hasattr(self, 'filter_backends'):
for backend in self.filter_backends:
backend().apply_filter(request, self.metadata_layer, qgs_request, self)
save_options = QgsVectorFileWriter.SaveVectorOptions()
save_options.driverName = 'ESRI Shapefile'
save_options.fileEncoding = 'utf-8'
# Make a selection based on the request
self._selection_responde_download_mode(qgs_request, save_options)
file_path = os.path.join(tmp_dir.name, filename)
error_code, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
self.metadata_layer.qgis_layer,
file_path,
self.metadata_layer.qgis_layer.transformContext(),
save_options
)
# Restore the original subset string and select no features
self.metadata_layer.qgis_layer.selectByIds([])
self.metadata_layer.qgis_layer.setSubsetString(original_subset_string)
if error_code != QgsVectorFileWriter.NoError:
tmp_dir.cleanup()
return HttpResponse(status=500, reason=error_message)
# Check for extra fields to add
self._add_extrafields(request, file_path, filename)
filenames = ["{}{}".format(filename, ftype)
for ftype in self.shp_extentions]
zip_filename = "{}.zip".format(filename)
# Open BytesIO to grab in-memory ZIP contents
s = io.BytesIO()
# The zip compressor
zf = zipfile.ZipFile(s, "w")
for fpath in filenames:
# Add file, at correct path
ftoadd = os.path.join(tmp_dir.name, fpath)
if os.path.exists(ftoadd):
zf.write(ftoadd, fpath)
# Must close zip for all contents to be written
zf.close()
tmp_dir.cleanup()
# Grab ZIP file from in-memory, make response with correct MIME-type
response = HttpResponse(
s.getvalue(), content_type="application/x-zip-compressed")
response['Content-Disposition'] = 'attachment; filename=%s' % zip_filename
response.set_cookie('fileDownload', 'true')
return response
def processAlgorithm(self, parameters, context, feedback):
"""
Here is where the processing itself takes place.
"""
source_pts = self.parameterAsSource(parameters, self.INPUT_POINTS,
context)
input_field = self.parameterAsString(parameters, self.INPUT_FIELD,
context)
source_dem = self.parameterAsRasterLayer(parameters, self.INPUT_DEM,
context)
out_directory = self.parameterAsString(parameters, self.OUTPUT_DIR,
context)
out_type_nr = self.parameterAsInt(parameters, self.OUTPUT_TYPE,
context)
out_type = QgsVectorFileWriter.supportedFormatExtensions(
)[:2][out_type_nr]
to_gpkg = out_type == 'gpkg'
load_results = self.parameterAsBool(parameters, self.LOAD_RESULTS,
context)
if source_pts is None:
raise QgsProcessingException(
self.invalidSourceError(parameters, self.INPUT_POINTS))
if source_dem is None:
raise QgsProcessingException(
self.invalidSourceError(parameters, self.INPUT_DEM))
feedback.pushInfo("Input data loaded! Creating catchments...")
feedback.setProgress(1)
unique_field = input_field if input_field else ""
if unique_field:
field_idx = source_pts.fields().lookupField(unique_field)
unique_values = source_pts.uniqueValues(field_idx)
else:
unique_values = [f.id() for f in source_pts.getFeatures()]
feedback.pushInfo(f"Creating directory: {out_directory}")
mkdir(out_directory)
bname = f"catchment{'s' if to_gpkg else ''}"
output_basename = os.path.join(out_directory, bname)
# Compute the number of steps to display within the progress bar and
# get features from source
total_nr = len(unique_values)
total = 100. / total_nr if source_pts.featureCount() else 1
output_layers = []
for i, unique_value in enumerate(unique_values):
# Stop the algorithm if cancel button has been clicked
if feedback.isCanceled():
break
table = f"catchment_{unique_value}" if to_gpkg else ""
file_mod = "" if to_gpkg else f"_{unique_value}"
filename = f"{output_basename}{file_mod}"
destination = f"{filename}.{out_type}"
output_uri = destination + (f"|layername={table}"
if to_gpkg else "")
feedback.pushInfo(
self.tr('Creating layer: {}').format(destination))
if unique_field:
req_filter = f"{QgsExpression.quotedColumnRef(unique_field)}={QgsExpression.quotedValue(unique_value)}"
req = QgsFeatureRequest().setFilterExpression(req_filter)
else:
req = QgsFeatureRequest(unique_value) # feature id
for source_pt in source_pts.getFeatures(req):
if feedback.isCanceled():
break
# Get x and y coordinate from point feature
geom = source_pt.geometry()
p = geom.asPoint()
x = p.x()
y = p.y()
feedback.pushInfo(
'Creating upslope area for point ({:.2f}, {:.2f}) - {} of {}'
.format(x, y, i + 1, total_nr))
# Calculate catchment raster from point feature
catchraster = processing.run(
"saga:upslopearea",
{
'TARGET': None,
'TARGET_PT_X': x,
'TARGET_PT_Y': y,
'ELEVATION': source_dem,
'SINKROUTE': None,
'METHOD': 0,
'CONVERGE': 1.1,
'AREA': 'TEMPORARY_OUTPUT'
},
context=context,
feedback=feedback,
)
# Polygonize raster catchment
catchpoly = processing.run(
"gdal:polygonize",
{
'INPUT': catchraster["AREA"],
'BAND': 1,
'FIELD': 'DN',
'EIGHT_CONNECTEDNESS': False,
'OUTPUT': 'TEMPORARY_OUTPUT'
},
context=context,
feedback=feedback,
)
# Select features having DN = 100
catchpoly_lyr = QgsProcessingUtils.mapLayerFromString(
catchpoly["OUTPUT"], context=context)
catchpoly_lyr.selectByExpression('"DN"=100')
options = QgsVectorFileWriter.SaveVectorOptions()
options.driverName = "GPKG" if to_gpkg else "ESRI Shapefile"
options.layerName = table
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
options.onlySelectedFeatures = True
trans_context = QgsCoordinateTransformContext()
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
catchpoly_lyr, destination, trans_context, options)
if write_result != 0:
feedback.pushInfo(f"Initial write failed: {error_message}")
# retry with option for creating the dataset
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteFile
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
catchpoly_lyr, destination, trans_context, options)
feedback.pushInfo(
f"Final write attempt: {write_result} == 0 -> SUCCESS or {error_message}"
)
output_layer = QgsProcessingUtils.mapLayerFromString(
output_uri, context=context)
output_layers.append(output_uri)
if load_results:
context.temporaryLayerStore().addMapLayer(output_layer)
context.addLayerToLoadOnCompletion(
output_layer.id(),
QgsProcessingContext.LayerDetails(
table if to_gpkg else f"catchment {unique_value}",
context.project(), self.OUTPUT_LAYERS))
feedback.setProgress(int((i + 1) * total))
return {
self.OUTPUT_DIR: out_directory,
self.OUTPUT_LAYERS: output_layers
}
def sendOutputFile(self, handler):
format_dict = WFSFormats[self.format]
# read the GML
gml_path = join(self.tempdir, '{}.gml'.format(self.filename))
output_layer = QgsVectorLayer(gml_path, 'qgis_server_wfs_features',
'ogr')
# Temporary file where to write the output
temporary = QTemporaryFile(
join(QDir.tempPath(),
'request-WFS-XXXXXX.{}'.format(format_dict['filenameExt'])))
temporary.open()
output_file = temporary.fileName()
temporary.close()
if output_layer.isValid():
try:
# create save options
options = QgsVectorFileWriter.SaveVectorOptions()
# driver name
options.driverName = format_dict['ogrProvider']
# file encoding
options.fileEncoding = 'utf-8'
# coordinate transformation
if format_dict['forceCRS']:
options.ct = QgsCoordinateTransform(
output_layer.crs(),
QgsCoordinateReferenceSystem(format_dict['forceCRS']),
QgsProject.instance())
# datasource options
if format_dict['ogrDatasourceOptions']:
options.datasourceOptions = format_dict[
'ogrDatasourceOptions']
# write file
if Qgis.QGIS_VERSION_INT >= 31003:
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
output_layer, output_file,
QgsProject.instance().transformContext(), options)
else:
write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat(
output_layer, output_file, options)
if write_result != QgsVectorFileWriter.NoError:
handler.appendBody(b'')
self.logger.critical(error_message)
return False
except Exception as e:
handler.appendBody(b'')
self.logger.critical(str(e))
return False
if format_dict['zip']:
# compress files
import zipfile
# noinspection PyBroadException
try:
import zlib # NOQA
compression = zipfile.ZIP_DEFLATED
except Exception:
compression = zipfile.ZIP_STORED
# create the zip file
zip_file_path = join(self.tempdir, '%s.zip' % self.filename)
with zipfile.ZipFile(zip_file_path, 'w') as zf:
# add all files
zf.write(join(
self.tempdir,
'%s.%s' % (self.filename, format_dict['filenameExt'])),
compress_type=compression,
arcname='%s.%s' %
(self.typename, format_dict['filenameExt']))
for e in format_dict['extToZip']:
file_path = join(self.tempdir,
'%s.%s' % (self.filename, e))
if exists(file_path):
zf.write(file_path,
compress_type=compression,
arcname='%s.%s' % (self.typename, e))
zf.close()
f = QFile(zip_file_path)
if f.open(QFile.ReadOnly):
ba = f.readAll()
handler.appendBody(ba)
return True
else:
# return the file created without zip
f = QFile(output_file)
if f.open(QFile.ReadOnly):
ba = f.readAll()
handler.appendBody(ba)
return True
handler.appendBody(b'')
self.logger.critical('Error no output file')
return False
def exportLayer(layer,
fields=None,
to_shapefile=False,
path=None,
force=False,
logger=None):
logger = logger or feedback
filepath, _, ext = lyr_utils.getLayerSourceInfo(layer)
lyr_name, safe_name = lyr_utils.getLayerTitleAndName(layer)
fields = fields or []
if layer.type() == layer.VectorLayer:
if to_shapefile and (force or layer.fields().count() != len(fields)
or ext != EXT_SHAPEFILE):
# Export with Shapefile extension
ext = EXT_SHAPEFILE
elif force or ext != EXT_GEOPACKAGE or layer.fields().count() != len(fields) \
or not isSingleTableGpkg(filepath):
# Export with GeoPackage extension
ext = EXT_GEOPACKAGE
else:
# No need to export
logger.logInfo(
f"No need to export layer {lyr_name} stored at {filepath}")
return filepath
# Perform GeoPackage or Shapefile export
attrs = [
i for i, f in enumerate(layer.fields())
if len(fields) == 0 or f.name() in fields
]
output = path or tempFileInSubFolder(safe_name + ext)
encoding = "UTF-8"
driver = "ESRI Shapefile" if ext == EXT_SHAPEFILE else "GPKG"
options = None
if hasattr(QgsVectorFileWriter, 'SaveVectorOptions'):
# QGIS v3.x has the SaveVectorOptions object
options = QgsVectorFileWriter.SaveVectorOptions()
options.fileEncoding = encoding
options.attributes = attrs
options.driverName = driver
# Make sure that we are using the latest (non-deprecated) write method
if hasattr(QgsVectorFileWriter, 'writeAsVectorFormatV3'):
# Use writeAsVectorFormatV3 for QGIS versions >= 3.20 to avoid DeprecationWarnings
result = QgsVectorFileWriter.writeAsVectorFormatV3(
layer, output, QgsCoordinateTransformContext(),
options) # noqa
elif hasattr(QgsVectorFileWriter, 'writeAsVectorFormatV2'):
# Use writeAsVectorFormatV2 for QGIS versions >= 3.10.3 to avoid DeprecationWarnings
result = QgsVectorFileWriter.writeAsVectorFormatV2(
layer, output, QgsCoordinateTransformContext(),
options) # noqa
else:
# Use writeAsVectorFormat for QGIS versions < 3.10.3 for backwards compatibility
result = QgsVectorFileWriter.writeAsVectorFormat(
layer,
output,
fileEncoding=encoding,
attributes=attrs,
driverName=driver) # noqa
# Check if first item in result tuple is an error code
if result[0] == QgsVectorFileWriter.NoError:
logger.logInfo(f"Layer {lyr_name} exported to {output}")
else:
# Dump the result tuple as-is when there are errors (the tuple size depends on the QGIS version)
logger.logError(
f"Layer {lyr_name} failed to export.\n\tResult object: {str(result)}"
)
return output
else:
# Export raster
if force or not filepath.lower().endswith("tif"):
output = path or tempFileInSubFolder(safe_name + ".tif")
writer = QgsRasterFileWriter(output)
writer.setOutputFormat("GTiff")
writer.writeRaster(layer.pipe(), layer.width(), layer.height(),
layer.extent(), layer.crs())
del writer
logger.logInfo(f"Layer {lyr_name} exported to {output}")
return output
else:
logger.logInfo(
f"No need to export layer {lyr_name} stored at {filepath}")
return filepath
def _init_ext_layer(self, geom_str, idx, crs):
"""given non map of feat, init a qgis layer
:map_feat: {geom_string: list_of_feat}
"""
ext = self.ext
driver_name = ext.upper() # might not needed for
layer_name = self._layer_name(geom_str, idx)
# sqlite max connection 64
# if xyz space -> more than 64 vlayer,
# then create new fname
# fname = make_unique_full_path(ext=ext)
fname = make_fixed_full_path(self._layer_fname(), ext=ext)
if geom_str:
geomz = geom_str if geom_str.endswith("Z") else "{}Z".format(
geom_str)
else:
geomz = "NoGeometry"
vlayer = QgsVectorLayer("{geom}?crs={crs}&index=yes".format(geom=geomz,
crs=crs),
layer_name,
"memory") # this should be done in main thread
# QgsVectorFileWriter.writeAsVectorFormat(vlayer, fname, "UTF-8", vlayer.sourceCrs(),
# driver_name)
db_layer_name = self._db_layer_name(geom_str, idx)
options = QgsVectorFileWriter.SaveVectorOptions()
options.fileEncoding = "UTF-8"
options.driverName = driver_name
options.ct = QgsCoordinateTransform(vlayer.sourceCrs(),
vlayer.sourceCrs(),
QgsProject.instance())
options.layerName = db_layer_name
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer # update mode
if hasattr(QgsVectorFileWriter, "writeAsVectorFormatV2"):
err = QgsVectorFileWriter.writeAsVectorFormatV2(
vlayer, fname, vlayer.transformContext(), options)
else:
err = QgsVectorFileWriter.writeAsVectorFormat(
vlayer, fname, options)
if err[0] == QgsVectorFileWriter.ErrCreateDataSource:
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteFile
if hasattr(QgsVectorFileWriter, "writeAsVectorFormatV2"):
err = QgsVectorFileWriter.writeAsVectorFormatV2(
vlayer, fname, vlayer.transformContext(), options)
else:
err = QgsVectorFileWriter.writeAsVectorFormat(
vlayer, fname, options)
if err[0] != QgsVectorFileWriter.NoError:
raise Exception("%s: %s" % err)
self._update_constraint_trigger(fname, db_layer_name)
uri = "%s|layername=%s" % (fname, db_layer_name)
vlayer = QgsVectorLayer(uri, layer_name, "ogr")
self._save_meta_vlayer(vlayer)
return vlayer
def convert_to_gpkg(self, target_path):
"""
Convert a layer to geopackage in the target path and adjust its datasource. If
a layer is already a geopackage, the dataset will merely be copied to the target
path.
:param layer: The layer to copy
:param target_path: A path to a folder into which the data will be copied
:param keep_existent: if True and target file already exists, keep it as it is
"""
if not self.layer.type(
) == QgsMapLayer.VectorLayer or not self.layer.isValid():
return
file_path = self.filename
suffix = ""
uri_parts = self.layer.source().split("|", 1)
if len(uri_parts) > 1:
suffix = uri_parts[1]
dest_file = ""
new_source = ""
# check if the source is a geopackage, and merely copy if it's the case
if (os.path.isfile(file_path)
and self.layer.dataProvider().storageType() == "GPKG"):
source_path, file_name = os.path.split(file_path)
dest_file = os.path.join(target_path, file_name)
if not os.path.isfile(dest_file):
shutil.copy(os.path.join(source_path, file_name), dest_file)
if self.provider_metadata is not None:
metadata = self.metadata
metadata["path"] = dest_file
new_source = self.provider_metadata.encodeUri(metadata)
if new_source == "":
new_source = os.path.join(target_path, file_name)
if suffix != "":
new_source = "{}|{}".format(new_source, suffix)
layer_subset_string = self.layer.subsetString()
if new_source == "":
pattern = re.compile("[\W_]+") # NOQA
cleaned_name = pattern.sub("", self.layer.name())
dest_file = os.path.join(target_path,
"{}.gpkg".format(cleaned_name))
suffix = 0
while os.path.isfile(dest_file):
suffix += 1
dest_file = os.path.join(
target_path, "{}_{}.gpkg".format(cleaned_name, suffix))
# clone vector layer and strip it of filter, joins, and virtual fields
source_layer = self.layer.clone()
source_layer.setSubsetString("")
source_layer_joins = source_layer.vectorJoins()
for join in source_layer_joins:
source_layer.removeJoin(join.joinLayerId())
fields = source_layer.fields()
virtual_field_count = 0
for i in range(0, len(fields)):
if fields.fieldOrigin(i) == QgsFields.OriginExpression:
source_layer.removeExpressionField(i - virtual_field_count)
virtual_field_count += 1
options = QgsVectorFileWriter.SaveVectorOptions()
options.fileEncoding = "UTF-8"
options.driverName = "GPKG"
(error,
returned_dest_file) = QgsVectorFileWriter.writeAsVectorFormatV2(
source_layer, dest_file, QgsCoordinateTransformContext(),
options)
if error != QgsVectorFileWriter.NoError:
return
if returned_dest_file:
new_source = returned_dest_file
else:
new_source = dest_file
self._change_data_source(new_source, "ogr")
if layer_subset_string:
self.layer.setSubsetString(layer_subset_string)
return dest_file
def exportLayer(layer,
fields=None,
toShapefile=False,
path=None,
force=False,
log=None):
filename = layer.source().split("|")[0]
destFilename = layer.name()
fields = fields or []
if layer.type() == layer.VectorLayer:
if toShapefile and (
force or layer.fields().count() != len(fields) or
(os.path.splitext(filename.lower())[1] != EXT_SHAPEFILE)):
# Export with Shapefile extension
ext = EXT_SHAPEFILE
elif force or os.path.splitext(filename.lower())[1] != EXT_GEOPACKAGE or \
layer.fields().count() != len(fields) or not isSingleTableGpkg(filename):
# Export with GeoPackage extension
ext = EXT_GEOPACKAGE
else:
# No need to export
if log is not None:
log.logInfo(
QCoreApplication.translate(
"GeocatBridge",
f"No need to export layer {destFilename} stored at {filename}"
))
return filename
# Perform GeoPackage or Shapefile export
attrs = [
i for i, f in enumerate(layer.fields())
if len(fields) == 0 or f.name() in fields
]
output = path or tempFilenameInTempFolder(destFilename + ext)
if Qgis.QGIS_VERSION_INT < 31003:
# Use writeAsVectorFormat for QGIS versions < 3.10.3 for backwards compatibility
QgsVectorFileWriter.writeAsVectorFormat(
layer,
output,
fileEncoding="UTF-8",
attributes=attrs,
driverName="ESRI Shapefile" if ext == EXT_SHAPEFILE else "")
else:
# Use writeAsVectorFormatV2 for QGIS versions >= 3.10.3 to avoid DeprecationWarnings
transform_ctx = QgsProject.instance().transformContext()
options = QgsVectorFileWriter.SaveVectorOptions()
options.fileEncoding = "UTF-8"
options.attributes = attrs
options.driverName = "ESRI Shapefile" if ext == EXT_SHAPEFILE else ""
QgsVectorFileWriter.writeAsVectorFormatV2(layer, output,
transform_ctx, options)
if log is not None:
log.logInfo(
QCoreApplication.translate(
"GeocatBridge",
f"Layer {destFilename} exported to {output}"))
return output
else:
# Export raster
if force or not filename.lower().endswith("tif"):
output = path or tempFilenameInTempFolder(destFilename + ".tif")
writer = QgsRasterFileWriter(output)
writer.setOutputFormat("GTiff")
writer.writeRaster(layer.pipe(), layer.width(), layer.height(),
layer.extent(), layer.crs())
del writer
if log is not None:
log.logInfo(
QCoreApplication.translate(
"GeocatBridge",
f"Layer {destFilename} exported to {output}"))
return output
else:
if log is not None:
log.logInfo(
QCoreApplication.translate(
"GeocatBridge",
f"No need to export layer {destFilename} stored at {filename}"
))
return filename
def write_layer_to_gpkg(layer, gpkgfile, layername):
options = QgsVectorFileWriter.SaveVectorOptions()
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer
options.layerName = layername
context = QgsProject.instance().transformContext()
QgsVectorFileWriter.writeAsVectorFormatV2(layer, gpkgfile, context, options)
def convert_vector_layer(
layer, # pylint: disable=too-many-locals,too-many-branches,too-many-statements
project,
data_folder,
feedback,
conversion_results: ConversionResults,
change_source_on_error: bool = False,
verbose_log=False):
"""
Converts a vector layer to a standard format
"""
if layer.customProperty('original_uri'):
uri = layer.customProperty('original_uri')
if verbose_log:
feedback.pushDebugInfo(
f'Original layer URI from custom properties is {uri}')
else:
uri = layer.source()
if verbose_log:
feedback.pushDebugInfo(
'Original layer URI not found in custom properties')
source = QgsProviderRegistry.instance().decodeUri(
layer.providerType(), uri)
if verbose_log:
feedback.pushInfo('')
# older versions of QGIS didn't correctly strip out the subset from the layerName:
if 'subset' not in source:
if '|subset=' in source['layerName']:
if verbose_log:
feedback.pushDebugInfo(
'Stripping out subset string from layerName: {}'.
format(source['layerName']))
layer_name = source['layerName']
parts = layer_name.split('|subset=')
if len(parts) == 2:
source['layerName'] = parts[0]
if verbose_log:
feedback.pushDebugInfo('Cleaned layer name: {}'.format(
source['layerName']))
elif verbose_log:
feedback.reportError('Failed to strip subset string!')
elif '|subset=' in source['path']:
path = source['path']
if verbose_log:
feedback.pushDebugInfo(
'Stripping out subset string from path: {}'.format(
source['path']))
parts = path.split('|subset=')
if len(parts) == 2:
source['path'] = parts[0]
if verbose_log:
feedback.pushDebugInfo('Cleaned path: {}'.format(
source['path']))
elif verbose_log:
feedback.reportError('Failed to strip subset string!')
# convert to Geopackage
source_uri = QgsProviderRegistry.instance().encodeUri(
layer.providerType(), {
'path': source['path'],
'layerName': source['layerName']
})
# Sometimes the case varies in ArcMap documents, so when comparing to previously converted layers
# we use a case-insensitive path/layername which is normalized
result_key = QgsProviderRegistry.instance().encodeUri(
layer.providerType(), {
'path': pathlib.Path(
source['path']).resolve().as_posix().lower(),
'layerName': source['layerName'].lower()
})
if verbose_log:
feedback.pushDebugInfo('Converting layer: {} ( {} )'.format(
source['path'], source['layerName']))
feedback.pushDebugInfo(f'Cached result key: {result_key}')
provider_options = QgsDataProvider.ProviderOptions()
provider_options.transformContext = project.transformContext()
subset = layer.subsetString()
# have we maybe already converted this layer??
if result_key in conversion_results.layer_map:
previous_results = conversion_results.layer_map[result_key]
if previous_results.get('error'):
if verbose_log:
feedback.pushDebugInfo(
'Already tried to convert this layer, but failed last time, skipping...'
)
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr', provider_options)
else:
if verbose_log:
feedback.pushDebugInfo(
'Already converted this layer, reusing previous converted path: {} layername: {}'
.format(previous_results['destPath'],
previous_results['destLayer']))
layer.setDataSource(
QgsProviderRegistry.instance().encodeUri(
'ogr', {
'path': previous_results['destPath'],
'layerName': previous_results['destLayer']
}), layer.name(), 'ogr', provider_options)
if verbose_log:
feedback.pushDebugInfo('new source {}'.format(
layer.dataProvider().dataSourceUri()))
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
return previous_results
source_layer = QgsVectorLayer(source_uri, '', layer.providerType())
path = pathlib.Path(source['path'])
dest_file_name = ((pathlib.Path(data_folder) /
path.stem).with_suffix('.gpkg')).as_posix()
if dest_file_name not in conversion_results.created_databases:
# about to use a new file -- let's double-check that it doesn't already exist. We don't want
# to put layers into a database which we didn't make for this project
counter = 1
while pathlib.Path(dest_file_name).exists():
counter += 1
dest_file_name = ((pathlib.Path(data_folder) /
(path.stem + '_' + str(counter))
).with_suffix('.gpkg')).as_posix()
if dest_file_name in conversion_results.created_databases:
break
if verbose_log:
feedback.pushDebugInfo(
'Creating new destination file {}'.format(dest_file_name))
elif verbose_log:
feedback.pushDebugInfo(
'Reusing existing destination file {}'.format(dest_file_name))
# now this filename is ok for other layers to be stored in for this conversion
conversion_results.created_databases.add(dest_file_name)
layer_name_candidate = source['layerName']
counter = 1
while QgsVectorFileWriter.targetLayerExists(dest_file_name,
layer_name_candidate):
counter += 1
layer_name_candidate = '{}_{}'.format(source['layerName'], counter)
if verbose_log:
feedback.pushDebugInfo(
'Target layer name is {}'.format(layer_name_candidate))
if not source_layer.isValid():
if verbose_log:
feedback.reportError('Source layer is not valid')
if path.exists():
if verbose_log:
feedback.pushDebugInfo('File path DOES exist')
test_layer = QgsVectorLayer(path.as_posix())
sub_layers = test_layer.dataProvider().subLayers()
feedback.pushDebugInfo(
f'Readable layers from "{path.as_posix()}" are:')
for sub_layer in sub_layers:
_, name, count, geom_type, _, _ = sub_layer.split(
QgsDataProvider.sublayerSeparator())
feedback.pushDebugInfo(
f'- "{name}" ({count} features, geometry type {geom_type})'
)
if path.exists() and path.suffix.lower() == '.mdb':
try:
source['layerName'].encode('ascii')
except UnicodeDecodeError:
error = f'''MDB layers with unicode names are not supported by QGIS -- cannot convert "{source['layerName']}"'''
if verbose_log:
feedback.reportError(error)
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr',
provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
conversion_results.layer_map[result_key] = {'error': error}
return conversion_results.layer_map[result_key]
# maybe a non-spatial table, which can't be read with GDAL < 3.2
source_layer = None
if verbose_log:
feedback.pushDebugInfo('Layer type is {}'.format(
QgsWkbTypes.displayString(layer.wkbType())))
if layer.wkbType() == QgsWkbTypes.NoGeometry:
if verbose_log:
feedback.pushDebugInfo(
'Attempting fallback for non-spatial tables')
try:
source_layer = ConversionUtils.convert_mdb_table_to_memory_layer(
str(path), source['layerName'])
if verbose_log:
feedback.pushDebugInfo('Fallback succeeded!')
except Exception as e: # nopep8, pylint: disable=broad-except
if verbose_log:
feedback.reportError('Fallback failed: {}'.format(
str(e)))
source_layer = None
elif verbose_log:
feedback.reportError(
'Nothing else to try, conversion failed')
if not source_layer:
# here we fake things. We don't leave the original path to the mdb layer intact in the converted
# project, as this can cause massive issues with QGIS as it attempts to re-read this path constantly
# rather we "pretend" that the conversion was ok and set the broken layer's path to what the gpkg
# converted version WOULD have been! It'll still be broken in the converted project (obviously),
# but QGIS will no longer try endless to read the MDB and get all hung up on this...
conversion_results.layer_map[result_key] = {
'sourcePath':
source['path'],
'sourceLayer':
source['layerName'],
'destPath':
dest_file_name,
'destLayer':
layer_name_candidate,
'error':
'Could not open {} ({}) for conversion'.format(
source_uri, source['layerName'])
}
if change_source_on_error:
if verbose_log:
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr',
provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(
subset))
layer.setSubsetString(subset)
if verbose_log:
feedback.pushDebugInfo('new source {}'.format(
layer.dataProvider().dataSourceUri()))
return conversion_results.layer_map[result_key]
else:
if not path.exists():
error = 'The referenced file {} does NOT exist!'.format(
str(path))
else:
error = 'The referenced file exists, but could not open {} ({}) for conversion'.format(
source_uri, source['layerName'])
if verbose_log:
feedback.reportError(error)
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr', provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
conversion_results.layer_map[result_key] = {'error': error}
return conversion_results.layer_map[result_key]
if verbose_log:
feedback.pushDebugInfo('Source is valid, converting')
options = QgsVectorFileWriter.SaveVectorOptions()
options.layerName = layer_name_candidate
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer if pathlib.Path(
dest_file_name).exists(
) else QgsVectorFileWriter.CreateOrOverwriteFile
options.feedback = feedback
error, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
source_layer, dest_file_name, project.transformContext(), options)
if error != QgsVectorFileWriter.NoError:
if verbose_log:
feedback.reportError('Failed: {}'.format(error_message))
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr', provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
conversion_results.layer_map[result_key] = {'error': error_message}
return conversion_results.layer_map[result_key]
if verbose_log:
feedback.pushDebugInfo('Success!')
provider_options = QgsDataProvider.ProviderOptions()
provider_options.transformContext = project.transformContext()
subset = layer.subsetString()
layer.setDataSource(
QgsProviderRegistry.instance().encodeUri(
'ogr', {
'path': dest_file_name,
'layerName': options.layerName
}), layer.name(), 'ogr', provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
if verbose_log:
feedback.pushDebugInfo('new source {}'.format(
layer.dataProvider().dataSourceUri()))
conversion_results.layer_map[result_key] = {
'sourcePath': source['path'],
'sourceLayer': source['layerName'],
'destPath': dest_file_name,
'destLayer': options.layerName
}
return conversion_results.layer_map[result_key]
