def __init__(self,
uri='',
providerOptions=QgsDataProvider.ProviderOptions()):
super().__init__(uri)
# Use the memory layer to parse the uri
mlayer = QgsVectorLayer(uri, 'ml', 'memory')
self.setNativeTypes(mlayer.dataProvider().nativeTypes())
self._uri = uri
self._fields = mlayer.fields()
self._wkbType = mlayer.wkbType()
self._features = {}
self._extent = QgsRectangle()
self._extent.setMinimal()
self._subset_string = ''
self._crs = mlayer.crs()
self._spatialindex = None
self._provider_options = providerOptions
if 'index=yes' in self._uri:
self.createSpatialIndex()
def replace_apikey_for_layer(layer):
source = layer.source()
if is_planet_layer(
source
) and not PLANET_CURRENT_MOSAIC in layer.customPropertyKeys():
client = PlanetClient.getInstance()
if client.has_api_key():
newsource = source.split(
"api_key=")[0] + "api_key=" + client.api_key()
newsource = newsource.replace(PLANET_ROOT_URL_PLACEHOLDER,
PLANET_ROOT_URL)
else:
newsource = source.split("api_key=")[0] + "api_key="
newsource = newsource.replace(PLANET_ROOT_URL,
PLANET_ROOT_URL_PLACEHOLDER)
layer.setDataSource(newsource, layer.name(),
layer.dataProvider().name(),
QgsDataProvider.ProviderOptions())
layer.triggerRepaint()
def update_layer_geometry(self):
"""
This function needs to be called every time a point is added or deleted from the layer
If needed, a new layer is created with the apropiate geometry
Point if only one point, LineString otherwise
"""
options = QgsDataProvider.ProviderOptions()
waypoints = self.find_waypoints_in_mission()
if (len(waypoints) != 1) and self.mission_layer.geometryType(
) == QgsWkbTypes.PointGeometry:
self.mission_layer.setDataSource("LineString?crs=epsg:4326",
self.mission_name, "memory",
options)
self.set_mission_renderer(self.get_default_track_renderer())
elif len(waypoints) == 1 and self.mission_layer.geometryType(
) == QgsWkbTypes.LineGeometry:
self.mission_layer.setDataSource("Point?crs=epsg:4326",
self.mission_name, "memory",
options)
self.set_mission_renderer(self.get_default_track_renderer())
def move_position(self, end_point):
"""
Translates landmark layer to end_point
:param end_point: Where to move the landmark point
"""
# If layer is not stored in memory, change its datasource to memory
if self.lm_layer.source()[0] == "/":
temp_feature = next(self.lm_layer.dataProvider().getFeatures()
) # iterator with only 1 feature in list
options = QgsDataProvider.ProviderOptions()
self.lm_layer.setDataSource("Point?crs=epsg:4326",
self.lm_layer.name(), "memory",
options)
self.lm_layer.dataProvider().addFeatures([temp_feature])
self.lm_feature = next(self.lm_layer.dataProvider().getFeatures())
dx = end_point.x() - self.lm_point.x()
dy = end_point.y() - self.lm_point.y()
self.lm_layer.startEditing()
self.lm_layer.translateFeature(self.lm_feature.id(), dx, dy)
self.lm_layer.commitChanges()
def write_raster(raster_layer, raster_writer, write_path):
"""Write raster to specified file and update the layer's data source."""
dp = raster_layer.dataProvider()
pipe = QgsRasterPipe()
if not pipe.set(dp.clone()):
raise PackagingError(
f"Couldn't set raster pipe projector for layer {write_path}")
projector = QgsRasterProjector()
projector.setCrs(raster_layer.crs(), raster_layer.crs())
if not pipe.insert(2, projector):
raise PackagingError(
f"Couldn't set raster pipe provider for layer {write_path}")
res = raster_writer.writeRaster(pipe, dp.xSize(), dp.ySize(), dp.extent(),
raster_layer.crs())
if not res == QgsRasterFileWriter.NoError:
raise PackagingError(
f"Couldn't save raster {write_path} - write error: {res}")
provider_opts = QgsDataProvider.ProviderOptions()
provider_opts.layerName = raster_layer.name()
raster_layer.setDataSource(write_path, raster_layer.name(), "gdal",
provider_opts)
def createProvider(cls, uri, providerOptions, flags=QgsDataProvider.ReadFlags()):
return PyProvider(uri, providerOptions, flags)
def __init__(self, layer):
QgsDataProvider.__init__(
self, "dummyURI")
def __init__(self, layer):
QgsDataProvider.__init__(self, "dummyURI")
def testStyles(self):
"""Test that styles for rasters and vectors are kept when setDataSource is called"""
options = QgsDataProvider.ProviderOptions()
temp_dir = QTemporaryDir()
p = QgsProject.instance()
for f in (
'bad_layer_raster_test.tfw',
'bad_layer_raster_test.tiff',
'bad_layer_raster_test.tiff.aux.xml',
'bad_layers_test.gpkg',
'good_layers_test.qgs'):
copyfile(os.path.join(TEST_DATA_DIR, 'projects', f), os.path.join(temp_dir.path(), f))
project_path = os.path.join(temp_dir.path(), 'good_layers_test.qgs')
p = QgsProject().instance()
self.assertTrue(p.read(project_path))
self.assertEqual(p.count(), 3)
ms = self.getBaseMapSettings()
point_a = list(p.mapLayersByName('point_a'))[0]
point_b = list(p.mapLayersByName('point_b'))[0]
raster = list(p.mapLayersByName('bad_layer_raster_test'))[0]
self.assertTrue(point_a.isValid())
self.assertTrue(point_b.isValid())
self.assertTrue(raster.isValid())
ms.setExtent(QgsRectangle(2.81861, 41.98138, 2.81952, 41.9816))
ms.setLayers([point_a, point_b, raster])
image = renderMapToImage(ms)
print(os.path.join(temp_dir.path(), 'expected.png'))
self.assertTrue(image.save(os.path.join(temp_dir.path(), 'expected.png'), 'PNG'))
point_a_source = point_a.publicSource()
point_b_source = point_b.publicSource()
raster_source = raster.publicSource()
point_a.setDataSource(point_a_source, point_a.name(), 'ogr', options)
point_b.setDataSource(point_b_source, point_b.name(), 'ogr', options)
raster.setDataSource(raster_source, raster.name(), 'gdal', options)
self.assertTrue(image.save(os.path.join(temp_dir.path(), 'actual.png'), 'PNG'))
self.assertTrue(filecmp.cmp(os.path.join(temp_dir.path(), 'actual.png'), os.path.join(temp_dir.path(), 'expected.png')), False)
# Now build a bad project
bad_project_path = os.path.join(temp_dir.path(), 'bad_layers_test.qgs')
with open(project_path, 'r') as infile:
with open(bad_project_path, 'w+') as outfile:
outfile.write(infile.read().replace('./bad_layers_test.', './bad_layers_test-BAD_SOURCE.').replace('bad_layer_raster_test.tiff', 'bad_layer_raster_test-BAD_SOURCE.tiff'))
self.assertTrue(p.read(bad_project_path))
self.assertEqual(p.count(), 3)
point_a = list(p.mapLayersByName('point_a'))[0]
point_b = list(p.mapLayersByName('point_b'))[0]
raster = list(p.mapLayersByName('bad_layer_raster_test'))[0]
self.assertFalse(point_a.isValid())
self.assertFalse(point_b.isValid())
self.assertFalse(raster.isValid())
point_a.setDataSource(point_a_source, point_a.name(), 'ogr', options)
point_b.setDataSource(point_b_source, point_b.name(), 'ogr', options)
raster.setDataSource(raster_source, raster.name(), 'gdal', options)
self.assertTrue(image.save(os.path.join(temp_dir.path(), 'actual_fixed.png'), 'PNG'))
self.assertTrue(filecmp.cmp(os.path.join(temp_dir.path(), 'actual_fixed.png'), os.path.join(temp_dir.path(), 'expected.png')), False)
def refresh_geocity_oapif_layers_for_current_atlas_feature(id):
""" Change dataSourceUri for OAPIF layers listed in QGIS project
so that the source is filtered server side. Then reload the
single feature which have $id = id. Thus, new feature are avalailable
for print despite QGIS SERVER cache and no full reload of endpoint
feature is required (which would cause a performance leak on endpoint).
Refreshing of virtual layer only ensure api structure evolution
would be reflected correctly.
QGIS SERVER atlas filtering does not work (3.22.0) if primary key is not defined correctly
which is the case for OAPIF sources. Thus, we're force to use virtual layers.
Parameters
----------
id : int
Feature to print
"""
project = QgsProject.instance()
prefix_url = getenv("PREFIX_URL", "")
for layer in QgsProject.instance().mapLayers().values():
uri = layer.dataProvider().uri()
# refresh and filter OAPIF virtual layer
if layer.dataProvider().name() == "OAPIF" and layer.isValid():
# only for geocity endpoints listed in project
if uri.param("typename") in [
"permits",
"permits_poly",
"permits_line",
"permits_point",
]:
try:
# replace url in order to filter for the required feature only
uri.removeParam("url")
uri.setParam(
"url",
f"http://web:9000/{prefix_url}wfs3/?permit_request_id={id}",
)
Logger().info(
"qgis-printatlas - uri: " + uri.uri(expandAuthConfig=False)
)
layer.setDataSource(
uri.uri(expandAuthConfig=False),
uri.param("typename"),
"OAPIF",
QgsDataProvider.ProviderOptions(),
)
layer.dataProvider().reloadData()
layer.updateFields()
layer.dataProvider().updateExtents()
layer.triggerRepaint()
Logger().info(
"qgis-printatlas - refreshed data source: "
+ uri.param("typename")
)
except:
Logger().critical(
"(skipped) qgis-printatlas: " + uri.param("typename")
)
# refresh virtual layers that we use as a bug workaround in QGIS OAPIF provider which does not set PKEY column correctly in 3.22.0
if layer.dataProvider().name() == "virtual" and layer.isValid():
try:
layer.dataProvider().reloadData()
layer.dataProvider().updateExtents()
layer.updateFields()
layer.triggerRepaint()
Logger().info(
"qgis-printatlas - refreshed virtual layer: "
+ uri.param("typename")
)
except:
Logger().critical(
"(skipped) qgis-printatlas: " + uri.param("typename")
)
def test_project_relations(self):
"""Tests that a project with bad layers and relations can be saved with relations"""
temp_dir = QTemporaryDir()
p = QgsProject.instance()
for ext in ('qgs', 'gpkg'):
copyfile(
os.path.join(TEST_DATA_DIR, 'projects',
'relation_reference_test.%s' % ext),
os.path.join(temp_dir.path(),
'relation_reference_test.%s' % ext))
# Load the good project
project_path = os.path.join(temp_dir.path(),
'relation_reference_test.qgs')
p.removeAllMapLayers()
self.assertTrue(p.read(project_path))
point_a = list(p.mapLayersByName('point_a'))[0]
point_b = list(p.mapLayersByName('point_b'))[0]
point_a_source = point_a.publicSource()
point_b_source = point_b.publicSource()
self.assertTrue(point_a.isValid())
self.assertTrue(point_b.isValid())
# Check relations
def _check_relations():
relation = list(p.relationManager().relations().values())[0]
self.assertTrue(relation.isValid())
self.assertEqual(relation.referencedLayer().id(), point_b.id())
self.assertEqual(relation.referencingLayer().id(), point_a.id())
_check_relations()
# Now build a bad project
bad_project_path = os.path.join(temp_dir.path(),
'relation_reference_test_bad.qgs')
with open(project_path, 'r') as infile:
with open(bad_project_path, 'w+') as outfile:
outfile.write(infile.read().replace(
'./relation_reference_test.gpkg',
'./relation_reference_test-BAD_SOURCE.gpkg'))
# Load the bad project
p.removeAllMapLayers()
self.assertTrue(p.read(bad_project_path))
point_a = list(p.mapLayersByName('point_a'))[0]
point_b = list(p.mapLayersByName('point_b'))[0]
self.assertFalse(point_a.isValid())
self.assertFalse(point_b.isValid())
# This fails because relations are not valid anymore
with self.assertRaises(AssertionError):
_check_relations()
# Changing data source, relations should be restored:
options = QgsDataProvider.ProviderOptions()
point_a.setDataSource(point_a_source, 'point_a', 'ogr', options)
point_b.setDataSource(point_b_source, 'point_b', 'ogr', options)
self.assertTrue(point_a.isValid())
self.assertTrue(point_b.isValid())
# Check if relations were restored
_check_relations()
# Reload the bad project
p.removeAllMapLayers()
self.assertTrue(p.read(bad_project_path))
point_a = list(p.mapLayersByName('point_a'))[0]
point_b = list(p.mapLayersByName('point_b'))[0]
self.assertFalse(point_a.isValid())
self.assertFalse(point_b.isValid())
# This fails because relations are not valid anymore
with self.assertRaises(AssertionError):
_check_relations()
# Save the bad project
bad_project_path2 = os.path.join(temp_dir.path(),
'relation_reference_test_bad2.qgs')
p.write(bad_project_path2)
# Now fix the bad project
bad_project_path_fixed = os.path.join(
temp_dir.path(), 'relation_reference_test_bad_fixed.qgs')
with open(bad_project_path2, 'r') as infile:
with open(bad_project_path_fixed, 'w+') as outfile:
outfile.write(infile.read().replace(
'./relation_reference_test-BAD_SOURCE.gpkg',
'./relation_reference_test.gpkg'))
# Load the fixed project
p.removeAllMapLayers()
self.assertTrue(p.read(bad_project_path_fixed))
point_a = list(p.mapLayersByName('point_a'))[0]
point_b = list(p.mapLayersByName('point_b'))[0]
point_a_source = point_a.publicSource()
point_b_source = point_b.publicSource()
self.assertTrue(point_a.isValid())
self.assertTrue(point_b.isValid())
_check_relations()
def createProvider(cls,
uri,
providerOptions,
flags=QgsDataProvider.ReadFlags()):
return EarthEngineRasterDataProvider(uri, providerOptions, flags)
def convert_vector_layer(
layer, # pylint: disable=too-many-locals,too-many-branches,too-many-statements
project,
data_folder,
feedback,
conversion_results: ConversionResults,
change_source_on_error: bool = False,
verbose_log=False):
"""
Converts a vector layer to a standard format
"""
if layer.customProperty('original_uri'):
uri = layer.customProperty('original_uri')
if verbose_log:
feedback.pushDebugInfo(
f'Original layer URI from custom properties is {uri}')
else:
uri = layer.source()
if verbose_log:
feedback.pushDebugInfo(
'Original layer URI not found in custom properties')
source = QgsProviderRegistry.instance().decodeUri(
layer.providerType(), uri)
if verbose_log:
feedback.pushInfo('')
# older versions of QGIS didn't correctly strip out the subset from the layerName:
if 'subset' not in source:
if '|subset=' in source['layerName']:
if verbose_log:
feedback.pushDebugInfo(
'Stripping out subset string from layerName: {}'.
format(source['layerName']))
layer_name = source['layerName']
parts = layer_name.split('|subset=')
if len(parts) == 2:
source['layerName'] = parts[0]
if verbose_log:
feedback.pushDebugInfo('Cleaned layer name: {}'.format(
source['layerName']))
elif verbose_log:
feedback.reportError('Failed to strip subset string!')
elif '|subset=' in source['path']:
path = source['path']
if verbose_log:
feedback.pushDebugInfo(
'Stripping out subset string from path: {}'.format(
source['path']))
parts = path.split('|subset=')
if len(parts) == 2:
source['path'] = parts[0]
if verbose_log:
feedback.pushDebugInfo('Cleaned path: {}'.format(
source['path']))
elif verbose_log:
feedback.reportError('Failed to strip subset string!')
# convert to Geopackage
source_uri = QgsProviderRegistry.instance().encodeUri(
layer.providerType(), {
'path': source['path'],
'layerName': source['layerName']
})
# Sometimes the case varies in ArcMap documents, so when comparing to previously converted layers
# we use a case-insensitive path/layername which is normalized
result_key = QgsProviderRegistry.instance().encodeUri(
layer.providerType(), {
'path': pathlib.Path(
source['path']).resolve().as_posix().lower(),
'layerName': source['layerName'].lower()
})
if verbose_log:
feedback.pushDebugInfo('Converting layer: {} ( {} )'.format(
source['path'], source['layerName']))
feedback.pushDebugInfo(f'Cached result key: {result_key}')
provider_options = QgsDataProvider.ProviderOptions()
provider_options.transformContext = project.transformContext()
subset = layer.subsetString()
# have we maybe already converted this layer??
if result_key in conversion_results.layer_map:
previous_results = conversion_results.layer_map[result_key]
if previous_results.get('error'):
if verbose_log:
feedback.pushDebugInfo(
'Already tried to convert this layer, but failed last time, skipping...'
)
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr', provider_options)
else:
if verbose_log:
feedback.pushDebugInfo(
'Already converted this layer, reusing previous converted path: {} layername: {}'
.format(previous_results['destPath'],
previous_results['destLayer']))
layer.setDataSource(
QgsProviderRegistry.instance().encodeUri(
'ogr', {
'path': previous_results['destPath'],
'layerName': previous_results['destLayer']
}), layer.name(), 'ogr', provider_options)
if verbose_log:
feedback.pushDebugInfo('new source {}'.format(
layer.dataProvider().dataSourceUri()))
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
return previous_results
source_layer = QgsVectorLayer(source_uri, '', layer.providerType())
path = pathlib.Path(source['path'])
dest_file_name = ((pathlib.Path(data_folder) /
path.stem).with_suffix('.gpkg')).as_posix()
if dest_file_name not in conversion_results.created_databases:
# about to use a new file -- let's double-check that it doesn't already exist. We don't want
# to put layers into a database which we didn't make for this project
counter = 1
while pathlib.Path(dest_file_name).exists():
counter += 1
dest_file_name = ((pathlib.Path(data_folder) /
(path.stem + '_' + str(counter))
).with_suffix('.gpkg')).as_posix()
if dest_file_name in conversion_results.created_databases:
break
if verbose_log:
feedback.pushDebugInfo(
'Creating new destination file {}'.format(dest_file_name))
elif verbose_log:
feedback.pushDebugInfo(
'Reusing existing destination file {}'.format(dest_file_name))
# now this filename is ok for other layers to be stored in for this conversion
conversion_results.created_databases.add(dest_file_name)
layer_name_candidate = source['layerName']
counter = 1
while QgsVectorFileWriter.targetLayerExists(dest_file_name,
layer_name_candidate):
counter += 1
layer_name_candidate = '{}_{}'.format(source['layerName'], counter)
if verbose_log:
feedback.pushDebugInfo(
'Target layer name is {}'.format(layer_name_candidate))
if not source_layer.isValid():
if verbose_log:
feedback.reportError('Source layer is not valid')
if path.exists():
if verbose_log:
feedback.pushDebugInfo('File path DOES exist')
test_layer = QgsVectorLayer(path.as_posix())
sub_layers = test_layer.dataProvider().subLayers()
feedback.pushDebugInfo(
f'Readable layers from "{path.as_posix()}" are:')
for sub_layer in sub_layers:
_, name, count, geom_type, _, _ = sub_layer.split(
QgsDataProvider.sublayerSeparator())
feedback.pushDebugInfo(
f'- "{name}" ({count} features, geometry type {geom_type})'
)
if path.exists() and path.suffix.lower() == '.mdb':
try:
source['layerName'].encode('ascii')
except UnicodeDecodeError:
error = f'''MDB layers with unicode names are not supported by QGIS -- cannot convert "{source['layerName']}"'''
if verbose_log:
feedback.reportError(error)
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr',
provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
conversion_results.layer_map[result_key] = {'error': error}
return conversion_results.layer_map[result_key]
# maybe a non-spatial table, which can't be read with GDAL < 3.2
source_layer = None
if verbose_log:
feedback.pushDebugInfo('Layer type is {}'.format(
QgsWkbTypes.displayString(layer.wkbType())))
if layer.wkbType() == QgsWkbTypes.NoGeometry:
if verbose_log:
feedback.pushDebugInfo(
'Attempting fallback for non-spatial tables')
try:
source_layer = ConversionUtils.convert_mdb_table_to_memory_layer(
str(path), source['layerName'])
if verbose_log:
feedback.pushDebugInfo('Fallback succeeded!')
except Exception as e: # nopep8, pylint: disable=broad-except
if verbose_log:
feedback.reportError('Fallback failed: {}'.format(
str(e)))
source_layer = None
elif verbose_log:
feedback.reportError(
'Nothing else to try, conversion failed')
if not source_layer:
# here we fake things. We don't leave the original path to the mdb layer intact in the converted
# project, as this can cause massive issues with QGIS as it attempts to re-read this path constantly
# rather we "pretend" that the conversion was ok and set the broken layer's path to what the gpkg
# converted version WOULD have been! It'll still be broken in the converted project (obviously),
# but QGIS will no longer try endless to read the MDB and get all hung up on this...
conversion_results.layer_map[result_key] = {
'sourcePath':
source['path'],
'sourceLayer':
source['layerName'],
'destPath':
dest_file_name,
'destLayer':
layer_name_candidate,
'error':
'Could not open {} ({}) for conversion'.format(
source_uri, source['layerName'])
}
if change_source_on_error:
if verbose_log:
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr',
provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(
subset))
layer.setSubsetString(subset)
if verbose_log:
feedback.pushDebugInfo('new source {}'.format(
layer.dataProvider().dataSourceUri()))
return conversion_results.layer_map[result_key]
else:
if not path.exists():
error = 'The referenced file {} does NOT exist!'.format(
str(path))
else:
error = 'The referenced file exists, but could not open {} ({}) for conversion'.format(
source_uri, source['layerName'])
if verbose_log:
feedback.reportError(error)
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr', provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
conversion_results.layer_map[result_key] = {'error': error}
return conversion_results.layer_map[result_key]
if verbose_log:
feedback.pushDebugInfo('Source is valid, converting')
options = QgsVectorFileWriter.SaveVectorOptions()
options.layerName = layer_name_candidate
options.actionOnExistingFile = QgsVectorFileWriter.CreateOrOverwriteLayer if pathlib.Path(
dest_file_name).exists(
) else QgsVectorFileWriter.CreateOrOverwriteFile
options.feedback = feedback
error, error_message = QgsVectorFileWriter.writeAsVectorFormatV2(
source_layer, dest_file_name, project.transformContext(), options)
if error != QgsVectorFileWriter.NoError:
if verbose_log:
feedback.reportError('Failed: {}'.format(error_message))
feedback.pushDebugInfo('Restoring stored URI')
layer.setDataSource(uri, layer.name(), 'ogr', provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
conversion_results.layer_map[result_key] = {'error': error_message}
return conversion_results.layer_map[result_key]
if verbose_log:
feedback.pushDebugInfo('Success!')
provider_options = QgsDataProvider.ProviderOptions()
provider_options.transformContext = project.transformContext()
subset = layer.subsetString()
layer.setDataSource(
QgsProviderRegistry.instance().encodeUri(
'ogr', {
'path': dest_file_name,
'layerName': options.layerName
}), layer.name(), 'ogr', provider_options)
if subset:
if verbose_log:
feedback.pushDebugInfo(
'Resetting subset string: {}'.format(subset))
layer.setSubsetString(subset)
if verbose_log:
feedback.pushDebugInfo('new source {}'.format(
layer.dataProvider().dataSourceUri()))
conversion_results.layer_map[result_key] = {
'sourcePath': source['path'],
'sourceLayer': source['layerName'],
'destPath': dest_file_name,
'destLayer': options.layerName
}
return conversion_results.layer_map[result_key]