def test_source_escaping(self):
# the source contains ':'
source = QUrl.fromLocalFile(os.path.join(self.testDataDir, "delimitedtext/test.csv")).toString() + "?type=csv&geomType=none&subsetIndex=no&watchFile=no"
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "delimitedtext")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
def test_qgisExpressionFunctions(self):
QgsProject.instance().setTitle('project')
self.assertEqual(QgsProject.instance().title(), 'project')
df = QgsVirtualLayerDefinition()
df.setQuery("SELECT format('hello %1', 'world') as a, year(todate('2016-01-02')) as b, title('This') as t, var('project_title') as c")
l = QgsVectorLayer(df.toString(), "testq", "virtual")
self.assertEqual(l.isValid(), True)
for f in l.getFeatures():
self.assertEqual(f.attributes(), ['hello world', 2016, 'This', 'project'])
def test_layer_with_accents(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "françéà", "ogr", False)
self.assertEqual(l1.isValid(), True)
QgsProject.instance().addMapLayer(l1)
df = QgsVirtualLayerDefinition()
df.setQuery('select * from "françéà"')
vl = QgsVectorLayer(df.toString(), "testq", "virtual")
self.assertEqual(vl.isValid(), True)
ids = [f.id() for f in vl.getFeatures()]
self.assertEqual(len(ids), 4)
QgsProject.instance().removeMapLayer(l1.id())
def test_source_escaping2(self):
def create_test_db(dbfile):
if os.path.exists(dbfile):
os.remove(dbfile)
con = spatialite_connect(dbfile)
cur = con.cursor()
cur.execute("SELECT InitSpatialMetadata(1)")
cur.execute("CREATE TABLE test (id INTEGER, name TEXT)")
cur.execute("SELECT AddGeometryColumn('test', 'geometry', 4326, 'POINT', 'XY')")
sql = "INSERT INTO test (id, name, geometry) "
sql += "VALUES (1, 'toto',GeomFromText('POINT(0 0)',4326))"
cur.execute(sql)
con.close()
# the source contains ',' and single quotes
fn = os.path.join(tempfile.gettempdir(), "test,.db")
create_test_db(fn)
source = "dbname='%s' table=\"test\" (geometry) sql=" % fn
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "spatialite")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
# the source contains ':' and single quotes
fn = os.path.join(tempfile.gettempdir(), "test:.db")
create_test_db(fn)
source = "dbname='%s' table=\"test\" (geometry) sql=" % fn
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "spatialite")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
def __init__(self, db, sql, parent):
super().__init__(db, sql, parent)
tf = QTemporaryFile()
tf.open()
path = tf.fileName()
tf.close()
df = QgsVirtualLayerDefinition()
df.setFilePath(path)
df.setQuery(sql)
self.subtask = QgsVirtualLayerTask(df)
self.addSubTask(self.subtask, [], QgsTask.ParentDependsOnSubTask)
def getQueryGeometryName(sqlite_file):
# introspect the file
with sqlite3_connection(sqlite_file) as conn:
c = conn.cursor()
for r in c.execute("SELECT url FROM _meta"):
d = QgsVirtualLayerDefinition.fromUrl(QUrl(r[0]))
if d.hasDefinedGeometry():
return d.geometryField()
return None
def toSqlLayer(self, sql, geomCol, uniqueCol, layerName="QueryLayer", layerType=None, avoidSelectById=False, _filter=""):
df = QgsVirtualLayerDefinition()
df.setQuery(sql)
if uniqueCol is not None:
uniqueCol = uniqueCol.strip('"').replace('""', '"')
df.setUid(uniqueCol)
if geomCol is not None:
df.setGeometryField(geomCol)
vl = QgsVectorLayer(df.toString(), layerName, "virtual")
if _filter:
vl.setSubsetString(_filter)
return vl
def test_no_geometry(self):
df = QgsVirtualLayerDefinition()
df.addSource("vtab", os.path.join(self.testDataDir, "france_parts.shp"), "ogr")
df.setGeometryWkbType(QgsWkbTypes.NoGeometry)
l2 = QgsVectorLayer(df.toString(), "vtab2", "virtual", False)
self.assertEqual(l2.isValid(), True)
self.assertEqual(l2.dataProvider().wkbType(), 100) # NoGeometry
def test(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "françéà", "ogr", QgsVectorLayer.LayerOptions(False))
self.assertEqual(l1.isValid(), True)
QgsProject.instance().addMapLayer(l1)
df = QgsVirtualLayerDefinition()
df.setQuery('select * from "françéà"')
self.task = QgsVirtualLayerTask(df)
ids = [f.id() for f in self.task.layer().getFeatures()]
self.assertEqual(len(ids), 0)
self.task.taskCompleted.connect(self.onSuccess)
self.task.taskTerminated.connect(self.onFail)
QgsApplication.taskManager().addTask(self.task)
while not self._success and not self._fail:
QCoreApplication.processEvents()
self.assertTrue(self._success)
self.assertFalse(self._fail)
self.assertEqual(len(self.ids), 4)
# Test exception
self._success = False
self._fail = False
df.setQuery('select *')
self.task = QgsVirtualLayerTask(df)
self.task.taskCompleted.connect(self.onSuccess)
self.task.taskTerminated.connect(self.onFail)
QgsApplication.taskManager().addTask(self.task)
while not self._success and not self._fail:
QCoreApplication.processEvents()
self.assertFalse(self._success)
self.assertTrue(self._fail)
self.assertEqual(self._exceptionText, 'Query preparation error on PRAGMA table_info(_tview): no tables specified', self._exceptionText)
def testFiltersWithoutUid(self):
ml = QgsVectorLayer("Point?srid=EPSG:4326&field=a:int", "mem_no_uid", "memory")
self.assertEqual(ml.isValid(), True)
QgsProject.instance().addMapLayer(ml)
# a memory layer with 10 features
ml.startEditing()
for i in range(10):
f = QgsFeature(ml.fields())
f.setGeometry(QgsGeometry.fromWkt('POINT({} 0)'.format(i)))
f.setAttributes([i])
ml.addFeatures([f])
ml.commitChanges()
df = QgsVirtualLayerDefinition()
df.setQuery('select * from mem_no_uid')
vl = QgsVectorLayer(df.toString(), "vl", "virtual")
self.assertEqual(vl.isValid(), True)
# make sure the returned id with a filter is the same as
# if there is no filter
req = QgsFeatureRequest().setFilterRect(QgsRectangle(4.5, -1, 5.5, 1))
fids = [f.id() for f in vl.getFeatures(req)]
self.assertEqual(fids, [5])
req = QgsFeatureRequest().setFilterExpression("a = 5")
fids = [f.id() for f in vl.getFeatures(req)]
self.assertEqual(fids, [5])
req = QgsFeatureRequest().setFilterFid(5)
a = [(f.id(), f['a']) for f in vl.getFeatures(req)]
self.assertEqual(a, [(5, 5)])
req = QgsFeatureRequest().setFilterFids([5, 6, 8])
a = [(f.id(), f['a']) for f in vl.getFeatures(req)]
self.assertEqual(a, [(5, 5), (6, 6), (8, 8)])
QgsProject.instance().removeMapLayer(ml)
def test(self):
l1 = QgsVectorLayer(os.path.join(self.testDataDir, "france_parts.shp"), "françéà", "ogr", QgsVectorLayer.LayerOptions(False))
self.assertEqual(l1.isValid(), True)
QgsProject.instance().addMapLayer(l1)
df = QgsVirtualLayerDefinition()
df.setQuery('select * from "françéà"')
self.task = QgsVirtualLayerTask(df)
ids = [f.id() for f in self.task.layer().getFeatures()]
self.assertEqual(len(ids), 0)
self.task.taskCompleted.connect(self.onSuccess)
self.task.taskTerminated.connect(self.onFail)
QgsApplication.taskManager().addTask(self.task)
while not self.success and not self.fail:
QCoreApplication.processEvents()
self.assertTrue(self.success)
self.assertFalse(self.fail)
self.assertEqual(len(self.ids), 4)
def setUpClass(cls):
"""Run before all tests"""
# Create the layer for the common provider tests
shp = os.path.join(TEST_DATA_DIR, 'provider/shapefile.shp')
d = QgsVirtualLayerDefinition()
d.addSource("vtab1", shp, "ogr")
d.setUid("pk")
cls.vl = QgsVectorLayer(d.toString(), u'test', u'virtual')
assert (cls.vl.isValid())
cls.provider = cls.vl.dataProvider()
def _get_cursor_columns(self, c):
tf = QTemporaryFile()
tf.open()
tmp = tf.fileName()
tf.close()
df = QgsVirtualLayerDefinition()
df.setFilePath(tmp)
df.setQuery(c.sql)
p = QgsVectorLayer(df.toString(), "vv", "virtual")
if not p.isValid():
return []
f = [f.name() for f in p.fields()]
if p.geometryType() != QgsWkbTypes.NullGeometry:
gn = getQueryGeometryName(tmp)
if gn:
f += [gn]
return f
def __init__(self, db, sql, parent=None, layer=None, path=None):
t = QTime()
t.start()
if not layer:
tf = QTemporaryFile()
tf.open()
path = tf.fileName()
tf.close()
df = QgsVirtualLayerDefinition()
df.setFilePath(path)
df.setQuery(sql)
layer = QgsVectorLayer(df.toString(), "vv", "virtual")
self._secs = t.elapsed() / 1000.0
data = []
header = []
if not layer.isValid():
raise DbError(layer.dataProvider().error().summary(), sql)
else:
header = [f.name() for f in layer.fields()]
has_geometry = False
if layer.geometryType() != QgsWkbTypes.NullGeometry:
gn = getQueryGeometryName(path)
if gn:
has_geometry = True
header += [gn]
for f in layer.getFeatures():
a = f.attributes()
if has_geometry:
if f.hasGeometry():
a += [f.geometry().asWkt()]
else:
a += [None]
data += [a]
self._secs = 0
self._affectedRows = len(data)
BaseTableModel.__init__(self, header, data, parent)
def __init__(self, db, sql, parent=None):
# create a virtual layer with non-geometry results
t = QTime()
t.start()
tf = QTemporaryFile()
tf.open()
tmp = tf.fileName()
tf.close()
df = QgsVirtualLayerDefinition()
df.setFilePath(tmp)
df.setQuery(sql)
p = QgsVectorLayer(df.toString(), "vv", "virtual")
self._secs = t.elapsed() / 1000.0
if not p.isValid():
data = []
header = []
raise DbError(p.dataProvider().error().summary(), sql)
else:
header = [f.name() for f in p.fields()]
has_geometry = False
if p.geometryType() != QgsWkbTypes.NullGeometry:
gn = getQueryGeometryName(tmp)
if gn:
has_geometry = True
header += [gn]
data = []
for f in p.getFeatures():
a = f.attributes()
if has_geometry:
if f.hasGeometry():
a += [f.geometry().exportToWkt()]
else:
a += [None]
data += [a]
self._secs = 0
self._affectedRows = len(data)
BaseTableModel.__init__(self, header, data, parent)
def processAlgorithm(self, context, feedback):
layers = self.getParameterValue(self.INPUT_DATASOURCES)
query = self.getParameterValue(self.INPUT_QUERY)
uid_field = self.getParameterValue(self.INPUT_UID_FIELD)
geometry_field = self.getParameterValue(self.INPUT_GEOMETRY_FIELD)
geometry_type = self.getParameterValue(self.INPUT_GEOMETRY_TYPE)
geometry_crs = self.getParameterValue(self.INPUT_GEOMETRY_CRS)
df = QgsVirtualLayerDefinition()
layerIdx = 1
if layers:
for layerSource in layers.split(';'):
layer = dataobjects.getLayerFromString(layerSource)
if layer:
df.addSource('input{}'.format(layerIdx), layer.id())
layerIdx += 1
if query == '':
raise GeoAlgorithmExecutionException(
self.
tr('Empty SQL. Please enter valid SQL expression and try again.'
))
else:
df.setQuery(query)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWkbTypes.NullGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs:
crs = QgsCoordinateReferenceSystem(geometry_crs)
if crs.isValid():
df.setGeometrySrid(crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise GeoAlgorithmExecutionException(
vLayer.dataProvider().error().message())
writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(
vLayer.fields().toList(),
vLayer.wkbType() if geometry_type != 1 else 1, vLayer.crs(),
context)
features = QgsProcessingUtils.getFeatures(vLayer, context)
total = 100.0 / QgsProcessingUtils.featureCount(vLayer, context)
outFeat = QgsFeature()
for current, inFeat in enumerate(features):
outFeat.setAttributes(inFeat.attributes())
if geometry_type != 1:
outFeat.setGeometry(inFeat.geometry())
writer.addFeature(outFeat)
feedback.setProgress(int(current * total))
del writer
def processAlgorithm(self, progress):
layers = self.getParameterValue(self.INPUT_DATASOURCES)
query = self.getParameterValue(self.INPUT_QUERY)
uid_field = self.getParameterValue(self.INPUT_UID_FIELD)
geometry_field = self.getParameterValue(self.INPUT_GEOMETRY_FIELD)
geometry_type = self.getParameterValue(self.INPUT_GEOMETRY_TYPE)
geometry_crs = self.getParameterValue(self.INPUT_GEOMETRY_CRS)
df = QgsVirtualLayerDefinition()
layerIdx = 1
if layers:
for layerSource in layers.split(';'):
layer = dataobjects.getObjectFromUri(layerSource)
if layer:
df.addSource('input{}'.format(layerIdx), layer.id())
if query == '':
raise GeoAlgorithmExecutionException(
self.
tr('Empty SQL. Please enter valid SQL expression and try again.'
))
else:
df.setQuery(query)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWKBTypes.NoGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs:
crs = QgsCoordinateReferenceSystem(geometry_crs)
if crs.isValid():
df.setGeometrySrid(crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise GeoAlgorithmExecutionException(
vLayer.dataProvider().error().message())
writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(
vLayer.pendingFields().toList(),
# create a point layer (without any points) if 'no geometry' is chosen
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs())
features = vector.features(vLayer)
outFeat = QgsFeature()
for inFeat in features:
outFeat.setAttributes(inFeat.attributes())
if geometry_type != 1:
outFeat.setGeometry(inFeat.geometry())
writer.addFeature(outFeat)
del writer
def test_query_with_accents(self):
# shapefile with accents and latin1 encoding
df = QgsVirtualLayerDefinition()
df.addSource("vtab", os.path.join(self.testDataDir, "france_parts.shp"), "ogr", "ISO-8859-1")
df.setQuery("SELECT * FROM vtab WHERE TYPE_1 = 'Région'")
vl = QgsVectorLayer(df.toString(), "testq", "virtual")
self.assertEqual(vl.isValid(), True)
ids = [f.id() for f in vl.getFeatures()]
self.assertEqual(len(ids), 4)
# the same shapefile with a wrong encoding
df.addSource("vtab", os.path.join(self.testDataDir, "france_parts.shp"), "ogr", "UTF-8")
df.setQuery("SELECT * FROM vtab WHERE TYPE_1 = 'Région'")
vl2 = QgsVectorLayer(df.toString(), "testq", "virtual")
self.assertEqual(vl2.isValid(), True)
ids = [f.id() for f in vl2.getFeatures()]
self.assertEqual(ids, [])
def processAlgorithm(self, parameters, context, feedback):
layers = self.parameterAsLayerList(parameters, self.INPUT_DATASOURCES,
context)
query = self.parameterAsString(parameters, self.INPUT_QUERY, context)
uid_field = self.parameterAsString(parameters, self.INPUT_UID_FIELD,
context)
geometry_field = self.parameterAsString(parameters,
self.INPUT_GEOMETRY_FIELD,
context)
geometry_type = self.parameterAsEnum(parameters,
self.INPUT_GEOMETRY_TYPE, context)
geometry_crs = self.parameterAsCrs(parameters, self.INPUT_GEOMETRY_CRS,
context)
df = QgsVirtualLayerDefinition()
for layerIdx, layer in enumerate(layers):
df.addSource('input{}'.format(layerIdx + 1), layer.id())
if query == '':
raise QgsProcessingException(
self.
tr('Empty SQL. Please enter valid SQL expression and try again.'
))
else:
localContext = self.createExpressionContext(parameters, context)
expandedQuery = QgsExpression.replaceExpressionText(
query, localContext)
df.setQuery(expandedQuery)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWkbTypes.NoGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs.isValid():
df.setGeometrySrid(geometry_crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise QgsProcessingException(
vLayer.dataProvider().error().message())
(sink, dest_id) = self.parameterAsSink(
parameters, self.OUTPUT, context, vLayer.fields(),
vLayer.wkbType() if geometry_type != 1 else 1, vLayer.crs())
if sink is None:
raise QgsProcessingException(
self.invalidSinkError(parameters, self.OUTPUT))
features = vLayer.getFeatures()
total = 100.0 / vLayer.featureCount() if vLayer.featureCount() else 0
for current, inFeat in enumerate(features):
if feedback.isCanceled():
break
sink.addFeature(inFeat, QgsFeatureSink.FastInsert)
feedback.setProgress(int(current * total))
return {self.OUTPUT: dest_id}
def processAlgorithm(self, parameters, context, feedback):
layers = self.getParameterValue(self.INPUT_DATASOURCES)
query = self.getParameterValue(self.INPUT_QUERY)
uid_field = self.getParameterValue(self.INPUT_UID_FIELD)
geometry_field = self.getParameterValue(self.INPUT_GEOMETRY_FIELD)
geometry_type = self.getParameterValue(self.INPUT_GEOMETRY_TYPE)
geometry_crs = self.getParameterValue(self.INPUT_GEOMETRY_CRS)
df = QgsVirtualLayerDefinition()
layerIdx = 1
if layers:
for layerSource in layers.split(';'):
layer = QgsProcessingUtils.mapLayerFromString(layerSource, context)
if layer:
df.addSource('input{}'.format(layerIdx), layer.id())
layerIdx += 1
if query == '':
raise GeoAlgorithmExecutionException(
self.tr('Empty SQL. Please enter valid SQL expression and try again.'))
else:
df.setQuery(query)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWkbTypes.NullGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs:
crs = QgsCoordinateReferenceSystem(geometry_crs)
if crs.isValid():
df.setGeometrySrid(crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise GeoAlgorithmExecutionException(vLayer.dataProvider().error().message())
writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(vLayer.fields(),
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs(), context)
features = QgsProcessingUtils.getFeatures(vLayer, context)
total = 100.0 / vLayer.featureCount() if vLayer.featureCount() else 0
outFeat = QgsFeature()
for current, inFeat in enumerate(features):
outFeat.setAttributes(inFeat.attributes())
if geometry_type != 1:
outFeat.setGeometry(inFeat.geometry())
writer.addFeature(outFeat, QgsFeatureSink.FastInsert)
feedback.setProgress(int(current * total))
del writer
def setUpClass(cls):
"""Run before all tests"""
# Create the layer for the common provider tests
shp = os.path.join(TEST_DATA_DIR, "provider/shapefile.shp")
d = QgsVirtualLayerDefinition()
d.addSource("vtab1", shp, "ogr")
d.setUid("pk")
cls.vl = QgsVectorLayer(d.toString(), "test", "virtual")
assert cls.vl.isValid()
cls.provider = cls.vl.dataProvider()
shp_poly = os.path.join(TEST_DATA_DIR, "provider/shapefile_poly.shp")
d = QgsVirtualLayerDefinition()
d.addSource("vtab2", shp_poly, "ogr")
d.setUid("pk")
cls.poly_vl = QgsVectorLayer(d.toString(), "test_poly", "virtual")
assert cls.poly_vl.isValid()
cls.poly_provider = cls.poly_vl.dataProvider()
def test_query_with_accents(self):
# shapefile with accents and latin1 encoding
df = QgsVirtualLayerDefinition()
df.addSource("vtab", os.path.join(self.testDataDir,
"france_parts.shp"), "ogr",
"ISO-8859-1")
df.setQuery("SELECT * FROM vtab WHERE TYPE_1 = 'Région'")
vl = QgsVectorLayer(df.toString(), "testq", "virtual")
self.assertEqual(vl.isValid(), True)
ids = [f.id() for f in vl.getFeatures()]
self.assertEqual(len(ids), 4)
# the same shapefile with a wrong encoding
df.addSource("vtab", os.path.join(self.testDataDir,
"france_parts.shp"), "ogr", "UTF-8")
df.setQuery("SELECT * FROM vtab WHERE TYPE_1 = 'Région'")
vl2 = QgsVectorLayer(df.toString(), "testq", "virtual")
self.assertEqual(vl2.isValid(), True)
ids = [f.id() for f in vl2.getFeatures()]
self.assertEqual(ids, [])
def processAlgorithm(self, parameters, context, feedback):
layers = self.parameterAsLayerList(parameters, self.INPUT_DATASOURCES,
context)
query = self.parameterAsString(parameters, self.INPUT_QUERY, context)
uid_field = self.parameterAsString(parameters, self.INPUT_UID_FIELD,
context)
geometry_field = self.parameterAsString(parameters,
self.INPUT_GEOMETRY_FIELD,
context)
geometry_type = self.parameterAsEnum(parameters,
self.INPUT_GEOMETRY_TYPE, context)
geometry_crs = self.parameterAsCrs(parameters, self.INPUT_GEOMETRY_CRS,
context)
df = QgsVirtualLayerDefinition()
for layerIdx, layer in enumerate(layers):
# Issue https://github.com/qgis/QGIS/issues/24041
# When using this algorithm from the graphic modeler, it may try to
# access (thanks the QgsVirtualLayerProvider) to memory layer that
# belongs to temporary QgsMapLayerStore, not project.
# So, we write them to disk is this is the case.
if context.project() and not context.project().mapLayer(
layer.id()):
basename = "memorylayer." + QgsVectorFileWriter.supportedFormatExtensions(
)[0]
tmp_path = QgsProcessingUtils.generateTempFilename(basename)
QgsVectorFileWriter.writeAsVectorFormat(
layer, tmp_path,
layer.dataProvider().encoding())
df.addSource('input{}'.format(layerIdx + 1), tmp_path, "ogr")
else:
df.addSource('input{}'.format(layerIdx + 1), layer.id())
if query == '':
raise QgsProcessingException(
self.
tr('Empty SQL. Please enter valid SQL expression and try again.'
))
else:
localContext = self.createExpressionContext(parameters, context)
expandedQuery = QgsExpression.replaceExpressionText(
query, localContext)
df.setQuery(expandedQuery)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWkbTypes.NoGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs.isValid():
df.setGeometrySrid(geometry_crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise QgsProcessingException(
vLayer.dataProvider().error().message())
if vLayer.wkbType() == QgsWkbTypes.Unknown:
raise QgsProcessingException(self.tr("Cannot find geometry field"))
(sink, dest_id) = self.parameterAsSink(
parameters, self.OUTPUT, context, vLayer.fields(),
vLayer.wkbType() if geometry_type != 1 else 1, vLayer.crs())
if sink is None:
raise QgsProcessingException(
self.invalidSinkError(parameters, self.OUTPUT))
features = vLayer.getFeatures()
total = 100.0 / vLayer.featureCount() if vLayer.featureCount() else 0
for current, inFeat in enumerate(features):
if feedback.isCanceled():
break
sink.addFeature(inFeat, QgsFeatureSink.FastInsert)
feedback.setProgress(int(current * total))
return {self.OUTPUT: dest_id}
def processAlgorithm(self, progress):
layers = self.getParameterValue(self.INPUT_DATASOURCES)
query = self.getParameterValue(self.INPUT_QUERY)
uid_field = self.getParameterValue(self.INPUT_UID_FIELD)
geometry_field = self.getParameterValue(self.INPUT_GEOMETRY_FIELD)
geometry_type = self.getParameterValue(self.INPUT_GEOMETRY_TYPE)
geometry_crs = self.getParameterValue(self.INPUT_GEOMETRY_CRS)
df = QgsVirtualLayerDefinition()
layerIdx = 1
if layers:
for layerSource in layers.split(';'):
layer = dataobjects.getObjectFromUri(layerSource)
if layer:
df.addSource('input{}'.format(layerIdx), layer.id())
if query == '':
raise GeoAlgorithmExecutionException(
self.tr('Empty SQL. Please enter valid SQL expression and try again.'))
else:
df.setQuery(query)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWkbTypes.NullGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs:
crs = QgsCoordinateReferenceSystem(geometry_crs)
if crs.isValid():
df.setGeometrySrid(crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise GeoAlgorithmExecutionException(vLayer.dataProvider().error().message())
writer = self.getOutputFromName(self.OUTPUT_LAYER).getVectorWriter(
vLayer.fields().toList(),
# Create a point layer (without any points) if 'no geometry' is chosen
vLayer.wkbType() if geometry_type != 1 else 1,
vLayer.crs())
features = vector.features(vLayer)
total = 100.0 / len(features)
outFeat = QgsFeature()
for current, inFeat in enumerate(features):
outFeat.setAttributes(inFeat.attributes())
if geometry_type != 1:
outFeat.setGeometry(inFeat.geometry())
writer.addFeature(outFeat)
progress.setPercentage(int(current * total))
del writer
def processAlgorithm(self, parameters, context, feedback):
layers = self.parameterAsLayerList(parameters, self.INPUT_DATASOURCES, context)
query = self.parameterAsString(parameters, self.INPUT_QUERY, context)
uid_field = self.parameterAsString(parameters, self.INPUT_UID_FIELD, context)
geometry_field = self.parameterAsString(parameters, self.INPUT_GEOMETRY_FIELD, context)
geometry_type = self.parameterAsEnum(parameters, self.INPUT_GEOMETRY_TYPE, context)
geometry_crs = self.parameterAsCrs(parameters, self.INPUT_GEOMETRY_CRS, context)
df = QgsVirtualLayerDefinition()
for layerIdx, layer in enumerate(layers):
df.addSource('input{}'.format(layerIdx + 1), layer.id())
if query == '':
raise QgsProcessingException(
self.tr('Empty SQL. Please enter valid SQL expression and try again.'))
else:
localContext = self.createExpressionContext(parameters, context)
expandedQuery = QgsExpression.replaceExpressionText(query, localContext)
df.setQuery(expandedQuery)
if uid_field:
df.setUid(uid_field)
if geometry_type == 1: # no geometry
df.setGeometryWkbType(QgsWkbTypes.NoGeometry)
else:
if geometry_field:
df.setGeometryField(geometry_field)
if geometry_type > 1:
df.setGeometryWkbType(geometry_type - 1)
if geometry_crs.isValid():
df.setGeometrySrid(geometry_crs.postgisSrid())
vLayer = QgsVectorLayer(df.toString(), "temp_vlayer", "virtual")
if not vLayer.isValid():
raise QgsProcessingException(vLayer.dataProvider().error().message())
(sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context,
vLayer.fields(), vLayer.wkbType() if geometry_type != 1 else 1, vLayer.crs())
if sink is None:
raise QgsProcessingException(self.invalidSinkError(parameters, self.OUTPUT))
features = vLayer.getFeatures()
total = 100.0 / vLayer.featureCount() if vLayer.featureCount() else 0
for current, inFeat in enumerate(features):
if feedback.isCanceled():
break
sink.addFeature(inFeat, QgsFeatureSink.FastInsert)
feedback.setProgress(int(current * total))
return {self.OUTPUT: dest_id}
def test1(self):
d = QgsVirtualLayerDefinition()
self.assertEqual(d.toString(), "")
d.setFilePath("/file")
self.assertEqual(d.toString(), "file:///file")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), "/file")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).filePath(), "/file")
d.setFilePath(os.path.join("C:/", "file"))
self.assertEqual(d.toString(), "file:///C:/file")
# self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), os.path.join('C:/', 'file'))
d.setQuery("SELECT * FROM mytable")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), "SELECT * FROM mytable")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).query(), "SELECT * FROM mytable")
q = "SELECT * FROM tableéé /*:int*/"
d.setQuery(q)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), q)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).query(), q)
s1 = "file://foo&bar=okié"
d.addSource("name", s1, "provider", "utf8")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[0].source(), s1)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).sourceLayers()[0].source(), s1)
n1 = "éé ok"
d.addSource(n1, s1, "provider")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[1].name(), n1)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).sourceLayers()[1].name(), n1)
d.addSource("ref1", "id0001")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[2].reference(), "id0001")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).sourceLayers()[2].reference(), "id0001"
)
s = "dbname='C:\\tt' table=\"test\" (geometry) sql="
d.addSource("nn", s, "spatialite")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[3].source(), s)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).sourceLayers()[3].source(), s)
d.setGeometryField("geom")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryField(), "geom")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).geometryField(), "geom")
d.setGeometryWkbType(QgsWkbTypes.Point)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryWkbType(), QgsWkbTypes.Point)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).geometryWkbType(), QgsWkbTypes.Point)
f = QgsFields()
f.append(QgsField("a", QVariant.Int))
f.append(QgsField("f", QVariant.Double))
f.append(QgsField("s", QVariant.String))
d.setFields(f)
f2 = QgsVirtualLayerDefinition.fromUrl(d.toUrl()).fields()
self.assertEqual(f[0].name(), f2[0].name())
self.assertEqual(f[0].type(), f2[0].type())
self.assertEqual(f[1].name(), f2[1].name())
self.assertEqual(f[1].type(), f2[1].type())
self.assertEqual(f[2].name(), f2[2].name())
self.assertEqual(f[2].type(), f2[2].type())
def test1(self):
d = QgsVirtualLayerDefinition()
self.assertEqual(d.toString(), "")
d.setFilePath("/file")
self.assertEqual(d.toString(), "file:///file")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), "/file")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).filePath(), "/file")
d.setFilePath(os.path.join('C:/', 'file'))
self.assertEqual(d.toString(), "file:///C:/file")
# self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), os.path.join('C:/', 'file'))
d.setQuery("SELECT * FROM mytable")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(),
"SELECT * FROM mytable")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).query(),
"SELECT * FROM mytable")
q = "SELECT * FROM tableéé /*:int*/"
d.setQuery(q)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), q)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(d.toString())).query(),
q)
s1 = "file://foo&bar=okié"
d.addSource("name", s1, "provider", "utf8")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(
d.toUrl()).sourceLayers()[0].source(), s1)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).sourceLayers()[0].source(), s1)
n1 = "éé ok"
d.addSource(n1, s1, "provider")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(
d.toUrl()).sourceLayers()[1].name(), n1)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).sourceLayers()[1].name(), n1)
d.addSource("ref1", "id0001")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(
d.toUrl()).sourceLayers()[2].reference(), "id0001")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).sourceLayers()[2].reference(), "id0001")
s = "dbname='C:\\tt' table=\"test\" (geometry) sql="
d.addSource("nn", s, "spatialite")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(
d.toUrl()).sourceLayers()[3].source(), s)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).sourceLayers()[3].source(), s)
d.setGeometryField("geom")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryField(),
"geom")
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).geometryField(), "geom")
d.setGeometryWkbType(QgsWkbTypes.Point)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryWkbType(),
QgsWkbTypes.Point)
self.assertEqual(
QgsVirtualLayerDefinition.fromUrl(strToUrl(
d.toString())).geometryWkbType(), QgsWkbTypes.Point)
f = QgsFields()
f.append(QgsField("a", QVariant.Int))
f.append(QgsField("f", QVariant.Double))
f.append(QgsField("s", QVariant.String))
d.setFields(f)
f2 = QgsVirtualLayerDefinition.fromUrl(d.toUrl()).fields()
self.assertEqual(f[0].name(), f2[0].name())
self.assertEqual(f[0].type(), f2[0].type())
self.assertEqual(f[1].name(), f2[1].name())
self.assertEqual(f[1].type(), f2[1].type())
self.assertEqual(f[2].name(), f2[2].name())
self.assertEqual(f[2].type(), f2[2].type())
def test1(self):
d = QgsVirtualLayerDefinition()
self.assertEqual(d.toString(), "")
d.setFilePath("/file")
self.assertEqual(d.toString(), "/file")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), "/file")
d.setFilePath("C:\\file")
self.assertEqual(d.toString(), "C:%5Cfile")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).filePath(), "C:\\file")
d.setQuery("SELECT * FROM mytable")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), "SELECT * FROM mytable")
q = u"SELECT * FROM tableéé /*:int*/"
d.setQuery(q)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).query(), q)
s1 = u"file://foo&bar=okié"
d.addSource("name", s1, "provider", "utf8")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[0].source(), s1)
n1 = u"éé ok"
d.addSource(n1, s1, "provider")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[1].name(), n1)
d.addSource("ref1", "id0001")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).sourceLayers()[2].reference(), "id0001")
d.setGeometryField("geom")
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryField(), "geom")
d.setGeometryWkbType(QgsWKBTypes.Point)
self.assertEqual(QgsVirtualLayerDefinition.fromUrl(d.toUrl()).geometryWkbType(), QgsWKBTypes.Point)
f = QgsFields()
f.append(QgsField("a", QVariant.Int))
f.append(QgsField("f", QVariant.Double))
f.append(QgsField("s", QVariant.String))
d.setFields(f)
f2 = QgsVirtualLayerDefinition.fromUrl(d.toUrl()).fields()
self.assertEqual(f[0].name(), f2[0].name())
self.assertEqual(f[0].type(), f2[0].type())
self.assertEqual(f[1].name(), f2[1].name())
self.assertEqual(f[1].type(), f2[1].type())
self.assertEqual(f[2].name(), f2[2].name())
self.assertEqual(f[2].type(), f2[2].type())
def setUpClass(cls):
"""Run before all tests"""
# Create the layer for the common provider tests
shp = os.path.join(TEST_DATA_DIR, 'provider/shapefile.shp')
d = QgsVirtualLayerDefinition()
d.addSource("vtab1", shp, "ogr")
d.setUid("pk")
cls.vl = QgsVectorLayer(d.toString(), 'test', 'virtual')
assert (cls.vl.isValid())
cls.source = cls.vl.dataProvider()
shp_poly = os.path.join(TEST_DATA_DIR, 'provider/shapefile_poly.shp')
d = QgsVirtualLayerDefinition()
d.addSource("vtab2", shp_poly, "ogr")
d.setUid("pk")
cls.poly_vl = QgsVectorLayer(d.toString(), 'test_poly', 'virtual')
assert (cls.poly_vl.isValid())
cls.poly_provider = cls.poly_vl.dataProvider()
