def test_split_features(self):
from qc_tool.vector.helper import GapTable
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
cursor.execute(
"CREATE TABLE gap_mylayer (fid SERIAL PRIMARY KEY, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO gap_mylayer (geom) VALUES (ST_MakeEnvelope(-6.8, -1.6, -1.2, -1.1, 4326)),"
" (ST_Union(ST_Union(ST_MakeEnvelope(0.2, 0.2, 8, 1.8, 4326),"
" ST_MakeEnvelope(0.2, 3.2, 9.2, 4.2, 4326)),"
" ST_MakeEnvelope(0.1, 0.2, 5, 4.2, 4326)));")
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326,
max_vertices=5)
partitioned_layer._create_polygon_dump()
gap_table = GapTable(partitioned_layer, "myboundary", None)
gap_table._create_split_geom()
count = gap_table._split_features()
self.assertEqual(1, count)
cursor.execute(
"SELECT fid, ST_AsText(geom) FROM gap_mylayer ORDER BY fid;")
self.assertListEqual([
(1,
'POLYGON((-6.8 -1.6,-6.8 -1.1,-1.2 -1.1,-1.2 -1.6,-6.8 -1.6))'),
(3,
'POLYGON((5 4.2,5 3.2,5 1.8,5 0.2,0.2 0.2,0.1 0.2,0.1 4.2,0.2 4.2,5 4.2))'
), (4, 'POLYGON((5 0.2,5 1.8,8 1.8,8 0.2,5 0.2))'),
(5, 'POLYGON((5 3.2,5 4.2,9.2 4.2,9.2 3.2,5 3.2))')
], cursor.fetchall())
def test_extract_extent(self):
from qc_tool.vector.helper import PartitionedLayer
partitioned_layer = PartitionedLayer(self.connection, "mylayer",
"xfid")
xmin, ymin, xmax, ymax = partitioned_layer.extract_extent()
self.assertListEqual([-1.1, -2.2, 11.3, 11.4],
[xmin, ymin, xmax, ymax])
def test_subtract_partition(self):
from qc_tool.vector.helper import GapTable
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
cursor.execute(
"CREATE TABLE gap_mylayer (fid SERIAL PRIMARY KEY, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO gap_mylayer (geom) VALUES (ST_MakeEnvelope(0, 0, 1, 1, 4326)),"
" (ST_MakeEnvelope(0, 2, 2, 3, 4326)),"
" (ST_MakeEnvelope(0, 3, 2, 4, 4326));")
cursor.execute(
"CREATE TABLE interior_mylayer (partition_id integer, geom geometry(MultiPolygon, 4326));"
)
cursor.execute(
"INSERT INTO interior_mylayer VALUES (1, ST_Multi(ST_MakeEnvelope(-1, -1, 10, 10, 4326))),"
" (3, ST_Multi(ST_Union(ST_MakeEnvelope(0, 2, 1, 4, 4326),"
" ST_MakeEnvelope(1, 3, 2, 4, 4326))));")
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326)
partitioned_layer._create_polygon_dump()
gap_table = GapTable(partitioned_layer, "myboundary", None)
gap_table._subtract_partition(3)
cursor.execute(
"SELECT fid, ST_AsText(geom) FROM gap_mylayer ORDER BY fid;")
self.assertListEqual([(1, 'POLYGON((0 0,0 1,1 1,1 0,0 0))'),
(4, 'POLYGON((1 3,2 3,2 2,1 2,1 3))')],
cursor.fetchall())
def test_split_partitions(self):
from qc_tool.vector.helper import PartitionedLayer
cursor = self.connection.cursor()
cursor.execute("CREATE TABLE partition_mylayer (partition_id integer,"
" superpartition_id integer,"
" num_vertices integer,"
" geom geometry(Polygon, 4326));")
cursor.execute(
"INSERT INTO partition_mylayer VALUES (2, 1, 4, ST_MakeEnvelope(0, 0, 4, 1, 4326)),"
" (3, 1, 5, ST_MakeEnvelope(0, 0, 6, 1, 4326));")
partitioned_layer = PartitionedLayer(self.connection,
"mylayer",
"xfid",
srid=4326,
max_vertices=4)
split_count = partitioned_layer._split_partitions()
self.assertEqual(1, split_count)
cursor = self.connection.cursor()
cursor.execute(
"SELECT partition_id, superpartition_id, num_vertices, ST_AsText(geom) FROM partition_mylayer ORDER BY partition_id;"
)
self.assertListEqual(
[(2, 1, 4, "POLYGON((0 0,0 1,4 1,4 0,0 0))"),
(3, 1, 5, "POLYGON((0 0,0 1,6 1,6 0,0 0))"),
(None, 3, None, "POLYGON((0 0,0 1,3 1,3 0,0 0))"),
(None, 3, None, "POLYGON((3 0,3 1,6 1,6 0,3 0))")],
cursor.fetchall())
def test_delete_superitems(self):
from qc_tool.vector.helper import PartitionedLayer
cursor = self.connection.cursor()
cursor.execute(
"CREATE TABLE partition_mylayer (partition_id integer, superpartition_id integer);"
)
cursor.execute("INSERT INTO partition_mylayer VALUES (2, 1),"
" (3, 1),"
" (4, 3),"
" (5, 3);")
cursor.execute(
"CREATE TABLE feature_mylayer (fid integer, partition_id integer);"
)
cursor.execute("INSERT INTO feature_mylayer VALUES (1, 2),"
" (2, 3),"
" (3, 3),"
" (4, 4),"
" (5, 5),"
" (6, 5);")
partitioned_layer = PartitionedLayer(self.connection, "mylayer",
"xfid")
partitioned_layer._delete_superitems()
cursor.execute(
"SELECT partition_id, superpartition_id FROM partition_mylayer ORDER BY partition_id;"
)
self.assertListEqual([(2, 1), (4, 3), (5, 3)], cursor.fetchall())
cursor.execute(
"SELECT fid, partition_id FROM feature_mylayer ORDER BY fid;")
self.assertListEqual([(1, 2), (4, 4), (5, 5), (6, 5)],
cursor.fetchall())
def test_expand_box(self):
from qc_tool.vector.helper import PartitionedLayer
partitioned_layer = PartitionedLayer(self.connection,
"mylayer",
"xfid",
grid_size=2)
xmin, ymin, xmax, ymax = partitioned_layer.expand_box(
-1.1, -2.2, 11.3, 11.4)
self.assertListEqual([-4, -6, 14, 14], [xmin, ymin, xmax, ymax])
def test_fill_area(self):
from qc_tool.vector.helper import PartitionedLayer
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import _MetaTable
from qc_tool.vector.helper import ComplexChangeProperty
partitioned_layer = PartitionedLayer(self.connection, "mylayer",
"xfid")
neighbour_table = NeighbourTable(partitioned_layer)
meta_table = _MetaTable(self.connection, "mylayer", "xfid")
meta_table._create_meta_table()
complex_change_property = ComplexChangeProperty(
neighbour_table, "code1", "code2", "area")
complex_change_property._prepare_meta_table()
cursor = self.connection.cursor()
cursor.execute("DELETE FROM meta_mylayer;")
cursor.execute("INSERT INTO meta_mylayer VALUES (1, NULL, NULL),"
" (2, 2, NULL),"
" (3, 2, NULL),"
" (4, 2, 4),"
" (5, NULL, 4),"
" (6, NULL, 4),"
" (7, NULL, NULL),"
" (8, NULL, NULL);")
complex_change_property._fill_area("cc_id_initial")
complex_change_property._fill_area("cc_id_final")
cursor.execute("SELECT fid, cc_area FROM meta_mylayer ORDER BY fid;")
self.assertListEqual([(1, None), (2, 14.), (3, 14.), (4, 56.),
(5, 56.), (6, 56.), (7, None), (8, None)],
cursor.fetchall())
def test_empty_exterior(self):
from qc_tool.vector.helper import _ExteriorTable
from qc_tool.vector.helper import _InteriorTable
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
cursor.execute(
"CREATE TABLE partition_mylayer (partition_id integer, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO partition_mylayer VALUES (1, ST_MakeEnvelope(0, 0, 1, 1, 4326));"
)
cursor.execute(
"CREATE TABLE interior_mylayer (partition_id integer, geom geometry(MultiPolygon, 4326));"
)
cursor.execute(
"INSERT INTO interior_mylayer VALUES (1, ST_Multi(ST_MakeEnvelope(0, 0, 1, 1, 4326)));"
)
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326)
interior_table = _InteriorTable(partitioned_layer)
exterior_table = _ExteriorTable(interior_table)
exterior_table._create_exterior_table()
exterior_table._fill()
cursor.execute(
"SELECT partition_id, ST_AsText(geom) FROM exterior_mylayer ORDER BY partition_id;"
)
self.assertListEqual([], cursor.fetchall())
def test(self):
from qc_tool.vector.helper import _ExteriorTable
from qc_tool.vector.helper import _InteriorTable
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
cursor.execute(
"CREATE TABLE partition_mylayer (partition_id integer, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO partition_mylayer VALUES (2, ST_MakeEnvelope(-10, -10, 1, 10, 4326)),"
" (5, ST_MakeEnvelope(10, 1, 20, 2, 4326));")
cursor.execute(
"CREATE TABLE interior_mylayer (partition_id integer, geom geometry(MultiPolygon, 4326));"
)
cursor.execute(
"INSERT INTO interior_mylayer VALUES (2, ST_Multi(ST_MakeEnvelope(-10, -10, 1, 9, 4326))),"
" (5, ST_Union(ST_MakeEnvelope(10, 1, 11, 2, 4326),"
" ST_MakeEnvelope(19, 1, 20, 2, 4326)));")
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326)
interior_table = _InteriorTable(partitioned_layer)
exterior_table = _ExteriorTable(interior_table)
exterior_table._create_exterior_table()
exterior_table._fill()
cursor.execute(
"SELECT partition_id, ST_AsText(geom) FROM exterior_mylayer ORDER BY partition_id;"
)
self.assertListEqual(
[(2, 'MULTIPOLYGON(((-10 9,-10 10,1 10,1 9,-10 9)))'),
(5, 'MULTIPOLYGON(((11 2,19 2,19 1,11 1,11 2)))')],
cursor.fetchall())
def test_fill_initial_partition(self):
from qc_tool.vector.helper import PartitionedLayer
partitioned_layer = PartitionedLayer(self.connection,
"mylayer",
"xfid",
srid=4326)
partitioned_layer._create_partition_table()
initial_partition_id = partitioned_layer._fill_initial_partition(
-4, -6, 14, 14)
cursor = self.connection.cursor()
cursor.execute(
"SELECT partition_id, superpartition_id, num_vertices, ST_AsText(geom) FROM partition_mylayer;"
)
self.assertListEqual(
[(1, None, None, "POLYGON((-4 -6,-4 14,14 14,14 -6,-4 -6))")],
cursor.fetchall())
def test(self):
from qc_tool.vector.helper import MarginalProperty
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
cursor.execute("CREATE TABLE meta_mylayer (fid integer);")
cursor.execute(
"CREATE TABLE feature_mylayer (fid integer, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO feature_mylayer VALUES (2, ST_MakeEnvelope(1, 1, 2, 2, 4326)),"
" (2, ST_MakeEnvelope(3, 1, 4, 2, 4326)),"
" (3, ST_MakeEnvelope(5, 1, 6, 2, 4326)),"
" (4, ST_MakeEnvelope(7, 1, 8, 2, 4326));")
cursor.execute(
"CREATE TABLE exterior_mylayer (geom geometry(Polygon, 4326));")
cursor.execute(
"INSERT INTO exterior_mylayer VALUES (ST_MakeEnvelope(3, 2, 4, 3, 4326)),"
" (ST_MakeEnvelope(5, 2, 6, 3, 4326));")
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326)
marginal_property = MarginalProperty(partitioned_layer)
marginal_property._prepare_meta_table()
cursor.execute("INSERT INTO meta_mylayer (fid) VALUES (2), (3), (4);")
marginal_property._fill()
cursor.execute(
"SELECT fid, is_marginal FROM meta_mylayer ORDER BY fid;")
self.assertListEqual([(2, True), (3, True), (4, False)],
cursor.fetchall())
def test_fill_subpartitions(self):
from qc_tool.vector.helper import PartitionedLayer
cursor = self.connection.cursor()
cursor.execute("CREATE TABLE partition_mylayer (partition_id integer,"
" superpartition_id integer,"
" num_vertices integer,"
" geom geometry(Polygon, 4326));")
cursor.execute(
"INSERT INTO partition_mylayer VALUES (2, 1, NULL, ST_MakeEnvelope(-10, 0, 0, 3, 4326)),"
" (3, 1, NULL, ST_MakeEnvelope(0, 0, 10, 3, 4326)),"
" (4, 3, NULL, ST_MakeEnvelope(0, 0, 5, 3, 4326)),"
" (5, 3, NULL, ST_MakeEnvelope(5, 0, 10, 3, 4326));")
cursor.execute(
"CREATE TABLE feature_mylayer (fid integer, partition_id integer, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO feature_mylayer VALUES (1, 2, ST_MakeEnvelope(-5, 1, -3, 2, 4326)),"
" (2, 3, ST_MakeEnvelope(4, 1, 7, 2, 4326)),"
" (3, 3, ST_MakeEnvelope(5, 1, 6, 2, 4326));")
partitioned_layer = PartitionedLayer(self.connection, "mylayer",
"xfid")
partitioned_layer._create_polygon_dump()
partitioned_layer._fill_subpartitions()
cursor.execute(
"SELECT fid, partition_id, ST_AsText(geom) FROM feature_mylayer ORDER BY fid, partition_id;"
)
self.assertListEqual([(1, 2, "POLYGON((-5 1,-5 2,-3 2,-3 1,-5 1))"),
(2, 3, "POLYGON((4 1,4 2,7 2,7 1,4 1))"),
(2, 4, "POLYGON((4 1,4 2,5 2,5 1,4 1))"),
(2, 5, "POLYGON((5 2,7 2,7 1,5 1,5 2))"),
(3, 5, "POLYGON((5 1,5 2,6 2,6 1,5 1))")],
cursor.fetchall())
def test_split_geom(self):
from qc_tool.vector.helper import GapTable
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326)
partitioned_layer._create_polygon_dump()
gap_table = GapTable(partitioned_layer, "myboundary", None)
gap_table._create_split_geom()
cursor.execute(
"SELECT ST_AsText(split_geom(ST_MakeEnvelope(0.6, -1.1, 9.2, 6, 4326), 1.0));"
)
self.assertListEqual(
[('POLYGON((0.6 6,5 6,5 -1.1,0.6 -1.1,0.6 6))', ),
('POLYGON((5 -1.1,5 6,9.2 6,9.2 -1.1,5 -1.1))', )],
cursor.fetchall())
cursor.execute(
"SELECT ST_AsText(split_geom(ST_MakeEnvelope(0.2, 0.1, 0.6, 0.9, 4326), 1.0));"
)
self.assertListEqual([], cursor.fetchall())
def test_fill_initial_features(self):
from qc_tool.vector.helper import GapTable
from qc_tool.vector.helper import PartitionedLayer
cursor = self.params["connection_manager"].get_connection().cursor()
cursor.execute(
"CREATE TABLE myboundary (geom geometry(Polygon, 4326));")
cursor.execute(
"INSERT INTO myboundary VALUES (ST_MakeEnvelope(0, 0, 1, 1, 4326)),"
" (ST_MakeEnvelope(2, 2, 3, 3, 4326));")
partitioned_layer = PartitionedLayer(cursor.connection,
"mylayer",
"xfid",
srid=4326)
partitioned_layer._create_polygon_dump()
gap_table = GapTable(partitioned_layer, "myboundary", None)
gap_table._create_gap_table()
gap_table._fill_initial_features()
cursor.execute(
"SELECT fid, ST_AsText(geom) FROM gap_mylayer ORDER BY fid;")
self.assertListEqual([(1, 'POLYGON((0 0,0 1,1 1,1 0,0 0))'),
(2, 'POLYGON((2 2,2 3,3 3,3 2,2 2))')],
cursor.fetchall())
def test_update_npoints(self):
from qc_tool.vector.helper import PartitionedLayer
cursor = self.connection.cursor()
cursor.execute("CREATE TABLE partition_mylayer (partition_id integer,"
" num_vertices integer,"
" geom geometry(Polygon, 4326));")
cursor.execute(
"INSERT INTO partition_mylayer VALUES (1, 1, ST_MakeEnvelope(0, 0, 1, 1, 4326)),"
" (2, NULL, ST_MakeEnvelope(0, 0, 1, 1, 4326)),"
" (3, NULL, ST_MakeEnvelope(0, 0, 1, 1, 4326));")
cursor.execute(
"CREATE TABLE feature_mylayer (partition_id integer, geom geometry(Polygon, 4326));"
)
cursor.execute(
"INSERT INTO feature_mylayer VALUES (2, ST_MakeEnvelope(0, 0, 1, 1, 4326)),"
" (3, ST_MakeEnvelope(0, 0, 1, 1, 4326)),"
" (3, ST_MakeEnvelope(0, 0, 1, 1, 4326));")
partitioned_layer = PartitionedLayer(self.connection, "mylayer",
"xfid")
partitioned_layer._update_npoints()
cursor.execute(
"SELECT partition_id, num_vertices FROM partition_mylayer ORDER BY partition_id;"
)
self.assertListEqual([(1, 1), (2, 5), (3, 10)], cursor.fetchall())
def test_fill_initial_features(self):
from qc_tool.vector.helper import PartitionedLayer
partitioned_layer = PartitionedLayer(self.connection,
"mylayer",
"xfid",
srid=4326)
partitioned_layer._create_polygon_dump()
partitioned_layer._create_feature_table()
partitioned_layer._fill_initial_features(1)
cursor = self.connection.cursor()
cursor.execute(
"SELECT fid, partition_id, ST_AsText(geom) FROM feature_mylayer ORDER BY fid;"
)
self.assertListEqual(
[(1, 1, "POLYGON((-1.1 -2.2,-1.1 1,1 1,1 -2.2,-1.1 -2.2))"),
(2, 1, "POLYGON((10 10,10 11.4,11.3 11.4,11.3 10,10 10))")],
cursor.fetchall())
def test_fill(self):
from qc_tool.vector.helper import PartitionedLayer
from qc_tool.vector.helper import NeighbourTable
partitioned_layer = PartitionedLayer(self.connection,
"mylayer",
"xfid",
srid=4326)
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table._create_neighbour_table()
neighbour_table._fill()
cursor = self.connection.cursor()
cursor.execute(
"SELECT fida, fidb, dim FROM neighbour_mylayer ORDER BY fida, fidb;"
)
self.assertListEqual([(1, 2, 2), (2, 1, 2), (2, 3, 1), (3, 2, 1),
(4, 5, 1), (5, 4, 1)], cursor.fetchall())
def run_check(params, status):
from qc_tool.vector.helper import create_pg_has_comment
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import PartitionedLayer
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info("The delivery has been excluded from vector.neighbour check because the vector data source does not contain a single object of interest.")
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
log.debug("Started neighbour check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"])
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table.make()
create_pg_has_comment(cursor.connection)
# Prepare parameters for sql query.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"neighbour_table": neighbour_table.neighbour_table_name,
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_pairs_table": "s{:02d}_{:s}_exception_pairs".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_pairs_table": "s{:02d}_{:s}_error_pairs".format(params["step_nr"], layer_def["pg_layer_name"]),
"pair_clause": " AND ".join("layer.{0:s} = other.{0:s}".format(code_column_name)
for code_column_name in params["code_column_names"]),
"exception_where": "\n".join(params["exception_where"]),
"error_where": "\n".join(params["error_where"])}
# Create exception pairs table.
sql = ("CREATE TABLE {exception_pairs_table} AS\n"
"SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n"
" INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n"
"WHERE\n"
" ({exception_where})\n"
" AND ({pair_clause});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create error pairs table.
sql = ("CREATE TABLE {error_pairs_table} AS\n"
"SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n"
" INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n"
" LEFT JOIN {exception_pairs_table} AS excp ON layer.{fid_name} = excp.fida AND other.{fid_name} = excp.fidb\n"
"WHERE\n"
" excp.fida IS NULL\n"
" AND ({error_where})\n"
" AND ({pair_clause});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create exception table.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {exception_pairs_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception items.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create error table.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {error_pairs_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed("Layer {:s} has error features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("Neighbour check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import GapTable
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import PartitionedLayer
if "boundary" not in params["layer_defs"]:
status.info("Check cancelled due to boundary not being available.")
return
for layer_def in do_layers(params):
log.debug("Started gap check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(params["connection_manager"].get_connection(),
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
# Update boundary with negative tolerance buffer
boundary_table_name = params["layer_defs"]["boundary"]["pg_layer_name"]
sql_params = {"boundary_table": boundary_table_name,
"tolerance": TOLERANCE}
with params["connection_manager"].get_connection().cursor() as cursor:
sql = "UPDATE {boundary_table} SET geom = ST_Multi(ST_BUFFER(geom, -{tolerance}));"
sql = sql.format(**sql_params)
cursor.execute(sql)
gap_table = GapTable(partitioned_layer,
params["layer_defs"]["boundary"]["pg_layer_name"],
params["du_column_name"])
gap_table.make()
# Prepare parameters used in sql clauses.
sql_params = {"gap_table": gap_table.gap_table_name,
"gap_warning_table": "s{:02d}_{:s}_gap_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
with params["connection_manager"].get_connection().cursor() as cursor:
# Create table of warning items.
sql = ("CREATE TABLE {gap_warning_table} AS\n"
"SELECT geom FROM {gap_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report warning items.
if cursor.rowcount > 0:
status.info("Layer {:s} has {:d} gaps.".format(layer_def["pg_layer_name"], cursor.rowcount))
status.add_full_table(sql_params["gap_warning_table"])
if params["du_column_name"] is not None:
sql_params = {"fid_column": layer_def["pg_fid_name"],
"layer_table": layer_def["pg_layer_name"],
"du_column": params["du_column_name"],
"boundary_table": params["layer_defs"]["boundary"]["pg_layer_name"],
"du_warning_table": "s{:02d}_{:s}_du_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
# Create table of excessive items.
sql = ("CREATE TABLE {du_warning_table} AS\n"
"SELECT layer.{fid_column}\n"
"FROM\n"
" {layer_table} AS layer\n"
" LEFT JOIN (SELECT DISTINCT {du_column} FROM {boundary_table}) AS dut ON layer.{du_column} = dut.{du_column}\n"
"WHERE\n"
" dut.{du_column} IS NULL;")
with params["connection_manager"].get_connection().cursor() as cursor:
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report excessive items.
if cursor.rowcount > 0:
status.info("Layer {:s} has {:d} feature(s) of unknown boundary unit."
.format(layer_def["pg_layer_name"], cursor.rowcount))
status.add_error_table(sql_params["du_warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("MMU check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import PartitionedLayer
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info("The delivery has been excluded from vector.overlap check because the vector data source does not contain a single object of interest.")
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
log.debug("Started overlap check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"])
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table.make()
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"neighbour_table": neighbour_table.neighbour_table_name,
"overlap_detail_table": "s{:02d}_{:s}_detail".format(params["step_nr"], layer_def["pg_layer_name"]),
"overlap_suspect_table": "s{:02d}_{:s}_suspect".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"])}
# FIXME:
# It may happen during partitioning, that the splitted geometries may get shifted a bit.
# The NeighbourTable then reports two neighbouring geometries as overlapping with ST_Dimension()=2.
# In order to avoid reporting such misleading overlaps we verify the overlap by generating anew
# intersection from original geometries.
# If some overlaps are found actually, they are propagated into error table.
# So, the order of building the tables are reversed, the content of error table is extracted
# from the overlap detail table.
# Create suspects table.
sql = ("CREATE TABLE {overlap_suspect_table} AS\n"
"(SELECT fida, fidb\n"
"FROM {neighbour_table}\n"
"WHERE\n"
"fida < fidb\n"
"AND dim >= 2);")
sql = sql.format(**sql_params)
log.debug(sql)
cursor.execute(sql)
if cursor.rowcount > 0:
# Create overlap detail table.
sql = ("CREATE TABLE {overlap_detail_table} AS\n"
"(SELECT fida, fidb, polygon_dump(ST_Intersection(layer_a.geom, layer_b.geom)) AS geom\n"
"FROM {overlap_suspect_table}\n"
"INNER JOIN {layer_name} AS layer_a ON {overlap_suspect_table}.fida = layer_a.{fid_name}\n"
"INNER JOIN {layer_name} AS layer_b ON {overlap_suspect_table}.fidb = layer_b.{fid_name});\n")
sql = sql.format(**sql_params)
log.debug("SQL QUERY:")
log.debug(sql)
cursor.execute(sql)
if cursor.rowcount > 0:
# Report overlap detail table.
status.add_full_table(sql_params["overlap_detail_table"])
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {overlap_detail_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
status.failed("Layer {:s} has overlapping pairs in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("Overlap check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def test_setup_srid(self):
from qc_tool.vector.helper import PartitionedLayer
partitioned_layer = PartitionedLayer(self.connection, "mylayer",
"xfid")
self.assertEqual(4326, partitioned_layer.srid)