def test_split_features(self): from qc_tool.vector.helper import GapTable from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() cursor.execute( "CREATE TABLE gap_mylayer (fid SERIAL PRIMARY KEY, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO gap_mylayer (geom) VALUES (ST_MakeEnvelope(-6.8, -1.6, -1.2, -1.1, 4326))," " (ST_Union(ST_Union(ST_MakeEnvelope(0.2, 0.2, 8, 1.8, 4326)," " ST_MakeEnvelope(0.2, 3.2, 9.2, 4.2, 4326))," " ST_MakeEnvelope(0.1, 0.2, 5, 4.2, 4326)));") partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326, max_vertices=5) partitioned_layer._create_polygon_dump() gap_table = GapTable(partitioned_layer, "myboundary", None) gap_table._create_split_geom() count = gap_table._split_features() self.assertEqual(1, count) cursor.execute( "SELECT fid, ST_AsText(geom) FROM gap_mylayer ORDER BY fid;") self.assertListEqual([ (1, 'POLYGON((-6.8 -1.6,-6.8 -1.1,-1.2 -1.1,-1.2 -1.6,-6.8 -1.6))'), (3, 'POLYGON((5 4.2,5 3.2,5 1.8,5 0.2,0.2 0.2,0.1 0.2,0.1 4.2,0.2 4.2,5 4.2))' ), (4, 'POLYGON((5 0.2,5 1.8,8 1.8,8 0.2,5 0.2))'), (5, 'POLYGON((5 3.2,5 4.2,9.2 4.2,9.2 3.2,5 3.2))') ], cursor.fetchall())
def test_extract_extent(self): from qc_tool.vector.helper import PartitionedLayer partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid") xmin, ymin, xmax, ymax = partitioned_layer.extract_extent() self.assertListEqual([-1.1, -2.2, 11.3, 11.4], [xmin, ymin, xmax, ymax])
def test_subtract_partition(self): from qc_tool.vector.helper import GapTable from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() cursor.execute( "CREATE TABLE gap_mylayer (fid SERIAL PRIMARY KEY, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO gap_mylayer (geom) VALUES (ST_MakeEnvelope(0, 0, 1, 1, 4326))," " (ST_MakeEnvelope(0, 2, 2, 3, 4326))," " (ST_MakeEnvelope(0, 3, 2, 4, 4326));") cursor.execute( "CREATE TABLE interior_mylayer (partition_id integer, geom geometry(MultiPolygon, 4326));" ) cursor.execute( "INSERT INTO interior_mylayer VALUES (1, ST_Multi(ST_MakeEnvelope(-1, -1, 10, 10, 4326)))," " (3, ST_Multi(ST_Union(ST_MakeEnvelope(0, 2, 1, 4, 4326)," " ST_MakeEnvelope(1, 3, 2, 4, 4326))));") partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326) partitioned_layer._create_polygon_dump() gap_table = GapTable(partitioned_layer, "myboundary", None) gap_table._subtract_partition(3) cursor.execute( "SELECT fid, ST_AsText(geom) FROM gap_mylayer ORDER BY fid;") self.assertListEqual([(1, 'POLYGON((0 0,0 1,1 1,1 0,0 0))'), (4, 'POLYGON((1 3,2 3,2 2,1 2,1 3))')], cursor.fetchall())
def test_split_partitions(self): from qc_tool.vector.helper import PartitionedLayer cursor = self.connection.cursor() cursor.execute("CREATE TABLE partition_mylayer (partition_id integer," " superpartition_id integer," " num_vertices integer," " geom geometry(Polygon, 4326));") cursor.execute( "INSERT INTO partition_mylayer VALUES (2, 1, 4, ST_MakeEnvelope(0, 0, 4, 1, 4326))," " (3, 1, 5, ST_MakeEnvelope(0, 0, 6, 1, 4326));") partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid", srid=4326, max_vertices=4) split_count = partitioned_layer._split_partitions() self.assertEqual(1, split_count) cursor = self.connection.cursor() cursor.execute( "SELECT partition_id, superpartition_id, num_vertices, ST_AsText(geom) FROM partition_mylayer ORDER BY partition_id;" ) self.assertListEqual( [(2, 1, 4, "POLYGON((0 0,0 1,4 1,4 0,0 0))"), (3, 1, 5, "POLYGON((0 0,0 1,6 1,6 0,0 0))"), (None, 3, None, "POLYGON((0 0,0 1,3 1,3 0,0 0))"), (None, 3, None, "POLYGON((3 0,3 1,6 1,6 0,3 0))")], cursor.fetchall())
def test_delete_superitems(self): from qc_tool.vector.helper import PartitionedLayer cursor = self.connection.cursor() cursor.execute( "CREATE TABLE partition_mylayer (partition_id integer, superpartition_id integer);" ) cursor.execute("INSERT INTO partition_mylayer VALUES (2, 1)," " (3, 1)," " (4, 3)," " (5, 3);") cursor.execute( "CREATE TABLE feature_mylayer (fid integer, partition_id integer);" ) cursor.execute("INSERT INTO feature_mylayer VALUES (1, 2)," " (2, 3)," " (3, 3)," " (4, 4)," " (5, 5)," " (6, 5);") partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid") partitioned_layer._delete_superitems() cursor.execute( "SELECT partition_id, superpartition_id FROM partition_mylayer ORDER BY partition_id;" ) self.assertListEqual([(2, 1), (4, 3), (5, 3)], cursor.fetchall()) cursor.execute( "SELECT fid, partition_id FROM feature_mylayer ORDER BY fid;") self.assertListEqual([(1, 2), (4, 4), (5, 5), (6, 5)], cursor.fetchall())
def test_expand_box(self): from qc_tool.vector.helper import PartitionedLayer partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid", grid_size=2) xmin, ymin, xmax, ymax = partitioned_layer.expand_box( -1.1, -2.2, 11.3, 11.4) self.assertListEqual([-4, -6, 14, 14], [xmin, ymin, xmax, ymax])
def test_fill_area(self): from qc_tool.vector.helper import PartitionedLayer from qc_tool.vector.helper import NeighbourTable from qc_tool.vector.helper import _MetaTable from qc_tool.vector.helper import ComplexChangeProperty partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid") neighbour_table = NeighbourTable(partitioned_layer) meta_table = _MetaTable(self.connection, "mylayer", "xfid") meta_table._create_meta_table() complex_change_property = ComplexChangeProperty( neighbour_table, "code1", "code2", "area") complex_change_property._prepare_meta_table() cursor = self.connection.cursor() cursor.execute("DELETE FROM meta_mylayer;") cursor.execute("INSERT INTO meta_mylayer VALUES (1, NULL, NULL)," " (2, 2, NULL)," " (3, 2, NULL)," " (4, 2, 4)," " (5, NULL, 4)," " (6, NULL, 4)," " (7, NULL, NULL)," " (8, NULL, NULL);") complex_change_property._fill_area("cc_id_initial") complex_change_property._fill_area("cc_id_final") cursor.execute("SELECT fid, cc_area FROM meta_mylayer ORDER BY fid;") self.assertListEqual([(1, None), (2, 14.), (3, 14.), (4, 56.), (5, 56.), (6, 56.), (7, None), (8, None)], cursor.fetchall())
def test_empty_exterior(self): from qc_tool.vector.helper import _ExteriorTable from qc_tool.vector.helper import _InteriorTable from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() cursor.execute( "CREATE TABLE partition_mylayer (partition_id integer, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO partition_mylayer VALUES (1, ST_MakeEnvelope(0, 0, 1, 1, 4326));" ) cursor.execute( "CREATE TABLE interior_mylayer (partition_id integer, geom geometry(MultiPolygon, 4326));" ) cursor.execute( "INSERT INTO interior_mylayer VALUES (1, ST_Multi(ST_MakeEnvelope(0, 0, 1, 1, 4326)));" ) partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326) interior_table = _InteriorTable(partitioned_layer) exterior_table = _ExteriorTable(interior_table) exterior_table._create_exterior_table() exterior_table._fill() cursor.execute( "SELECT partition_id, ST_AsText(geom) FROM exterior_mylayer ORDER BY partition_id;" ) self.assertListEqual([], cursor.fetchall())
def test(self): from qc_tool.vector.helper import _ExteriorTable from qc_tool.vector.helper import _InteriorTable from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() cursor.execute( "CREATE TABLE partition_mylayer (partition_id integer, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO partition_mylayer VALUES (2, ST_MakeEnvelope(-10, -10, 1, 10, 4326))," " (5, ST_MakeEnvelope(10, 1, 20, 2, 4326));") cursor.execute( "CREATE TABLE interior_mylayer (partition_id integer, geom geometry(MultiPolygon, 4326));" ) cursor.execute( "INSERT INTO interior_mylayer VALUES (2, ST_Multi(ST_MakeEnvelope(-10, -10, 1, 9, 4326)))," " (5, ST_Union(ST_MakeEnvelope(10, 1, 11, 2, 4326)," " ST_MakeEnvelope(19, 1, 20, 2, 4326)));") partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326) interior_table = _InteriorTable(partitioned_layer) exterior_table = _ExteriorTable(interior_table) exterior_table._create_exterior_table() exterior_table._fill() cursor.execute( "SELECT partition_id, ST_AsText(geom) FROM exterior_mylayer ORDER BY partition_id;" ) self.assertListEqual( [(2, 'MULTIPOLYGON(((-10 9,-10 10,1 10,1 9,-10 9)))'), (5, 'MULTIPOLYGON(((11 2,19 2,19 1,11 1,11 2)))')], cursor.fetchall())
def test_fill_initial_partition(self): from qc_tool.vector.helper import PartitionedLayer partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid", srid=4326) partitioned_layer._create_partition_table() initial_partition_id = partitioned_layer._fill_initial_partition( -4, -6, 14, 14) cursor = self.connection.cursor() cursor.execute( "SELECT partition_id, superpartition_id, num_vertices, ST_AsText(geom) FROM partition_mylayer;" ) self.assertListEqual( [(1, None, None, "POLYGON((-4 -6,-4 14,14 14,14 -6,-4 -6))")], cursor.fetchall())
def test(self): from qc_tool.vector.helper import MarginalProperty from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() cursor.execute("CREATE TABLE meta_mylayer (fid integer);") cursor.execute( "CREATE TABLE feature_mylayer (fid integer, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO feature_mylayer VALUES (2, ST_MakeEnvelope(1, 1, 2, 2, 4326))," " (2, ST_MakeEnvelope(3, 1, 4, 2, 4326))," " (3, ST_MakeEnvelope(5, 1, 6, 2, 4326))," " (4, ST_MakeEnvelope(7, 1, 8, 2, 4326));") cursor.execute( "CREATE TABLE exterior_mylayer (geom geometry(Polygon, 4326));") cursor.execute( "INSERT INTO exterior_mylayer VALUES (ST_MakeEnvelope(3, 2, 4, 3, 4326))," " (ST_MakeEnvelope(5, 2, 6, 3, 4326));") partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326) marginal_property = MarginalProperty(partitioned_layer) marginal_property._prepare_meta_table() cursor.execute("INSERT INTO meta_mylayer (fid) VALUES (2), (3), (4);") marginal_property._fill() cursor.execute( "SELECT fid, is_marginal FROM meta_mylayer ORDER BY fid;") self.assertListEqual([(2, True), (3, True), (4, False)], cursor.fetchall())
def test_fill_subpartitions(self): from qc_tool.vector.helper import PartitionedLayer cursor = self.connection.cursor() cursor.execute("CREATE TABLE partition_mylayer (partition_id integer," " superpartition_id integer," " num_vertices integer," " geom geometry(Polygon, 4326));") cursor.execute( "INSERT INTO partition_mylayer VALUES (2, 1, NULL, ST_MakeEnvelope(-10, 0, 0, 3, 4326))," " (3, 1, NULL, ST_MakeEnvelope(0, 0, 10, 3, 4326))," " (4, 3, NULL, ST_MakeEnvelope(0, 0, 5, 3, 4326))," " (5, 3, NULL, ST_MakeEnvelope(5, 0, 10, 3, 4326));") cursor.execute( "CREATE TABLE feature_mylayer (fid integer, partition_id integer, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO feature_mylayer VALUES (1, 2, ST_MakeEnvelope(-5, 1, -3, 2, 4326))," " (2, 3, ST_MakeEnvelope(4, 1, 7, 2, 4326))," " (3, 3, ST_MakeEnvelope(5, 1, 6, 2, 4326));") partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid") partitioned_layer._create_polygon_dump() partitioned_layer._fill_subpartitions() cursor.execute( "SELECT fid, partition_id, ST_AsText(geom) FROM feature_mylayer ORDER BY fid, partition_id;" ) self.assertListEqual([(1, 2, "POLYGON((-5 1,-5 2,-3 2,-3 1,-5 1))"), (2, 3, "POLYGON((4 1,4 2,7 2,7 1,4 1))"), (2, 4, "POLYGON((4 1,4 2,5 2,5 1,4 1))"), (2, 5, "POLYGON((5 2,7 2,7 1,5 1,5 2))"), (3, 5, "POLYGON((5 1,5 2,6 2,6 1,5 1))")], cursor.fetchall())
def test_split_geom(self): from qc_tool.vector.helper import GapTable from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326) partitioned_layer._create_polygon_dump() gap_table = GapTable(partitioned_layer, "myboundary", None) gap_table._create_split_geom() cursor.execute( "SELECT ST_AsText(split_geom(ST_MakeEnvelope(0.6, -1.1, 9.2, 6, 4326), 1.0));" ) self.assertListEqual( [('POLYGON((0.6 6,5 6,5 -1.1,0.6 -1.1,0.6 6))', ), ('POLYGON((5 -1.1,5 6,9.2 6,9.2 -1.1,5 -1.1))', )], cursor.fetchall()) cursor.execute( "SELECT ST_AsText(split_geom(ST_MakeEnvelope(0.2, 0.1, 0.6, 0.9, 4326), 1.0));" ) self.assertListEqual([], cursor.fetchall())
def test_fill_initial_features(self): from qc_tool.vector.helper import GapTable from qc_tool.vector.helper import PartitionedLayer cursor = self.params["connection_manager"].get_connection().cursor() cursor.execute( "CREATE TABLE myboundary (geom geometry(Polygon, 4326));") cursor.execute( "INSERT INTO myboundary VALUES (ST_MakeEnvelope(0, 0, 1, 1, 4326))," " (ST_MakeEnvelope(2, 2, 3, 3, 4326));") partitioned_layer = PartitionedLayer(cursor.connection, "mylayer", "xfid", srid=4326) partitioned_layer._create_polygon_dump() gap_table = GapTable(partitioned_layer, "myboundary", None) gap_table._create_gap_table() gap_table._fill_initial_features() cursor.execute( "SELECT fid, ST_AsText(geom) FROM gap_mylayer ORDER BY fid;") self.assertListEqual([(1, 'POLYGON((0 0,0 1,1 1,1 0,0 0))'), (2, 'POLYGON((2 2,2 3,3 3,3 2,2 2))')], cursor.fetchall())
def test_update_npoints(self): from qc_tool.vector.helper import PartitionedLayer cursor = self.connection.cursor() cursor.execute("CREATE TABLE partition_mylayer (partition_id integer," " num_vertices integer," " geom geometry(Polygon, 4326));") cursor.execute( "INSERT INTO partition_mylayer VALUES (1, 1, ST_MakeEnvelope(0, 0, 1, 1, 4326))," " (2, NULL, ST_MakeEnvelope(0, 0, 1, 1, 4326))," " (3, NULL, ST_MakeEnvelope(0, 0, 1, 1, 4326));") cursor.execute( "CREATE TABLE feature_mylayer (partition_id integer, geom geometry(Polygon, 4326));" ) cursor.execute( "INSERT INTO feature_mylayer VALUES (2, ST_MakeEnvelope(0, 0, 1, 1, 4326))," " (3, ST_MakeEnvelope(0, 0, 1, 1, 4326))," " (3, ST_MakeEnvelope(0, 0, 1, 1, 4326));") partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid") partitioned_layer._update_npoints() cursor.execute( "SELECT partition_id, num_vertices FROM partition_mylayer ORDER BY partition_id;" ) self.assertListEqual([(1, 1), (2, 5), (3, 10)], cursor.fetchall())
def test_fill_initial_features(self): from qc_tool.vector.helper import PartitionedLayer partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid", srid=4326) partitioned_layer._create_polygon_dump() partitioned_layer._create_feature_table() partitioned_layer._fill_initial_features(1) cursor = self.connection.cursor() cursor.execute( "SELECT fid, partition_id, ST_AsText(geom) FROM feature_mylayer ORDER BY fid;" ) self.assertListEqual( [(1, 1, "POLYGON((-1.1 -2.2,-1.1 1,1 1,1 -2.2,-1.1 -2.2))"), (2, 1, "POLYGON((10 10,10 11.4,11.3 11.4,11.3 10,10 10))")], cursor.fetchall())
def test_fill(self): from qc_tool.vector.helper import PartitionedLayer from qc_tool.vector.helper import NeighbourTable partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid", srid=4326) neighbour_table = NeighbourTable(partitioned_layer) neighbour_table._create_neighbour_table() neighbour_table._fill() cursor = self.connection.cursor() cursor.execute( "SELECT fida, fidb, dim FROM neighbour_mylayer ORDER BY fida, fidb;" ) self.assertListEqual([(1, 2, 2), (2, 1, 2), (2, 3, 1), (3, 2, 1), (4, 5, 1), (5, 4, 1)], cursor.fetchall())
def run_check(params, status): from qc_tool.vector.helper import create_pg_has_comment from qc_tool.vector.helper import do_layers from qc_tool.vector.helper import get_failed_items_message from qc_tool.vector.helper import NeighbourTable from qc_tool.vector.helper import PartitionedLayer # Check if the current delivery is excluded from vector checks if "skip_vector_checks" in params: if params["skip_vector_checks"]: status.info("The delivery has been excluded from vector.neighbour check because the vector data source does not contain a single object of interest.") return cursor = params["connection_manager"].get_connection().cursor() for layer_def in do_layers(params): log.debug("Started neighbour check for the layer {:s}.".format(layer_def["pg_layer_name"])) # Prepare support data. partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"]) neighbour_table = NeighbourTable(partitioned_layer) neighbour_table.make() create_pg_has_comment(cursor.connection) # Prepare parameters for sql query. sql_params = {"fid_name": layer_def["pg_fid_name"], "layer_name": layer_def["pg_layer_name"], "neighbour_table": neighbour_table.neighbour_table_name, "exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]), "exception_pairs_table": "s{:02d}_{:s}_exception_pairs".format(params["step_nr"], layer_def["pg_layer_name"]), "error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"]), "error_pairs_table": "s{:02d}_{:s}_error_pairs".format(params["step_nr"], layer_def["pg_layer_name"]), "pair_clause": " AND ".join("layer.{0:s} = other.{0:s}".format(code_column_name) for code_column_name in params["code_column_names"]), "exception_where": "\n".join(params["exception_where"]), "error_where": "\n".join(params["error_where"])} # Create exception pairs table. sql = ("CREATE TABLE {exception_pairs_table} AS\n" "SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n" "FROM\n" " {layer_name} AS layer\n" " INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n" " INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n" "WHERE\n" " ({exception_where})\n" " AND ({pair_clause});") sql = sql.format(**sql_params) cursor.execute(sql) # Create error pairs table. sql = ("CREATE TABLE {error_pairs_table} AS\n" "SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n" "FROM\n" " {layer_name} AS layer\n" " INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n" " INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n" " LEFT JOIN {exception_pairs_table} AS excp ON layer.{fid_name} = excp.fida AND other.{fid_name} = excp.fidb\n" "WHERE\n" " excp.fida IS NULL\n" " AND ({error_where})\n" " AND ({pair_clause});") sql = sql.format(**sql_params) cursor.execute(sql) # Create exception table. sql = ("CREATE TABLE {exception_table} AS\n" "SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n" "FROM {exception_pairs_table};") sql = sql.format(**sql_params) cursor.execute(sql) # Report exception items. items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"]) if items_message is not None: status.info("Layer {:s} has exception features with {:s}: {:s}." .format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message)) status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"]) # Create error table. sql = ("CREATE TABLE {error_table} AS\n" "SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n" "FROM {error_pairs_table};") sql = sql.format(**sql_params) cursor.execute(sql) # Report error items. items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"]) if items_message is not None: status.failed("Layer {:s} has error features with {:s}: {:s}." .format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message)) status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"]) log.info("Neighbour check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status): from qc_tool.vector.helper import do_layers from qc_tool.vector.helper import GapTable from qc_tool.vector.helper import get_failed_items_message from qc_tool.vector.helper import PartitionedLayer if "boundary" not in params["layer_defs"]: status.info("Check cancelled due to boundary not being available.") return for layer_def in do_layers(params): log.debug("Started gap check for the layer {:s}.".format(layer_def["pg_layer_name"])) # Prepare support data. partitioned_layer = PartitionedLayer(params["connection_manager"].get_connection(), layer_def["pg_layer_name"], layer_def["pg_fid_name"]) # Update boundary with negative tolerance buffer boundary_table_name = params["layer_defs"]["boundary"]["pg_layer_name"] sql_params = {"boundary_table": boundary_table_name, "tolerance": TOLERANCE} with params["connection_manager"].get_connection().cursor() as cursor: sql = "UPDATE {boundary_table} SET geom = ST_Multi(ST_BUFFER(geom, -{tolerance}));" sql = sql.format(**sql_params) cursor.execute(sql) gap_table = GapTable(partitioned_layer, params["layer_defs"]["boundary"]["pg_layer_name"], params["du_column_name"]) gap_table.make() # Prepare parameters used in sql clauses. sql_params = {"gap_table": gap_table.gap_table_name, "gap_warning_table": "s{:02d}_{:s}_gap_warning".format(params["step_nr"], layer_def["pg_layer_name"])} with params["connection_manager"].get_connection().cursor() as cursor: # Create table of warning items. sql = ("CREATE TABLE {gap_warning_table} AS\n" "SELECT geom FROM {gap_table};") sql = sql.format(**sql_params) cursor.execute(sql) # Report warning items. if cursor.rowcount > 0: status.info("Layer {:s} has {:d} gaps.".format(layer_def["pg_layer_name"], cursor.rowcount)) status.add_full_table(sql_params["gap_warning_table"]) if params["du_column_name"] is not None: sql_params = {"fid_column": layer_def["pg_fid_name"], "layer_table": layer_def["pg_layer_name"], "du_column": params["du_column_name"], "boundary_table": params["layer_defs"]["boundary"]["pg_layer_name"], "du_warning_table": "s{:02d}_{:s}_du_warning".format(params["step_nr"], layer_def["pg_layer_name"])} # Create table of excessive items. sql = ("CREATE TABLE {du_warning_table} AS\n" "SELECT layer.{fid_column}\n" "FROM\n" " {layer_table} AS layer\n" " LEFT JOIN (SELECT DISTINCT {du_column} FROM {boundary_table}) AS dut ON layer.{du_column} = dut.{du_column}\n" "WHERE\n" " dut.{du_column} IS NULL;") with params["connection_manager"].get_connection().cursor() as cursor: sql = sql.format(**sql_params) cursor.execute(sql) # Report excessive items. if cursor.rowcount > 0: status.info("Layer {:s} has {:d} feature(s) of unknown boundary unit." .format(layer_def["pg_layer_name"], cursor.rowcount)) status.add_error_table(sql_params["du_warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"]) log.info("MMU check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status): from qc_tool.vector.helper import do_layers from qc_tool.vector.helper import get_failed_items_message from qc_tool.vector.helper import NeighbourTable from qc_tool.vector.helper import PartitionedLayer # Check if the current delivery is excluded from vector checks if "skip_vector_checks" in params: if params["skip_vector_checks"]: status.info("The delivery has been excluded from vector.overlap check because the vector data source does not contain a single object of interest.") return cursor = params["connection_manager"].get_connection().cursor() for layer_def in do_layers(params): log.debug("Started overlap check for the layer {:s}.".format(layer_def["pg_layer_name"])) # Prepare support data. partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"]) neighbour_table = NeighbourTable(partitioned_layer) neighbour_table.make() sql_params = {"fid_name": layer_def["pg_fid_name"], "layer_name": layer_def["pg_layer_name"], "neighbour_table": neighbour_table.neighbour_table_name, "overlap_detail_table": "s{:02d}_{:s}_detail".format(params["step_nr"], layer_def["pg_layer_name"]), "overlap_suspect_table": "s{:02d}_{:s}_suspect".format(params["step_nr"], layer_def["pg_layer_name"]), "error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"])} # FIXME: # It may happen during partitioning, that the splitted geometries may get shifted a bit. # The NeighbourTable then reports two neighbouring geometries as overlapping with ST_Dimension()=2. # In order to avoid reporting such misleading overlaps we verify the overlap by generating anew # intersection from original geometries. # If some overlaps are found actually, they are propagated into error table. # So, the order of building the tables are reversed, the content of error table is extracted # from the overlap detail table. # Create suspects table. sql = ("CREATE TABLE {overlap_suspect_table} AS\n" "(SELECT fida, fidb\n" "FROM {neighbour_table}\n" "WHERE\n" "fida < fidb\n" "AND dim >= 2);") sql = sql.format(**sql_params) log.debug(sql) cursor.execute(sql) if cursor.rowcount > 0: # Create overlap detail table. sql = ("CREATE TABLE {overlap_detail_table} AS\n" "(SELECT fida, fidb, polygon_dump(ST_Intersection(layer_a.geom, layer_b.geom)) AS geom\n" "FROM {overlap_suspect_table}\n" "INNER JOIN {layer_name} AS layer_a ON {overlap_suspect_table}.fida = layer_a.{fid_name}\n" "INNER JOIN {layer_name} AS layer_b ON {overlap_suspect_table}.fidb = layer_b.{fid_name});\n") sql = sql.format(**sql_params) log.debug("SQL QUERY:") log.debug(sql) cursor.execute(sql) if cursor.rowcount > 0: # Report overlap detail table. status.add_full_table(sql_params["overlap_detail_table"]) # Create table of error items. sql = ("CREATE TABLE {error_table} AS\n" "SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n" "FROM {overlap_detail_table};") sql = sql.format(**sql_params) cursor.execute(sql) # Report error items. items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"]) status.failed("Layer {:s} has overlapping pairs in features with {:s}: {:s}." .format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message)) status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"]) log.info("Overlap check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def test_setup_srid(self): from qc_tool.vector.helper import PartitionedLayer partitioned_layer = PartitionedLayer(self.connection, "mylayer", "xfid") self.assertEqual(4326, partitioned_layer.srid)