def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"warning_where": params["warning_where"],
"warning_table": "s{:02d}_{:s}_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
# Create table of warning items.
sql = ("CREATE TABLE {warning_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name} AS layer\n"
"WHERE\n"
" ({warning_where})\n"
" AND NOT ST_IsEmpty(ST_Buffer(geom, %(buffer)s));")
sql = sql.format(**sql_params)
cursor.execute(sql, {"buffer": -params["mxmw"] / 2})
# Report warning features.
items_message = get_failed_items_message(cursor, sql_params["warning_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has warning features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.singlepart check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
error_table_name = "s{:02d}_{:s}_error".format(
params["step_nr"], layer_def["pg_layer_name"])
sql = SQL.format(error_table_name, layer_def["pg_fid_name"],
layer_def["pg_layer_name"])
cursor.execute(sql)
if cursor.rowcount > 0:
failed_items_message = get_failed_items_message(
cursor, error_table_name, layer_def["pg_fid_name"])
status.aborted(
"Layer {:s} has multipart geometries in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"],
layer_def["fid_display_name"], failed_items_message))
status.add_error_table(error_table_name,
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"warning_where":
params["warning_where"],
"warning_table":
"s{:02d}_{:s}_warning".format(params["step_nr"],
layer_def["pg_layer_name"])
}
# Create table of warning items.
sql = (
"CREATE TABLE {warning_table} AS\n"
"SELECT {fid_name}\n"
"FROM\n"
" (SELECT {fid_name}, geom\n"
" FROM\n"
" (SELECT\n"
" {fid_name},\n"
" ST_Boundary(ST_OrientedEnvelope(geom)) AS env,\n"
" geom\n"
" FROM {layer_name} AS layer\n"
" WHERE {warning_where}\n"
" ) AS tenv\n"
" WHERE\n"
" greatest(ST_Distance(ST_PointN(env, 1), ST_PointN(env, 2)),\n"
" ST_Distance(ST_PointN(env, 2), ST_PointN(env, 3))) < %(mml)s\n"
" ) AS tdist\n"
"WHERE\n"
" ST_Length(ST_ApproximateMedialAxis(ST_MakePolygon(ST_ExteriorRing(geom)))) <= %(mml)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, {"mml": params["mml"]})
# Report warning features.
items_message = get_failed_items_message(cursor,
sql_params["warning_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.info(
"Layer {:s} has warning features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["warning_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.area check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"area_column_name":
params["area_column_name"],
"error_table":
"s{:02d}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
sql_execute_params = {
"unit": params["unit"],
"tolerance": params["tolerance"]
}
# Create table of error items.
sql = (
"CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE abs({area_column_name} - ST_Area(geom) / %(unit)s) > %(tolerance)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_execute_params)
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_inspire_check
from qc_tool.vector.helper import do_layers
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.inspire check because the vector data source does not contain a single object of interest."
)
return
for layer_def in do_layers(params):
# Locate a 'metadata' subdirectory inside the delivery.
# Metadata directory name is case-insensitive, 'Metadata' and 'metadata' are both allowed.
metadata_dirs = [
d for d in params["unzip_dir"].glob('**/*')
if d.is_dir() and str(d).lower().endswith(METADATA_DIRNAME)
]
if len(metadata_dirs) == 0:
status.info(
"The delivery does not contain the expected '{:s}' folder".
format(METADATA_DIRNAME))
return
elif len(metadata_dirs) > 1:
status.info(
"Multiple folders named '{:s}' were found in the delivery.",
"Only one '{:s}' folder is allowed.".format(METADATA_DIRNAME))
return
else:
metadata_dir = metadata_dirs[0]
# Verify if there is one INSPIRE metadata file to check.
xml_filepath = locate_xml_file(metadata_dir, layer_def["src_filepath"])
if xml_filepath is None:
status.info(
"The delivery does not contain the expected metadata file '{:s}/{:s}.xml'"
.format(metadata_dir.stem, layer_def["src_filepath"].stem))
return
# Validate the xml file using INSPIRE validator service
export_prefix = "s{:02d}_{:s}_inspire".format(
params["step_nr"], layer_def["src_filepath"].stem)
do_inspire_check(xml_filepath, export_prefix, params["output_dir"],
status)
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"area_column_name":
params["area_column_name"],
"error_where":
params["error_where"],
"error_table":
"s{:02d}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name} AS layer\n"
"WHERE\n"
" ({error_where})\n"
" AND {area_column_name} > %(mxmu)s;")
sql = sql.format(**sql_params)
cursor.execute(sql, {"mxmu": params["mxmu"]})
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"error_table":
"s{:02d}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
sql_exec_params = {"limit": -params["limit"]}
# Create table of error items.
sql = (
"CREATE TABLE {error_table} AS"
" SELECT DISTINCT unnest(ARRAY[ta.{fid_name}, tb.{fid_name}]) AS {fid_name}"
" FROM {layer_name} ta, {layer_name} tb"
" WHERE"
" ta.{fid_name} < tb.{fid_name}"
" AND ta.wkb_geometry && tb.wkb_geometry"
" AND (NOT ST_Relate(ta.wkb_geometry, tb.wkb_geometry, '**T***T**')"
" OR NOT ST_IsEmpty(ST_Buffer(ST_Intersection(ta.wkb_geometry, tb.wkb_geometry), %(limit)s)));"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_exec_params)
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has overlapping pairs in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"],
layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
for unique_key in params["unique_keys"]:
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"unique_column_name":
unique_key,
"error_table_name":
"s{:02d}_{:s}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"],
unique_key)
}
sql = (
"CREATE TABLE {error_table_name} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN (SELECT {unique_column_name}\n"
" FROM {layer_name}\n"
" GROUP BY {unique_column_name}\n"
" HAVING count({unique_column_name}) > 1) AS ut ON layer.{unique_column_name} = ut.{unique_column_name};"
)
sql = sql.format(**sql_params)
cursor.execute(sql)
if cursor.rowcount > 0:
failed_items_message = get_failed_items_message(
cursor, sql_params["error_table_name"],
layer_def["pg_fid_name"])
status.failed(
"The column {:s}.{:s} has non-unique values in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], unique_key,
layer_def["fid_display_name"],
failed_items_message))
status.add_error_table(sql_params["error_table_name"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
import osgeo.ogr as ogr
import osgeo.osr as osr
from qc_tool.vector.helper import do_layers
for layer_def in do_layers(params):
place = layer_def["place"]
ds = ogr.Open(str(layer_def["src_filepath"]))
layer = ds.GetLayerByName(layer_def["src_layer_name"])
srs = layer.GetSpatialRef()
if srs is None:
status.aborted(
"Layer {:s} has missing spatial reference system.".format(
layer_def["src_layer_name"]))
else:
srs.AutoIdentifyEPSG()
# Get epsg code from authority clause.
authority_name = srs.GetAuthorityName(None)
authority_code = srs.GetAuthorityCode(None)
if authority_name == "EPSG":
# Compare epsg code using the root-level epsg authority in the srs WKT of the layer.
try:
authority_code = int(authority_code)
except ValueError:
status.aborted(
"Layer {:s} has non integer epsg code {:s}".format(
layer_def["src_layer_name"], authority_code))
else:
expected_authority_code = params["epsg"][place]
if authority_code != expected_authority_code:
status.aborted(
"Layer {:s} has illegal epsg code {:d}."
"Expected epsg code for {:s} is {:d}".format(
layer_def["src_layer_name"], authority_code,
place, expected_authority_code))
else:
# the setting is strict and no epsg code has been found in the srs of the layer.
status.aborted(
"Layer {:s} does not have an epsg code or epsg code could not be detected, srs: {:s}."
.format(layer_def["src_layer_name"], srs.ExportToWkt()))
def run_check(params, status):
import osgeo.ogr as ogr
import osgeo.osr as osr
from qc_tool.vector.helper import do_layers
for layer_def in do_layers(params):
ds = ogr.Open(str(layer_def["src_filepath"]))
layer = ds.GetLayerByName(layer_def["src_layer_name"])
srs = layer.GetSpatialRef()
if srs is None:
status.aborted("Layer {:s} has missing spatial reference system.".format(layer_def["src_layer_name"]))
return
# Search EPSG authority code
srs.AutoIdentifyEPSG()
authority_name = srs.GetAuthorityName(None)
authority_code = srs.GetAuthorityCode(None)
if authority_name != "EPSG":
status.failed("Layer {:s} has missing EPSG authority.".format(layer_def["src_layer_name"]))
return
if authority_code is None:
status.failed("Layer {:s} has missing EPSG code.".format(layer_def["src_layer_name"]))
return
# Compare EPSG code against boundary EPSG code.
if "boundary" not in params["layer_defs"]:
status.cancelled("Check cancelled due to missing boundary.")
return
ds = ogr.Open(str(params["layer_defs"]["boundary"]["src_filepath"]))
layer = ds.GetLayerByName(params["layer_defs"]["boundary"]["src_layer_name"])
srs = layer.GetSpatialRef()
boundary_authority_code = srs.GetAuthorityCode(None)
if authority_code != boundary_authority_code:
status.cancelled("Check cancelled while layer {:s} has epsg code {:s} different from boundary epsg code {:s}."
.format(layer_def["src_layer_name"], authority_code, boundary_authority_code))
return
status.add_params({"layer_srs_epsg": int(authority_code)})
def run_check(params, status):
import osgeo.ogr as ogr
from qc_tool.vector.helper import do_layers
ogr.UseExceptions()
filepaths = set(layer_def["src_filepath"] for layer_def in do_layers(params))
for filepath in filepaths:
ds_extension = filepath.suffix
if ds_extension not in params["drivers"]:
status.aborted("The source file has forbidden extension: {:s}.".format(repr(ds_extension)))
else:
ds_open = None
try:
ds_open = ogr.Open(str(filepath))
except:
pass
if ds_open is None:
status.aborted("The source file can not be opened.")
else:
drivername = ds_open.GetDriver().GetName()
if drivername != params["drivers"][ds_extension]:
status.aborted("The file format is invalid.")
def run_check(params, status):
from qc_tool.vector.helper import create_pg_has_comment
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import PartitionedLayer
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info("The delivery has been excluded from vector.neighbour check because the vector data source does not contain a single object of interest.")
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
log.debug("Started neighbour check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"])
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table.make()
create_pg_has_comment(cursor.connection)
# Prepare parameters for sql query.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"neighbour_table": neighbour_table.neighbour_table_name,
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_pairs_table": "s{:02d}_{:s}_exception_pairs".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_pairs_table": "s{:02d}_{:s}_error_pairs".format(params["step_nr"], layer_def["pg_layer_name"]),
"pair_clause": " AND ".join("layer.{0:s} = other.{0:s}".format(code_column_name)
for code_column_name in params["code_column_names"]),
"exception_where": "\n".join(params["exception_where"]),
"error_where": "\n".join(params["error_where"])}
# Create exception pairs table.
sql = ("CREATE TABLE {exception_pairs_table} AS\n"
"SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n"
" INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n"
"WHERE\n"
" ({exception_where})\n"
" AND ({pair_clause});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create error pairs table.
sql = ("CREATE TABLE {error_pairs_table} AS\n"
"SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n"
" INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n"
" LEFT JOIN {exception_pairs_table} AS excp ON layer.{fid_name} = excp.fida AND other.{fid_name} = excp.fidb\n"
"WHERE\n"
" excp.fida IS NULL\n"
" AND ({error_where})\n"
" AND ({pair_clause});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create exception table.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {exception_pairs_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception items.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create error table.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {error_pairs_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed("Layer {:s} has error features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("Neighbour check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status):
import osgeo.ogr as ogr
import osgeo.osr as osr
from qc_tool.vector.helper import do_layers
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.epsg check because the vector data source does not contain a single object of interest."
)
return
for layer_def in do_layers(params):
ds = ogr.Open(str(layer_def["src_filepath"]))
layer = ds.GetLayerByName(layer_def["src_layer_name"])
srs = layer.GetSpatialRef()
if srs is None:
status.aborted(
"Layer {:s} has missing spatial reference system.".format(
layer_def["src_layer_name"]))
else:
# Get epsg code from authority clause.
srs.AutoIdentifyEPSG()
authority_name = srs.GetAuthorityName(None)
authority_code = srs.GetAuthorityCode(None)
if authority_name == "EPSG":
# Compare epsg code using the root-level epsg authority in the srs WKT of the layer.
try:
authority_code = int(authority_code)
except ValueError:
status.aborted(
"Layer {:s} has non integer epsg code {:s}".format(
layer_def["src_layer_name"], authority_code))
else:
if authority_code != params["epsg"]:
status.aborted(
"Layer {:s} has illegal epsg code {:d}.".format(
layer_def["src_layer_name"], authority_code))
else:
status.add_params({"detected_epsg": authority_code})
elif params.get("auto_identify_epsg", False):
# Parameter auto_identify_epsg can be used for less-strict checking of .prj files.
# There is a built-in function in GDAL 2.3 with matching logic.
is_detected = False
expected_srs = osr.SpatialReference()
expected_srs.ImportFromEPSG(params["epsg"])
if srs.IsSame(expected_srs):
# The auto-detected epsg is made available for other checks.
status.add_params({"detected_epsg": params["epsg"]})
else:
status.aborted(
"Layer {:s} does not have an epsg code and the epsg code can not be detected, srs: {:s}."
.format(layer_def["src_layer_name"],
srs.ExportToWkt()))
else:
# the setting is strict and no epsg code has been found in the srs of the layer.
status.aborted(
"Layer {:s} has epsg code missing, srs: {:s}.".format(
layer_def["src_layer_name"], srs.ExportToWkt()))
def run_check(params, status):
import osgeo.ogr as ogr
from qc_tool.vector.helper import do_layers
OGR_TYPES = {ogr.OFTBinary: "binary",
ogr.OFTDate: "date",
ogr.OFTDateTime: "datetime",
ogr.OFTInteger: "integer",
ogr.OFTInteger64: "integer64",
ogr.OFTInteger64List: "list-of-integer64",
ogr.OFTIntegerList: "list-of-integer",
ogr.OFTReal: "real",
ogr.OFTRealList: "list-of-real",
ogr.OFTString: "string",
ogr.OFTStringList: "list-of-string",
ogr.OFTTime: "time",
ogr.OFTWideString: "wide-string",
ogr.OFTWideStringList: "list-of-wide-string"}
ALLOWED_TYPES = {ogr.OFTInteger: "numeric",
ogr.OFTInteger64: "numeric",
ogr.OFTReal: "numeric",
ogr.OFTString: "string",
ogr.OFTWideString: "string"}
for layer_def in do_layers(params):
ds = ogr.Open(str(layer_def["src_filepath"]))
layer = ds.GetLayerByName(layer_def["src_layer_name"])
year1 = layer_def["year1"]
year2 = layer_def["year2"]
status.info("year1: {0}, year2: {1}".format(year1, year2))
# check product_attrs, replacing YEAR1 with actual year and YEAR2 with
product_attrs = {}
for attr_name, attr_type_names in params["attributes"].items():
if year1 is not None and "YEAR1" in attr_name:
new_attr_name = attr_name.replace("YEAR1", layer_def["year1"])
product_attrs[new_attr_name] = attr_type_names
elif year2 is not None and "YEAR2" in attr_name:
new_attr_name = attr_name.replace("YEAR2", layer_def["year2"])
product_attrs[new_attr_name] = attr_type_names
else:
product_attrs[attr_name] = attr_type_names
extra_attrs = {}
for field_defn in layer.schema:
field_type = field_defn.GetType()
field_name = field_defn.name
if field_name not in product_attrs:
# extra field
continue
if field_type not in OGR_TYPES:
# Field type is unknown.
del product_attrs[field_name]
elif field_type not in ALLOWED_TYPES:
# Field type is not allowed.
del product_attrs[field_name]
elif ALLOWED_TYPES[field_type] not in product_attrs[field_name]:
# Field does not match a type in product definition.
extra_attrs[field_name] = ALLOWED_TYPES[field_type]
else:
# Field matches product definition.
del product_attrs[field_name]
missing_attrs = product_attrs
if len(extra_attrs) > 0:
extra_attr_infos = ["{:s}({:s})".format(attr, str(extra_attrs[attr])) for attr in sorted(extra_attrs.keys())]
status.aborted("Layer {:s} has attributes in incorrect format: {:s}.".format(
layer_def["src_layer_name"], ", ".join(extra_attr_infos)))
if len(missing_attrs) > 0:
status.aborted("Layer {:s} has missing attributes: {:s}."
.format(layer_def["src_layer_name"],
", ".join("{:s}({:s})".format(attr_name, ",".join(missing_attrs[attr_name]))
for attr_name in sorted(missing_attrs.keys()))))
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.enum check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
for column_name, allowed_codes in params["column_defs"]:
# Prepare clause excluding features with non-null value of specific column.
if "exclude_column_name" in params:
exclude_clause = "AND {:s} IS NULL".format(
params["exclude_column_name"])
else:
exclude_clause = ""
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"column_name":
column_name,
"exclude_clause":
exclude_clause,
"error_table":
"s{:02d}_{:s}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"],
column_name)
}
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" ({column_name} IS NULL"
" OR {column_name} NOT IN %s)"
" {exclude_clause};")
sql = sql.format(**sql_params)
cursor.execute(sql, [tuple(allowed_codes)])
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has column {:s} with invalid codes in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], column_name,
layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
for column_name, allowed_codes in params["column_defs"]:
# check product_attrs, replacing YEAR1 with actual year and YEAR2 with
if layer_def["year1"] is not None and "year1" in column_name:
column_name = column_name.replace("year1", layer_def["year1"])
elif layer_def["year2"] is not None and "year2" in column_name:
column_name = column_name.replace("year2", layer_def["year2"])
# Prepare clause excluding features with non-null value of specific column.
if "exclude_column_name" in params:
exclude_clause = "AND {:s} IS NULL".format(
params["exclude_column_name"])
else:
exclude_clause = ""
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"column_name":
column_name,
"exclude_clause":
exclude_clause,
"error_table":
"s{:02d}_{:s}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"],
column_name)
}
# Create table of error items.
if None in allowed_codes:
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" ({column_name} IS NULL"
" OR {column_name} NOT IN %s)"
" {exclude_clause};")
else:
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" ({column_name} IS NULL"
" OR {column_name} NOT IN %s)"
" {exclude_clause};")
sql = sql.format(**sql_params)
cursor.execute(sql, [tuple(allowed_codes)])
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
invalid_codes_message = get_invalid_codes_message(
cursor, sql_params["error_table"],
layer_def["pg_layer_name"], layer_def["pg_fid_name"],
sql_params["column_name"])
status.failed(
"Layer {:s} has column {:s} with invalid codes in features with {:s}: {:s}. Invalid codes are: {:s}."
.format(layer_def["pg_layer_name"], column_name.upper(),
layer_def["fid_display_name"], items_message,
invalid_codes_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"initial_code_column_name": params["initial_code_column_name"],
"final_code_column_name": params["final_code_column_name"],
"chtype_column_name": params.get("chtype_column_name", ""),
"general_table": "s{:02d}_{:s}_general".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"]),
"technical_change_flag": TECHNICAL_CHANGE_FLAG}
# Create table of general items.
sql = ("CREATE TABLE {general_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name}\n"
"WHERE\n"
" {initial_code_column_name} != {final_code_column_name};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create table of exception items.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n")
if sql_params["chtype_column_name"] != "":
sql += " AND layer.{chtype_column_name} = '{technical_change_flag}';"
else:
sql += " AND FALSE;"
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception items.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
" LEFT JOIN {exception_table} AS exc ON layer.{fid_name} = exc.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n"
" AND exc.{fid_name} IS NULL;")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed("Layer {:s} has error features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.geometry check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"error_table":
"s{:02d}_{:s}_invalid".format(params["step_nr"],
layer_def["pg_layer_name"]),
"detail_table":
"s{:02d}_{:s}_detail".format(params["step_nr"],
layer_def["pg_layer_name"])
}
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE NOT ST_IsValid(geom);")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report items with invalid geometry.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.aborted(
"Layer {:s} has invalid geometry in features with {:s}: {:s}.".
format(layer_def["pg_layer_name"],
layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
# Create table of descriptions of invalid geometries.
sql = (
"CREATE TABLE {detail_table} AS"
" SELECT"
" {fid_name},"
" (ST_IsValidDetail(geom)).reason AS reason,"
" ST_SetSRID((ST_IsValidDetail(geom)).location, ST_SRID(geom)) AS location"
" FROM {layer_name}"
" WHERE NOT ST_IsValid(geom);")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report table of descriptions.
status.add_full_table(sql_params["detail_table"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import PartitionedLayer
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info("The delivery has been excluded from vector.overlap check because the vector data source does not contain a single object of interest.")
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
log.debug("Started overlap check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"])
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table.make()
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"neighbour_table": neighbour_table.neighbour_table_name,
"overlap_detail_table": "s{:02d}_{:s}_detail".format(params["step_nr"], layer_def["pg_layer_name"]),
"overlap_suspect_table": "s{:02d}_{:s}_suspect".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"])}
# FIXME:
# It may happen during partitioning, that the splitted geometries may get shifted a bit.
# The NeighbourTable then reports two neighbouring geometries as overlapping with ST_Dimension()=2.
# In order to avoid reporting such misleading overlaps we verify the overlap by generating anew
# intersection from original geometries.
# If some overlaps are found actually, they are propagated into error table.
# So, the order of building the tables are reversed, the content of error table is extracted
# from the overlap detail table.
# Create suspects table.
sql = ("CREATE TABLE {overlap_suspect_table} AS\n"
"(SELECT fida, fidb\n"
"FROM {neighbour_table}\n"
"WHERE\n"
"fida < fidb\n"
"AND dim >= 2);")
sql = sql.format(**sql_params)
log.debug(sql)
cursor.execute(sql)
if cursor.rowcount > 0:
# Create overlap detail table.
sql = ("CREATE TABLE {overlap_detail_table} AS\n"
"(SELECT fida, fidb, polygon_dump(ST_Intersection(layer_a.geom, layer_b.geom)) AS geom\n"
"FROM {overlap_suspect_table}\n"
"INNER JOIN {layer_name} AS layer_a ON {overlap_suspect_table}.fida = layer_a.{fid_name}\n"
"INNER JOIN {layer_name} AS layer_b ON {overlap_suspect_table}.fidb = layer_b.{fid_name});\n")
sql = sql.format(**sql_params)
log.debug("SQL QUERY:")
log.debug(sql)
cursor.execute(sql)
if cursor.rowcount > 0:
# Report overlap detail table.
status.add_full_table(sql_params["overlap_detail_table"])
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {overlap_detail_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
status.failed("Layer {:s} has overlapping pairs in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("Overlap check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import GapTable
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import PartitionedLayer
if "boundary" not in params["layer_defs"]:
status.info("Check cancelled due to boundary not being available.")
return
for layer_def in do_layers(params):
log.debug("Started gap check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(params["connection_manager"].get_connection(),
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
# Update boundary with negative tolerance buffer
boundary_table_name = params["layer_defs"]["boundary"]["pg_layer_name"]
sql_params = {"boundary_table": boundary_table_name,
"tolerance": TOLERANCE}
with params["connection_manager"].get_connection().cursor() as cursor:
sql = "UPDATE {boundary_table} SET geom = ST_Multi(ST_BUFFER(geom, -{tolerance}));"
sql = sql.format(**sql_params)
cursor.execute(sql)
gap_table = GapTable(partitioned_layer,
params["layer_defs"]["boundary"]["pg_layer_name"],
params["du_column_name"])
gap_table.make()
# Prepare parameters used in sql clauses.
sql_params = {"gap_table": gap_table.gap_table_name,
"gap_warning_table": "s{:02d}_{:s}_gap_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
with params["connection_manager"].get_connection().cursor() as cursor:
# Create table of warning items.
sql = ("CREATE TABLE {gap_warning_table} AS\n"
"SELECT geom FROM {gap_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report warning items.
if cursor.rowcount > 0:
status.info("Layer {:s} has {:d} gaps.".format(layer_def["pg_layer_name"], cursor.rowcount))
status.add_full_table(sql_params["gap_warning_table"])
if params["du_column_name"] is not None:
sql_params = {"fid_column": layer_def["pg_fid_name"],
"layer_table": layer_def["pg_layer_name"],
"du_column": params["du_column_name"],
"boundary_table": params["layer_defs"]["boundary"]["pg_layer_name"],
"du_warning_table": "s{:02d}_{:s}_du_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
# Create table of excessive items.
sql = ("CREATE TABLE {du_warning_table} AS\n"
"SELECT layer.{fid_column}\n"
"FROM\n"
" {layer_table} AS layer\n"
" LEFT JOIN (SELECT DISTINCT {du_column} FROM {boundary_table}) AS dut ON layer.{du_column} = dut.{du_column}\n"
"WHERE\n"
" dut.{du_column} IS NULL;")
with params["connection_manager"].get_connection().cursor() as cursor:
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report excessive items.
if cursor.rowcount > 0:
status.info("Layer {:s} has {:d} feature(s) of unknown boundary unit."
.format(layer_def["pg_layer_name"], cursor.rowcount))
status.add_error_table(sql_params["du_warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("MMU check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status):
import osgeo.ogr as ogr
from qc_tool.vector.helper import do_layers
OGR_TYPES = {
ogr.OFTBinary: "binary",
ogr.OFTDate: "date",
ogr.OFTDateTime: "datetime",
ogr.OFTInteger: "integer",
ogr.OFTInteger64: "integer64",
ogr.OFTInteger64List: "list-of-integer64",
ogr.OFTIntegerList: "list-of-integer",
ogr.OFTReal: "real",
ogr.OFTRealList: "list-of-real",
ogr.OFTString: "string",
ogr.OFTStringList: "list-of-string",
ogr.OFTTime: "time",
ogr.OFTWideString: "wide-string",
ogr.OFTWideStringList: "list-of-wide-string"
}
ALLOWED_TYPES = {
ogr.OFTInteger: "integer",
ogr.OFTInteger64: "integer",
ogr.OFTReal: "real",
ogr.OFTString: "string",
ogr.OFTWideString: "string"
}
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.attribute check because the vector data source does not contain a single object of interest."
)
return
for layer_def in do_layers(params):
ds = ogr.Open(str(layer_def["src_filepath"]))
layer = ds.GetLayerByName(layer_def["src_layer_name"])
required_attrs = {
attr_name.lower(): attr_type_name.lower()
for attr_name, attr_type_name in params["required"].items()
}
ignored_attrs = params["ignored"].copy()
extra_attrs = {}
bad_type_attrs = {}
for field_defn in layer.schema:
field_name = field_defn.name.lower()
field_type = field_defn.GetType()
if field_name in ignored_attrs:
# Ignored attribute.
ignored_attrs.remove(field_name)
elif field_name in required_attrs:
# Required attribute.
if field_type not in OGR_TYPES:
# Attribute type is unknown.
bad_type_attrs.update({field_name: "unknown-type"})
elif field_type not in ALLOWED_TYPES:
# Attribute type is not allowed.
bad_type_attrs.update({field_name: OGR_TYPES[field_type]})
elif ALLOWED_TYPES[field_type] != required_attrs[field_name]:
# Attribute type does not match the type in product definition.
bad_type_attrs.update(
{field_name: ALLOWED_TYPES[field_type]})
del required_attrs[field_name]
else:
# Extra attribute.
extra_attrs.update({field_name: OGR_TYPES[field_type]})
# The attributes remaining in required_attrs are missing.
if len(required_attrs) > 0:
status.aborted("Layer {:s} has missing attributes: {:s}.".format(
layer_def["src_layer_name"], ", ".join(
"{:s}({:s})".format(attr_name, required_attrs[attr_name])
for attr_name in sorted(required_attrs.keys()))))
if len(extra_attrs) > 0:
status.failed("Layer {:s} has extra attributes: {:s}.".format(
layer_def["src_layer_name"], ", ".join(
"{:s}({:s})".format(attr_name, extra_attrs[attr_name])
for attr_name in sorted(extra_attrs.keys()))))
if len(bad_type_attrs) > 0:
status.aborted(
"Layer {:s} has attributes with bad type: {:s}.".format(
layer_def["src_layer_name"],
", ".join("{:s}({:s})".format(attr_name,
bad_type_attrs[attr_name])
for attr_name in sorted(bad_type_attrs.keys()))))
def run_check(params, status):
"""
Check compactness of linear and patchy features.
Features marked as linear must have compactness less then threshold.
Features marked as patchy must have compactness greater then threshold.
Features not marked as linear or patchy are not validated.
Compactness is measured using Polsby-Popper method,
see `Measuring District Compactness in PostGIS <https://www.azavea.com/blog/2016/07/11/measuring-district-compactness-postgis/>`__.
"""
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"area_column_name":
params["area_column_name"],
"code_column_name":
params["code_column_name"],
"linear_error_table":
"s{:02d}_{:s}_linear_error".format(params["step_nr"],
layer_def["pg_layer_name"]),
"patchy_error_table":
"s{:02d}_{:s}_patchy_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
sql_execute_params = {
"threshold": params["threshold"],
"linear_code": params["linear_code"],
"patchy_code": params["patchy_code"]
}
# Create table of error items of linear features.
sql = (
"CREATE TABLE {linear_error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" {code_column_name} = %(linear_code)s"
" AND 4 * pi() * {area_column_name} / power(ST_Perimeter(geom), 2) > %(threshold)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_execute_params)
# Report error items of linear features
items_message = get_failed_items_message(
cursor, sql_params["linear_error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error linear features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["linear_error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
# Create table of error items of patchy features.
sql = (
"CREATE TABLE {patchy_error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" {code_column_name} = %(patchy_code)s"
" AND 4 * pi() * {area_column_name} / power(ST_Perimeter(geom), 2) <= %(threshold)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_execute_params)
# Report error items of patchy features
items_message = get_failed_items_message(
cursor, sql_params["patchy_error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error patchy features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["patchy_error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"general_where": params["general_where"],
"exception_where": params["exception_where"],
"general_table": "s{:02d}_{:s}_general".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"warning_table": "s{:02d}_{:s}_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
# Create table of general items.
sql = ("CREATE TABLE {general_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name} AS layer\n"
"WHERE\n"
" ({general_where})\n"
" OR ST_NumGeometries(ST_Buffer(geom, %(buffer)s)) = 1;")
sql = sql.format(**sql_params)
cursor.execute(sql, {"buffer": -params["mmw"] / 2})
# Create table of exception items.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n"
" AND ({exception_where});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception features.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create table of warning items.
sql = ("CREATE TABLE {warning_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
" LEFT JOIN {exception_table} AS exc ON layer.{fid_name} = exc.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n"
" AND exc.{fid_name} IS NULL;")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report warning features.
items_message = get_failed_items_message(cursor, sql_params["warning_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has warning features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])