def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"warning_where": params["warning_where"],
"warning_table": "s{:02d}_{:s}_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
# Create table of warning items.
sql = ("CREATE TABLE {warning_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name} AS layer\n"
"WHERE\n"
" ({warning_where})\n"
" AND NOT ST_IsEmpty(ST_Buffer(geom, %(buffer)s));")
sql = sql.format(**sql_params)
cursor.execute(sql, {"buffer": -params["mxmw"] / 2})
# Report warning features.
items_message = get_failed_items_message(cursor, sql_params["warning_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has warning features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.singlepart check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
error_table_name = "s{:02d}_{:s}_error".format(
params["step_nr"], layer_def["pg_layer_name"])
sql = SQL.format(error_table_name, layer_def["pg_fid_name"],
layer_def["pg_layer_name"])
cursor.execute(sql)
if cursor.rowcount > 0:
failed_items_message = get_failed_items_message(
cursor, error_table_name, layer_def["pg_fid_name"])
status.aborted(
"Layer {:s} has multipart geometries in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"],
layer_def["fid_display_name"], failed_items_message))
status.add_error_table(error_table_name,
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"warning_where":
params["warning_where"],
"warning_table":
"s{:02d}_{:s}_warning".format(params["step_nr"],
layer_def["pg_layer_name"])
}
# Create table of warning items.
sql = (
"CREATE TABLE {warning_table} AS\n"
"SELECT {fid_name}\n"
"FROM\n"
" (SELECT {fid_name}, geom\n"
" FROM\n"
" (SELECT\n"
" {fid_name},\n"
" ST_Boundary(ST_OrientedEnvelope(geom)) AS env,\n"
" geom\n"
" FROM {layer_name} AS layer\n"
" WHERE {warning_where}\n"
" ) AS tenv\n"
" WHERE\n"
" greatest(ST_Distance(ST_PointN(env, 1), ST_PointN(env, 2)),\n"
" ST_Distance(ST_PointN(env, 2), ST_PointN(env, 3))) < %(mml)s\n"
" ) AS tdist\n"
"WHERE\n"
" ST_Length(ST_ApproximateMedialAxis(ST_MakePolygon(ST_ExteriorRing(geom)))) <= %(mml)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, {"mml": params["mml"]})
# Report warning features.
items_message = get_failed_items_message(cursor,
sql_params["warning_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.info(
"Layer {:s} has warning features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["warning_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.area check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"area_column_name":
params["area_column_name"],
"error_table":
"s{:02d}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
sql_execute_params = {
"unit": params["unit"],
"tolerance": params["tolerance"]
}
# Create table of error items.
sql = (
"CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE abs({area_column_name} - ST_Area(geom) / %(unit)s) > %(tolerance)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_execute_params)
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"error_table":
"s{:02d}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
sql_exec_params = {"limit": -params["limit"]}
# Create table of error items.
sql = (
"CREATE TABLE {error_table} AS"
" SELECT DISTINCT unnest(ARRAY[ta.{fid_name}, tb.{fid_name}]) AS {fid_name}"
" FROM {layer_name} ta, {layer_name} tb"
" WHERE"
" ta.{fid_name} < tb.{fid_name}"
" AND ta.wkb_geometry && tb.wkb_geometry"
" AND (NOT ST_Relate(ta.wkb_geometry, tb.wkb_geometry, '**T***T**')"
" OR NOT ST_IsEmpty(ST_Buffer(ST_Intersection(ta.wkb_geometry, tb.wkb_geometry), %(limit)s)));"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_exec_params)
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has overlapping pairs in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"],
layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"area_column_name":
params["area_column_name"],
"error_where":
params["error_where"],
"error_table":
"s{:02d}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name} AS layer\n"
"WHERE\n"
" ({error_where})\n"
" AND {area_column_name} > %(mxmu)s;")
sql = sql.format(**sql_params)
cursor.execute(sql, {"mxmu": params["mxmu"]})
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
for unique_key in params["unique_keys"]:
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"unique_column_name":
unique_key,
"error_table_name":
"s{:02d}_{:s}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"],
unique_key)
}
sql = (
"CREATE TABLE {error_table_name} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN (SELECT {unique_column_name}\n"
" FROM {layer_name}\n"
" GROUP BY {unique_column_name}\n"
" HAVING count({unique_column_name}) > 1) AS ut ON layer.{unique_column_name} = ut.{unique_column_name};"
)
sql = sql.format(**sql_params)
cursor.execute(sql)
if cursor.rowcount > 0:
failed_items_message = get_failed_items_message(
cursor, sql_params["error_table_name"],
layer_def["pg_fid_name"])
status.failed(
"The column {:s}.{:s} has non-unique values in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], unique_key,
layer_def["fid_display_name"],
failed_items_message))
status.add_error_table(sql_params["error_table_name"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.enum check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
for column_name, allowed_codes in params["column_defs"]:
# Prepare clause excluding features with non-null value of specific column.
if "exclude_column_name" in params:
exclude_clause = "AND {:s} IS NULL".format(
params["exclude_column_name"])
else:
exclude_clause = ""
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"column_name":
column_name,
"exclude_clause":
exclude_clause,
"error_table":
"s{:02d}_{:s}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"],
column_name)
}
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" ({column_name} IS NULL"
" OR {column_name} NOT IN %s)"
" {exclude_clause};")
sql = sql.format(**sql_params)
cursor.execute(sql, [tuple(allowed_codes)])
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has column {:s} with invalid codes in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], column_name,
layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info(
"The delivery has been excluded from vector.geometry check because the vector data source does not contain a single object of interest."
)
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"error_table":
"s{:02d}_{:s}_invalid".format(params["step_nr"],
layer_def["pg_layer_name"]),
"detail_table":
"s{:02d}_{:s}_detail".format(params["step_nr"],
layer_def["pg_layer_name"])
}
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE NOT ST_IsValid(geom);")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report items with invalid geometry.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
status.aborted(
"Layer {:s} has invalid geometry in features with {:s}: {:s}.".
format(layer_def["pg_layer_name"],
layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
# Create table of descriptions of invalid geometries.
sql = (
"CREATE TABLE {detail_table} AS"
" SELECT"
" {fid_name},"
" (ST_IsValidDetail(geom)).reason AS reason,"
" ST_SetSRID((ST_IsValidDetail(geom)).location, ST_SRID(geom)) AS location"
" FROM {layer_name}"
" WHERE NOT ST_IsValid(geom);")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report table of descriptions.
status.add_full_table(sql_params["detail_table"])
def run_check(params, status):
"""
Check compactness of linear and patchy features.
Features marked as linear must have compactness less then threshold.
Features marked as patchy must have compactness greater then threshold.
Features not marked as linear or patchy are not validated.
Compactness is measured using Polsby-Popper method,
see `Measuring District Compactness in PostGIS <https://www.azavea.com/blog/2016/07/11/measuring-district-compactness-postgis/>`__.
"""
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"area_column_name":
params["area_column_name"],
"code_column_name":
params["code_column_name"],
"linear_error_table":
"s{:02d}_{:s}_linear_error".format(params["step_nr"],
layer_def["pg_layer_name"]),
"patchy_error_table":
"s{:02d}_{:s}_patchy_error".format(params["step_nr"],
layer_def["pg_layer_name"])
}
sql_execute_params = {
"threshold": params["threshold"],
"linear_code": params["linear_code"],
"patchy_code": params["patchy_code"]
}
# Create table of error items of linear features.
sql = (
"CREATE TABLE {linear_error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" {code_column_name} = %(linear_code)s"
" AND 4 * pi() * {area_column_name} / power(ST_Perimeter(geom), 2) > %(threshold)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_execute_params)
# Report error items of linear features
items_message = get_failed_items_message(
cursor, sql_params["linear_error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error linear features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["linear_error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
# Create table of error items of patchy features.
sql = (
"CREATE TABLE {patchy_error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" {code_column_name} = %(patchy_code)s"
" AND 4 * pi() * {area_column_name} / power(ST_Perimeter(geom), 2) <= %(threshold)s;"
)
sql = sql.format(**sql_params)
cursor.execute(sql, sql_execute_params)
# Report error items of patchy features
items_message = get_failed_items_message(
cursor, sql_params["patchy_error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed(
"Layer {:s} has error patchy features with {:s}: {:s}.".format(
layer_def["pg_layer_name"], layer_def["fid_display_name"],
items_message))
status.add_error_table(sql_params["patchy_error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import PartitionedLayer
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info("The delivery has been excluded from vector.overlap check because the vector data source does not contain a single object of interest.")
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
log.debug("Started overlap check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"])
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table.make()
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"neighbour_table": neighbour_table.neighbour_table_name,
"overlap_detail_table": "s{:02d}_{:s}_detail".format(params["step_nr"], layer_def["pg_layer_name"]),
"overlap_suspect_table": "s{:02d}_{:s}_suspect".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"])}
# FIXME:
# It may happen during partitioning, that the splitted geometries may get shifted a bit.
# The NeighbourTable then reports two neighbouring geometries as overlapping with ST_Dimension()=2.
# In order to avoid reporting such misleading overlaps we verify the overlap by generating anew
# intersection from original geometries.
# If some overlaps are found actually, they are propagated into error table.
# So, the order of building the tables are reversed, the content of error table is extracted
# from the overlap detail table.
# Create suspects table.
sql = ("CREATE TABLE {overlap_suspect_table} AS\n"
"(SELECT fida, fidb\n"
"FROM {neighbour_table}\n"
"WHERE\n"
"fida < fidb\n"
"AND dim >= 2);")
sql = sql.format(**sql_params)
log.debug(sql)
cursor.execute(sql)
if cursor.rowcount > 0:
# Create overlap detail table.
sql = ("CREATE TABLE {overlap_detail_table} AS\n"
"(SELECT fida, fidb, polygon_dump(ST_Intersection(layer_a.geom, layer_b.geom)) AS geom\n"
"FROM {overlap_suspect_table}\n"
"INNER JOIN {layer_name} AS layer_a ON {overlap_suspect_table}.fida = layer_a.{fid_name}\n"
"INNER JOIN {layer_name} AS layer_b ON {overlap_suspect_table}.fidb = layer_b.{fid_name});\n")
sql = sql.format(**sql_params)
log.debug("SQL QUERY:")
log.debug(sql)
cursor.execute(sql)
if cursor.rowcount > 0:
# Report overlap detail table.
status.add_full_table(sql_params["overlap_detail_table"])
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {overlap_detail_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
status.failed("Layer {:s} has overlapping pairs in features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("Overlap check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"initial_code_column_name": params["initial_code_column_name"],
"final_code_column_name": params["final_code_column_name"],
"chtype_column_name": params.get("chtype_column_name", ""),
"general_table": "s{:02d}_{:s}_general".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"]),
"technical_change_flag": TECHNICAL_CHANGE_FLAG}
# Create table of general items.
sql = ("CREATE TABLE {general_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name}\n"
"WHERE\n"
" {initial_code_column_name} != {final_code_column_name};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create table of exception items.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n")
if sql_params["chtype_column_name"] != "":
sql += " AND layer.{chtype_column_name} = '{technical_change_flag}';"
else:
sql += " AND FALSE;"
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception items.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create table of error items.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
" LEFT JOIN {exception_table} AS exc ON layer.{fid_name} = exc.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n"
" AND exc.{fid_name} IS NULL;")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed("Layer {:s} has error features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
for column_name, allowed_codes in params["column_defs"]:
# check product_attrs, replacing YEAR1 with actual year and YEAR2 with
if layer_def["year1"] is not None and "year1" in column_name:
column_name = column_name.replace("year1", layer_def["year1"])
elif layer_def["year2"] is not None and "year2" in column_name:
column_name = column_name.replace("year2", layer_def["year2"])
# Prepare clause excluding features with non-null value of specific column.
if "exclude_column_name" in params:
exclude_clause = "AND {:s} IS NULL".format(
params["exclude_column_name"])
else:
exclude_clause = ""
# Prepare parameters used in sql clauses.
sql_params = {
"fid_name":
layer_def["pg_fid_name"],
"layer_name":
layer_def["pg_layer_name"],
"column_name":
column_name,
"exclude_clause":
exclude_clause,
"error_table":
"s{:02d}_{:s}_{:s}_error".format(params["step_nr"],
layer_def["pg_layer_name"],
column_name)
}
# Create table of error items.
if None in allowed_codes:
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" ({column_name} IS NULL"
" OR {column_name} NOT IN %s)"
" {exclude_clause};")
else:
sql = ("CREATE TABLE {error_table} AS"
" SELECT {fid_name}"
" FROM {layer_name}"
" WHERE"
" ({column_name} IS NULL"
" OR {column_name} NOT IN %s)"
" {exclude_clause};")
sql = sql.format(**sql_params)
cursor.execute(sql, [tuple(allowed_codes)])
# Report error items.
items_message = get_failed_items_message(cursor,
sql_params["error_table"],
layer_def["pg_fid_name"])
if items_message is not None:
invalid_codes_message = get_invalid_codes_message(
cursor, sql_params["error_table"],
layer_def["pg_layer_name"], layer_def["pg_fid_name"],
sql_params["column_name"])
status.failed(
"Layer {:s} has column {:s} with invalid codes in features with {:s}: {:s}. Invalid codes are: {:s}."
.format(layer_def["pg_layer_name"], column_name.upper(),
layer_def["fid_display_name"], items_message,
invalid_codes_message))
status.add_error_table(sql_params["error_table"],
layer_def["pg_layer_name"],
layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
# Prepare parameters used in sql clauses.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"general_where": params["general_where"],
"exception_where": params["exception_where"],
"general_table": "s{:02d}_{:s}_general".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"warning_table": "s{:02d}_{:s}_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
# Create table of general items.
sql = ("CREATE TABLE {general_table} AS\n"
"SELECT {fid_name}\n"
"FROM {layer_name} AS layer\n"
"WHERE\n"
" ({general_where})\n"
" OR ST_NumGeometries(ST_Buffer(geom, %(buffer)s)) = 1;")
sql = sql.format(**sql_params)
cursor.execute(sql, {"buffer": -params["mmw"] / 2})
# Create table of exception items.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n"
" AND ({exception_where});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception features.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create table of warning items.
sql = ("CREATE TABLE {warning_table} AS\n"
"SELECT layer.{fid_name}\n"
"FROM\n"
" {layer_name} AS layer\n"
" LEFT JOIN {general_table} AS gen ON layer.{fid_name} = gen.{fid_name}\n"
" LEFT JOIN {exception_table} AS exc ON layer.{fid_name} = exc.{fid_name}\n"
"WHERE\n"
" gen.{fid_name} IS NULL\n"
" AND exc.{fid_name} IS NULL;")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report warning features.
items_message = get_failed_items_message(cursor, sql_params["warning_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has warning features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["warning_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
def run_check(params, status):
from qc_tool.vector.helper import create_pg_has_comment
from qc_tool.vector.helper import do_layers
from qc_tool.vector.helper import get_failed_items_message
from qc_tool.vector.helper import NeighbourTable
from qc_tool.vector.helper import PartitionedLayer
# Check if the current delivery is excluded from vector checks
if "skip_vector_checks" in params:
if params["skip_vector_checks"]:
status.info("The delivery has been excluded from vector.neighbour check because the vector data source does not contain a single object of interest.")
return
cursor = params["connection_manager"].get_connection().cursor()
for layer_def in do_layers(params):
log.debug("Started neighbour check for the layer {:s}.".format(layer_def["pg_layer_name"]))
# Prepare support data.
partitioned_layer = PartitionedLayer(cursor.connection, layer_def["pg_layer_name"], layer_def["pg_fid_name"])
neighbour_table = NeighbourTable(partitioned_layer)
neighbour_table.make()
create_pg_has_comment(cursor.connection)
# Prepare parameters for sql query.
sql_params = {"fid_name": layer_def["pg_fid_name"],
"layer_name": layer_def["pg_layer_name"],
"neighbour_table": neighbour_table.neighbour_table_name,
"exception_table": "s{:02d}_{:s}_exception".format(params["step_nr"], layer_def["pg_layer_name"]),
"exception_pairs_table": "s{:02d}_{:s}_exception_pairs".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_table": "s{:02d}_{:s}_error".format(params["step_nr"], layer_def["pg_layer_name"]),
"error_pairs_table": "s{:02d}_{:s}_error_pairs".format(params["step_nr"], layer_def["pg_layer_name"]),
"pair_clause": " AND ".join("layer.{0:s} = other.{0:s}".format(code_column_name)
for code_column_name in params["code_column_names"]),
"exception_where": "\n".join(params["exception_where"]),
"error_where": "\n".join(params["error_where"])}
# Create exception pairs table.
sql = ("CREATE TABLE {exception_pairs_table} AS\n"
"SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n"
" INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n"
"WHERE\n"
" ({exception_where})\n"
" AND ({pair_clause});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create error pairs table.
sql = ("CREATE TABLE {error_pairs_table} AS\n"
"SELECT layer.{fid_name} AS fida, other.{fid_name} AS fidb\n"
"FROM\n"
" {layer_name} AS layer\n"
" INNER JOIN {neighbour_table} AS neib ON layer.{fid_name} = neib.fida\n"
" INNER JOIN {layer_name} AS other ON neib.fidb = other.{fid_name}\n"
" LEFT JOIN {exception_pairs_table} AS excp ON layer.{fid_name} = excp.fida AND other.{fid_name} = excp.fidb\n"
"WHERE\n"
" excp.fida IS NULL\n"
" AND ({error_where})\n"
" AND ({pair_clause});")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Create exception table.
sql = ("CREATE TABLE {exception_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {exception_pairs_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report exception items.
items_message = get_failed_items_message(cursor, sql_params["exception_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.info("Layer {:s} has exception features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["exception_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
# Create error table.
sql = ("CREATE TABLE {error_table} AS\n"
"SELECT DISTINCT unnest(ARRAY[fida, fidb]) AS {fid_name}\n"
"FROM {error_pairs_table};")
sql = sql.format(**sql_params)
cursor.execute(sql)
# Report error items.
items_message = get_failed_items_message(cursor, sql_params["error_table"], layer_def["pg_fid_name"])
if items_message is not None:
status.failed("Layer {:s} has error features with {:s}: {:s}."
.format(layer_def["pg_layer_name"], layer_def["fid_display_name"], items_message))
status.add_error_table(sql_params["error_table"], layer_def["pg_layer_name"], layer_def["pg_fid_name"])
log.info("Neighbour check for the layer {:s} has been finished.".format(layer_def["pg_layer_name"]))