def _get_check(import_def, model_def, check_type, check_props):
table = f"{import_def['target']}_new"
if check_type == 'count':
if 'count_minimum' in check_props:
count_method = assert_count_minimum(check_props['count_minimum'])
else:
raise NotImplementedError
return [('count', f"select count(*) from {table}", count_method)]
elif check_type == 'geometrie':
model_repr = get_model_repr(model_def)
geo_fields = [
field for field in model_repr
if isinstance(field, GeoFieldRepresentation)
]
collect_checks = []
for field in geo_fields:
if not field.allow_null:
collect_checks.append((
f'filled_geometrie_{field.field_name}',
f"select count(*) from {table} where {field.field_name} is null",
assert_count_zero()))
collect_checks.append((
f'valid_geometrie_{field.field_name}',
f"select count(*) from {table} where ST_IsValid({field.field_name}) = false",
assert_count_zero()))
collect_checks.append(
(f'geometrie_type_{field.field_name}',
f"select count(*) from {table} where "
f"ST_GeometryType({field.field_name}) <> '{field.psql_type}'",
assert_count_zero()))
return collect_checks
else:
raise NotImplementedError
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_minimum
sql_checks = [
('count', "select count(*) from openbare_verlichting_new",
assert_count_minimum(129410)),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from overlastgebieden_new", assert_count_minimum(110)),
('columns', """
select column_name from information_schema.columns where
table_schema = 'public' and table_name = 'overlastgebieden_new'
""", lambda x: x == [('ogc_fid',), ('wkb_geometry',), ('oov_naam',), ('type',), ('url',)]),
('geometrie', """
select count(*) from overlastgebieden_new where
wkb_geometry is null or ST_IsValid(wkb_geometry) = false or ST_GeometryType(wkb_geometry) <> 'ST_Polygon'
""",
assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, all_valid_url, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from biz_view_new", assert_count_minimum(48)),
('columns', """
select column_name from information_schema.columns where
table_schema = 'public' and table_name = 'biz_data_new'
""", lambda x: x == [("biz_id",), ("naam",), ("biz_type",), ("heffingsgrondslag",), ("website",), ("heffing",),
("bijdrageplichtigen",), ("verordening",), ("wkb_geometry",)]),
('website', "select website from biz_view_new where website is not NULL", all_valid_url),
('verordening', "select verordening from biz_view_new where verordening is not NULL", all_valid_url),
('geometrie', """
select count(*) from biz_view_new where
geometrie is null or ST_IsValid(geometrie) = false or ST_GeometryType(geometrie) <> 'ST_Polygon'
""",
assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from bekendmakingen_new", assert_count_minimum(1000)),
('columns', """
select column_name from information_schema.columns where
table_schema = 'public' and table_name = 'bekendmakingen_new'
""", lambda x: x == [('ogc_fid',), ('wkb_geometry',), ('id',), ('oid_',), ('categorie',),
('onderwerp',), ('titel',), ('beschrijving',), ('url',),
('postcodehuisnummer',), ('plaats',), ('straat',), ('datum',), ('overheid',)]),
('geometrie', """
select count(*) from bekendmakingen_new where
wkb_geometry is null or ST_IsValid(wkb_geometry) = false or ST_GeometryType(wkb_geometry) <> 'ST_Point'
""",
assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from vezips_new", assert_count_minimum(75)),
('columns', """
select column_name from information_schema.columns where
table_schema = 'public' and table_name = 'vezips_new'
""", lambda x: x == [('ogc_fid', ), ('wkb_geometry', ), ('soortcode', ),
('vezip_nummer', ), ('vezip_type', ),
('standplaats', ), ('jaar_aanleg', ),
('venstertijden', ), ('toegangssysteem', ),
('camera', ), ('beheerorganisatie', ),
('bijzonderheden', )]),
('geometrie', """
select count(*) from vezips_new where
wkb_geometry is null or ST_IsValid(wkb_geometry) = false or ST_GeometryType(wkb_geometry) <> 'ST_Point'
""", assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, all_valid_url, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from milieuzones_new", assert_count_minimum(5)),
('columns', """
select column_name from information_schema.columns where
table_schema = 'public' and table_name = 'milieuzones_new'
""", lambda x: x == [("ogc_fid",), ("wkb_geometry",), ("id",), ("verkeerstype",), ("vanafdatum",),("color",)]),
('geometrie', """
select count(*) from milieuzones_new where
wkb_geometry is null or ST_GeometryType(wkb_geometry) <> 'ST_MultiPolygon'
""",
assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from oplaadpalen_new",
assert_count_minimum(10)),
('geometrie', """
select count(*) from oplaadpalen_new where
wkb_geometry is null or ST_IsValid(wkb_geometry) = false or ST_GeometryType(wkb_geometry) <> 'ST_Point'
""", assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, all_valid_url, assert_count_minimum, assert_count_zero
sql_checks = [
('count', "select count(*) from winkgeb_new", assert_count_minimum(75)),
# ('columns', """
# select column_name from information_schema.columns where
# table_schema = 'public' and table_name = 'vezips_new'
# """, lambda x: x == [('ogc_fid',), ('wkb_geometry',), ('soortcode',), ('vezip_nummer',), ('vezip_type',),
# ('standplaats',), ('jaar_aanleg',), ('venstertijden',), ('toegangssysteem',),
# ('beheerorganisatie',), ('bijzonderheden',)]),
('geometrie', """
select count(*) from winkgeb_new where
wkb_geometry is null or ST_GeometryType(wkb_geometry) not in ('ST_Polygon', 'ST_MultiPolygon')
""", assert_count_zero()),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_zero, assert_count_minimum
sql_checks = [
('count', "select count(*) from hoofdroutes_new", assert_count_minimum(3)),
('geometrie', """
select count(*) from hoofdroutes_new where
wkb_geometry is null or ST_IsValid(wkb_geometry) = false or ST_GeometryType(wkb_geometry) <> 'ST_MultiLineString'
""", assert_count_zero()),
('columns', """
select column_name from information_schema.columns where
table_schema = 'public' and table_name = 'hoofdroutes_new'
""", lambda x: x == [("ogc_fid", ), ("wkb_geometry", ), ("id", ),
("name", ), ("route", ), ("type", )]),
]
if __name__ == '__main__':
main(sql_checks)
#!/usr/bin/env python
from utils.check_imported_data import main, assert_count_zero, assert_count_minimum
sql_checks = [
('count', "select count(*) from parkeerzones_new",
assert_count_minimum(100)),
('count_uitz', "select count(*) from parkeerzones_uitz_new",
assert_count_minimum(60)),
('geometrie', """
select count(*) from parkeerzones_new where
wkb_geometry is null or ST_GeometryType(wkb_geometry) <> 'ST_Polygon'
""", assert_count_zero()),
('geometrie', """
select count(*) from parkeerzones_uitz_new where
wkb_geometry is null or ST_GeometryType(wkb_geometry) <> 'ST_Polygon'
""", assert_count_zero()),
# ('columns', """
#select column_name from information_schema.columns where
#table_schema = 'public' and table_name = 'hoofdroutes_new'
# """, lambda x: x == [("ogc_fid",), ("wkb_geometry",), ("id",), ("name",), ("route",), ("type",)]),
]
if __name__ == '__main__':
main(sql_checks)
