def test_valid_invalid_key_yaml(self):
y = '''
all:
select:
- has space
- has_underscore
- has:colon
- UPPERCASE
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
y = '''
all:
select:
- na?me
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Invalid OSM key: na?me")
y = '''
all:
select:
-
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Missing OSM key")
def test_reserved_table_names(self):
# these table names are used by the ogr2ogr gpkg importer
y = '''
points:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Theme name reserved: points")
y = '''
rtree_something:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Theme name reserved: rtree_something")
y = '''
gpkg_something:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Theme name reserved: gpkg_something")
def test_minimal_yaml(self):
# the shortest valid feature selection
y = '''
all:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.geom_types('all'), ['points', 'lines', 'polygons'])
def test_minimal_yaml(self):
# the shortest valid feature selection
y = '''
all:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.geom_types('all'),['points','lines','polygons'])
def test_minimal_yaml(self):
# the shortest valid feature selection
y = """
all:
select:
- name
"""
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.geom_types("all"), ["points", "lines", "polygons"])
def test_zip_readme(self):
y = '''
buildings:
select:
- column1
where: column2 IS NOT NULL
other:
select:
- column3
'''
f = FeatureSelection(y)
self.assertMultiLineEqual(f.zip_readme('buildings'),ZIP_README)
def test_zip_readme(self):
y = '''
buildings:
select:
- column1
where: column2 IS NOT NULL
other:
select:
- column3
'''
f = FeatureSelection(y)
self.assertMultiLineEqual(f.zip_readme('buildings'), ZIP_README)
def test_sql_list(self):
y = '''
waterways:
types:
- polygons
select:
- name
where:
- name IS NOT NULL
- name = 'some building'
'''
f = FeatureSelection(y)
self.assertEqual(f.filter_clause('waterways'),"name IS NOT NULL OR name = 'some building'")
def test_sql_list(self):
y = """
waterways:
types:
- polygons
select:
- name
where:
- name IS NOT NULL
- name = 'some building'
"""
f = FeatureSelection(y)
self.assertEqual(f.filter_clause("waterways"), "name IS NOT NULL OR name = 'some building'")
def test_key_union_and_filters(self):
y = """
waterways:
types:
- lines
- polygons
select:
- name
- waterway
buildings:
types:
- points
- lines
- polygons
select:
- name
- building
where: building IS NOT NULL
"""
f = FeatureSelection(y)
self.assertCountEqual(f.themes, ["buildings", "waterways"])
self.assertCountEqual(f.geom_types("waterways"), ["lines", "polygons"])
self.assertCountEqual(f.key_selections("waterways"), ["name", "waterway"])
self.assertEqual(f.filter_clause("waterways"), '"name" IS NOT NULL OR "waterway" IS NOT NULL')
self.assertCountEqual(f.key_union(), ["building", "name", "waterway"])
self.assertCountEqual(f.key_union("points"), ["building", "name"])
self.assertEqual(f.filter_clause("buildings"), "building IS NOT NULL")
def test_key_union_and_filters(self):
y = '''
waterways:
types:
- lines
- polygons
select:
- name
- waterway
buildings:
types:
- points
- lines
- polygons
select:
- name
- building
where: building IS NOT NULL
'''
f = FeatureSelection(y)
self.assertCountEqual(f.themes, ['buildings', 'waterways'])
self.assertCountEqual(f.geom_types('waterways'), ['lines', 'polygons'])
self.assertCountEqual(f.key_selections('waterways'),
['name', 'waterway'])
self.assertEqual(f.filter_clause('waterways'),
'"name" IS NOT NULL OR "waterway" IS NOT NULL')
self.assertCountEqual(f.key_union(), ['building', 'name', 'waterway'])
self.assertCountEqual(f.key_union('points'), ['building', 'name'])
self.assertEqual(f.filter_clause('buildings'), 'building IS NOT NULL')
def test_sqls(self):
y = """
buildings:
types:
- points
- polygons
select:
- name
- addr:housenumber
"""
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEqual(
create_sqls[0],
"CREATE TABLE buildings_points(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,"
'\ngeom POINT,\nosm_id TEXT,"name" TEXT,"addr:housenumber" TEXT\n);\n'
'INSERT INTO buildings_points(geom, osm_id,"name","addr:housenumber") '
'select geom, osm_id,"name","addr:housenumber" from points '
'WHERE ("name" IS NOT NULL OR "addr:housenumber" IS NOT NULL);\n',
)
self.assertEqual(
create_sqls[1],
"CREATE TABLE buildings_polygons(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom "
'MULTIPOLYGON,\nosm_id TEXT,osm_way_id TEXT,"name" TEXT,"addr:housenumber" '
'TEXT\n);\nINSERT INTO buildings_polygons(geom, osm_id,osm_way_id,"name",'
'"addr:housenumber") select geom, osm_id,osm_way_id,"name","addr:housenumber" '
'from multipolygons WHERE ("name" IS NOT NULL OR "addr:housenumber" '
"IS NOT NULL);\n",
)
def clean_config(self):
config = self.cleaned_data.get('config')
service_type = self.cleaned_data.get('export_provider_type').type_name
if service_type in ['wms', 'wmts', 'tms']:
from eventkit_cloud.utils.mapproxy import MapproxyGeopackage, \
ConfigurationError
service = MapproxyGeopackage(
layer=self.cleaned_data.get('layer'),
service_type=self.cleaned_data.get('export_provider_type'),
config=config)
try:
service.get_check_config()
except ConfigurationError as e:
raise forms.ValidationError(str(e))
elif service_type in ['osm', 'osm-generic']:
if not config:
raise forms.ValidationError(
"Configuration is required for OSM data providers")
from eventkit_cloud.feature_selection.feature_selection import FeatureSelection
feature_selection = FeatureSelection(config)
feature_selection.valid
if feature_selection.errors:
raise forms.ValidationError(
"Invalid configuration: {0}".format(
feature_selection.errors))
return config
def test_unsafe_yaml(self):
y = '''
!!python/object:feature_selection.feature_selection.FeatureSelection
a: 0
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(1, len(f.errors))
def clean_config(self):
config = self.cleaned_data.get("config")
service_type = self.cleaned_data.get("export_provider_type").type_name
if service_type in ["wms", "wmts", "tms", "arcgis-raster"]:
from eventkit_cloud.utils.mapproxy import ConfigurationError, MapproxyGeopackage
service = MapproxyGeopackage(
layer=self.cleaned_data.get("layer"),
service_type=self.cleaned_data.get("export_provider_type"),
config=config,
)
try:
service.get_check_config()
except ConfigurationError as e:
raise forms.ValidationError(str(e))
elif service_type in ["osm", "osm-generic"]:
if not config:
raise forms.ValidationError(
"Configuration is required for OSM data providers")
from eventkit_cloud.feature_selection.feature_selection import FeatureSelection
feature_selection = FeatureSelection(clean_config_as_str(config))
if feature_selection.errors:
raise forms.ValidationError(
"Invalid configuration: {0}".format(
feature_selection.errors))
elif service_type in ["ogcapi-process"]:
if not config:
raise forms.ValidationError(
"Configuration is required for OGC API Process")
cleaned_config = clean_config(config)
assert isinstance(cleaned_config, dict)
ogcapi_process = cleaned_config.get("ogcapi_process")
if not ogcapi_process:
raise forms.ValidationError(
"OGC API Process requires an ogcapi_process key with valid configuration"
)
area = ogcapi_process.get("area")
if not area:
raise forms.ValidationError(
"OGC API Process requires an area key with a name and a type."
)
if not area.get("name"):
raise forms.ValidationError(
"OGC API Process requires the name of the field to submit the area."
)
if area.get("type") not in ["geojson", "bbox", "wkt"]:
raise forms.ValidationError(
"OGC API Process requires an area type of geojson, bbox, or wkt."
)
if not ogcapi_process.get("id"):
raise forms.ValidationError(
"OGC API Process requires a process id.")
return config
def test_key_union_and_filters(self):
y = '''
waterways:
types:
- lines
- polygons
select:
- name
- waterway
buildings:
types:
- points
- lines
- polygons
select:
- name
- building
where: building IS NOT NULL
'''
f = FeatureSelection(y)
self.assertCountEqual(f.themes,['buildings','waterways'])
self.assertCountEqual(f.geom_types('waterways'),['lines','polygons'])
self.assertCountEqual(f.key_selections('waterways'),['name','waterway'])
self.assertEqual(f.filter_clause('waterways'),'"name" IS NOT NULL OR "waterway" IS NOT NULL')
self.assertCountEqual(f.key_union(), ['building','name','waterway'])
self.assertCountEqual(f.key_union('points'), ['building','name'])
self.assertEqual(f.filter_clause('buildings'),'building IS NOT NULL')
def test_enforces_subset_columns(self):
y = """
buildings:
types:
- polygons
select:
- column1
where: column2 IS NOT NULL
other:
types:
- points
select:
- column3
"""
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.key_union(), ["column1", "column2", "column3"])
self.assertEqual(f.key_union("points"), ["column3"])
def test_enforces_subset_columns(self):
y = '''
buildings:
types:
- polygons
select:
- column1
where: column2 IS NOT NULL
other:
types:
- points
select:
- column3
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.key_union(), ['column1', 'column2', 'column3'])
self.assertEqual(f.key_union('points'), ['column3'])
def test_enforces_subset_columns(self):
y = '''
buildings:
types:
- polygons
select:
- column1
where: column2 IS NOT NULL
other:
types:
- points
select:
- column3
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.key_union(), ['column1','column2','column3'])
self.assertEqual(f.key_union('points'), ['column3'])
def test_dash_spacing_yaml(self):
# top level is a list and not a dict
y = '''
all:
select:
-name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
def test_theme_names(self):
y = '''
A Theme Name:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.themes, ["A Theme Name"])
self.assertEqual(f.slug_themes, ["a_theme_name"])
def test_no_select_yaml(self):
# top level is a list and not a dict
y = '''
all:
-select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Each theme must have a 'select' key")
def test_unspecified_yaml(self):
# top level is a list and not a dict
y = '''
- all:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "YAML must be dict, not list")
def test_malformed_yaml(self):
# if it's not a valid YAML document
# TODO: errors for if yaml indentation is incorrect
y = '''
all
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
def test_passes_sqlvalidator_errors(self):
y = """
buildings:
select:
- name
- addr:housenumber
where: addr:housenumber IS NOT NULL
"""
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "SQL WHERE Invalid: identifier with colon : must be in double quotes.")
def test_zindex(self):
y = '''
roads:
types:
- lines
select:
- highway
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEqual(
create_sqls[0],
'CREATE TABLE roads_lines(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom MULTILINESTRING,\nosm_id TEXT,"highway" TEXT,"z_index" TEXT\n);\nINSERT INTO roads_lines(geom, osm_id,"highway","z_index") select geom, osm_id,"highway","z_index" from lines WHERE ("highway" IS NOT NULL);\n'
)
def test_empty_feature_selection(self):
y = '''
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)