def test_sql_empty_list(self):
y = '''
waterways:
types:
- polygons
select:
- name
where: []
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0],
"if 'where' key is specified, it must not be empty")
y = '''
waterways:
types:
- polygons
select:
- name
where:
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0],
"if 'where' key is specified, it must not be empty")
def test_overpass_filter(self):
y = '''
buildings:
types:
- points
select:
- column1
where: column2 IS NOT NULL
other1:
types:
- points
- polygons
select:
- column1
- irrelevant
where: column2 IS NOT NULL AND column3 IN ('foo','bar')
other2:
types:
- lines
select:
- column5:key
'''
f = FeatureSelection(y)
nodes, ways, relations = f.overpass_filter()
self.assertCountEqual(nodes,["[column3~'foo|bar']","[column2]"])
# force quoting of strings to handle keys with colons
self.assertCountEqual(ways,["['column5:key']","[column3~'foo|bar']","[column2]"])
self.assertCountEqual(relations,["[column3~'foo|bar']","[column2]"])
def test_no_select_yaml(self):
# top level is a list and not a dict
y = '''
all:
-select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Each theme must have a 'select' key")
y = '''
theme_0:
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Each theme must have a 'select' key")
y = '''
theme_0:
select:
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "'select' cannot be empty")
def test_valid_invalid_key_yaml(self):
y = '''
all:
select:
- has space
- has_underscore
- has:colon
- UPPERCASE
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
y = '''
all:
select:
- na?me
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Invalid OSM key: na?me")
y = '''
all:
select:
-
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Missing OSM key")
def test_reserved_table_names(self):
# these table names are used by the ogr2ogr gpkg importer
y = '''
points:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Theme name reserved: points")
y = '''
rtree_something:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Theme name reserved: rtree_something")
y = '''
gpkg_something:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Theme name reserved: gpkg_something")
def test_overpass_filter(self):
y = '''
buildings:
types:
- points
select:
- column1
where: column2 IS NOT NULL
other1:
types:
- points
- polygons
select:
- column1
- irrelevant
where: column2 IS NOT NULL AND column3 IN ('foo','bar')
other2:
types:
- lines
select:
- column5
'''
f = FeatureSelection(y)
nodes, ways, relations = f.overpass_filter()
self.assertEquals(nodes, ["[column3~'foo|bar']", "[column2]"])
self.assertEquals(ways,
["[column3~'foo|bar']", "[column5]", "[column2]"])
self.assertEquals(relations, ["[column3~'foo|bar']", "[column2]"])
def test_minimal_yaml(self):
# the shortest valid feature selection
y = '''
all:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.geom_types('all'), ['points', 'lines', 'polygons'])
def test_zip_readme(self):
y = '''
buildings:
select:
- column1
where: column2 IS NOT NULL
other:
select:
- column3
'''
f = FeatureSelection(y)
self.assertMultiLineEqual(f.zip_readme('buildings'), ZIP_README)
def test_sql_list(self):
y = '''
waterways:
types:
- polygons
select:
- name
where:
- name IS NOT NULL
- name = 'some building'
'''
f = FeatureSelection(y)
self.assertEquals(f.filter_clause('waterways'),"name IS NOT NULL OR name = 'some building'")
def test_key_union_and_filters(self):
y = '''
waterways:
types:
- lines
- polygons
select:
- name
- waterway
buildings:
types:
- points
- lines
- polygons
select:
- name
- building
where: building IS NOT NULL
'''
f = FeatureSelection(y)
self.assertEquals(f.themes, ['buildings', 'waterways'])
self.assertEquals(f.geom_types('waterways'), ['lines', 'polygons'])
self.assertEquals(f.key_selections('waterways'), ['name', 'waterway'])
self.assertEquals(f.filter_clause('waterways'),
'"name" IS NOT NULL OR "waterway" IS NOT NULL')
self.assertEquals(f.key_union(), ['building', 'name', 'waterway'])
self.assertEquals(f.key_union('points'), ['building', 'name'])
self.assertEquals(f.filter_clause('buildings'), 'building IS NOT NULL')
def setUp(self):
self.user = User.objects.create_user(username='******',
email='*****@*****.**',
password='******',
is_superuser=True)
token = Token.objects.create(user=self.user)
self.client.credentials(HTTP_AUTHORIZATION='Token ' + token.key,
HTTP_ACCEPT='application/json; version=1.0',
HTTP_ACCEPT_LANGUAGE='en',
HTTP_HOST='testserver')
self.request_data = {
'name': 'TestHDXRegion',
'export_formats': ["shp"],
'the_geom': {
'type':
'Polygon',
'coordinates': [[[-17.464, 14.727], [-17.449, 14.727],
[-17.449, 14.740], [-17.464, 14.740],
[-17.464, 14.727]]]
},
'feature_selection': FeatureSelection.example_raw("simple"),
'dataset_prefix': 'hdx_test_',
'locations': ['SEN'],
'is_private': True,
'buffer_aoi': True,
'schedule_period': 'daily',
'schedule_hour': 0,
'subnational': True,
'extra_notes': '',
'license': ''
}
def validate(self, data): # noqa
f = FeatureSelection(data.get('feature_selection'))
if not f.valid:
raise serializers.ValidationError({'feature_selection': f.errors})
return data
def feature_selection_object(self):
"""
a valid FeatureSelection object based off the feature_selection column.
"""
fs = FeatureSelection(self.feature_selection)
# assert fs.valid, 'Feature selection is invalid'
return fs
def test_empty_yaml(self):
y = '''
{}
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "YAML cannot be empty")
def test_unsafe_yaml(self):
y = '''
!!python/object:feature_selection.feature_selection.FeatureSelection
a: 0
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(1, len(f.errors))
def test_enforces_subset_columns(self):
y = '''
buildings:
types:
- polygons
select:
- column1
where: column2 IS NOT NULL
other:
types:
- points
select:
- column3
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEquals(f.key_union(), ['column1', 'column2', 'column3'])
self.assertEquals(f.key_union('points'), ['column3'])
def test_dash_spacing_yaml(self):
# top level is a list and not a dict
y = '''
all:
select:
-name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
def test_malformed_yaml(self):
# if it's not a valid YAML document
# TODO: errors for if yaml indentation is incorrect
y = '''
all
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
def test_unspecified_yaml(self):
# top level is a list and not a dict
y = '''
- all:
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "YAML must be dict, not list")
def test_invalid_type(self):
y = '''
all:
types:
- multilines
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0],"types must be one or more of points, lines or polygons, got: multilines")
def test_passes_sqlvalidator_errors(self):
y = '''
buildings:
select:
- name
- addr:housenumber
where: addr:housenumber IS NOT NULL
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEquals(f.errors[0], "SQL (addr:housenumber IS NOT NULL) is invalid: identifier with colon : must be in double quotes.")
def test_theme_names(self):
y = '''
A Theme Name:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.themes, ["A Theme Name"])
self.assertEqual(f.slug_themes, ["a_theme_name"])
u = u'''
Å Theme Name:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEqual(f.themes, ["A Theme Name"])
self.assertEqual(f.slug_themes, ["a_theme_name"])
def test_unicode_osm_key(self):
# https://taginfo.openstreetmap.org/reports/characters_in_keys
y = '''
planet_osm_polygon:
types:
- polygons
select:
- buildingBuildings
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
def test_single_theme_note(self):
yaml = '''
all:
select:
- name
'''
h = HDXExportSet(dataset_prefix="hot_dakar",
name="Dakar Urban Area",
extent=DAKAR_GEOJSON_POLYGON,
feature_selection=FeatureSelection(yaml))
self.assertMultiLineEqual(h.hdx_note('all'), SINGLE_THEME_NOTE)
def test_zindex(self):
y = '''
roads:
types:
- lines
select:
- highway
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEquals(create_sqls[0],'CREATE TABLE roads_lines(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom MULTILINESTRING,\nosm_id TEXT,"highway" TEXT,"z_index" TEXT\n);\nINSERT INTO roads_lines(geom, osm_id,"highway","z_index") select geom, osm_id,"highway","z_index" from lines WHERE ("highway" IS NOT NULL);\n')
def setUp(self, ):
self.user1 = User.objects.create(username='******',
email='*****@*****.**',
password='******')
the_geom = Polygon.from_bbox(
(-10.80029, 6.3254236, -10.79809, 6.32752))
self.job = Job.objects.create(
name='TestJob',
user=self.user1,
the_geom=the_geom,
export_formats=['shp'],
feature_selection=FeatureSelection.example('simple'))
def test_filtered_note(self):
yaml = '''
some:
select:
- name
where: highway IS NOT NULL
'''
h = HDXExportSet(dataset_prefix="hot_dakar",
name="Dakar Urban Area",
extent=DAKAR_GEOJSON_POLYGON,
feature_selection=FeatureSelection(yaml))
self.assertMultiLineEqual(h.hdx_note('some'), SINGLE_FILTER_NOTE)
def test_duplicated_yaml(self):
y = '''
all:
select:
- name
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(f.errors[0], "Duplicate tag: name")
y = '''
t1:
select:
- name
t2:
select:
- name
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
def setUp(self,):
user = User.objects.create(
username='******', email='*****@*****.**', password='******')
the_geom = Polygon.from_bbox((-10.80029,6.3254236,-10.79809,6.32752))
self.fixture = {
'name': 'TestJob',
'description': 'Test Description',
'event': 'Nepal Activation',
'user': user,
'the_geom': the_geom,
'export_formats': ['shp'],
'feature_selection':FeatureSelection.example('simple')
}
def test_zindex(self):
y = '''
roads:
types:
- lines
select:
- highway
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEquals(
create_sqls[0],
'CREATE TABLE roads_lines AS SELECT geom,osm_id,"highway","z_index" FROM planet_osm_line WHERE (1)'
)