def test_sql_empty_list(self): y = ''' waterways: types: - polygons select: - name where: [] ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "if 'where' key is specified, it must not be empty") y = ''' waterways: types: - polygons select: - name where: ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "if 'where' key is specified, it must not be empty")
def test_overpass_filter(self): y = ''' buildings: types: - points select: - column1 where: column2 IS NOT NULL other1: types: - points - polygons select: - column1 - irrelevant where: column2 IS NOT NULL AND column3 IN ('foo','bar') other2: types: - lines select: - column5:key ''' f = FeatureSelection(y) nodes, ways, relations = f.overpass_filter() self.assertCountEqual(nodes,["[column3~'foo|bar']","[column2]"]) # force quoting of strings to handle keys with colons self.assertCountEqual(ways,["['column5:key']","[column3~'foo|bar']","[column2]"]) self.assertCountEqual(relations,["[column3~'foo|bar']","[column2]"])
def test_no_select_yaml(self): # top level is a list and not a dict y = ''' all: -select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Each theme must have a 'select' key") y = ''' theme_0: ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Each theme must have a 'select' key") y = ''' theme_0: select: ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "'select' cannot be empty")
def test_valid_invalid_key_yaml(self): y = ''' all: select: - has space - has_underscore - has:colon - UPPERCASE ''' f = FeatureSelection(y) self.assertTrue(f.valid) y = ''' all: select: - na?me ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Invalid OSM key: na?me") y = ''' all: select: - ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Missing OSM key")
def test_reserved_table_names(self): # these table names are used by the ogr2ogr gpkg importer y = ''' points: select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Theme name reserved: points") y = ''' rtree_something: select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Theme name reserved: rtree_something") y = ''' gpkg_something: select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Theme name reserved: gpkg_something")
def test_overpass_filter(self): y = ''' buildings: types: - points select: - column1 where: column2 IS NOT NULL other1: types: - points - polygons select: - column1 - irrelevant where: column2 IS NOT NULL AND column3 IN ('foo','bar') other2: types: - lines select: - column5 ''' f = FeatureSelection(y) nodes, ways, relations = f.overpass_filter() self.assertEquals(nodes, ["[column3~'foo|bar']", "[column2]"]) self.assertEquals(ways, ["[column3~'foo|bar']", "[column5]", "[column2]"]) self.assertEquals(relations, ["[column3~'foo|bar']", "[column2]"])
def test_minimal_yaml(self): # the shortest valid feature selection y = ''' all: select: - name ''' f = FeatureSelection(y) self.assertTrue(f.valid) self.assertEqual(f.geom_types('all'), ['points', 'lines', 'polygons'])
def test_zip_readme(self): y = ''' buildings: select: - column1 where: column2 IS NOT NULL other: select: - column3 ''' f = FeatureSelection(y) self.assertMultiLineEqual(f.zip_readme('buildings'), ZIP_README)
def test_sql_list(self): y = ''' waterways: types: - polygons select: - name where: - name IS NOT NULL - name = 'some building' ''' f = FeatureSelection(y) self.assertEquals(f.filter_clause('waterways'),"name IS NOT NULL OR name = 'some building'")
def test_key_union_and_filters(self): y = ''' waterways: types: - lines - polygons select: - name - waterway buildings: types: - points - lines - polygons select: - name - building where: building IS NOT NULL ''' f = FeatureSelection(y) self.assertEquals(f.themes, ['buildings', 'waterways']) self.assertEquals(f.geom_types('waterways'), ['lines', 'polygons']) self.assertEquals(f.key_selections('waterways'), ['name', 'waterway']) self.assertEquals(f.filter_clause('waterways'), '"name" IS NOT NULL OR "waterway" IS NOT NULL') self.assertEquals(f.key_union(), ['building', 'name', 'waterway']) self.assertEquals(f.key_union('points'), ['building', 'name']) self.assertEquals(f.filter_clause('buildings'), 'building IS NOT NULL')
def setUp(self): self.user = User.objects.create_user(username='******', email='*****@*****.**', password='******', is_superuser=True) token = Token.objects.create(user=self.user) self.client.credentials(HTTP_AUTHORIZATION='Token ' + token.key, HTTP_ACCEPT='application/json; version=1.0', HTTP_ACCEPT_LANGUAGE='en', HTTP_HOST='testserver') self.request_data = { 'name': 'TestHDXRegion', 'export_formats': ["shp"], 'the_geom': { 'type': 'Polygon', 'coordinates': [[[-17.464, 14.727], [-17.449, 14.727], [-17.449, 14.740], [-17.464, 14.740], [-17.464, 14.727]]] }, 'feature_selection': FeatureSelection.example_raw("simple"), 'dataset_prefix': 'hdx_test_', 'locations': ['SEN'], 'is_private': True, 'buffer_aoi': True, 'schedule_period': 'daily', 'schedule_hour': 0, 'subnational': True, 'extra_notes': '', 'license': '' }
def validate(self, data): # noqa f = FeatureSelection(data.get('feature_selection')) if not f.valid: raise serializers.ValidationError({'feature_selection': f.errors}) return data
def feature_selection_object(self): """ a valid FeatureSelection object based off the feature_selection column. """ fs = FeatureSelection(self.feature_selection) # assert fs.valid, 'Feature selection is invalid' return fs
def test_empty_yaml(self): y = ''' {} ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "YAML cannot be empty")
def test_unsafe_yaml(self): y = ''' !!python/object:feature_selection.feature_selection.FeatureSelection a: 0 ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(1, len(f.errors))
def test_enforces_subset_columns(self): y = ''' buildings: types: - polygons select: - column1 where: column2 IS NOT NULL other: types: - points select: - column3 ''' f = FeatureSelection(y) self.assertTrue(f.valid) self.assertEquals(f.key_union(), ['column1', 'column2', 'column3']) self.assertEquals(f.key_union('points'), ['column3'])
def test_dash_spacing_yaml(self): # top level is a list and not a dict y = ''' all: select: -name ''' f = FeatureSelection(y) self.assertFalse(f.valid)
def test_malformed_yaml(self): # if it's not a valid YAML document # TODO: errors for if yaml indentation is incorrect y = ''' all select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid)
def test_unspecified_yaml(self): # top level is a list and not a dict y = ''' - all: select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "YAML must be dict, not list")
def test_invalid_type(self): y = ''' all: types: - multilines select: - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0],"types must be one or more of points, lines or polygons, got: multilines")
def test_passes_sqlvalidator_errors(self): y = ''' buildings: select: - name - addr:housenumber where: addr:housenumber IS NOT NULL ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEquals(f.errors[0], "SQL (addr:housenumber IS NOT NULL) is invalid: identifier with colon : must be in double quotes.")
def test_theme_names(self): y = ''' A Theme Name: select: - name ''' f = FeatureSelection(y) self.assertTrue(f.valid) self.assertEqual(f.themes, ["A Theme Name"]) self.assertEqual(f.slug_themes, ["a_theme_name"]) u = u''' Å Theme Name: select: - name ''' f = FeatureSelection(y) self.assertTrue(f.valid) self.assertEqual(f.themes, ["A Theme Name"]) self.assertEqual(f.slug_themes, ["a_theme_name"])
def test_unicode_osm_key(self): # https://taginfo.openstreetmap.org/reports/characters_in_keys y = ''' planet_osm_polygon: types: - polygons select: - buildingBuildings ''' f = FeatureSelection(y) self.assertTrue(f.valid)
def test_single_theme_note(self): yaml = ''' all: select: - name ''' h = HDXExportSet(dataset_prefix="hot_dakar", name="Dakar Urban Area", extent=DAKAR_GEOJSON_POLYGON, feature_selection=FeatureSelection(yaml)) self.assertMultiLineEqual(h.hdx_note('all'), SINGLE_THEME_NOTE)
def test_zindex(self): y = ''' roads: types: - lines select: - highway ''' f = FeatureSelection(y) create_sqls, index_sqls = f.sqls self.assertEquals(create_sqls[0],'CREATE TABLE roads_lines(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom MULTILINESTRING,\nosm_id TEXT,"highway" TEXT,"z_index" TEXT\n);\nINSERT INTO roads_lines(geom, osm_id,"highway","z_index") select geom, osm_id,"highway","z_index" from lines WHERE ("highway" IS NOT NULL);\n')
def setUp(self, ): self.user1 = User.objects.create(username='******', email='*****@*****.**', password='******') the_geom = Polygon.from_bbox( (-10.80029, 6.3254236, -10.79809, 6.32752)) self.job = Job.objects.create( name='TestJob', user=self.user1, the_geom=the_geom, export_formats=['shp'], feature_selection=FeatureSelection.example('simple'))
def test_filtered_note(self): yaml = ''' some: select: - name where: highway IS NOT NULL ''' h = HDXExportSet(dataset_prefix="hot_dakar", name="Dakar Urban Area", extent=DAKAR_GEOJSON_POLYGON, feature_selection=FeatureSelection(yaml)) self.assertMultiLineEqual(h.hdx_note('some'), SINGLE_FILTER_NOTE)
def test_duplicated_yaml(self): y = ''' all: select: - name - name ''' f = FeatureSelection(y) self.assertFalse(f.valid) self.assertEqual(f.errors[0], "Duplicate tag: name") y = ''' t1: select: - name t2: select: - name ''' f = FeatureSelection(y) self.assertTrue(f.valid)
def setUp(self,): user = User.objects.create( username='******', email='*****@*****.**', password='******') the_geom = Polygon.from_bbox((-10.80029,6.3254236,-10.79809,6.32752)) self.fixture = { 'name': 'TestJob', 'description': 'Test Description', 'event': 'Nepal Activation', 'user': user, 'the_geom': the_geom, 'export_formats': ['shp'], 'feature_selection':FeatureSelection.example('simple') }
def test_zindex(self): y = ''' roads: types: - lines select: - highway ''' f = FeatureSelection(y) create_sqls, index_sqls = f.sqls self.assertEquals( create_sqls[0], 'CREATE TABLE roads_lines AS SELECT geom,osm_id,"highway","z_index" FROM planet_osm_line WHERE (1)' )