def test_sql_empty_list(self):
y = '''
waterways:
types:
- polygons
select:
- name
where: []
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(
f.errors[0],
"if 'where' key is specified, it must not be an empty list")
def test_passes_sqlvalidator_errors(self):
y = '''
buildings:
select:
- name
- addr:housenumber
where: addr:housenumber IS NOT NULL
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEquals(
f.errors[0],
"SQL (addr:housenumber IS NOT NULL) is invalid: identifier with colon : must be in double quotes."
)
def test_sql_list(self):
y = '''
waterways:
types:
- polygons
select:
- name
where:
- name IS NOT NULL
- name = 'some building'
'''
f = FeatureSelection(y)
self.assertEquals(f.filter_clause('waterways'),
"name IS NOT NULL OR name = 'some building'")
def test_zindex(self):
y = '''
roads:
types:
- lines
select:
- highway
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEquals(
create_sqls[0],
'CREATE TABLE roads_lines(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom MULTILINESTRING,\nosm_id TEXT,"highway" TEXT,"z_index" TEXT\n);\nINSERT INTO roads_lines(geom, osm_id,"highway","z_index") select geom, osm_id,"highway","z_index" from lines WHERE ("highway" IS NOT NULL);\n'
)
def test_invalid_type(self):
y = '''
all:
types:
- multilines
select:
- name
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
self.assertEqual(
f.errors[0],
"types must be one or more of points, lines or polygons, got: multilines"
)
def test_zindex(self):
y = '''
roads:
types:
- lines
select:
- highway
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEquals(
create_sqls[0],
'CREATE TABLE roads_lines AS SELECT geom,osm_id,"highway","z_index" FROM planet_osm_line WHERE (1)'
)
def test_sqls(self):
y = '''
buildings:
types:
- points
- polygons
select:
- name
- addr:housenumber
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEquals(create_sqls[0],'CREATE TABLE buildings_points(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom POINT,\nosm_id TEXT,"name" TEXT,"addr:housenumber" TEXT\n);\nINSERT INTO buildings_points(geom, osm_id,"name","addr:housenumber") select geom, osm_id,"name","addr:housenumber" from points WHERE ("name" IS NOT NULL OR "addr:housenumber" IS NOT NULL);\n')
self.assertEquals(create_sqls[1],'CREATE TABLE buildings_polygons(\nfid INTEGER PRIMARY KEY AUTOINCREMENT,\ngeom MULTIPOLYGON,\nosm_id TEXT,osm_way_id TEXT,"name" TEXT,"addr:housenumber" TEXT\n);\nINSERT INTO buildings_polygons(geom, osm_id,osm_way_id,"name","addr:housenumber") select geom, osm_id,osm_way_id,"name","addr:housenumber" from multipolygons WHERE ("name" IS NOT NULL OR "addr:housenumber" IS NOT NULL);\n')
def test_enforces_subset_columns(self):
y = '''
buildings:
types:
- polygons
select:
- column1
where: column2 IS NOT NULL
other:
types:
- points
select:
- column3
'''
f = FeatureSelection(y)
self.assertTrue(f.valid)
self.assertEquals(f.key_union(), ['column1', 'column2', 'column3'])
self.assertEquals(f.key_union('points'), ['column3'])
def test_sqls(self):
y = '''
buildings:
types:
- points
- polygons
select:
- name
- addr:housenumber
'''
f = FeatureSelection(y)
create_sqls, index_sqls = f.sqls
self.assertEquals(
create_sqls[0],
'CREATE TABLE buildings_points AS SELECT geom,osm_id,"name","addr:housenumber" FROM planet_osm_point WHERE (1)'
)
self.assertEquals(
create_sqls[1],
'CREATE TABLE buildings_polygons AS SELECT geom,osm_id,osm_way_id,"name","addr:housenumber" FROM planet_osm_polygon WHERE (1)'
)
def test_empty_feature_selection(self):
y = '''
'''
f = FeatureSelection(y)
self.assertFalse(f.valid)
def run_task_remote(run_uid): # noqa
run = ExportRun.objects.get(uid=run_uid)
run.status = 'RUNNING'
run.started_at = timezone.now()
run.save()
LOG.debug('Running ExportRun with id: {0}'.format(run_uid))
job = run.job
try:
stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT,
str(run_uid)) + '/'
os.makedirs(stage_dir, 6600)
run_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid)
if not os.path.exists(run_dir):
os.makedirs(run_dir)
aoi = GEOSGeometry(job.the_geom)
if job.buffer_aoi:
aoi = aoi.buffer(
0.02
) # 0.02 degrees is a reasonable amount for an admin 0 boundary
aoi = simplify_max_points(aoi)
try:
feature_selection = job.feature_selection_object
except Exception:
feature_selection = FeatureSelection(SIMPLE)
export_formats = map_names_to_formats(job.export_formats)
download_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid)
zipper = Zipper(job.name, stage_dir, download_dir, aoi)
def on_task_start(formatcls):
LOG.debug('Task Start: {0} for run: {1}'.format(
formatcls.name, run_uid))
if formatcls in export_formats:
task = ExportTask.objects.get(run__uid=run_uid,
name=formatcls.name)
task.status = 'RUNNING'
task.started_at = timezone.now()
task.save()
def on_task_success(formatcls, results):
LOG.debug('Task Success: {0} for run: {1}'.format(
formatcls.name, run_uid))
if formatcls in export_formats:
task = ExportTask.objects.get(run__uid=run_uid,
name=formatcls.name)
zipfiles = zipper.run(results)
task.filesize_bytes = sum(
os.stat(zipfile).st_size for zipfile in zipfiles)
task.filenames = [
os.path.basename(zipfile) for zipfile in zipfiles
]
task.status = 'SUCCESS'
task.finished_at = timezone.now()
task.save()
r = RunManager(export_formats,
aoi,
feature_selection,
stage_dir,
map_creator_dir=settings.OSMAND_MAP_CREATOR_DIR,
garmin_splitter=settings.GARMIN_SPLITTER,
garmin_mkgmap=settings.GARMIN_MKGMAP,
per_theme=True,
on_task_start=on_task_start,
on_task_success=on_task_success,
overpass_api_url=settings.OVERPASS_API_URL)
r.run()
public_dir = settings.HOSTNAME + os.path.join(
settings.EXPORT_MEDIA_ROOT, run_uid)
if settings.SYNC_TO_HDX and HDXExportRegion.objects.filter(
job_id=run.job_id).exists():
LOG.debug("Adding resources to HDX")
region = HDXExportRegion.objects.get(job_id=run.job_id)
export_set = region.hdx_dataset
export_set.sync_resources(zipper.zipped_resources, public_dir)
if run.job.hdx_export_region_set.count() == 0:
# not associated with an HDX Export Regon; send mail
send_completion_notification(run)
else:
send_hdx_completion_notification(
run, run.job.hdx_export_region_set.first())
run.status = 'COMPLETED'
LOG.debug('Finished ExportRun with id: {0}'.format(run_uid))
except Exception as e:
client.captureException(extra={
'run_uid': run_uid,
})
run.status = 'FAILED'
LOG.warn('ExportRun {0} failed: {1}'.format(run_uid, e))
LOG.warn(traceback.format_exc())
send_error_notification(run)
if run.job.hdx_export_region_set.count() >= 0:
send_hdx_error_notification(run,
run.job.hdx_export_region_set.first())
finally:
shutil.rmtree(stage_dir)
run.finished_at = timezone.now()
run.save()
def validate_feature_selection(value):
f = FeatureSelection(value)
if not f.valid:
raise ValidationError(f.errors)
def feature_selection_object(self):
"""
a valid FeatureSelection object based off the feature_selection text column.
"""
return FeatureSelection(self.feature_selection)
- highway
where: highway IS NOT NULL
"""
if __name__ == '__main__':
import json
import pprint
from hdx.configuration import Configuration
import requests
Configuration.create(
hdx_site=os.getenv('HDX_SITE', 'demo'),
hdx_key=os.getenv('HDX_API_KEY'),
)
logging.basicConfig()
f_s = FeatureSelection(F_S)
extent = open('hdx_exports/adm0/GIN_adm0.geojson').read()
h = HDXExportSet(dataset_prefix='demodata_test',
name='Geopreview Test',
extent=extent,
feature_selection=f_s,
locations=['GIN'])
h.sync_resources([
Artifact(['hotosm_roads_gpkg.zip'], 'geopackage', theme='roads'),
Artifact(['hotosm_roads_lines_shp.zip'], 'shp', theme='roads')
], 'http://exports-staging.hotosm.org/downloads/4fa2e396-a6bf-4476-829b-c88b953af42c'
)
#pp = pprint.PrettyPrinter(indent=4)
#headers = {
# 'Authorization':os.getenv('HDX_API_KEY'),
TEST_FEATURE_SELECTION = FeatureSelection("""
buildings:
types:
- polygons
select:
- name
- building
- building_levels
- building_materials
- addr_housenumber
- addr_street
- addr_city
- office
where: building IS NOT NULL
roads:
types:
- lines
- polygons
select:
- name
- highway
- surface
- smoothness
- width
- lanes
- oneway
- bridge
- layer
where: highway IS NOT NULL
""")
Some Buildings:
types:
- polygons
select:
- building
where: building is not null
waterways:
types:
- lines
- polygons
select:
- natural
where: natural in ('waterway')
'''
BASIC_FEATURE_SELECTION = FeatureSelection(yaml)
SINGLE_THEME_NOTE = """
OpenStreetMap exports for use in GIS applications.
This theme includes all OpenStreetMap features in this area matching:
"name" IS NOT NULL
Features may have these attributes:
- [name](http://wiki.openstreetmap.org/wiki/Key:name)
This dataset is one of many [OpenStreetMap exports on
HDX](/dataset?tags=openstreetmap).
See the [Humanitarian OpenStreetMap Team](http://hotosm.org/) website for more