def test_geom_types(self):
y = '''
buildings:
types:
- points
select:
- name
'''
m = Mapping(y)
self.assertTrue(m.themes[0].points)
self.assertFalse(m.themes[0].lines)
self.assertFalse(m.themes[0].polygons)
y = '''
buildings:
types:
- lines
select:
- name
'''
m = Mapping(y)
self.assertFalse(m.themes[0].points)
self.assertTrue(m.themes[0].lines)
self.assertFalse(m.themes[0].polygons)
y = '''
buildings:
types:
- polygons
select:
- name
'''
m = Mapping(y)
self.assertFalse(m.themes[0].points)
self.assertFalse(m.themes[0].lines)
self.assertTrue(m.themes[0].polygons)
def test_default_osm_id(self):
y = '''
buildings:
types:
- polygons
select:
- building
'''
m = Mapping(y)
self.assertTrue(m.themes[0].osm_id)
self.assertFalse('osm_id' in m.themes[0].keys)
m = Mapping(y, default_osm_id=False)
self.assertFalse(m.themes[0].osm_id)
def test_mapping(self):
y = '''
buildings:
types:
- points
select:
- column1
where: column2 IS NOT NULL
other1:
types:
- points
- polygons
select:
- column1
- irrelevant
where: column2 IS NOT NULL AND column3 IN ('foo','bar')
other2:
types:
- lines
select:
- column5:key
'''
mapping = Mapping(y)
nodes, ways, relations = Overpass.filters(mapping)
self.assertCountEqual(nodes, ["['column3'~'foo|bar']", "['column2']"])
# force quoting of strings to handle keys with colons
self.assertCountEqual(
ways, ["['column5:key']", "['column3'~'foo|bar']", "['column2']"])
self.assertCountEqual(relations,
["['column3'~'foo|bar']", "['column2']"])
def sync_region(region, files=[], public_dir=''):
export_set = HDXExportSet(Mapping(region.feature_selection),
region.dataset_prefix, region.name,
region.extra_notes)
datasets = export_set.datasets(region.is_private, region.subnational,
region.update_frequency, region.locations,
files, public_dir)
sync_datasets(datasets, len(files) > 0)
def datasets(self): # noqa
export_set = HDXExportSet(
Mapping(self.feature_selection),
self.dataset_prefix,
self.name,
self.extra_notes
)
return export_set.dataset_links(settings.HDX_URL_PREFIX)
def test_no_select(self):
y = '''
buildings:
types:
- polygons
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
def test_extra(self):
y = '''
buildings:
foo:
bar: baz
select:
- building
'''
m = Mapping(y)
self.assertEqual(m.themes[0].extra, {'foo': {'bar': 'baz'}})
def test_key_selections(self):
y = '''
buildings:
types:
- polygons
select:
- addr:housenumber
'''
m = Mapping(y)
self.assertTrue('addr:housenumber' in m.themes[0].keys)
def test_wrong_yaml_list(self):
y = '''
buildings:
types: polygons
select:
- building
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
def test_wrong_toplevel_themes(self):
y = '''
- buildings:
types:
- polygons
select:
- building
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
def test_invalid_yaml_parse(self):
y = '''
buildings:
- types:
- polygons
select:
- building
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
def test_duplicate_key(self):
y = '''
buildings:
types:
- polygons
select:
- building
- building
'''
m = Mapping(y)
self.assertTrue(len(m.themes[0].keys) == 1)
def test_default_matcher(self):
y = '''
buildings:
types:
- polygons
select:
- addr:housenumber
'''
m = Mapping(y)
self.assertTrue(m.themes[0].matches(GeomType.POLYGON,
{'addr:housenumber': '1234'}))
def test_gt(self):
y = '''
buildings:
types:
- polygons
select:
- building
where: height > 20
'''
m = Mapping(y)
self.assertTrue(m.themes[0].matches(GeomType.POLYGON, {'height': 21}))
self.assertFalse(m.themes[0].matches(GeomType.POLYGON, {'height': 20}))
def test_invalid_sql(self):
y = '''
buildings:
types:
- polygons
select:
- building
where: XXX aaa
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
def test_wrong_yaml_parse(self):
y = '''
buildings:
- types:
- polygons
- select:
- building
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
self.assertTrue('must be YAML dict' in errors[0])
def test_nonlist_matcher(self):
y = '''
buildings:
types:
- polygons
select:
- addr:housenumber
where: building = 'yes'
'''
m = Mapping(y)
self.assertTrue(m.themes[0].matches(GeomType.POLYGON,
{'building': 'yes'}))
def test_basic_mapping(self):
y = '''
buildings:
select:
- name
'''
m = Mapping(y)
self.assertEqual(len(m.themes), 1)
theme = m.themes[0]
self.assertEqual(theme.name, 'buildings')
self.assertTrue(theme.points)
self.assertTrue(theme.lines)
self.assertTrue(theme.polygons)
self.assertTrue('name' in theme.keys)
def test_multiple_matchers(self):
y = '''
buildings:
types:
- polygons
select:
- addr:housenumber
where:
- building = 'yes'
- amenity = 'parking'
'''
m = Mapping(y)
self.assertTrue(m.themes[0].matches(GeomType.POLYGON,
{'building': 'yes'}))
self.assertTrue(m.themes[0].matches(GeomType.POLYGON,
{'amenity': 'parking'}))
def validate_feature_selection(value):
from osm_export_tool.mapping import Mapping
m, errors = Mapping.validate(value)
if not m:
raise ValidationError(errors)
def test_empty_yaml(self):
y = '''
'''
m, errors = Mapping.validate(y)
self.assertTrue(m is None)
self.assertTrue(len(errors) == 1)
def run_task(run_uid, run, stage_dir, download_dir):
LOG.debug('Running ExportRun with id: {0}'.format(run_uid))
job = run.job
valid_name = get_valid_filename(job.name)
geom = load_geometry(job.simplified_geom.json)
export_formats = job.export_formats
mapping = Mapping(job.feature_selection)
def start_task(name):
task = ExportTask.objects.get(run__uid=run_uid, name=name)
task.status = 'RUNNING'
task.started_at = timezone.now()
task.save()
def finish_task(name, created_files, planet_file=False):
LOG.debug('Task Finish: {0} for run: {1}'.format(name, run_uid))
task = ExportTask.objects.get(run__uid=run_uid, name=name)
task.status = 'SUCCESS'
task.finished_at = timezone.now()
# assumes each file only has one part (all are zips or PBFs)
task.filenames = [basename(file.parts[0]) for file in created_files]
if planet_file is False:
total_bytes = 0
for file in created_files:
total_bytes += file.size()
task.filesize_bytes = total_bytes
task.save()
is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists()
is_partner_export = PartnerExportRegion.objects.filter(
job_id=run.job_id).exists()
planet_file = False
polygon_centroid = False
if is_hdx_export:
planet_file = HDXExportRegion.objects.get(
job_id=run.job_id).planet_file
if is_partner_export:
export_region = PartnerExportRegion.objects.get(job_id=run.job_id)
planet_file = export_region.planet_file
polygon_centroid = export_region.polygon_centroid
# Run PDC special task.
if export_region.group.name == "PDC" and planet_file is True and polygon_centroid is True:
params = {
"PLANET_FILE": settings.PLANET_FILE,
"MAPPING": mapping,
"STAGE_DIR": stage_dir,
"DOWNLOAD_DIR": download_dir,
"VALID_NAME": valid_name
}
if "geopackage" not in export_formats:
raise ValueError("geopackage must be the export format")
paths = run_pdc_task(params)
start_task("geopackage")
target = join(download_dir, "{}.gpkg".format(valid_name))
shutil.move(paths["geopackage"], target)
os.chmod(target, 0o644)
finish_task("geopackage",
[osm_export_tool.File("gpkg", [target], '')],
planet_file)
send_completion_notification(run)
run.status = 'COMPLETED'
run.finished_at = timezone.now()
run.save()
LOG.debug('Finished ExportRun with id: {0}'.format(run_uid))
return
if is_hdx_export:
geopackage = None
shp = None
kml = None
tabular_outputs = []
if 'geopackage' in export_formats:
geopackage = tabular.MultiGeopackage(join(stage_dir, valid_name),
mapping)
tabular_outputs.append(geopackage)
start_task('geopackage')
if 'shp' in export_formats:
shp = tabular.Shapefile(join(stage_dir, valid_name), mapping)
tabular_outputs.append(shp)
start_task('shp')
if 'kml' in export_formats:
kml = tabular.Kml(join(stage_dir, valid_name), mapping)
tabular_outputs.append(kml)
start_task('kml')
if planet_file:
h = tabular.Handler(tabular_outputs,
mapping,
polygon_centroid=polygon_centroid)
source = OsmiumTool('osmium',
settings.PLANET_FILE,
geom,
join(stage_dir, 'extract.osm.pbf'),
tempdir=stage_dir)
else:
h = tabular.Handler(tabular_outputs,
mapping,
clipping_geom=geom,
polygon_centroid=polygon_centroid)
mapping_filter = mapping
if job.unfiltered:
mapping_filter = None
source = Overpass(settings.OVERPASS_API_URL,
geom,
join(stage_dir, 'overpass.osm.pbf'),
tempdir=stage_dir,
use_curl=True,
mapping=mapping_filter)
LOG.debug('Source start for run: {0}'.format(run_uid))
source_path = source.path()
LOG.debug('Source end for run: {0}'.format(run_uid))
h.apply_file(source_path, locations=True, idx='sparse_file_array')
all_zips = []
def add_metadata(z, theme):
columns = []
for key in theme.keys:
columns.append(
'{0} http://wiki.openstreetmap.org/wiki/Key:{0}'.format(
key))
columns = '\n'.join(columns)
readme = ZIP_README.format(criteria=theme.matcher.to_sql(),
columns=columns)
z.writestr("README.txt", readme)
if geopackage:
geopackage.finalize()
zips = []
for theme in mapping.themes:
destination = join(
download_dir,
valid_name + '_' + slugify(theme.name) + '_gpkg.zip')
matching_files = [
f for f in geopackage.files
if 'theme' in f.extra and f.extra['theme'] == theme.name
]
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED,
True) as z:
add_metadata(z, theme)
for file in matching_files:
for part in file.parts:
z.write(part, os.path.basename(part))
zips.append(
osm_export_tool.File('geopackage', [destination],
{'theme': theme.name}))
finish_task('geopackage', zips)
all_zips += zips
if shp:
shp.finalize()
zips = []
for file in shp.files:
# for HDX geopreview to work
# each file (_polygons, _lines) is a separate zip resource
# the zipfile must end with only .zip (not .shp.zip)
destination = join(
download_dir,
os.path.basename(file.parts[0]).replace('.', '_') + '.zip')
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED,
True) as z:
theme = [
t for t in mapping.themes
if t.name == file.extra['theme']
][0]
add_metadata(z, theme)
for part in file.parts:
z.write(part, os.path.basename(part))
zips.append(
osm_export_tool.File('shp', [destination],
{'theme': file.extra['theme']}))
finish_task('shp', zips)
all_zips += zips
if kml:
kml.finalize()
zips = []
for file in kml.files:
destination = join(
download_dir,
os.path.basename(file.parts[0]).replace('.', '_') + '.zip')
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED,
True) as z:
theme = [
t for t in mapping.themes
if t.name == file.extra['theme']
][0]
add_metadata(z, theme)
for part in file.parts:
z.write(part, os.path.basename(part))
zips.append(
osm_export_tool.File('kml', [destination],
{'theme': file.extra['theme']}))
finish_task('kml', zips)
all_zips += zips
if 'garmin_img' in export_formats:
start_task('garmin_img')
garmin_files = nontabular.garmin(source_path,
settings.GARMIN_SPLITTER,
settings.GARMIN_MKGMAP,
tempdir=stage_dir)
zipped = create_package(join(download_dir,
valid_name + '_gmapsupp_img.zip'),
garmin_files,
boundary_geom=geom,
output_name='garmin_img')
all_zips.append(zipped)
finish_task('garmin_img', [zipped])
if settings.SYNC_TO_HDX:
print("Syncing to HDX")
region = HDXExportRegion.objects.get(job_id=run.job_id)
public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT,
run_uid)
sync_region(region, all_zips, public_dir)
send_hdx_completion_notification(run,
run.job.hdx_export_region_set.first())
else:
geopackage = None
shp = None
kml = None
tabular_outputs = []
if 'geopackage' in export_formats:
geopackage = tabular.Geopackage(join(stage_dir, valid_name),
mapping)
tabular_outputs.append(geopackage)
start_task('geopackage')
if 'shp' in export_formats:
shp = tabular.Shapefile(join(stage_dir, valid_name), mapping)
tabular_outputs.append(shp)
start_task('shp')
if 'kml' in export_formats:
kml = tabular.Kml(join(stage_dir, valid_name), mapping)
tabular_outputs.append(kml)
start_task('kml')
if planet_file:
h = tabular.Handler(tabular_outputs,
mapping,
polygon_centroid=polygon_centroid)
source = OsmiumTool('osmium',
settings.PLANET_FILE,
geom,
join(stage_dir, 'extract.osm.pbf'),
tempdir=stage_dir,
mapping=mapping)
else:
h = tabular.Handler(tabular_outputs,
mapping,
clipping_geom=geom,
polygon_centroid=polygon_centroid)
mapping_filter = mapping
if job.unfiltered:
mapping_filter = None
source = Overpass(settings.OVERPASS_API_URL,
geom,
join(stage_dir, 'overpass.osm.pbf'),
tempdir=stage_dir,
use_curl=True,
mapping=mapping_filter)
LOG.debug('Source start for run: {0}'.format(run_uid))
source_path = source.path()
LOG.debug('Source end for run: {0}'.format(run_uid))
h.apply_file(source_path, locations=True, idx='sparse_file_array')
bundle_files = []
if geopackage:
geopackage.finalize()
zipped = create_package(join(download_dir,
valid_name + '_gpkg.zip'),
geopackage.files,
boundary_geom=geom)
bundle_files += geopackage.files
finish_task('geopackage', [zipped])
if shp:
shp.finalize()
zipped = create_package(join(download_dir,
valid_name + '_shp.zip'),
shp.files,
boundary_geom=geom)
bundle_files += shp.files
finish_task('shp', [zipped])
if kml:
kml.finalize()
zipped = create_package(join(download_dir,
valid_name + '_kml.zip'),
kml.files,
boundary_geom=geom)
bundle_files += kml.files
finish_task('kml', [zipped])
if 'garmin_img' in export_formats:
start_task('garmin_img')
garmin_files = nontabular.garmin(source_path,
settings.GARMIN_SPLITTER,
settings.GARMIN_MKGMAP,
tempdir=stage_dir)
bundle_files += garmin_files
zipped = create_package(join(download_dir,
valid_name + '_gmapsupp_img.zip'),
garmin_files,
boundary_geom=geom)
finish_task('garmin_img', [zipped])
if 'mwm' in export_formats:
start_task('mwm')
mwm_dir = join(stage_dir, 'mwm')
if not exists(mwm_dir):
os.makedirs(mwm_dir)
mwm_files = nontabular.mwm(source_path, mwm_dir,
settings.GENERATE_MWM,
settings.GENERATOR_TOOL)
bundle_files += mwm_files
zipped = create_package(join(download_dir,
valid_name + '_mwm.zip'),
mwm_files,
boundary_geom=geom)
finish_task('mwm', [zipped])
if 'osmand_obf' in export_formats:
start_task('osmand_obf')
osmand_files = nontabular.osmand(source_path,
settings.OSMAND_MAP_CREATOR_DIR,
tempdir=stage_dir)
bundle_files += osmand_files
zipped = create_package(join(download_dir,
valid_name + '_Osmand2_obf.zip'),
osmand_files,
boundary_geom=geom)
finish_task('osmand_obf', [zipped])
if 'mbtiles' in export_formats:
start_task('mbtiles')
mbtiles_files = nontabular.mbtiles(
geom, join(stage_dir, valid_name + '.mbtiles'),
job.mbtiles_source, job.mbtiles_minzoom, job.mbtiles_maxzoom)
bundle_files += mbtiles_files
zipped = create_package(join(download_dir,
valid_name + '_mbtiles.zip'),
mbtiles_files,
boundary_geom=geom)
finish_task('mbtiles', [zipped])
if 'osm_pbf' in export_formats:
bundle_files += [
osm_export_tool.File('osm_pbf', [source_path], '')
]
if 'bundle' in export_formats:
start_task('bundle')
zipped = create_posm_bundle(
join(download_dir, valid_name + '-bundle.tar.gz'),
bundle_files, job.name, valid_name, job.description, geom)
finish_task('bundle', [zipped])
# do this last so we can do a mv instead of a copy
if 'osm_pbf' in export_formats:
start_task('osm_pbf')
target = join(download_dir, valid_name + '.osm.pbf')
shutil.move(source_path, target)
os.chmod(target, 0o644)
finish_task('osm_pbf', [osm_export_tool.File('pbf', [target], '')],
planet_file)
send_completion_notification(run)
run.status = 'COMPLETED'
run.finished_at = timezone.now()
run.save()
LOG.debug('Finished ExportRun with id: {0}'.format(run_uid))
[
[-155.077815, 19.722514],
[-155.087643, 19.722514],
[-155.087643, 19.715929],
[-155.077815, 19.715929],
[-155.077815, 19.722514]
]
]]
}"""
geom = load_geometry(GEOJSON)
tempdir = 'tmp'
with open('../osm_export_tool/mappings/default.yml', 'r') as f:
mapping_txt = f.read()
mapping = Mapping(mapping_txt)
source = Overpass('http://overpass.hotosm.org',
geom,
join(tempdir, 'extract.osm.pbf'),
tempdir=tempdir,
mapping=mapping,
use_existing=False)
shp = tabular.Shapefile("tmp/example", mapping)
gpkg = tabular.Geopackage("tmp/example", mapping)
kml = tabular.Kml("tmp/example", mapping)
tabular_outputs = [shp, gpkg, kml]
h = tabular.Handler(tabular_outputs, mapping)