def run_task(run_uid, run, stage_dir, download_dir): LOG.debug('Running ExportRun with id: {0}'.format(run_uid)) job = run.job valid_name = get_valid_filename(job.name) geom = load_geometry(job.simplified_geom.json) export_formats = job.export_formats mapping = Mapping(job.feature_selection) def start_task(name): task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'RUNNING' task.started_at = timezone.now() task.save() def finish_task(name, created_files, planet_file=False): LOG.debug('Task Finish: {0} for run: {1}'.format(name, run_uid)) task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'SUCCESS' task.finished_at = timezone.now() # assumes each file only has one part (all are zips or PBFs) task.filenames = [basename(file.parts[0]) for file in created_files] if planet_file is False: total_bytes = 0 for file in created_files: total_bytes += file.size() task.filesize_bytes = total_bytes task.save() is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists() is_partner_export = PartnerExportRegion.objects.filter( job_id=run.job_id).exists() planet_file = False polygon_centroid = False if is_hdx_export: planet_file = HDXExportRegion.objects.get( job_id=run.job_id).planet_file if is_partner_export: export_region = PartnerExportRegion.objects.get(job_id=run.job_id) planet_file = export_region.planet_file polygon_centroid = export_region.polygon_centroid # Run PDC special task. if export_region.group.name == "PDC" and planet_file is True and polygon_centroid is True: params = { "PLANET_FILE": settings.PLANET_FILE, "MAPPING": mapping, "STAGE_DIR": stage_dir, "DOWNLOAD_DIR": download_dir, "VALID_NAME": valid_name } if "geopackage" not in export_formats: raise ValueError("geopackage must be the export format") paths = run_pdc_task(params) start_task("geopackage") target = join(download_dir, "{}.gpkg".format(valid_name)) shutil.move(paths["geopackage"], target) os.chmod(target, 0o644) finish_task("geopackage", [osm_export_tool.File("gpkg", [target], '')], planet_file) send_completion_notification(run) run.status = 'COMPLETED' run.finished_at = timezone.now() run.save() LOG.debug('Finished ExportRun with id: {0}'.format(run_uid)) return if is_hdx_export: geopackage = None shp = None kml = None tabular_outputs = [] if 'geopackage' in export_formats: geopackage = tabular.MultiGeopackage(join(stage_dir, valid_name), mapping) tabular_outputs.append(geopackage) start_task('geopackage') if 'shp' in export_formats: shp = tabular.Shapefile(join(stage_dir, valid_name), mapping) tabular_outputs.append(shp) start_task('shp') if 'kml' in export_formats: kml = tabular.Kml(join(stage_dir, valid_name), mapping) tabular_outputs.append(kml) start_task('kml') if planet_file: h = tabular.Handler(tabular_outputs, mapping, polygon_centroid=polygon_centroid) source = OsmiumTool('osmium', settings.PLANET_FILE, geom, join(stage_dir, 'extract.osm.pbf'), tempdir=stage_dir) else: h = tabular.Handler(tabular_outputs, mapping, clipping_geom=geom, polygon_centroid=polygon_centroid) mapping_filter = mapping if job.unfiltered: mapping_filter = None source = Overpass(settings.OVERPASS_API_URL, geom, join(stage_dir, 'overpass.osm.pbf'), tempdir=stage_dir, use_curl=True, mapping=mapping_filter) LOG.debug('Source start for run: {0}'.format(run_uid)) source_path = source.path() LOG.debug('Source end for run: {0}'.format(run_uid)) h.apply_file(source_path, locations=True, idx='sparse_file_array') all_zips = [] def add_metadata(z, theme): columns = [] for key in theme.keys: columns.append( '{0} http://wiki.openstreetmap.org/wiki/Key:{0}'.format( key)) columns = '\n'.join(columns) readme = ZIP_README.format(criteria=theme.matcher.to_sql(), columns=columns) z.writestr("README.txt", readme) if geopackage: geopackage.finalize() zips = [] for theme in mapping.themes: destination = join( download_dir, valid_name + '_' + slugify(theme.name) + '_gpkg.zip') matching_files = [ f for f in geopackage.files if 'theme' in f.extra and f.extra['theme'] == theme.name ] with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: add_metadata(z, theme) for file in matching_files: for part in file.parts: z.write(part, os.path.basename(part)) zips.append( osm_export_tool.File('geopackage', [destination], {'theme': theme.name})) finish_task('geopackage', zips) all_zips += zips if shp: shp.finalize() zips = [] for file in shp.files: # for HDX geopreview to work # each file (_polygons, _lines) is a separate zip resource # the zipfile must end with only .zip (not .shp.zip) destination = join( download_dir, os.path.basename(file.parts[0]).replace('.', '_') + '.zip') with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: theme = [ t for t in mapping.themes if t.name == file.extra['theme'] ][0] add_metadata(z, theme) for part in file.parts: z.write(part, os.path.basename(part)) zips.append( osm_export_tool.File('shp', [destination], {'theme': file.extra['theme']})) finish_task('shp', zips) all_zips += zips if kml: kml.finalize() zips = [] for file in kml.files: destination = join( download_dir, os.path.basename(file.parts[0]).replace('.', '_') + '.zip') with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: theme = [ t for t in mapping.themes if t.name == file.extra['theme'] ][0] add_metadata(z, theme) for part in file.parts: z.write(part, os.path.basename(part)) zips.append( osm_export_tool.File('kml', [destination], {'theme': file.extra['theme']})) finish_task('kml', zips) all_zips += zips if 'garmin_img' in export_formats: start_task('garmin_img') garmin_files = nontabular.garmin(source_path, settings.GARMIN_SPLITTER, settings.GARMIN_MKGMAP, tempdir=stage_dir) zipped = create_package(join(download_dir, valid_name + '_gmapsupp_img.zip'), garmin_files, boundary_geom=geom, output_name='garmin_img') all_zips.append(zipped) finish_task('garmin_img', [zipped]) if settings.SYNC_TO_HDX: print("Syncing to HDX") region = HDXExportRegion.objects.get(job_id=run.job_id) public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid) sync_region(region, all_zips, public_dir) send_hdx_completion_notification(run, run.job.hdx_export_region_set.first()) else: geopackage = None shp = None kml = None tabular_outputs = [] if 'geopackage' in export_formats: geopackage = tabular.Geopackage(join(stage_dir, valid_name), mapping) tabular_outputs.append(geopackage) start_task('geopackage') if 'shp' in export_formats: shp = tabular.Shapefile(join(stage_dir, valid_name), mapping) tabular_outputs.append(shp) start_task('shp') if 'kml' in export_formats: kml = tabular.Kml(join(stage_dir, valid_name), mapping) tabular_outputs.append(kml) start_task('kml') if planet_file: h = tabular.Handler(tabular_outputs, mapping, polygon_centroid=polygon_centroid) source = OsmiumTool('osmium', settings.PLANET_FILE, geom, join(stage_dir, 'extract.osm.pbf'), tempdir=stage_dir, mapping=mapping) else: h = tabular.Handler(tabular_outputs, mapping, clipping_geom=geom, polygon_centroid=polygon_centroid) mapping_filter = mapping if job.unfiltered: mapping_filter = None source = Overpass(settings.OVERPASS_API_URL, geom, join(stage_dir, 'overpass.osm.pbf'), tempdir=stage_dir, use_curl=True, mapping=mapping_filter) LOG.debug('Source start for run: {0}'.format(run_uid)) source_path = source.path() LOG.debug('Source end for run: {0}'.format(run_uid)) h.apply_file(source_path, locations=True, idx='sparse_file_array') bundle_files = [] if geopackage: geopackage.finalize() zipped = create_package(join(download_dir, valid_name + '_gpkg.zip'), geopackage.files, boundary_geom=geom) bundle_files += geopackage.files finish_task('geopackage', [zipped]) if shp: shp.finalize() zipped = create_package(join(download_dir, valid_name + '_shp.zip'), shp.files, boundary_geom=geom) bundle_files += shp.files finish_task('shp', [zipped]) if kml: kml.finalize() zipped = create_package(join(download_dir, valid_name + '_kml.zip'), kml.files, boundary_geom=geom) bundle_files += kml.files finish_task('kml', [zipped]) if 'garmin_img' in export_formats: start_task('garmin_img') garmin_files = nontabular.garmin(source_path, settings.GARMIN_SPLITTER, settings.GARMIN_MKGMAP, tempdir=stage_dir) bundle_files += garmin_files zipped = create_package(join(download_dir, valid_name + '_gmapsupp_img.zip'), garmin_files, boundary_geom=geom) finish_task('garmin_img', [zipped]) if 'mwm' in export_formats: start_task('mwm') mwm_dir = join(stage_dir, 'mwm') if not exists(mwm_dir): os.makedirs(mwm_dir) mwm_files = nontabular.mwm(source_path, mwm_dir, settings.GENERATE_MWM, settings.GENERATOR_TOOL) bundle_files += mwm_files zipped = create_package(join(download_dir, valid_name + '_mwm.zip'), mwm_files, boundary_geom=geom) finish_task('mwm', [zipped]) if 'osmand_obf' in export_formats: start_task('osmand_obf') osmand_files = nontabular.osmand(source_path, settings.OSMAND_MAP_CREATOR_DIR, tempdir=stage_dir) bundle_files += osmand_files zipped = create_package(join(download_dir, valid_name + '_Osmand2_obf.zip'), osmand_files, boundary_geom=geom) finish_task('osmand_obf', [zipped]) if 'mbtiles' in export_formats: start_task('mbtiles') mbtiles_files = nontabular.mbtiles( geom, join(stage_dir, valid_name + '.mbtiles'), job.mbtiles_source, job.mbtiles_minzoom, job.mbtiles_maxzoom) bundle_files += mbtiles_files zipped = create_package(join(download_dir, valid_name + '_mbtiles.zip'), mbtiles_files, boundary_geom=geom) finish_task('mbtiles', [zipped]) if 'osm_pbf' in export_formats: bundle_files += [ osm_export_tool.File('osm_pbf', [source_path], '') ] if 'bundle' in export_formats: start_task('bundle') zipped = create_posm_bundle( join(download_dir, valid_name + '-bundle.tar.gz'), bundle_files, job.name, valid_name, job.description, geom) finish_task('bundle', [zipped]) # do this last so we can do a mv instead of a copy if 'osm_pbf' in export_formats: start_task('osm_pbf') target = join(download_dir, valid_name + '.osm.pbf') shutil.move(source_path, target) os.chmod(target, 0o644) finish_task('osm_pbf', [osm_export_tool.File('pbf', [target], '')], planet_file) send_completion_notification(run) run.status = 'COMPLETED' run.finished_at = timezone.now() run.save() LOG.debug('Finished ExportRun with id: {0}'.format(run_uid))
use_existing=False) shp = tabular.Shapefile("tmp/example", mapping) gpkg = tabular.Geopackage("tmp/example", mapping) kml = tabular.Kml("tmp/example", mapping) tabular_outputs = [shp, gpkg, kml] h = tabular.Handler(tabular_outputs, mapping) h.apply_file(source.path(), locations=True, idx='sparse_file_array') for output in tabular_outputs: output.finalize() osmand_files = nontabular.osmand(source.path(), 'tools/OsmAndMapCreator-main', tempdir=tempdir) garmin_files = nontabular.garmin(source.path(), 'tools/splitter-r583/splitter.jar', 'tools/mkgmap-r3890/mkgmap.jar', tempdir=tempdir) #mwm_files = nontabular.mwm(source.path(),join(tempdir,'mwm'),'generate_mwm.sh','/usr/local/bin/generator_tool','/usr/bin/osmconvert') #mbtiles_files = nontabular.mbtiles(geom,join(tempdir,'output.mbtiles'),'http://tile.openstreetmap.org/{z}/{x}/{y}.png',14,14) files = [] files += shp.files files += gpkg.files files += kml.files files += osmand_files files += garmin_files #files += mbtiles_files