def convert(self, file_format, out_file, in_file, params=None):
"""
:param file_format: The OGR format (e.g. KML, GPKG, ESRI Shapefile)
:param out_file: The full path to the converted file.
:param in_file: The full path to the source file.
:param params: any arbitrary parameters passed as a string.
:return: The `out_file`, or raises an exception.
"""
if not params:
params = ""
cmd = "ogr2ogr -f '{format}' {params} {out_file} {in_file}".format(
format=file_format, in_file=in_file, out_file=out_file, params=params)
logger.info('Running: {}'.format(cmd))
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(cmd, shell=True, executable='/bin/bash',
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error('%s', task_process.stderr)
raise Exception("ogr2ogr process failed with returncode: {0}".format(task_process.exitcode))
if self.requires_zip(file_format):
logger.debug("Requires zip: {0}".format(out_file))
out_file = create_zip_file(out_file, get_zip_name(out_file))
logger.debug('ogr2ogr returned: {0}'.format(task_process.exitcode))
return out_file
def merge_geotiffs(in_files, out_file, task_uid=None):
"""
:param in_files: A list of geotiffs.
:param out_file: A location for the result of the merge.
:param task_uid: A task uid to manage the subprocess.
:return: The out_file path.
"""
cmd_template = Template("gdalwarp $in_ds $out_ds")
cmd = cmd_template.safe_substitute({
'in_ds': ' '.join(in_files),
'out_ds': out_file
})
logger.debug("GDAL merge cmd: {0}".format(cmd))
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error('{0}'.format(task_process.stderr))
raise Exception(
"GeoTIFF merge process failed with return code {0}".format(
task_process.exitcode))
return out_file
def convert(self,):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config()
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(max_repeat=int(conf_dict.get('max_repeat', 5)))
mapproxy.cache.geopackage.GeopackageCache.load_tile_metadata = load_tile_metadata
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
auth_requests.patch_https(self.name)
auth_requests.patch_mapproxy_opener_cache(slug=self.name)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(verbose=True, task_uid=self.task_uid, progress_store=progress_store)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(billiard=True, target=seeder.seed,
kwargs={"tasks": seed_configuration.seeds(['seed']),
"concurrency": get_concurrency(conf_dict),
"progress_logger": progress_logger})
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
if task_process.exitcode != 0:
raise Exception("The Raster Service failed to complete, please contact an administrator.")
except Exception:
logger.error("Export failed for url {}.".format(self.service_url))
raise
finally:
connections.close_all()
return self.gpkgfile
def convert(self):
"""
Convert the raw osm to pbf.
"""
convert_cmd = self.cmd.safe_substitute({
"osm": self.osm,
"pbf": self.pbffile
})
if self.debug:
print("Running: %s" % convert_cmd)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(convert_cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error("{0}".format(task_process.stderr))
logger.error("osmconvert failed with return code: {0}".format(
task_process.exitcode))
logger.error(
"osmconvert most commonly fails due to lack of memory.")
raise Exception("Osmconvert Failed.")
if self.debug:
print("Osmconvert returned: %s" % task_process.exitcode)
return self.pbffile
def convert(dataset=None, fmt=None, task_uid=None):
"""
Uses gdalwarp or ogr2ogr to convert a raster or vector dataset into another format.
If the dataset is already in the output format, returns the unaltered original.
:param dataset: Raster or vector file to be converted
:param fmt: Short format (e.g. gpkg, gtiff) to convert into
:param task_uid: A task uid to update
:return: Converted dataset, same filename as input
"""
if not dataset:
raise Exception("Could not open input file: {0}".format(dataset))
meta = get_meta(dataset)
driver, is_raster = meta['driver'], meta['is_raster']
if not fmt or not driver or driver.lower() == fmt.lower():
return dataset
in_ds = os.path.join(os.path.dirname(dataset),
"old_{0}".format(os.path.basename(dataset)))
os.rename(dataset, in_ds)
band_type = ""
if is_raster:
cmd_template = Template(
"gdalwarp -overwrite -of $fmt $type $in_ds $out_ds")
# Geopackage raster only supports byte band type, so check for that
if fmt.lower() == 'gpkg':
band_type = "-ot byte"
else:
cmd_template = Template("ogr2ogr -overwrite -f $fmt $out_ds $in_ds")
cmd = cmd_template.safe_substitute({
'fmt': fmt,
'type': band_type,
'in_ds': in_ds,
'out_ds': dataset
})
logger.debug("GDAL convert cmd: %s", cmd)
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error('{0}'.format(task_process.stderr))
raise Exception(
"Conversion process failed with return code {0}".format(
task_process.exitcode))
return dataset
def get_coverage_with_gdal(self):
# Get username and password from url params, if possible
cred = auth_requests.get_cred(slug=self.name, url=self.service_url)
try:
# Isolate url params
self.params = "&" + self.service_url.split('?')[1].replace('&', '&')
self.service_url = self.service_url.split('?')[0]
finally:
self.service_url += "?"
# Create temporary WCS description XML file for gdal_translate
(wcs_xml_fd, self.wcs_xml_path) = tempfile.mkstemp()
wcs_xml_auth_string = self.wcs_xml_auth.safe_substitute({"userpwd": ":".join(cred)}) if cred else ""
wcs_xml_string = self.wcs_xml.safe_substitute({
'url': self.service_url,
'coverage': self.layer,
'params': self.params,
'auth': wcs_xml_auth_string
})
logger.debug("Creating temporary WCS XML at %s:\n%s", self.wcs_xml_path, wcs_xml_string)
os.write(wcs_xml_fd, wcs_xml_string.encode())
os.close(wcs_xml_fd)
if self.bbox:
convert_cmd = self.cmd.safe_substitute(
{'out': self.out, 'wcs': self.wcs_xml_path, 'minX': self.bbox[0], 'minY': self.bbox[1],
'maxX': self.bbox[2], 'maxY': self.bbox[3], 'fmt': self.format, 'type': self.band_type})
else:
convert_cmd = self.cmd.safe_substitute({'out': self.out, 'wcs': self.wcs_xml_path, 'fmt': self.format,
'type': self.band_type})
logger.debug('WCS command: %s' % convert_cmd)
try:
os.remove(self.out)
except OSError:
pass
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(convert_cmd, shell=True, executable='/bin/sh',
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error('%s', task_process.stderr)
raise Exception("WCS translation failed with code %s: \n%s\n%s",
task_process.exitcode,
convert_cmd,
wcs_xml_string)
if self.debug:
logger.debug('gdal_translate returned: %s', task_process.exitcode)
os.remove(self.wcs_xml_path)
def convert(self):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config()
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(
max_repeat=int(conf_dict.get("max_repeat", 5)), task_uid=self.task_uid
)
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
conf = yaml.safe_load(self.config) or dict()
cert_info = conf.get("cert_info")
auth_requests.patch_https(cert_info=cert_info)
cred_var = conf.get("cred_var")
auth_requests.patch_mapproxy_opener_cache(slug=self.name, cred_var=cred_var)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(
task_uid=self.task_uid,
progress_store=progress_store,
verbose=log_settings.get("verbose"),
silent=log_settings.get("silent"),
)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(
lambda: seeder.seed(
tasks=seed_configuration.seeds(["seed"]),
concurrency=get_concurrency(conf_dict),
progress_logger=progress_logger,
)
)
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
except CancelException:
raise
except Exception as e:
logger.error("Export failed for url {}.".format(self.service_url))
logger.error(e)
raise
finally:
connections.close_all()
return self.gpkgfile
def convert(dataset=None, fmt=None, task_uid=None):
"""
Uses gdalwarp or ogr2ogr to convert a raster or vector dataset into another format.
If the dataset is already in the output format, returns the unaltered original.
:param dataset: Raster or vector file to be converted
:param fmt: Short format (e.g. gpkg, gtiff) to convert into
:param task_uid: A task uid to update
:return: Converted dataset, same filename as input
"""
if not dataset:
raise Exception("Could not open input file: {0}".format(dataset))
meta = get_meta(dataset)
driver, is_raster = meta['driver'], meta['is_raster']
if not fmt or not driver or driver.lower() == fmt.lower():
return dataset
in_ds = os.path.join(os.path.dirname(dataset), "old_{0}".format(os.path.basename(dataset)))
os.rename(dataset, in_ds)
band_type = ""
if is_raster:
cmd_template = Template("gdalwarp -overwrite -of $fmt $type $in_ds $out_ds")
# Geopackage raster only supports byte band type, so check for that
if fmt.lower() == 'gpkg':
band_type = "-ot byte"
else:
cmd_template = Template("ogr2ogr -overwrite -f $fmt $out_ds $in_ds")
cmd = cmd_template.safe_substitute({'fmt': fmt,
'type': band_type,
'in_ds': in_ds,
'out_ds': dataset})
logger.debug("GDAL convert cmd: %s", cmd)
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd, shell=True, executable="/bin/bash",
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error('{0}'.format(task_process.stderr))
raise Exception("Conversion process failed with return code {0}".format(task_process.exitcode))
return dataset
def merge_geotiffs(in_files, out_file, task_uid=None):
"""
:param in_files: A list of geotiffs.
:param out_file: A location for the result of the merge.
:param task_uid: A task uid to track the conversion.
:return: The out_file path.
"""
cmd = get_task_command(convert_raster, in_files, out_file, task_uid=task_uid, driver="gtiff")
try:
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd)
except Exception as e:
logger.error(e)
raise Exception("GeoTIFF merge process failed.")
return out_file
def convert(self, ):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config(
)
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(
max_repeat=int(conf_dict.get('max_repeat', 5)))
mapproxy.cache.geopackage.GeopackageCache.load_tile_metadata = load_tile_metadata
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
auth_requests.patch_https(self.name)
auth_requests.patch_mapproxy_opener_cache(slug=self.name)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(verbose=True,
task_uid=self.task_uid,
progress_store=progress_store)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(billiard=True,
target=seeder.seed,
kwargs={
"tasks":
seed_configuration.seeds(['seed']),
"concurrency":
get_concurrency(conf_dict),
"progress_logger":
progress_logger
})
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
if task_process.exitcode != 0:
raise Exception(
"The Raster Service failed to complete, please contact an administrator."
)
except Exception:
logger.error("Export failed for url {}.".format(self.service_url))
raise
finally:
connections.close_all()
return self.gpkgfile
def merge_geotiffs(in_files, out_file, task_uid=None):
"""
:param in_files: A list of geotiffs.
:param out_file: A location for the result of the merge.
:param task_uid: A task uid to manage the subprocess.
:return: The out_file path.
"""
cmd_template = Template("gdalwarp $in_ds $out_ds")
cmd = cmd_template.safe_substitute({'in_ds': ' '.join(in_files),
'out_ds': out_file})
logger.debug("GDAL merge cmd: {0}".format(cmd))
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd, shell=True, executable="/bin/bash",
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error('{0}'.format(task_process.stderr))
raise Exception("GeoTIFF merge process failed with return code {0}".format(task_process.exitcode))
return out_file
def add_geojson_to_geopackage(geojson=None, gpkg=None, layer_name=None, task_uid=None, user_details=None):
"""Uses an ogr2ogr script to upload a geojson file.
Args:
geojson: A geojson string.
gpkg: Database dict from the django settings.
layer_name: A DB table.
task_uid: A task uid to update.
Returns:
True if the file is successfully uploaded.
"""
# This is just to make it easier to trace when user_details haven't been sent
if user_details is None:
user_details = {"username": "******"}
if not geojson or not gpkg:
raise Exception(
"A valid geojson: {0} was not provided\nor a geopackage: {1} was not accessible.".format(geojson, gpkg)
)
geojson_file = os.path.join(
os.path.dirname(gpkg), "{0}.geojson".format(os.path.splitext(os.path.basename(gpkg))[0])
)
from audit_logging.file_logging import logging_open
with logging_open(geojson_file, "w", user_details=user_details) as open_file:
open_file.write(geojson)
task_process = TaskProcess(task_uid=task_uid)
gpkg = convert(
driver="gpkg",
input_files=gpkg,
output_file=geojson_file,
creation_options=f"-nln {layer_name}",
executor=task_process.start_process,
)
return gpkg
def convert(
boundary=None,
input_file=None,
output_file=None,
src_srs=4326,
driver=None,
layers=None,
layer_name=None,
task_uid=None,
projection: int = 4326,
creation_options: list = None,
dataset_creation_options: list = None,
layer_creation_options: list = None,
is_raster: bool = True,
warp_params: dict = None,
translate_params: dict = None,
use_translate: bool = False,
access_mode: str = "overwrite",
config_options: List[Tuple[str]] = None,
distinct_field=None,
):
"""
Uses gdal to convert and clip a supported dataset file to a mask if boundary is passed in.
:param use_translate: A flag to force the use of translate instead of warp.
:param layer_creation_options: Data options specific to vector conversion.
:param dataset_creation_options: Data options specific to vector conversion.
:param translate_params: A dict of params to pass into gdal translate.
:param warp_params: A dict of params to pass into gdal warp.
:param is_raster: A explicit declaration that dataset is raster (for disambiguating mixed mode files...gpkg)
:param boundary: A geojson file or bbox (xmin, ymin, xmax, ymax) to serve as a cutline
:param input_file: A raster or vector file to be clipped
:param output_file: The dataset to put the clipped output in (if not specified will use in_dataset)
:param driver: Short name of output driver to use (defaults to input format)
:param layer_name: Table name in database for in_dataset
:param layers: A list of layers to include for translation.
:param task_uid: A task uid to update
:param projection: A projection as an int referencing an EPSG code (e.g. 4326 = EPSG:4326)
:param creation_options: Additional options to pass to the convert method (e.g. "-co SOMETHING")
:param config_options: A list of gdal configuration options as a tuple (option, value).
:return: Filename of clipped dataset
"""
if isinstance(input_file, str) and not use_translate:
input_file = [input_file]
meta_list = []
for _index, _file in enumerate(input_file):
input_file[_index], output_file = get_dataset_names(_file, output_file)
meta_list.append(get_meta(input_file[_index], is_raster))
src_src = f"EPSG:{src_srs}"
dst_src = f"EPSG:{projection}"
# Currently, when there are more than 1 files, they much each be the same driver, making the meta the same.
meta = meta_list[0]
if not driver:
driver = meta["driver"] or "gpkg"
# Geopackage raster only supports byte band type, so check for that
band_type = None
dstalpha = None
if driver.lower() == "gpkg":
band_type = gdal.GDT_Byte
if meta.get("nodata") is None and meta.get("is_raster"):
dstalpha = True
# Clip the dataset if a boundary is passed in.
temp_boundfile = None
geojson = None
bbox = None
if boundary:
# Strings are expected to be a file.
if isinstance(boundary, str):
if not os.path.isfile(boundary):
raise Exception(f"Called convert using a boundary of {boundary} but no such path exists.")
elif is_valid_bbox(boundary):
geojson = bbox2polygon(boundary)
bbox = boundary
elif isinstance(boundary, dict):
geojson = boundary
if geojson:
temp_boundfile = NamedTemporaryFile(suffix=".json")
temp_boundfile.write(json.dumps(geojson).encode())
temp_boundfile.flush()
boundary = temp_boundfile.name
if meta["is_raster"]:
cmd = get_task_command(
convert_raster,
input_file,
output_file,
driver=driver,
creation_options=creation_options,
band_type=band_type,
dst_alpha=dstalpha,
boundary=boundary,
src_srs=src_src,
dst_srs=dst_src,
task_uid=task_uid,
warp_params=warp_params,
translate_params=translate_params,
use_translate=use_translate,
config_options=config_options,
)
else:
cmd = get_task_command(
convert_vector,
input_file,
output_file,
driver=driver,
dataset_creation_options=dataset_creation_options,
layer_creation_options=layer_creation_options,
src_srs=src_src,
dst_srs=dst_src,
layers=layers,
layer_name=layer_name,
task_uid=task_uid,
boundary=boundary,
bbox=bbox,
access_mode=access_mode,
config_options=config_options,
distinct_field=distinct_field,
)
try:
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd)
except CancelException:
# If we don't allow cancel exception to propagate then the task won't exit properly.
# TODO: Allow retry state to be more informed.
raise
except Exception as e:
logger.error(e)
raise Exception("File conversion failed. Please try again or contact support.")
finally:
if temp_boundfile:
temp_boundfile.close()
if requires_zip(driver):
logger.debug(f"Requires zip: {output_file}")
output_file = create_zip_file(output_file, get_zip_name(output_file))
return output_file
def clip_dataset(boundary=None, in_dataset=None, out_dataset=None, fmt=None, table=None, task_uid=None):
"""
Uses gdalwarp or ogr2ogr to clip a supported dataset file to a mask.
:param boundary: A geojson file or bbox (xmin, ymin, xmax, ymax) to serve as a cutline
:param in_dataset: A raster or vector file to be clipped
:param out_dataset: The dataset to put the clipped output in (if not specified will use in_dataset)
:param fmt: Short name of output driver to use (defaults to input format)
:param table: Table name in database for in_dataset
:param task_uid: A task uid to update
:return: Filename of clipped dataset
"""
if not boundary:
raise Exception("Could not open boundary mask file: {0}".format(boundary))
if not in_dataset:
raise Exception("Could not open input dataset: {0}".format(in_dataset))
if not out_dataset:
out_dataset = in_dataset
# don't operate on the original file. If the renamed file already exists,
# then don't try to rename, since that file may not exist if this is a retry.
if out_dataset == in_dataset:
in_dataset = os.path.join(os.path.dirname(out_dataset), "old_{0}".format(os.path.basename(out_dataset)))
if not os.path.isfile(in_dataset):
logger.info("Renaming '{}' to '{}'".format(out_dataset, in_dataset))
os.rename(out_dataset, in_dataset)
meta = get_meta(in_dataset)
if not fmt:
fmt = meta['driver'] or 'gpkg'
band_type = ""
# Overwrite is added to the commands in the event that the dataset is retried. In general we want these to
# act idempotently.
if table:
cmd_template = Template("ogr2ogr -overwrite -f $fmt -clipsrc $boundary $out_ds $in_ds $table")
elif meta['is_raster']:
cmd_template = Template("gdalwarp -overwrite -cutline $boundary -crop_to_cutline $dstalpha -of $fmt $type $in_ds $out_ds")
# Geopackage raster only supports byte band type, so check for that
if fmt.lower() == 'gpkg':
band_type = "-ot byte"
else:
cmd_template = Template("ogr2ogr -overwrite -f $fmt -clipsrc $boundary $out_ds $in_ds")
temp_boundfile = None
if isinstance(boundary, list):
boundary = " ".join(str(i) for i in boundary) # ogr2ogr can handle bbox as params
if not table: # gdalwarp needs a file
temp_boundfile = NamedTemporaryFile()
bounds_template = Template('{"type":"MultiPolygon","coordinates":[[[[$xmin,$ymin],'
'[$xmax,$ymin],[$xmax,$y'
'max],[$xmin,$ymax],[$xmin,$ymin]]]]}')
geojson = bounds_template.safe_substitute({
'xmin': boundary[0],
'ymin': boundary[1],
'xmax': boundary[2],
'ymax': boundary[3]
})
temp_boundfile.write(geojson.encode())
temp_boundfile.flush()
boundary = temp_boundfile.name
try:
if meta.get('nodata') is None and not is_envelope(in_dataset):
dstalpha = "-dstalpha"
else:
dstalpha = ""
if table:
cmd = cmd_template.safe_substitute({'boundary': boundary,
'fmt': fmt,
'type': band_type,
'in_ds': in_dataset,
'out_ds': out_dataset,
'table': table})
else:
cmd = cmd_template.safe_substitute({'boundary': boundary,
'fmt': fmt,
'dstalpha': dstalpha,
'type': band_type,
'in_ds': in_dataset,
'out_ds': out_dataset})
logger.debug("GDAL clip cmd: %s", cmd)
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd, shell=True, executable="/bin/bash",
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
finally:
if temp_boundfile:
temp_boundfile.close()
if task_process.exitcode != 0:
logger.error('{0}'.format(task_process.stderr))
raise Exception("Cutline process failed with return code {0}".format(task_process.exitcode))
return out_dataset
def run(self, subtask_percentage=100, subtask_start=0, eta=None):
"""
Create the GeoPackage from the osm data.
"""
from eventkit_cloud.tasks.helpers import update_progress
if self.is_complete:
logger.debug("Skipping Geopackage, file exists")
return
keys_points = self.feature_selection.key_union("points")
keys_lines = self.feature_selection.key_union("lines")
keys_polygons = self.feature_selection.key_union("polygons")
osmconf = OSMConfig(self.stage_dir, points=keys_points, lines=keys_lines, polygons=keys_polygons)
conf = osmconf.create_osm_conf()
logger.debug(
f"Creating OSM gpkg using OSM_MAX_TMPFILE_SIZE {settings.OSM_MAX_TMPFILE_SIZE}"
f"from {self.input_pbf} to {self.output_gpkg}"
)
task_process = TaskProcess(task_uid=self.export_task_record_uid)
convert(
input_files=self.input_pbf,
output_file=self.output_gpkg,
driver="GPKG",
boundary=json.loads(self.aoi_geom.geojson),
config_options=[
("OSM_CONFIG_FILE", conf),
("OGR_INTERLEAVED_READING", "YES"),
("OSM_MAX_TMPFILE_SIZE", settings.OSM_MAX_TMPFILE_SIZE),
],
executor=task_process.start_process,
)
# Cancel the provider task if the geopackage has no data.
if not check_content_exists(self.output_gpkg):
return None
"""
Create the default osm gpkg schema
"""
conn = sqlite3.connect(self.output_gpkg)
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("select load_extension('mod_spatialite')")
cur.execute("CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)")
cur.execute("INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));", (self.aoi_geom.wkb,))
update_progress(self.export_task_record_uid, 30, subtask_percentage, subtask_start, eta=eta)
cur.executescript(SPATIAL_SQL)
self.update_zindexes(cur, self.feature_selection)
update_progress(self.export_task_record_uid, 42, subtask_percentage, subtask_start, eta=eta)
# add themes
create_sqls, index_sqls = self.feature_selection.sqls
for query in create_sqls:
logger.debug(query)
cur.executescript(query)
update_progress(self.export_task_record_uid, 50, subtask_percentage, subtask_start, eta=eta)
for query in index_sqls:
logger.debug(query)
cur.executescript(query)
"""
Remove points/lines/multipolygons tables
"""
cur.execute("DROP TABLE points")
cur.execute("DROP TABLE lines")
cur.execute("DROP TABLE multipolygons")
cur.execute("VACUUM;")
conn.commit()
conn.close()
if self.per_theme:
# this creates per-theme GPKGs
for theme in self.feature_selection.themes:
conn = sqlite3.connect(self.stage_dir + slugify(theme) + ".gpkg")
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("attach database ? as 'geopackage'", (self.output_gpkg,))
cur.execute("create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys")
cur.execute("create table gpkg_contents as select * from geopackage.gpkg_contents where 0")
cur.execute(
"create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0"
)
for geom_type in self.feature_selection.geom_types(theme):
for stmt in self.feature_selection.create_sql(theme, geom_type):
cur.executescript(stmt)
cur.execute("VACUUM;")
conn.commit()
conn.close()
update_progress(self.export_task_record_uid, 100, subtask_percentage, subtask_start, eta=eta)
return self.output_gpkg
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get("service")
params = self.config.get("params")
if not service:
raise Exception(
"A service key needs to be defined to include the scale of source in meters"
)
coverages = service.get("coverages", params.get("COVERAGE"))
coverages = str(coverages).split(",")
if not coverages:
logger.error(
"No coverages were specified for this provider, "
"please specify `coverages` under service or `COVERAGE` under params."
# NOQA
)
raise Exception("Data source incorrectly configured.")
scale = float(service.get("scale"))
params["service"] = "WCS"
width, height = get_dimensions(self.bbox, scale)
tile_bboxes = get_chunked_bbox(self.bbox, (width, height))
geotiffs = []
session = get_or_update_session(slug=self.slug, **self.config)
for idx, coverage in enumerate(coverages):
params["COVERAGE"] = coverage
file_path, ext = os.path.splitext(self.out)
try:
for (
_bbox_idx,
_tile_bbox,
) in enumerate(tile_bboxes):
outfile = "{0}-{1}-{2}{3}".format(file_path, idx,
_bbox_idx, ext)
try:
os.remove(outfile)
except OSError:
pass
# Setting this to arbitrarily high values improves the computed
# resolution but makes the requests slow down.
# If it is set in the config, use that value, otherwise compute approximate res based on scale
if self.config.get("tile_size", None) is None:
tile_x, tile_y = get_dimensions(_tile_bbox, scale)
params["width"] = tile_x
params["height"] = tile_y
else:
params["width"] = self.config.get("tile_size")
params["height"] = self.config.get("tile_size")
params["bbox"] = ",".join(map(str, _tile_bbox))
req = session.get(self.service_url,
params=params,
stream=True)
try:
size = int(req.headers.get("content-length"))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception(
"Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(
self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile,
"wb",
user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
task_process = TaskProcess(self.task_uid)
self.out = merge_geotiffs(geotiffs,
self.out,
executor=task_process.start_process)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
if not get_meta(self.out).get("is_raster"):
with open(self.out, "r") as output_file:
logger.error("Content of failed WCS request")
logger.error(output_file.read())
raise Exception("The service failed to return a proper response")
def get_coverage_with_gdal(self):
# Get username and password from url params, if possible
cred = auth_requests.get_cred(cred_var=self.name, url=self.service_url)
try:
# Isolate url params
self.params = "&" + self.service_url.split("?")[1].replace(
"&", "&")
self.service_url = self.service_url.split("?")[0]
finally:
self.service_url += "?"
# Create temporary WCS description XML file for gdal_translate
(wcs_xml_fd, self.wcs_xml_path) = tempfile.mkstemp()
wcs_xml_auth_string = self.wcs_xml_auth.safe_substitute(
{"userpwd": ":".join(cred)}) if cred else ""
wcs_xml_string = self.wcs_xml.safe_substitute({
"url":
self.service_url,
"coverage":
self.layer,
"params":
self.params,
"auth":
wcs_xml_auth_string
})
logger.debug("Creating temporary WCS XML at %s:\n%s",
self.wcs_xml_path, wcs_xml_string)
os.write(wcs_xml_fd, wcs_xml_string.encode())
os.close(wcs_xml_fd)
if self.bbox:
convert_cmd = self.cmd.safe_substitute({
"out": self.out,
"wcs": self.wcs_xml_path,
"minX": self.bbox[0],
"minY": self.bbox[1],
"maxX": self.bbox[2],
"maxY": self.bbox[3],
"fmt": self.format,
"type": self.band_type,
})
else:
convert_cmd = self.cmd.safe_substitute({
"out": self.out,
"wcs": self.wcs_xml_path,
"fmt": self.format,
"type": self.band_type
})
logger.debug("WCS command: %s" % convert_cmd)
try:
os.remove(self.out)
except OSError:
pass
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(convert_cmd,
shell=True,
executable="/bin/sh",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if task_process.exitcode != 0:
logger.error("%s", task_process.stderr)
raise Exception("WCS translation failed with code %s: \n%s\n%s",
task_process.exitcode, convert_cmd, wcs_xml_string)
if self.debug:
logger.debug("gdal_translate returned: %s", task_process.exitcode)
os.remove(self.wcs_xml_path)
def clip_dataset(boundary=None, in_dataset=None, out_dataset=None, fmt=None, table=None, task_uid=None):
"""
Uses gdalwarp or ogr2ogr to clip a supported dataset file to a mask.
:param boundary: A geojson file or bbox (xmin, ymin, xmax, ymax) to serve as a cutline
:param in_dataset: A raster or vector file to be clipped
:param out_dataset: The dataset to put the clipped output in (if not specified will use in_dataset)
:param fmt: Short name of output driver to use (defaults to input format)
:param table: Table name in database for in_dataset
:param task_uid: A task uid to update
:return: Filename of clipped dataset
"""
if not boundary:
raise Exception("Could not open boundary mask file: {0}".format(boundary))
if not in_dataset:
raise Exception("Could not open input dataset: {0}".format(in_dataset))
if not out_dataset:
out_dataset = in_dataset
# don't operate on the original file. If the renamed file already exists,
# then don't try to rename, since that file may not exist if this is a retry.
if out_dataset == in_dataset:
in_dataset = os.path.join(os.path.dirname(out_dataset), "old_{0}".format(os.path.basename(out_dataset)))
if not os.path.isfile(in_dataset):
logger.info("Renaming '{}' to '{}'".format(out_dataset, in_dataset))
os.rename(out_dataset, in_dataset)
meta = get_meta(in_dataset)
if not fmt:
fmt = meta['driver'] or 'gpkg'
band_type = ""
# Overwrite is added to the commands in the event that the dataset is retried. In general we want these to
# act idempotently.
if table:
cmd_template = Template("ogr2ogr -overwrite -f $fmt -clipsrc $boundary $out_ds $in_ds $table")
elif meta['is_raster']:
cmd_template = Template("gdalwarp -overwrite -cutline $boundary -crop_to_cutline $dstalpha -of $fmt $type $in_ds $out_ds")
# Geopackage raster only supports byte band type, so check for that
if fmt.lower() == 'gpkg':
band_type = "-ot byte"
else:
cmd_template = Template("ogr2ogr -overwrite -f $fmt -clipsrc $boundary $out_ds $in_ds")
temp_boundfile = None
if isinstance(boundary, list):
boundary = " ".join(str(i) for i in boundary) # ogr2ogr can handle bbox as params
if not table: # gdalwarp needs a file
temp_boundfile = NamedTemporaryFile()
bounds_template = Template('{"type":"MultiPolygon","coordinates":[[[[$xmin,$ymin],'
'[$xmax,$ymin],[$xmax,$y'
'max],[$xmin,$ymax],[$xmin,$ymin]]]]}')
geojson = bounds_template.safe_substitute({
'xmin': boundary[0],
'ymin': boundary[1],
'xmax': boundary[2],
'ymax': boundary[3]
})
temp_boundfile.write(geojson.encode())
temp_boundfile.flush()
boundary = temp_boundfile.name
try:
if meta.get('nodata') is None and not is_envelope(in_dataset):
dstalpha = "-dstalpha"
else:
dstalpha = ""
if table:
cmd = cmd_template.safe_substitute({'boundary': boundary,
'fmt': fmt,
'type': band_type,
'in_ds': in_dataset,
'out_ds': out_dataset,
'table': table})
else:
cmd = cmd_template.safe_substitute({'boundary': boundary,
'fmt': fmt,
'dstalpha': dstalpha,
'type': band_type,
'in_ds': in_dataset,
'out_ds': out_dataset})
logger.debug("GDAL clip cmd: %s", cmd)
task_process = TaskProcess(task_uid=task_uid)
task_process.start_process(cmd, shell=True, executable="/bin/bash",
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
finally:
if temp_boundfile:
temp_boundfile.close()
if task_process.exitcode != 0:
logger.error('{0}'.format(task_process.stderr))
raise Exception("Cutline process failed with return code {0}".format(task_process.exitcode))
return out_dataset