def results(self):
if self.per_theme:
results_list = []
for theme in self.feature_selection.themes:
results_list.append(Artifact([os.path.join(self.stage_dir, slugify(theme)) + ".gpkg"], Geopackage.name, theme=theme))
return results_list
else:
return [Artifact([self.output_gpkg], Geopackage.name)]
def results(self):
if self.per_theme:
results_list = []
for theme in self.feature_selection.themes:
results_list.append(
Artifact([os.path.join(self.stage_dir, slugify(theme)) + ".gpkg"], Geopackage.name, theme=theme))
return results_list
else:
return [Artifact([self.output_gpkg], Geopackage.name)]
def wrapper(*args, **kwargs):
# handle if first arg is self.
arg = type(args[0]).__name__ if inspect.isclass(args[0]) else str(
slugify(args[0]))
arg_string = ".".join([str(slugify(arg)) for arg in args[:1]])
keys = (copy.deepcopy(kwargs) if not key_fields else {
key_field: kwargs.get(key_field)
for key_field in key_fields
})
kwarg_string = ".".join([
str(slugify(kwarg))
for kwarg in itertools.chain.from_iterable(keys.items())
])
cache_key = f"{func.__name__}.{arg}.{arg_string}.{kwarg_string}"[:
249]
logger.debug("Getting or setting the cache_key %s", cache_key)
return_value = cache.get_or_set(cache_key,
lambda: func(*args, **kwargs),
timeout=timeout)
return return_value
def get_layers(self) -> LayersDescription:
cap_doc = self.get_capabilities(layer_id=self.layer)
if cap_doc and self.layer:
return {self.layer: {"name": str(self.layer), "url": str(cap_doc["url"])}}
if cap_doc.get("layers") or cap_doc.get("subLayers"):
# TODO: This logic is specific for feature layers,
# this will need to change or be subclassed to separate raster/feature services.
return {
slugify(layer["name"]): layer
for layer in (cap_doc.get("subLayers", []) or cap_doc.get("layers", []))
if "Feature" in layer["type"]
}
return {self.layer: {"name": str(self.layer), "url": self.service_url}}
def run(self, subtask_percentage=100, subtask_start=0, eta=None):
"""
Create the GeoPackage from the osm data.
"""
if self.is_complete:
logger.debug("Skipping Geopackage, file exists")
return
keys_points = self.feature_selection.key_union("points")
keys_lines = self.feature_selection.key_union("lines")
keys_polygons = self.feature_selection.key_union("polygons")
osmconf = OSMConfig(self.stage_dir,
points=keys_points,
lines=keys_lines,
polygons=keys_polygons)
conf = osmconf.create_osm_conf()
logger.debug(
f"Creating OSM gpkg using OSM_MAX_TMPFILE_SIZE {settings.OSM_MAX_TMPFILE_SIZE}"
f"from {self.input_pbf} to {self.output_gpkg}")
gdalutils.convert(
input_file=self.input_pbf,
output_file=self.output_gpkg,
driver="GPKG",
boundary=json.loads(self.aoi_geom.geojson),
config_options=[
("OSM_CONFIG_FILE", conf),
("OGR_INTERLEAVED_READING", "YES"),
("OSM_MAX_TMPFILE_SIZE", settings.OSM_MAX_TMPFILE_SIZE),
],
task_uid=self.export_task_record_uid,
)
# Cancel the provider task if the geopackage has no data.
if not check_content_exists(self.output_gpkg):
return None
"""
Create the default osm gpkg schema
"""
conn = sqlite3.connect(self.output_gpkg)
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("select load_extension('mod_spatialite')")
cur.execute(
"CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)"
)
cur.execute(
"INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));",
(self.aoi_geom.wkb, ))
update_progress(self.export_task_record_uid,
30,
subtask_percentage,
subtask_start,
eta=eta)
cur.executescript(SPATIAL_SQL)
self.update_zindexes(cur, self.feature_selection)
update_progress(self.export_task_record_uid,
42,
subtask_percentage,
subtask_start,
eta=eta)
# add themes
create_sqls, index_sqls = self.feature_selection.sqls
for query in create_sqls:
logger.debug(query)
cur.executescript(query)
update_progress(self.export_task_record_uid,
50,
subtask_percentage,
subtask_start,
eta=eta)
for query in index_sqls:
logger.debug(query)
cur.executescript(query)
"""
Remove points/lines/multipolygons tables
"""
cur.execute("DROP TABLE points")
cur.execute("DROP TABLE lines")
cur.execute("DROP TABLE multipolygons")
cur.execute("VACUUM;")
conn.commit()
conn.close()
if self.per_theme:
# this creates per-theme GPKGs
for theme in self.feature_selection.themes:
conn = sqlite3.connect(self.stage_dir + slugify(theme) +
".gpkg")
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("attach database ? as 'geopackage'",
(self.output_gpkg, ))
cur.execute(
"create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys"
)
cur.execute(
"create table gpkg_contents as select * from geopackage.gpkg_contents where 0"
)
cur.execute(
"create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0"
)
for geom_type in self.feature_selection.geom_types(theme):
for stmt in self.feature_selection.create_sql(
theme, geom_type):
cur.executescript(stmt)
cur.execute("VACUUM;")
conn.commit()
conn.close()
update_progress(self.export_task_record_uid,
100,
subtask_percentage,
subtask_start,
eta=eta)
return self.output_gpkg
def run(self):
"""
Create the GeoPackage from the osm data.
"""
if self.is_complete:
LOG.debug("Skipping Geopackage, file exists")
return
keys_points = self.feature_selection.key_union('points')
keys_lines = self.feature_selection.key_union('lines')
keys_polygons = self.feature_selection.key_union('polygons')
osmconf = OSMConfig(self.stage_dir, points=keys_points, lines=keys_lines, polygons=keys_polygons)
conf = osmconf.create_osm_conf()
ogr_cmd = self.ogr_cmd.safe_substitute({'gpkg': self.output_gpkg,
'osm': self.input_pbf, 'osmconf': conf})
LOG.debug('Running: %s' % ogr_cmd)
subprocess.check_call(ogr_cmd, shell=True, executable='/bin/bash')
"""
Create the default osm gpkg schema
"""
conn = sqlite3.connect(self.output_gpkg)
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("select load_extension('mod_spatialite')")
cur.execute("CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)");
cur.execute("INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));", (self.aoi_geom.wkb,))
cur.executescript(SPATIAL_SQL)
self.update_zindexes(cur, self.feature_selection)
# add themes
create_sqls, index_sqls = self.feature_selection.sqls
for query in create_sqls:
LOG.debug(query)
cur.executescript(query)
for query in index_sqls:
LOG.debug(query)
cur.executescript(query)
"""
Remove points/lines/multipolygons tables
"""
cur.execute("DROP TABLE points")
cur.execute("DROP TABLE lines")
cur.execute("DROP TABLE multipolygons")
conn.commit()
conn.close()
if self.per_theme:
# this creates per-theme GPKGs
for theme in self.feature_selection.themes:
conn = sqlite3.connect(self.stage_dir + slugify(theme) + ".gpkg")
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("attach database ? as 'geopackage'", (self.output_gpkg,))
cur.execute("create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys")
cur.execute("create table gpkg_contents as select * from geopackage.gpkg_contents where 0")
cur.execute("create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0")
for geom_type in self.feature_selection.geom_types(theme):
for stmt in self.feature_selection.create_sql(theme, geom_type):
cur.executescript(stmt)
conn.commit()
conn.close()
def run(self, subtask_percentage=100, subtask_start=0, eta=None):
"""
Create the GeoPackage from the osm data.
"""
# avoiding a circular import
from eventkit_cloud.tasks.export_tasks import update_progress
if self.is_complete:
LOG.debug("Skipping Geopackage, file exists")
return
keys_points = self.feature_selection.key_union('points')
keys_lines = self.feature_selection.key_union('lines')
keys_polygons = self.feature_selection.key_union('polygons')
osmconf = OSMConfig(self.stage_dir, points=keys_points, lines=keys_lines, polygons=keys_polygons)
conf = osmconf.create_osm_conf()
ogr_cmd = self.ogr_cmd.safe_substitute({'gpkg': self.output_gpkg,
'osm': self.input_pbf, 'osmconf': conf})
LOG.debug('Running: %s' % ogr_cmd)
subprocess.check_call(ogr_cmd, shell=True, executable='/bin/bash')
"""
Create the default osm gpkg schema
"""
conn = sqlite3.connect(self.output_gpkg)
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("select load_extension('mod_spatialite')")
cur.execute("CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)");
cur.execute("INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));", (self.aoi_geom.wkb,))
update_progress(self.export_task_record_uid, 30, subtask_percentage, subtask_start, eta=eta)
cur.executescript(SPATIAL_SQL)
self.update_zindexes(cur, self.feature_selection)
update_progress(self.export_task_record_uid, 42, subtask_percentage, subtask_start, eta=eta)
# add themes
create_sqls, index_sqls = self.feature_selection.sqls
for query in create_sqls:
LOG.debug(query)
cur.executescript(query)
update_progress(self.export_task_record_uid, 50, subtask_percentage, subtask_start, eta=eta)
for query in index_sqls:
LOG.debug(query)
cur.executescript(query)
"""
Remove points/lines/multipolygons tables
"""
cur.execute("DROP TABLE points")
cur.execute("DROP TABLE lines")
cur.execute("DROP TABLE multipolygons")
conn.commit()
conn.close()
if self.per_theme:
# this creates per-theme GPKGs
for theme in self.feature_selection.themes:
conn = sqlite3.connect(self.stage_dir + slugify(theme) + ".gpkg")
conn.enable_load_extension(True)
cur = conn.cursor()
cur.execute("attach database ? as 'geopackage'", (self.output_gpkg,))
cur.execute("create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys")
cur.execute("create table gpkg_contents as select * from geopackage.gpkg_contents where 0")
cur.execute(
"create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0")
for geom_type in self.feature_selection.geom_types(theme):
for stmt in self.feature_selection.create_sql(theme, geom_type):
cur.executescript(stmt)
conn.commit()
conn.close()
update_progress(self.export_task_record_uid, 100, subtask_percentage, subtask_start, eta=eta)
