def results(self): if self.per_theme: results_list = [] for theme in self.feature_selection.themes: results_list.append(Artifact([os.path.join(self.stage_dir, slugify(theme)) + ".gpkg"], Geopackage.name, theme=theme)) return results_list else: return [Artifact([self.output_gpkg], Geopackage.name)]
def results(self): if self.per_theme: results_list = [] for theme in self.feature_selection.themes: results_list.append( Artifact([os.path.join(self.stage_dir, slugify(theme)) + ".gpkg"], Geopackage.name, theme=theme)) return results_list else: return [Artifact([self.output_gpkg], Geopackage.name)]
def wrapper(*args, **kwargs): # handle if first arg is self. arg = type(args[0]).__name__ if inspect.isclass(args[0]) else str( slugify(args[0])) arg_string = ".".join([str(slugify(arg)) for arg in args[:1]]) keys = (copy.deepcopy(kwargs) if not key_fields else { key_field: kwargs.get(key_field) for key_field in key_fields }) kwarg_string = ".".join([ str(slugify(kwarg)) for kwarg in itertools.chain.from_iterable(keys.items()) ]) cache_key = f"{func.__name__}.{arg}.{arg_string}.{kwarg_string}"[: 249] logger.debug("Getting or setting the cache_key %s", cache_key) return_value = cache.get_or_set(cache_key, lambda: func(*args, **kwargs), timeout=timeout) return return_value
def get_layers(self) -> LayersDescription: cap_doc = self.get_capabilities(layer_id=self.layer) if cap_doc and self.layer: return {self.layer: {"name": str(self.layer), "url": str(cap_doc["url"])}} if cap_doc.get("layers") or cap_doc.get("subLayers"): # TODO: This logic is specific for feature layers, # this will need to change or be subclassed to separate raster/feature services. return { slugify(layer["name"]): layer for layer in (cap_doc.get("subLayers", []) or cap_doc.get("layers", [])) if "Feature" in layer["type"] } return {self.layer: {"name": str(self.layer), "url": self.service_url}}
def run(self, subtask_percentage=100, subtask_start=0, eta=None): """ Create the GeoPackage from the osm data. """ if self.is_complete: logger.debug("Skipping Geopackage, file exists") return keys_points = self.feature_selection.key_union("points") keys_lines = self.feature_selection.key_union("lines") keys_polygons = self.feature_selection.key_union("polygons") osmconf = OSMConfig(self.stage_dir, points=keys_points, lines=keys_lines, polygons=keys_polygons) conf = osmconf.create_osm_conf() logger.debug( f"Creating OSM gpkg using OSM_MAX_TMPFILE_SIZE {settings.OSM_MAX_TMPFILE_SIZE}" f"from {self.input_pbf} to {self.output_gpkg}") gdalutils.convert( input_file=self.input_pbf, output_file=self.output_gpkg, driver="GPKG", boundary=json.loads(self.aoi_geom.geojson), config_options=[ ("OSM_CONFIG_FILE", conf), ("OGR_INTERLEAVED_READING", "YES"), ("OSM_MAX_TMPFILE_SIZE", settings.OSM_MAX_TMPFILE_SIZE), ], task_uid=self.export_task_record_uid, ) # Cancel the provider task if the geopackage has no data. if not check_content_exists(self.output_gpkg): return None """ Create the default osm gpkg schema """ conn = sqlite3.connect(self.output_gpkg) conn.enable_load_extension(True) cur = conn.cursor() cur.execute("select load_extension('mod_spatialite')") cur.execute( "CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)" ) cur.execute( "INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));", (self.aoi_geom.wkb, )) update_progress(self.export_task_record_uid, 30, subtask_percentage, subtask_start, eta=eta) cur.executescript(SPATIAL_SQL) self.update_zindexes(cur, self.feature_selection) update_progress(self.export_task_record_uid, 42, subtask_percentage, subtask_start, eta=eta) # add themes create_sqls, index_sqls = self.feature_selection.sqls for query in create_sqls: logger.debug(query) cur.executescript(query) update_progress(self.export_task_record_uid, 50, subtask_percentage, subtask_start, eta=eta) for query in index_sqls: logger.debug(query) cur.executescript(query) """ Remove points/lines/multipolygons tables """ cur.execute("DROP TABLE points") cur.execute("DROP TABLE lines") cur.execute("DROP TABLE multipolygons") cur.execute("VACUUM;") conn.commit() conn.close() if self.per_theme: # this creates per-theme GPKGs for theme in self.feature_selection.themes: conn = sqlite3.connect(self.stage_dir + slugify(theme) + ".gpkg") conn.enable_load_extension(True) cur = conn.cursor() cur.execute("attach database ? as 'geopackage'", (self.output_gpkg, )) cur.execute( "create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys" ) cur.execute( "create table gpkg_contents as select * from geopackage.gpkg_contents where 0" ) cur.execute( "create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0" ) for geom_type in self.feature_selection.geom_types(theme): for stmt in self.feature_selection.create_sql( theme, geom_type): cur.executescript(stmt) cur.execute("VACUUM;") conn.commit() conn.close() update_progress(self.export_task_record_uid, 100, subtask_percentage, subtask_start, eta=eta) return self.output_gpkg
def run(self): """ Create the GeoPackage from the osm data. """ if self.is_complete: LOG.debug("Skipping Geopackage, file exists") return keys_points = self.feature_selection.key_union('points') keys_lines = self.feature_selection.key_union('lines') keys_polygons = self.feature_selection.key_union('polygons') osmconf = OSMConfig(self.stage_dir, points=keys_points, lines=keys_lines, polygons=keys_polygons) conf = osmconf.create_osm_conf() ogr_cmd = self.ogr_cmd.safe_substitute({'gpkg': self.output_gpkg, 'osm': self.input_pbf, 'osmconf': conf}) LOG.debug('Running: %s' % ogr_cmd) subprocess.check_call(ogr_cmd, shell=True, executable='/bin/bash') """ Create the default osm gpkg schema """ conn = sqlite3.connect(self.output_gpkg) conn.enable_load_extension(True) cur = conn.cursor() cur.execute("select load_extension('mod_spatialite')") cur.execute("CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)"); cur.execute("INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));", (self.aoi_geom.wkb,)) cur.executescript(SPATIAL_SQL) self.update_zindexes(cur, self.feature_selection) # add themes create_sqls, index_sqls = self.feature_selection.sqls for query in create_sqls: LOG.debug(query) cur.executescript(query) for query in index_sqls: LOG.debug(query) cur.executescript(query) """ Remove points/lines/multipolygons tables """ cur.execute("DROP TABLE points") cur.execute("DROP TABLE lines") cur.execute("DROP TABLE multipolygons") conn.commit() conn.close() if self.per_theme: # this creates per-theme GPKGs for theme in self.feature_selection.themes: conn = sqlite3.connect(self.stage_dir + slugify(theme) + ".gpkg") conn.enable_load_extension(True) cur = conn.cursor() cur.execute("attach database ? as 'geopackage'", (self.output_gpkg,)) cur.execute("create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys") cur.execute("create table gpkg_contents as select * from geopackage.gpkg_contents where 0") cur.execute("create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0") for geom_type in self.feature_selection.geom_types(theme): for stmt in self.feature_selection.create_sql(theme, geom_type): cur.executescript(stmt) conn.commit() conn.close()
def run(self, subtask_percentage=100, subtask_start=0, eta=None): """ Create the GeoPackage from the osm data. """ # avoiding a circular import from eventkit_cloud.tasks.export_tasks import update_progress if self.is_complete: LOG.debug("Skipping Geopackage, file exists") return keys_points = self.feature_selection.key_union('points') keys_lines = self.feature_selection.key_union('lines') keys_polygons = self.feature_selection.key_union('polygons') osmconf = OSMConfig(self.stage_dir, points=keys_points, lines=keys_lines, polygons=keys_polygons) conf = osmconf.create_osm_conf() ogr_cmd = self.ogr_cmd.safe_substitute({'gpkg': self.output_gpkg, 'osm': self.input_pbf, 'osmconf': conf}) LOG.debug('Running: %s' % ogr_cmd) subprocess.check_call(ogr_cmd, shell=True, executable='/bin/bash') """ Create the default osm gpkg schema """ conn = sqlite3.connect(self.output_gpkg) conn.enable_load_extension(True) cur = conn.cursor() cur.execute("select load_extension('mod_spatialite')") cur.execute("CREATE TABLE boundary (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, geom GEOMETRY)"); cur.execute("INSERT INTO boundary (geom) VALUES (GeomFromWKB(?,4326));", (self.aoi_geom.wkb,)) update_progress(self.export_task_record_uid, 30, subtask_percentage, subtask_start, eta=eta) cur.executescript(SPATIAL_SQL) self.update_zindexes(cur, self.feature_selection) update_progress(self.export_task_record_uid, 42, subtask_percentage, subtask_start, eta=eta) # add themes create_sqls, index_sqls = self.feature_selection.sqls for query in create_sqls: LOG.debug(query) cur.executescript(query) update_progress(self.export_task_record_uid, 50, subtask_percentage, subtask_start, eta=eta) for query in index_sqls: LOG.debug(query) cur.executescript(query) """ Remove points/lines/multipolygons tables """ cur.execute("DROP TABLE points") cur.execute("DROP TABLE lines") cur.execute("DROP TABLE multipolygons") conn.commit() conn.close() if self.per_theme: # this creates per-theme GPKGs for theme in self.feature_selection.themes: conn = sqlite3.connect(self.stage_dir + slugify(theme) + ".gpkg") conn.enable_load_extension(True) cur = conn.cursor() cur.execute("attach database ? as 'geopackage'", (self.output_gpkg,)) cur.execute("create table gpkg_spatial_ref_sys as select * from geopackage.gpkg_spatial_ref_sys") cur.execute("create table gpkg_contents as select * from geopackage.gpkg_contents where 0") cur.execute( "create table gpkg_geometry_columns as select * from geopackage.gpkg_geometry_columns where 0") for geom_type in self.feature_selection.geom_types(theme): for stmt in self.feature_selection.create_sql(theme, geom_type): cur.executescript(stmt) conn.commit() conn.close() update_progress(self.export_task_record_uid, 100, subtask_percentage, subtask_start, eta=eta)