def _on_connect(self, connection_name, path_or_url): debug("Connect to path_or_url: {}", path_or_url) self.reload_action.setText("{} ({})".format(self._reload_button_text, connection_name)) try: if self._current_reader and self._current_reader.source.source( ) != path_or_url: self._current_reader.shutdown() self._current_reader.progress_changed.disconnect() self._current_reader.max_progress_changed.disconnect() self._current_reader.title_changed.disconnect() self._current_reader.message_changed.disconnect() self._current_reader.show_progress_changed.disconnect() self._current_reader = None if not self._current_reader: reader = self._create_reader(path_or_url) reader.set_root_group_name(connection_name) self._current_reader = reader if self._current_reader: layers = self._current_reader.source.vector_layers() self.connections_dialog.set_layers(layers) self.connections_dialog.options.set_zoom( self._current_reader.source.min_zoom(), self._current_reader.source.max_zoom()) self.reload_action.setEnabled(True) else: self.connections_dialog.set_layers([]) self.reload_action.setEnabled(False) self.reload_action.setText(self._reload_button_text) except: QMessageBox.critical( None, "Unexpected Error", "An unexpected error occured. {}".format( str(sys.exc_info()[1])))
def _on_add_layer(self, path_or_url, selected_layers): debug("add layer: {}", path_or_url) crs_string = self._current_reader.source.crs() self._init_qgis_map(crs_string) scheme = self._current_reader.source.scheme() zoom = self._get_current_zoom() extent = self._get_visible_extent_as_tile_bounds(scheme=scheme, zoom=zoom) # if not self.tilejson.is_within_bounds(zoom=zoom, extent=extent): # pass # todo: something's wrong here. probably a CRS mismatch between _get_visible_extent and tilejson # print "not in bounds" # self._set_qgis_extent(self.tilejson) keep_dialog_open = self.connections_dialog.keep_dialog_open() if keep_dialog_open: dialog_owner = self.connections_dialog else: dialog_owner = self.iface.mainWindow() self.connections_dialog.close() self._create_progress_dialog(dialog_owner, on_cancel=self._cancel_load) self._load_tiles(path=path_or_url, options=self.connections_dialog.options, layers_to_load=selected_layers, bounds=extent) self._current_source_path = path_or_url self._current_layer_filter = selected_layers
def _on_connect(self, connection_name, path_or_url): debug("Connect to path_or_url: {}", path_or_url) self.reload_action.setText("{} ({})".format(self._reload_button_text, connection_name)) try: if self._current_reader: self._current_reader.source.close_connection() reader = self._create_reader(path_or_url) self._current_reader = reader if reader: layers = reader.source.vector_layers() self.connections_dialog.set_layers(layers) self.connections_dialog.options.set_zoom( reader.source.min_zoom(), reader.source.max_zoom()) self.reload_action.setEnabled(True) self._current_source_path = path_or_url else: self.connections_dialog.set_layers([]) self.reload_action.setEnabled(False) self.reload_action.setText(self._reload_button_text) self._current_source_path = None except: QMessageBox.critical( None, "Unexpected Error", "An unexpected error occured. {}".format( str(sys.exc_info()[1])))
def _apply_named_style(layer, geo_type): """ * Looks for a styles with the same name as the layer and if one is found, it is applied to the layer :param layer: :param layer_path: e.g. 'transportation.service' or 'transportation_name.path' :return: """ try: name = layer.name().split(VtReader._zoom_level_delimiter)[0].lower() styles = [ "{}.{}".format(name, geo_type.lower()), name ] for p in styles: style_name = "{}.qml".format(p).lower() if style_name in VtReader._styles: style_path = os.path.join(FileHelper.get_plugin_directory(), "styles/{}".format(style_name)) res = layer.loadNamedStyle(style_path) if res[1]: # Style loaded layer.setCustomProperty("layerStyle", style_path) if layer.customProperty("layerStyle") == style_path: debug("Style successfully applied: {}", style_name) break except: critical("Loading style failed: {}", sys.exc_info())
def load_tiles(self, zoom_level, tiles_to_load, max_tiles=None, for_each=None, limit_reacher_handler=None): """ * Loads the tiles for the specified zoom_level and bounds from the mbtiles file this source has been created with :param zoom_level: The zoom level which will be loaded :param bounds: If set, only tiles inside this tile boundary will be loaded :param max_tiles: The maximum number of tiles to be loaded :param for_each: A function which will be called for every row :param limit_reacher_handler: A function which will be called, if the potential nr of tiles is greater than the specified limit :return: """ self._cancelling = False debug("Reading tiles of zoom level {}", zoom_level) if max_tiles: center_tiles = get_tiles_from_center( max_tiles, tiles_to_load, should_cancel_func=lambda: self._cancelling) else: center_tiles = tiles_to_load where_clause = self._get_where_clause(tiles_to_load=center_tiles, zoom_level=zoom_level) sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles {};" sql = sql_command.format(where_clause) tile_data_tuples = [] rows = self._get_from_db(sql=sql) no_tiles_in_current_extent = not rows or len(rows) == 0 if no_tiles_in_current_extent: where_clause = self._get_where_clause(tiles_to_load=None, zoom_level=zoom_level) sql = sql_command.format(where_clause) rows = self._get_from_db(sql=sql) if rows: if max_tiles and len(rows) > max_tiles: if no_tiles_in_current_extent: rows = rows[:max_tiles] if no_tiles_in_current_extent: if limit_reacher_handler: limit_reacher_handler() self._progress_handler(max_progress=len(rows)) for index, row in enumerate(rows): if for_each: for_each() if self._cancelling or (max_tiles and len(tile_data_tuples) >= max_tiles): break tile, data = self._create_tile(row) tile_data_tuples.append((tile, data)) self._progress_handler(progress=index + 1) return tile_data_tuples
def _get_metadata_value(self, field_name, default=None): if field_name not in self._metadata_cache: debug("Loading metadata value '{}'", field_name) sql = "select value as '{0}' from metadata where name = '{0}'".format(field_name) value = self._get_single_value(sql_query=sql, field_name=field_name) if default and not value: value = default self._metadata_cache[field_name] = value return self._metadata_cache[field_name]
def _load_recently_used(self): recently_used = FileHelper.get_recently_used_file() if os.path.isfile(recently_used): with open(recently_used, 'r') as f: for line in f: line = line.rstrip("\n") if os.path.isfile(line): debug("recently used: {}".format(line)) self.recently_used.append(line)
def cache_tile(tile, file_name): if not tile.decoded_data: warn("Trying to cache a tile without data: {}", tile) file_path = os.path.join(FileHelper.get_cache_directory(), file_name) try: with open(file_path, 'wb') as f: pickle.dump(tile, f, pickle.HIGHEST_PROTOCOL) except: debug("Error while writing tile '{}' to cache", str(tile))
def _create_qgis_layers(self, merge_features, apply_styles): """ * Creates a hierarchy of groups and layers in qgis """ debug("Creating hierarchy in QGIS") self._assure_qgis_groups_exist() self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_path), msg="Creating layers...") layers = [] for index, layer_name_and_type in enumerate(self.feature_collections_by_layer_path): layer_name_and_zoom = layer_name_and_type[0] geo_type = layer_name_and_type[1] layer_name = layer_name_and_zoom.split(VtReader._zoom_level_delimiter)[0] zoom_level = layer_name_and_zoom.split(VtReader._zoom_level_delimiter)[1] QApplication.processEvents() if self.cancel_requested: break target_group = self._qgis_layer_groups_by_name[layer_name] feature_collections_by_tile_coord = self.feature_collections_by_layer_path[layer_name_and_type] file_name = self._get_geojson_filename(layer_name, geo_type, zoom_level) file_path = FileHelper.get_geojson_file_name(file_name) layer = None if os.path.isfile(file_path): # file exists already. add the features of the collection to the existing collection # get the layer from qgis and update its source layer = self._get_layer_by_source(layer_name_and_zoom, file_path) if layer: self._update_layer_source(file_path, feature_collections_by_tile_coord) layer.reload() if not layer: complete_collection = self._get_empty_feature_collection(zoom_level, layer_name) self._merge_feature_collections(current_feature_collection=complete_collection, feature_collections_by_tile_coord=feature_collections_by_tile_coord) with open(file_path, "w") as f: f.write(json.dumps(complete_collection)) layer = self._add_vector_layer_to_qgis(file_path, layer_name, zoom_level, target_group, merge_features, geo_type) if apply_styles: layers.append((layer_name_and_type, layer)) self._update_progress(progress=index+1) if apply_styles: self._update_progress(progress=0, max_progress=len(layers), msg="Styling layers...") for index, layer_path_tuple in enumerate(layers): QApplication.processEvents() if self.cancel_requested: break path_and_type = layer_path_tuple[0] geo_type = path_and_type[1] layer = layer_path_tuple[1] VtReader._apply_named_style(layer, geo_type) self._update_progress(progress=index+1)
def _get_from_db(self, sql): if not self.conn: debug("Not connected yet.") self._connect_to_db() try: debug("Execute SQL: {}", sql) cur = self.conn.cursor() cur.execute(sql) return cur.fetchall() except: critical("Getting data from db failed: {}", sys.exc_info())
def _connect_to_db(self): """ * Since an mbtile file is a sqlite database, we can connect to it """ debug("Connecting to: {}", self.path) try: self.conn = sqlite3.connect(self.path) self.conn.row_factory = sqlite3.Row debug("Successfully connected") except: critical("Db connection failed:", sys.exc_info())
def close_connection(self): """ * Closes the current db connection :return: """ if self.conn: try: self.conn.close() debug("Connection closed") except: warn("Closing connection failed: {}".format(sys.exc_info())) self.conn = None
def is_extent_within_bounds(self, extent, bounds): is_within = True if bounds: x_min_within = extent['x_min'] >= bounds['x_min'] y_min_within = extent['y_min'] >= bounds['y_min'] x_max_within = extent['x_max'] <= bounds['x_max'] y_max_within = extent['y_max'] <= bounds['y_max'] is_within = x_min_within and y_min_within and x_max_within and y_max_within else: debug( "Bounds not available on source. Assuming extent is within bounds" ) return is_within
def is_within_bounds(self, zoom, extent): bounds = self.bounds_tile(zoom) is_within = True if bounds: x_min_within = extent[0][0] >= bounds[0][0] y_min_within = extent[0][1] >= bounds[0][1] x_max_within = extent[1][0] <= bounds[1][0] y_max_within = extent[1][1] <= bounds[1][1] is_within = x_min_within and y_min_within and x_max_within and y_max_within debug("Extent {} is within bounds {}: {}", extent, bounds, is_within) else: debug("Assuming extent is within bounds") return is_within
def load_vector_tiles(self, zoom_level, path): self._file_path = path debug("Loading vector tiles: {}".format(path)) self.reinit() self._connect_to_db(path) mask_level = self._get_mask_layer_id() tile_data_tuples = self._load_tiles_from_db(zoom_level, mask_level) # if mask_level: # mask_layer_data = self._load_tiles_from_db(mask_level) # tile_data_tuples.extend(mask_layer_data) tiles = self._decode_all_tiles(tile_data_tuples) self._process_tiles(tiles) self._create_qgis_layer_hierarchy() print("Import complete!")
def get_all_tiles(bounds, is_cancel_requested_handler): tiles = [] width = bounds["width"] height = bounds["height"] x_min = bounds["x_min"] y_min = bounds["y_min"] debug("Preprocessing {} tiles", width*height) for x in range(width): if is_cancel_requested_handler(): break for y in range(height): col = x + x_min row = y + y_min tiles.append((col, row)) return tiles
def get_cached_tile(file_name): file_path = os.path.join(FileHelper.get_cache_directory(), file_name) tile = None try: if os.path.exists(file_path): age_in_seconds = int(time.time()) - os.path.getmtime(file_path) is_deprecated = age_in_seconds > FileHelper.max_cache_age_minutes * 60 if is_deprecated: os.remove(file_path) else: with open(file_path, 'rb') as f: tile = pickle.load(f) except: debug("Error while reading cache entry {}: {}", file_name, sys.exc_info()[1]) return tile
def _get_single_value(self, sql_query, field_name): """ * Helper function that can be used to safely load a single value from the db * Returns the value or None if result is empty or execution of query failed :param sql_query: :param field_name: :return: """ value = None try: rows = self._get_from_db(sql=sql_query) if rows: value = rows[0][field_name] debug("Value is: {}".format(value)) except: critical("Loading metadata value '{}' failed: {}", field_name, sys.exc_info()) return value
def _get_visible_extent_as_tile_bounds(self, scheme, zoom): extent = self._get_current_extent_as_wkt() splits = extent.split(", ") new_extent = map(lambda x: map(float, x.split(" ")), splits) min_extent = new_extent[0] max_extent = new_extent[1] min_proj = epsg3857_to_wgs84_lonlat(min_extent[0], min_extent[1]) max_proj = epsg3857_to_wgs84_lonlat(max_extent[0], max_extent[1]) bounds = [] bounds.extend(min_proj) bounds.extend(max_proj) tile_bounds = get_tile_bounds(zoom, bounds=bounds, scheme=scheme) debug("Current extent: {}", tile_bounds) return tile_bounds
def _get_visible_extent_as_tile_bounds(self, scheme, zoom): e = self.iface.mapCanvas().extent().asWktCoordinates().split(", ") new_extent = map(lambda x: map(float, x.split(" ")), e) min_extent = new_extent[0] max_extent = new_extent[1] min_proj = epsg3857_to_wgs84_lonlat(min_extent[0], min_extent[1]) max_proj = epsg3857_to_wgs84_lonlat(max_extent[0], max_extent[1]) bounds = [] bounds.extend(min_proj) bounds.extend(max_proj) tile_bounds = get_tile_bounds(zoom, bounds=bounds, scheme=scheme, source_crs="EPSG:4326") debug("Current extent: {}", tile_bounds) return tile_bounds
def _convert_layer(self, layer_name, tile): layer = tile.decoded_data[layer_name] self._get_metadata("json")["vector_layers"].append({"id": layer_name}) converted_layer = {"name": layer_name, "features": []} debug("current layer: {}", layer_name) for k in layer.keys(): if k != "features": converted_layer[k] = layer[k] for f in layer["features"]: geo_type = geo_types[f["type"]] geom_string = geo_type.upper() geometry = f["geometry"] is_polygon = geom_string == "POLYGON" is_multi_geometry = is_multi(geo_type, geometry) all_geometries = [] if is_multi_geometry: VtWriter.get_subarr(geometry, all_geometries) else: if all(VtWriter.is_coordinate_tuple(c) for c in geometry): all_geometries = [geometry] else: all_geometries = geometry for geom in all_geometries: new_feature = self._copy_feature(f) new_feature["geometry"] = self._create_wkt_geometry( geo_type, is_polygon, geom) try: single_feature_layer = { "name": "dummy", "features": [new_feature] } mapbox_vector_tile.encode(single_feature_layer, y_coord_down=True) converted_layer["features"].append(new_feature) except: debug("invalid geometry: {}", new_feature["geometry"]) pass return converted_layer
def _load_tiles(self, path, options, layers_to_load, bounds=None, ignore_limit=False): merge_tiles = options.merge_tiles_enabled() apply_styles = options.apply_styles_enabled() tile_limit = options.tile_number_limit() load_mask_layer = options.load_mask_layer_enabled() if ignore_limit: tile_limit = None manual_zoom = options.manual_zoom() cartographic_ordering = options.cartographic_ordering() if apply_styles: self._set_background_color() debug("Load: {}", path) reader = self._current_reader if reader: reader.enable_cartographic_ordering(enabled=cartographic_ordering) try: zoom = reader.source.max_zoom() if manual_zoom is not None: zoom = manual_zoom reader.load_tiles(zoom_level=zoom, layer_filter=layers_to_load, load_mask_layer=load_mask_layer, merge_tiles=merge_tiles, apply_styles=apply_styles, max_tiles=tile_limit, bounds=bounds, limit_reacher_handler=lambda: self. _show_limit_exceeded_message(tile_limit)) self.refresh_layers() debug("Loading complete!") except RuntimeError: QMessageBox.critical(None, "Unexpected exception", str(sys.exc_info()[1])) critical(str(sys.exc_info()[1]))
def add_menu(self): self.popupMenu = QMenu(self.iface.mainWindow()) default_action = self._create_action("Add Vector Tiles Layer", "icon.png", self.run) self.popupMenu.addAction(default_action) self.popupMenu.addAction( self._create_action("Open Mapbox Tiles...", "folder.svg", self._open_file_browser)) self.recent = self.popupMenu.addMenu("Open Recent") debug("Recently used: {}", self.recently_used) for path in self.recently_used: debug("Create action: {}".format(path)) self._add_recently_used(path) self.toolButton = QToolButton() self.toolButton.setMenu(self.popupMenu) self.toolButton.setDefaultAction(default_action) self.toolButton.setPopupMode(QToolButton.MenuButtonPopup) self.toolButtonAction = self.iface.addVectorToolBarWidget( self.toolButton)
def get_tiles_from_center(nr_of_tiles, available_tiles, should_cancel_func): if nr_of_tiles > len(available_tiles): nr_of_tiles = len(available_tiles) debug("Getting {} center-tiles from a total of {} tiles", nr_of_tiles, len(available_tiles)) if not nr_of_tiles or nr_of_tiles >= len(available_tiles) or len(available_tiles) == 0: return available_tiles min_x = min(map(lambda t: t[0], available_tiles)) min_y = min(map(lambda t: t[1], available_tiles)) max_x = max(map(lambda t: t[0], available_tiles)) max_y = max(map(lambda t: t[1], available_tiles)) center_tile_offset = (int(round((max_x-min_x)/2)), int(round((max_y-min_y)/2))) selected_tiles = set() center_tile = _sum_tiles((min_x, min_y), center_tile_offset) if center_tile in available_tiles: selected_tiles.add(center_tile) current_tile = center_tile nr_of_steps = 0 current_direction = 0 while len(selected_tiles) < nr_of_tiles: if should_cancel_func and should_cancel_func(): break # always after two direction changes, the step length has to be increased by one if current_direction % 2 == 0: nr_of_steps += 1 # go nr_of_steps steps into the current direction for s in xrange(nr_of_steps): current_tile = _sum_tiles(current_tile, _directions[current_direction]) if current_tile in available_tiles: selected_tiles.add(current_tile) if len(selected_tiles) >= nr_of_tiles: break current_direction = (current_direction + 1) % 4 debug("Center tiles completed") return selected_tiles
def load_tiles(self, zoom_level, tiles_to_load, max_tiles=None): self._cancelling = False debug("Reading tiles of zoom level {}", zoom_level) if max_tiles: center_tiles = get_tiles_from_center(nr_of_tiles=max_tiles, available_tiles=tiles_to_load, should_cancel_func=lambda: self._cancelling) else: center_tiles = tiles_to_load where_clause = self._get_where_clause(tiles_to_load=center_tiles, zoom_level=zoom_level) sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles {};" sql = sql_command.format(where_clause) tile_data_tuples = [] rows = self._get_from_db(sql=sql) no_tiles_in_current_extent = not rows or len(rows) == 0 if no_tiles_in_current_extent: where_clause = self._get_where_clause(tiles_to_load=None, zoom_level=zoom_level) sql = sql_command.format(where_clause) rows = self._get_from_db(sql=sql) if rows: if max_tiles and len(rows) > max_tiles: if no_tiles_in_current_extent: rows = rows[:max_tiles] if no_tiles_in_current_extent: self.tile_limit_reached.emit() self.max_progress_changed.emit(len(rows)) for index, row in enumerate(rows): if self._cancelling or (max_tiles and len(tile_data_tuples) >= max_tiles): break tile, data = self._create_tile(row) tile_data_tuples.append((tile, data)) self.progress_changed.emit(index+1) return tile_data_tuples
def _prepare_features_for_dissolvment(layer): _DISSOLVE_GROUP_FIELD = "dissolveGroup" layer.startEditing() layer.dataProvider().addAttributes([QgsField(_DISSOLVE_GROUP_FIELD, QVariant.String, len=36)]) layer.updateFields() # Create a dictionary of all features feature_dict = {f.id(): f for f in layer.getFeatures()} idx = layer.fieldNameIndex('class') # Build a spatial index index = QgsSpatialIndex() for f in feature_dict.values(): index.insertFeature(f) for f in feature_dict.values(): if f[_DISSOLVE_GROUP_FIELD]: continue f[_DISSOLVE_GROUP_FIELD] = str(uuid.uuid4()) FeatureMerger._assign_dissolve_group_to_neighbours_rec(_DISSOLVE_GROUP_FIELD, index, f, [], feature_dict, feature_handler=lambda feat: layer.updateFeature(feat), feature_class_attr_index=idx) layer.updateFeature(f) layer.commitChanges() debug('Dissolvement complete: {}', layer.name()) return
def export(self): self._cancel_requested = False self.conn = None try: self.conn = self._create_db() except: critical("db creation failed: {}", sys.exc_info()) if self.conn: try: with self.conn: tile_names = self._get_loaded_tile_names() if tile_names: tiles = self._load_tiles(tile_names) nr_tiles = len(tiles) for index, t in enumerate(tiles): if self._cancel_requested: break self._update_progress( title="Export tile {}/{}".format( index + 1, nr_tiles), show_dialog=True) QApplication.processEvents() self._update_bounds(t) debug("layers to export: {}", self.layers_to_export) self._save_tile(t) if not self._cancel_requested: layer_objects = map(lambda l: {"id": l}, self.layer_names) vector_layers = {"vector_layers": layer_objects} self.metadata["json"] = json.dumps(vector_layers) self._save_metadata() debug("export complete") self.iface.messageBar().pushInfo( u'Vector Tiles Reader', u'mbtiles export completed') else: debug("export cancelled") self.iface.messageBar().pushInfo( u'Vector Tiles Reader', u'mbtiles export cancelled') self.conn.close() except: if self.conn: self.conn.close() critical("Export failed: {}", sys.exc_info()) raise self._update_progress(show_dialog=False)
def load(self): debug("Loading TileJSON") success = False try: status, data = FileHelper.load_url(self.url) self.json = json.loads(data) if self.json: debug("TileJSON loaded") self._validate() success = True else: debug("Loading TileJSON failed") self.json = {} raise RuntimeError("TileJSON could not be loaded.") except: critical("Loading TileJSON failed ({}): {}", self.url, sys.exc_info()) return success
def is_mapbox_vector_tile(self): """ * A .mbtiles file is a Mapbox Vector Tile if the binary tile data is gzipped. :return: """ debug("Checking if file corresponds to Mapbox format (i.e. gzipped)") is_mapbox_pbf = False try: tile_data_tuples = self.load_tiles(max_tiles=1, zoom_level=None) if len(tile_data_tuples) == 1: undecoded_data = tile_data_tuples[0][1] if undecoded_data: is_mapbox_pbf = FileHelper.is_gzipped(undecoded_data) if is_mapbox_pbf: debug("File is valid mbtiles") else: debug("pbf is not gzipped") except: warn( "Something went wrong. This file doesn't seem to be a Mapbox Vector Tile. {}", sys.exc_info()) return is_mapbox_pbf
def load(self): debug("Loading TileJSON") success = False try: if os.path.isfile(self.url): with open(self.url, 'r') as f: data = f.read() else: status, data = FileHelper.load_url(self.url) self.json = json.loads(data) if self.json: debug("TileJSON loaded") self._validate() debug("TileJSON validated") success = True else: info("Parsing TileJSON failed") self.json = {} raise RuntimeError("TileJSON could not be loaded.") except: critical("Loading TileJSON failed ({}): {}", self.url, sys.exc_info()) return success