def _get_pool(): nr_processors = 4 try: nr_processors = mp.cpu_count() except NotImplementedError: info("CPU count cannot be retrieved. Falling back to default = 4") pool = mp.Pool(nr_processors) return pool
def load_tiles_async(self, bounds): """ * Loads the vector tiles from either a file or a URL and adds them to QGIS :param zoom_level: The zoom level to load :param bounds: :return: """ zoom_level = bounds["zoom"] info("Loading zoom level '{}', bounds: {}", zoom_level, bounds) self._loading_options["zoom_level"] = zoom_level self._loading_options["bounds"] = bounds _worker_thread = QThread(self.iface.mainWindow()) self.moveToThread(_worker_thread) _worker_thread.started.connect(self._load_tiles) _worker_thread.start()
def _continue_loading(self): zoom_level = self._loading_options["zoom_level"] merge_tiles = self._loading_options["merge_tiles"] apply_styles = self._loading_options["apply_styles"] clip_tiles = self._loading_options["clip_tiles"] if not self.cancel_requested: self._create_qgis_layers(merge_features=merge_tiles, apply_styles=apply_styles, clip_tiles=clip_tiles) self._update_progress(show_dialog=False) if self.cancel_requested: info("Import cancelled") self.cancelled.emit() else: info("Import complete") loaded_extent = self._get_extent(self._all_tiles, zoom_level) if not loaded_extent: loaded_extent = {} self.loading_finished.emit(zoom_level, loaded_extent)
def _create_qgis_layers(self, merge_features, apply_styles, clip_tiles): """ * Creates a hierarchy of groups and layers in qgis """ own_layers = get_loaded_layers_of_connection(self._connection["name"]) for l in own_layers: name = l.name() geo_type = l.customProperty("VectorTilesReader/geo_type") if (name, geo_type ) not in self.feature_collections_by_layer_name_and_geotype: if not bool(l.customProperty("VectorTilesReader/is_empty")): info("Clearing layer: {}", name) l.setCustomProperty("VectorTilesReader/is_empty", True) self._update_layer_source( l.source(), self._get_empty_feature_collection(0, l.name())) else: l.setCustomProperty("VectorTilesReader/is_empty", False) self._update_progress( progress=0, max_progress=len( self.feature_collections_by_layer_name_and_geotype), msg="Creating layers...") layer_filter = self._loading_options["layer_filter"] clipping_bounds = None if merge_features: clipping_bounds = self._loading_options["bounds"] new_layers = [] count = 0 for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype: count += 1 if self.cancel_requested: break if layer_filter and layer_name not in layer_filter: continue feature_collection = self.feature_collections_by_layer_name_and_geotype[ (layer_name, geo_type)] zoom_level = feature_collection["zoom_level"] file_name = self._get_geojson_filename(layer_name, geo_type) file_path = get_geojson_file_name(file_name) layer = None if os.path.isfile(file_path): # file exists already. add the features of the collection to the existing collection # get the layer from qgis and update its source for l in own_layers: if os.path.abspath(file_path).lower() == os.path.abspath( l.source()).lower(): layer = l break if layer: self._update_layer_source(file_path, feature_collection) layer.reload() if merge_features and geo_type in [ GeoTypes.LINE_STRING, GeoTypes.POLYGON ]: merger = FeatureMerger( should_cancel_func=lambda: self.cancel_requested) merger.merge_features(layer) if clip_tiles: clip_features( layer=layer, scheme=self._source.scheme(), bounds=clipping_bounds, should_cancel_func=lambda: self.cancel_requested) if not layer\ and (self._allowed_sources is None or file_path in self._allowed_sources)\ and (not layer_filter or layer_name in layer_filter): self._update_layer_source(file_path, feature_collection) layer = self._create_named_layer(json_src=file_path, layer_name=layer_name, geo_type=geo_type, zoom_level=zoom_level) if merge_features and geo_type in [ GeoTypes.LINE_STRING, GeoTypes.POLYGON ]: merger = FeatureMerger( should_cancel_func=lambda: self.cancel_requested) merger.merge_features(layer) if clip_tiles: clip_features( layer=layer, scheme=self._source.scheme(), bounds=clipping_bounds, should_cancel_func=lambda: self.cancel_requested) new_layers.append((layer_name, geo_type, layer)) self._update_progress(progress=count + 1) self._update_progress(msg="Refresh layers...") if len(new_layers) > 0 and not self.cancel_requested: self._update_progress(msg="Adding new layers...") only_layers = list( [layer_name_tuple[2] for layer_name_tuple in new_layers]) QgsMapLayerRegistry.instance().addMapLayers(only_layers, False) for name, geo_type, layer in new_layers: if self.cancel_requested: break self.add_layer_to_group.emit(layer) if apply_styles and not self.cancel_requested and new_layers: count = 0 conn_name = self.connection()["name"] styles_folder = get_style_folder(conn_name) styles = get_styles(conn_name) self._update_progress(progress=0, max_progress=len(new_layers), msg="Styling layers...") for name, geo_type, layer in new_layers: count += 1 if self.cancel_requested: break VtReader._apply_named_style(existing_styles=styles, style_dir=styles_folder, layer=layer, geo_type=geo_type) self._update_progress(progress=count)
def _decode_tiles(self, tiles_with_encoded_data): """ * Decodes the PBF data from all the specified tiles and reports the progress * If a tile is loaded from the cache, the decoded_data is already set and doesn't have to be encoded :param tiles_with_encoded_data: :return: """ clip_tiles = not self._loading_options["inspection_mode"] tiles_with_encoded_data = [(t[0], self._unzip(t[1]), clip_tiles) for t in tiles_with_encoded_data] if can_load_lib(): decoder_func = decode_tile_native else: decoder_func = decode_tile_python tiles = [] tile_data_tuples = [] if len(tiles_with_encoded_data) <= self._nr_tiles_to_process_serial: for t in tiles_with_encoded_data: tile, decoded_data = decoder_func(t) if decoded_data: tile_data_tuples.append((tile, decoded_data)) else: pool = self._get_pool() rs = pool.map_async(decoder_func, tiles_with_encoded_data, callback=tile_data_tuples.extend) pool.close() current_progress = 0 nr_of_tiles = len(tiles_with_encoded_data) self._update_progress( max_progress=nr_of_tiles, msg="Decoding {} tiles...".format(nr_of_tiles)) while not rs.ready() and not self.cancel_requested: QApplication.processEvents() remaining = rs._number_left index = nr_of_tiles - remaining progress = int(100.0 / nr_of_tiles * (index + 1)) if progress != current_progress: current_progress = progress self._update_progress(progress=progress) if self.cancel_requested: pool.terminate() pool.join() tile_data_tuples = sorted(tile_data_tuples, key=lambda t: t[0].id()) groups = groupby(tile_data_tuples, lambda t: t[0].id()) for key, group in groups: tile = None data = {} for t, decoded_data in list(group): if not decoded_data: continue if not tile: tile = t for layer_name in decoded_data: data[layer_name] = decoded_data[layer_name] tile.decoded_data = data tiles.append(tile) info("Decoding finished, {} tiles with data", len(tiles)) return tiles
def _load_tiles(self): # recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise self._source = self._create_source(self.connection()) try: if can_load_lib(): info("Native decoding supported!!!") else: bits = "32" if sys.maxsize > 2**32: bits = "64" info("Native decoding not supported: {}, {}bit", sys.platform, bits) self._feature_count = 0 self._all_tiles = [] bounds = self._loading_options["bounds"] clip_tiles = self._loading_options["clip_tiles"] max_tiles = self._loading_options["max_tiles"] layer_filter = self._loading_options["layer_filter"] info("Tile limit enabled: {}", max_tiles is not None and max_tiles > 0) self.cancel_requested = False self.feature_collections_by_layer_name_and_geotype = {} self._update_progress(show_dialog=True) self._clip_tiles_at_tile_bounds = clip_tiles zoom_level = self._get_clamped_zoom_level() all_tiles = get_all_tiles( bounds=bounds, is_cancel_requested_handler=lambda: self.cancel_requested, ) tiles_to_load = set() cached_tiles = [] tiles_to_ignore = set() source_name = self._source.name() scheme = self._source.scheme() for t in all_tiles: if self.cancel_requested or (max_tiles and len(cached_tiles) >= max_tiles): break decoded_data = get_cache_entry(cache_name=source_name, zoom_level=zoom_level, x=t[0], y=t[1]) if decoded_data: tile = VectorTile(scheme=scheme, zoom_level=zoom_level, x=t[0], y=t[1]) tile.decoded_data = decoded_data cached_tiles.append(tile) tiles_to_ignore.add((tile.column, tile.row)) else: tiles_to_load.add(t) remaining_nr_of_tiles = len(tiles_to_load) if max_tiles: if len(cached_tiles) + len(tiles_to_load) >= max_tiles: remaining_nr_of_tiles = clamp(max_tiles - len(cached_tiles), low=0) info("{} tiles in cache. Max. {} will be loaded additionally.", len(cached_tiles), remaining_nr_of_tiles) if len(cached_tiles) > 0: if not self.cancel_requested: self._process_tiles(cached_tiles, layer_filter) self._all_tiles.extend(cached_tiles) debug("Loading data for zoom level '{}' source '{}'", zoom_level, self._source.name()) if remaining_nr_of_tiles: tile_data_tuples = self._source.load_tiles( zoom_level=zoom_level, tiles_to_load=tiles_to_load, max_tiles=remaining_nr_of_tiles) if len(tile_data_tuples) > 0 and not self.cancel_requested: tiles = self._decode_tiles(tile_data_tuples) self._process_tiles(tiles, layer_filter) for t in tiles: cache_tile(cache_name=source_name, zoom_level=zoom_level, x=t.column, y=t.row, decoded_data=t.decoded_data) self._all_tiles.extend(tiles) self._continue_loading() except Exception as e: tb = "" if traceback: tb = traceback.format_exc() critical("An exception occured: {}, {}", e, tb) self.cancelled.emit()
def shutdown(self): info("Shutdown reader") self._source.progress_changed.disconnect() self._source.max_progress_changed.disconnect() self._source.message_changed.disconnect() self._source.close_connection()
def _create_qgis_layers(self, merge_features, apply_styles, clip_tiles): """ * Creates a hierarchy of groups and layers in qgis """ debug("Creating hierarchy in QGIS") qgis_layers = QgsMapLayerRegistry.instance().mapLayers() vt_qgis_name_layer_tuples = list( filter( lambda t: t[1].customProperty( "VectorTilesReader/vector_tile_source") == self._source. source(), iter(qgis_layers.items()))) own_layers = list(map(lambda t: t[1], vt_qgis_name_layer_tuples)) for l in own_layers: name = l.name() geo_type = l.customProperty("VectorTilesReader/geo_type") if (name, geo_type ) not in self.feature_collections_by_layer_name_and_geotype: if not bool(l.customProperty("VectorTilesReader/is_empty")): info("Clearing layer: {}", name) l.setCustomProperty("VectorTilesReader/is_empty", True) self._update_layer_source( l.source(), self._get_empty_feature_collection(0, l.name())) else: l.setCustomProperty("VectorTilesReader/is_empty", False) self._update_progress( progress=0, max_progress=len( self.feature_collections_by_layer_name_and_geotype), msg="Creating layers...") layer_filter = self._loading_options["layer_filter"] add_missing_layers = self._loading_options["add_missing_layers"] new_layers = [] count = 0 for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype: count += 1 if self.cancel_requested: break if layer_filter and layer_name not in layer_filter: continue feature_collection = self.feature_collections_by_layer_name_and_geotype[ (layer_name, geo_type)] zoom_level = feature_collection["zoom_level"] file_name = self._get_geojson_filename(layer_name, geo_type) file_path = get_geojson_file_name(file_name) layer = None if os.path.isfile(file_path): # file exists already. add the features of the collection to the existing collection # get the layer from qgis and update its source for l in own_layers: if os.path.abspath(file_path).lower() == os.path.abspath( l.source()).lower(): layer = l break if layer: self._update_layer_source(file_path, feature_collection) if merge_features and geo_type in [ GeoTypes.LINE_STRING, GeoTypes.POLYGON ]: FeatureMerger().merge_features(layer) if clip_tiles: clip_features(layer=layer, scheme=self._source.scheme()) if not layer and add_missing_layers and ( not layer_filter or layer_name in layer_filter): self._update_layer_source(file_path, feature_collection) layer = self._create_named_layer(json_src=file_path, layer_name=layer_name, geo_type=geo_type, zoom_level=zoom_level) if merge_features and geo_type in [ GeoTypes.LINE_STRING, GeoTypes.POLYGON ]: FeatureMerger().merge_features(layer) if clip_tiles: clip_features(layer=layer, scheme=self._source.scheme()) new_layers.append((layer_name, geo_type, layer)) self._update_progress(progress=count + 1) self._update_progress(msg="Refresh layers...") QgsMapLayerRegistry.instance().reloadAllLayers() if len(new_layers) > 0 and not self.cancel_requested: self._update_progress(msg="Adding new layers...") only_layers = list( [layer_name_tuple[2] for layer_name_tuple in new_layers]) QgsMapLayerRegistry.instance().addMapLayers(only_layers, False) for name, geo_type, layer in new_layers: if self.cancel_requested: break self.add_layer_to_group.emit(layer) if apply_styles and not self.cancel_requested: count = 0 self._update_progress(progress=0, max_progress=len(new_layers), msg="Styling layers...") for name, geo_type, layer in new_layers: count += 1 if self.cancel_requested: break VtReader._apply_named_style(layer, geo_type) self._update_progress(progress=count)