def _get_pool():
nr_processors = 4
try:
nr_processors = mp.cpu_count()
except NotImplementedError:
info("CPU count cannot be retrieved. Falling back to default = 4")
pool = mp.Pool(nr_processors)
return pool
def load_tiles_async(self, bounds):
"""
* Loads the vector tiles from either a file or a URL and adds them to QGIS
:param zoom_level: The zoom level to load
:param bounds:
:return:
"""
zoom_level = bounds["zoom"]
info("Loading zoom level '{}', bounds: {}", zoom_level, bounds)
self._loading_options["zoom_level"] = zoom_level
self._loading_options["bounds"] = bounds
_worker_thread = QThread(self.iface.mainWindow())
self.moveToThread(_worker_thread)
_worker_thread.started.connect(self._load_tiles)
_worker_thread.start()
def _continue_loading(self):
zoom_level = self._loading_options["zoom_level"]
merge_tiles = self._loading_options["merge_tiles"]
apply_styles = self._loading_options["apply_styles"]
clip_tiles = self._loading_options["clip_tiles"]
if not self.cancel_requested:
self._create_qgis_layers(merge_features=merge_tiles,
apply_styles=apply_styles,
clip_tiles=clip_tiles)
self._update_progress(show_dialog=False)
if self.cancel_requested:
info("Import cancelled")
self.cancelled.emit()
else:
info("Import complete")
loaded_extent = self._get_extent(self._all_tiles, zoom_level)
if not loaded_extent:
loaded_extent = {}
self.loading_finished.emit(zoom_level, loaded_extent)
def _create_qgis_layers(self, merge_features, apply_styles, clip_tiles):
"""
* Creates a hierarchy of groups and layers in qgis
"""
own_layers = get_loaded_layers_of_connection(self._connection["name"])
for l in own_layers:
name = l.name()
geo_type = l.customProperty("VectorTilesReader/geo_type")
if (name, geo_type
) not in self.feature_collections_by_layer_name_and_geotype:
if not bool(l.customProperty("VectorTilesReader/is_empty")):
info("Clearing layer: {}", name)
l.setCustomProperty("VectorTilesReader/is_empty", True)
self._update_layer_source(
l.source(),
self._get_empty_feature_collection(0, l.name()))
else:
l.setCustomProperty("VectorTilesReader/is_empty", False)
self._update_progress(
progress=0,
max_progress=len(
self.feature_collections_by_layer_name_and_geotype),
msg="Creating layers...")
layer_filter = self._loading_options["layer_filter"]
clipping_bounds = None
if merge_features:
clipping_bounds = self._loading_options["bounds"]
new_layers = []
count = 0
for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype:
count += 1
if self.cancel_requested:
break
if layer_filter and layer_name not in layer_filter:
continue
feature_collection = self.feature_collections_by_layer_name_and_geotype[
(layer_name, geo_type)]
zoom_level = feature_collection["zoom_level"]
file_name = self._get_geojson_filename(layer_name, geo_type)
file_path = get_geojson_file_name(file_name)
layer = None
if os.path.isfile(file_path):
# file exists already. add the features of the collection to the existing collection
# get the layer from qgis and update its source
for l in own_layers:
if os.path.abspath(file_path).lower() == os.path.abspath(
l.source()).lower():
layer = l
break
if layer:
self._update_layer_source(file_path, feature_collection)
layer.reload()
if merge_features and geo_type in [
GeoTypes.LINE_STRING, GeoTypes.POLYGON
]:
merger = FeatureMerger(
should_cancel_func=lambda: self.cancel_requested)
merger.merge_features(layer)
if clip_tiles:
clip_features(
layer=layer,
scheme=self._source.scheme(),
bounds=clipping_bounds,
should_cancel_func=lambda: self.cancel_requested)
if not layer\
and (self._allowed_sources is None or file_path in self._allowed_sources)\
and (not layer_filter or layer_name in layer_filter):
self._update_layer_source(file_path, feature_collection)
layer = self._create_named_layer(json_src=file_path,
layer_name=layer_name,
geo_type=geo_type,
zoom_level=zoom_level)
if merge_features and geo_type in [
GeoTypes.LINE_STRING, GeoTypes.POLYGON
]:
merger = FeatureMerger(
should_cancel_func=lambda: self.cancel_requested)
merger.merge_features(layer)
if clip_tiles:
clip_features(
layer=layer,
scheme=self._source.scheme(),
bounds=clipping_bounds,
should_cancel_func=lambda: self.cancel_requested)
new_layers.append((layer_name, geo_type, layer))
self._update_progress(progress=count + 1)
self._update_progress(msg="Refresh layers...")
if len(new_layers) > 0 and not self.cancel_requested:
self._update_progress(msg="Adding new layers...")
only_layers = list(
[layer_name_tuple[2] for layer_name_tuple in new_layers])
QgsMapLayerRegistry.instance().addMapLayers(only_layers, False)
for name, geo_type, layer in new_layers:
if self.cancel_requested:
break
self.add_layer_to_group.emit(layer)
if apply_styles and not self.cancel_requested and new_layers:
count = 0
conn_name = self.connection()["name"]
styles_folder = get_style_folder(conn_name)
styles = get_styles(conn_name)
self._update_progress(progress=0,
max_progress=len(new_layers),
msg="Styling layers...")
for name, geo_type, layer in new_layers:
count += 1
if self.cancel_requested:
break
VtReader._apply_named_style(existing_styles=styles,
style_dir=styles_folder,
layer=layer,
geo_type=geo_type)
self._update_progress(progress=count)
def _decode_tiles(self, tiles_with_encoded_data):
"""
* Decodes the PBF data from all the specified tiles and reports the progress
* If a tile is loaded from the cache, the decoded_data is already set and doesn't have to be encoded
:param tiles_with_encoded_data:
:return:
"""
clip_tiles = not self._loading_options["inspection_mode"]
tiles_with_encoded_data = [(t[0], self._unzip(t[1]), clip_tiles)
for t in tiles_with_encoded_data]
if can_load_lib():
decoder_func = decode_tile_native
else:
decoder_func = decode_tile_python
tiles = []
tile_data_tuples = []
if len(tiles_with_encoded_data) <= self._nr_tiles_to_process_serial:
for t in tiles_with_encoded_data:
tile, decoded_data = decoder_func(t)
if decoded_data:
tile_data_tuples.append((tile, decoded_data))
else:
pool = self._get_pool()
rs = pool.map_async(decoder_func,
tiles_with_encoded_data,
callback=tile_data_tuples.extend)
pool.close()
current_progress = 0
nr_of_tiles = len(tiles_with_encoded_data)
self._update_progress(
max_progress=nr_of_tiles,
msg="Decoding {} tiles...".format(nr_of_tiles))
while not rs.ready() and not self.cancel_requested:
QApplication.processEvents()
remaining = rs._number_left
index = nr_of_tiles - remaining
progress = int(100.0 / nr_of_tiles * (index + 1))
if progress != current_progress:
current_progress = progress
self._update_progress(progress=progress)
if self.cancel_requested:
pool.terminate()
pool.join()
tile_data_tuples = sorted(tile_data_tuples, key=lambda t: t[0].id())
groups = groupby(tile_data_tuples, lambda t: t[0].id())
for key, group in groups:
tile = None
data = {}
for t, decoded_data in list(group):
if not decoded_data:
continue
if not tile:
tile = t
for layer_name in decoded_data:
data[layer_name] = decoded_data[layer_name]
tile.decoded_data = data
tiles.append(tile)
info("Decoding finished, {} tiles with data", len(tiles))
return tiles
def _load_tiles(self):
# recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise
self._source = self._create_source(self.connection())
try:
if can_load_lib():
info("Native decoding supported!!!")
else:
bits = "32"
if sys.maxsize > 2**32:
bits = "64"
info("Native decoding not supported: {}, {}bit", sys.platform,
bits)
self._feature_count = 0
self._all_tiles = []
bounds = self._loading_options["bounds"]
clip_tiles = self._loading_options["clip_tiles"]
max_tiles = self._loading_options["max_tiles"]
layer_filter = self._loading_options["layer_filter"]
info("Tile limit enabled: {}", max_tiles is not None
and max_tiles > 0)
self.cancel_requested = False
self.feature_collections_by_layer_name_and_geotype = {}
self._update_progress(show_dialog=True)
self._clip_tiles_at_tile_bounds = clip_tiles
zoom_level = self._get_clamped_zoom_level()
all_tiles = get_all_tiles(
bounds=bounds,
is_cancel_requested_handler=lambda: self.cancel_requested,
)
tiles_to_load = set()
cached_tiles = []
tiles_to_ignore = set()
source_name = self._source.name()
scheme = self._source.scheme()
for t in all_tiles:
if self.cancel_requested or (max_tiles and
len(cached_tiles) >= max_tiles):
break
decoded_data = get_cache_entry(cache_name=source_name,
zoom_level=zoom_level,
x=t[0],
y=t[1])
if decoded_data:
tile = VectorTile(scheme=scheme,
zoom_level=zoom_level,
x=t[0],
y=t[1])
tile.decoded_data = decoded_data
cached_tiles.append(tile)
tiles_to_ignore.add((tile.column, tile.row))
else:
tiles_to_load.add(t)
remaining_nr_of_tiles = len(tiles_to_load)
if max_tiles:
if len(cached_tiles) + len(tiles_to_load) >= max_tiles:
remaining_nr_of_tiles = clamp(max_tiles -
len(cached_tiles),
low=0)
info("{} tiles in cache. Max. {} will be loaded additionally.",
len(cached_tiles), remaining_nr_of_tiles)
if len(cached_tiles) > 0:
if not self.cancel_requested:
self._process_tiles(cached_tiles, layer_filter)
self._all_tiles.extend(cached_tiles)
debug("Loading data for zoom level '{}' source '{}'", zoom_level,
self._source.name())
if remaining_nr_of_tiles:
tile_data_tuples = self._source.load_tiles(
zoom_level=zoom_level,
tiles_to_load=tiles_to_load,
max_tiles=remaining_nr_of_tiles)
if len(tile_data_tuples) > 0 and not self.cancel_requested:
tiles = self._decode_tiles(tile_data_tuples)
self._process_tiles(tiles, layer_filter)
for t in tiles:
cache_tile(cache_name=source_name,
zoom_level=zoom_level,
x=t.column,
y=t.row,
decoded_data=t.decoded_data)
self._all_tiles.extend(tiles)
self._continue_loading()
except Exception as e:
tb = ""
if traceback:
tb = traceback.format_exc()
critical("An exception occured: {}, {}", e, tb)
self.cancelled.emit()
def shutdown(self):
info("Shutdown reader")
self._source.progress_changed.disconnect()
self._source.max_progress_changed.disconnect()
self._source.message_changed.disconnect()
self._source.close_connection()
def _create_qgis_layers(self, merge_features, apply_styles, clip_tiles):
"""
* Creates a hierarchy of groups and layers in qgis
"""
debug("Creating hierarchy in QGIS")
qgis_layers = QgsMapLayerRegistry.instance().mapLayers()
vt_qgis_name_layer_tuples = list(
filter(
lambda t: t[1].customProperty(
"VectorTilesReader/vector_tile_source") == self._source.
source(), iter(qgis_layers.items())))
own_layers = list(map(lambda t: t[1], vt_qgis_name_layer_tuples))
for l in own_layers:
name = l.name()
geo_type = l.customProperty("VectorTilesReader/geo_type")
if (name, geo_type
) not in self.feature_collections_by_layer_name_and_geotype:
if not bool(l.customProperty("VectorTilesReader/is_empty")):
info("Clearing layer: {}", name)
l.setCustomProperty("VectorTilesReader/is_empty", True)
self._update_layer_source(
l.source(),
self._get_empty_feature_collection(0, l.name()))
else:
l.setCustomProperty("VectorTilesReader/is_empty", False)
self._update_progress(
progress=0,
max_progress=len(
self.feature_collections_by_layer_name_and_geotype),
msg="Creating layers...")
layer_filter = self._loading_options["layer_filter"]
add_missing_layers = self._loading_options["add_missing_layers"]
new_layers = []
count = 0
for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype:
count += 1
if self.cancel_requested:
break
if layer_filter and layer_name not in layer_filter:
continue
feature_collection = self.feature_collections_by_layer_name_and_geotype[
(layer_name, geo_type)]
zoom_level = feature_collection["zoom_level"]
file_name = self._get_geojson_filename(layer_name, geo_type)
file_path = get_geojson_file_name(file_name)
layer = None
if os.path.isfile(file_path):
# file exists already. add the features of the collection to the existing collection
# get the layer from qgis and update its source
for l in own_layers:
if os.path.abspath(file_path).lower() == os.path.abspath(
l.source()).lower():
layer = l
break
if layer:
self._update_layer_source(file_path, feature_collection)
if merge_features and geo_type in [
GeoTypes.LINE_STRING, GeoTypes.POLYGON
]:
FeatureMerger().merge_features(layer)
if clip_tiles:
clip_features(layer=layer,
scheme=self._source.scheme())
if not layer and add_missing_layers and (
not layer_filter or layer_name in layer_filter):
self._update_layer_source(file_path, feature_collection)
layer = self._create_named_layer(json_src=file_path,
layer_name=layer_name,
geo_type=geo_type,
zoom_level=zoom_level)
if merge_features and geo_type in [
GeoTypes.LINE_STRING, GeoTypes.POLYGON
]:
FeatureMerger().merge_features(layer)
if clip_tiles:
clip_features(layer=layer, scheme=self._source.scheme())
new_layers.append((layer_name, geo_type, layer))
self._update_progress(progress=count + 1)
self._update_progress(msg="Refresh layers...")
QgsMapLayerRegistry.instance().reloadAllLayers()
if len(new_layers) > 0 and not self.cancel_requested:
self._update_progress(msg="Adding new layers...")
only_layers = list(
[layer_name_tuple[2] for layer_name_tuple in new_layers])
QgsMapLayerRegistry.instance().addMapLayers(only_layers, False)
for name, geo_type, layer in new_layers:
if self.cancel_requested:
break
self.add_layer_to_group.emit(layer)
if apply_styles and not self.cancel_requested:
count = 0
self._update_progress(progress=0,
max_progress=len(new_layers),
msg="Styling layers...")
for name, geo_type, layer in new_layers:
count += 1
if self.cancel_requested:
break
VtReader._apply_named_style(layer, geo_type)
self._update_progress(progress=count)