def _apply_named_style(layer, geo_type):
"""
* Looks for a styles with the same name as the layer and if one is found, it is applied to the layer
:param layer:
:param layer_path: e.g. 'transportation.service' or 'transportation_name.path'
:return:
"""
try:
name = layer.name().split(VtReader._zoom_level_delimiter)[0].lower()
styles = [
"{}.{}".format(name, geo_type.lower()),
name
]
for p in styles:
style_name = "{}.qml".format(p).lower()
if style_name in VtReader._styles:
style_path = os.path.join(FileHelper.get_plugin_directory(), "styles/{}".format(style_name))
res = layer.loadNamedStyle(style_path)
if res[1]: # Style loaded
layer.setCustomProperty("layerStyle", style_path)
if layer.customProperty("layerStyle") == style_path:
debug("Style successfully applied: {}", style_name)
break
except:
critical("Loading style failed: {}", sys.exc_info())
def _connect_to_db(self):
"""
* Since an mbtile file is a sqlite database, we can connect to it
"""
debug("Connecting to: {}", self.path)
try:
self.conn = sqlite3.connect(self.path)
self.conn.row_factory = sqlite3.Row
debug("Successfully connected")
except:
critical("Db connection failed:", sys.exc_info())
def cache_tile(tile, file_name):
if not tile.decoded_data:
warn("Trying to cache a tile without data: {}", tile)
file_path = os.path.join(FileHelper.get_cache_directory(), file_name)
try:
with open(file_path, 'wb') as f:
pickle.dump(tile, f, pickle.HIGHEST_PROTOCOL)
except:
critical("Error while writing tile '{}' to cache: {}", str(tile),
sys.exc_info()[1])
def _get_from_db(self, sql):
if not self.conn:
debug("Not connected yet.")
self._connect_to_db()
try:
debug("Execute SQL: {}", sql)
cur = self.conn.cursor()
cur.execute(sql)
return cur.fetchall()
except:
critical("Getting data from db failed: {}", sys.exc_info())
def _create_reader(self, path_or_url):
# A lazy import is required because the vtreader depends on the external libs
from vt_reader import VtReader
reader = None
try:
reader = VtReader(self.iface,
path_or_url=path_or_url,
progress_handler=self.handle_progress_update)
except RuntimeError:
QMessageBox.critical(None, "Loading Error", str(sys.exc_info()[1]))
critical(str(sys.exc_info()[1]))
return reader
def export(self):
self._cancel_requested = False
self.conn = None
try:
self.conn = self._create_db()
except:
critical("db creation failed: {}", sys.exc_info())
if self.conn:
try:
with self.conn:
tile_names = self._get_loaded_tile_names()
if tile_names:
tiles = self._load_tiles(tile_names)
nr_tiles = len(tiles)
for index, t in enumerate(tiles):
if self._cancel_requested:
break
self._update_progress(
title="Export tile {}/{}".format(
index + 1, nr_tiles),
show_dialog=True)
QApplication.processEvents()
self._update_bounds(t)
debug("layers to export: {}",
self.layers_to_export)
self._save_tile(t)
if not self._cancel_requested:
layer_objects = map(lambda l: {"id": l},
self.layer_names)
vector_layers = {"vector_layers": layer_objects}
self.metadata["json"] = json.dumps(vector_layers)
self._save_metadata()
debug("export complete")
self.iface.messageBar().pushInfo(
u'Vector Tiles Reader',
u'mbtiles export completed')
else:
debug("export cancelled")
self.iface.messageBar().pushInfo(
u'Vector Tiles Reader',
u'mbtiles export cancelled')
self.conn.close()
except:
if self.conn:
self.conn.close()
critical("Export failed: {}", sys.exc_info())
raise
self._update_progress(show_dialog=False)
def load(self):
debug("Loading TileJSON")
success = False
try:
status, data = FileHelper.load_url(self.url)
self.json = json.loads(data)
if self.json:
debug("TileJSON loaded")
self._validate()
success = True
else:
debug("Loading TileJSON failed")
self.json = {}
raise RuntimeError("TileJSON could not be loaded.")
except:
critical("Loading TileJSON failed ({}): {}", self.url, sys.exc_info())
return success
def _get_single_value(self, sql_query, field_name):
"""
* Helper function that can be used to safely load a single value from the db
* Returns the value or None if result is empty or execution of query failed
:param sql_query:
:param field_name:
:return:
"""
value = None
try:
rows = self._get_from_db(sql=sql_query)
if rows:
value = rows[0][field_name]
debug("Value is: {}".format(value))
except:
critical("Loading metadata value '{}' failed: {}", field_name, sys.exc_info())
return value
def _export_tiles(self):
from vt_writer import VtWriter
file_name = QFileDialog.getSaveFileName(
None, "Export Vector Tiles", FileHelper.get_home_directory(),
"mbtiles (*.mbtiles)")
if file_name:
self.export_action.setDisabled(True)
try:
self._current_writer = VtWriter(
self.iface,
file_name,
progress_handler=self.handle_progress_update)
self._create_progress_dialog(self.iface.mainWindow(),
on_cancel=self._cancel_export)
self._current_writer.export()
except:
critical("Error during export: {}", sys.exc_info())
self.export_action.setEnabled(True)
def _load_tiles(self,
path,
options,
layers_to_load,
bounds=None,
ignore_limit=False):
merge_tiles = options.merge_tiles_enabled()
apply_styles = options.apply_styles_enabled()
tile_limit = options.tile_number_limit()
load_mask_layer = options.load_mask_layer_enabled()
if ignore_limit:
tile_limit = None
manual_zoom = options.manual_zoom()
cartographic_ordering = options.cartographic_ordering()
if apply_styles:
self._set_background_color()
debug("Load: {}", path)
reader = self._current_reader
if reader:
reader.enable_cartographic_ordering(enabled=cartographic_ordering)
try:
zoom = reader.source.max_zoom()
if manual_zoom is not None:
zoom = manual_zoom
reader.load_tiles(zoom_level=zoom,
layer_filter=layers_to_load,
load_mask_layer=load_mask_layer,
merge_tiles=merge_tiles,
apply_styles=apply_styles,
max_tiles=tile_limit,
bounds=bounds,
limit_reacher_handler=lambda: self.
_show_limit_exceeded_message(tile_limit))
self.refresh_layers()
debug("Loading complete!")
except RuntimeError:
QMessageBox.critical(None, "Unexpected exception",
str(sys.exc_info()[1]))
critical(str(sys.exc_info()[1]))
def _create_reader(self, path_or_url):
# A lazy import is required because the vtreader depends on the external libs
from vt_reader import VtReader
reader = None
try:
reader = VtReader(self.iface, path_or_url=path_or_url)
reader.progress_changed.connect(self.reader_progress_changed)
reader.max_progress_changed.connect(
self.reader_max_progress_changed)
reader.show_progress_changed.connect(
self.reader_show_progress_changed)
reader.title_changed.connect(self.reader_title_changed)
reader.message_changed.connect(self.reader_message_changed)
reader.loading_finished.connect(self.reader_loading_finished)
reader.tile_limit_reached.connect(
self.reader_limit_exceeded_message)
reader.cancelled.connect(self.reader_cancelled)
except RuntimeError:
QMessageBox.critical(None, "Loading Error", str(sys.exc_info()[1]))
critical(str(sys.exc_info()[1]))
return reader
def load(self):
debug("Loading TileJSON")
success = False
try:
if os.path.isfile(self.url):
with open(self.url, 'r') as f:
data = f.read()
else:
status, data = FileHelper.load_url(self.url)
self.json = json.loads(data)
if self.json:
debug("TileJSON loaded")
self._validate()
debug("TileJSON validated")
success = True
else:
info("Parsing TileJSON failed")
self.json = {}
raise RuntimeError("TileJSON could not be loaded.")
except:
critical("Loading TileJSON failed ({}): {}", self.url, sys.exc_info())
return success
def _load_tiles(self):
try:
# recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise
self.source = self._create_source(self.source.source())
zoom_level = self._loading_options["zoom_level"]
bounds = self._loading_options["bounds"]
load_mask_layer = self._loading_options["load_mask_layer"]
merge_tiles = self._loading_options["merge_tiles"]
clip_tiles = self._loading_options["clip_tiles"]
apply_styles = self._loading_options["apply_styles"]
max_tiles = self._loading_options["max_tiles"]
layer_filter = self._loading_options["layer_filter"]
self.cancel_requested = False
self.feature_collections_by_layer_name_and_geotype = {}
self._qgis_layer_groups_by_name = {}
self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name())))
self._clip_tiles_at_tile_bounds = clip_tiles
min_zoom = self.source.min_zoom()
max_zoom = self.source.max_zoom()
zoom_level = clamp(zoom_level, low=min_zoom, high=max_zoom)
all_tiles = get_all_tiles(
bounds=bounds,
is_cancel_requested_handler=lambda: self.cancel_requested,
)
tiles_to_load = set()
tiles = []
tiles_to_ignore = set()
for t in all_tiles:
if self.cancel_requested or (max_tiles and len(tiles) >= max_tiles):
break
file_name = self._get_tile_cache_name(zoom_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
tiles_to_ignore.add((tile.column, tile.row))
else:
tiles_to_load.add(t)
remaining_nr_of_tiles = len(tiles_to_load)
if max_tiles:
if len(tiles) + len(tiles_to_load) >= max_tiles:
remaining_nr_of_tiles = max_tiles - len(tiles)
if remaining_nr_of_tiles < 0:
remaining_nr_of_tiles = 0
debug("{} cache hits. {} may potentially be loaded.", len(tiles), remaining_nr_of_tiles)
debug("Loading data for zoom level '{}' source '{}'", zoom_level, self.source.name())
tile_data_tuples = []
if remaining_nr_of_tiles > 0:
tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level,
tiles_to_load=tiles_to_load,
max_tiles=remaining_nr_of_tiles)
if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0):
QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!")
if load_mask_layer:
mask_level = self.source.mask_level()
if mask_level is not None and mask_level != zoom_level:
debug("Mapping {} tiles to mask level", len(all_tiles))
scheme = self.source.scheme()
mask_tiles = map(
lambda t: change_zoom(zoom_level, int(mask_level), t, scheme),
all_tiles)
debug("Mapping done")
mask_tiles_to_load = set()
for t in mask_tiles:
file_name = self._get_tile_cache_name(mask_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
mask_tiles_to_load.add(t)
debug("Loading mask layer (zoom_level={})", mask_level)
tile_data_tuples = []
if len(mask_tiles_to_load) > 0:
mask_layer_data = self.source.load_tiles(zoom_level=mask_level,
tiles_to_load=mask_tiles_to_load,
max_tiles=max_tiles)
debug("Mask layer loaded")
tile_data_tuples.extend(mask_layer_data)
if tile_data_tuples and len(tile_data_tuples) > 0:
if not self.cancel_requested:
decoded_tiles = self._decode_tiles(tile_data_tuples)
tiles.extend(decoded_tiles)
if len(tiles) > 0:
if not self.cancel_requested:
self._process_tiles(tiles, layer_filter)
if not self.cancel_requested:
self._create_qgis_layers(merge_features=merge_tiles,
apply_styles=apply_styles)
self._update_progress(show_dialog=False)
if self.cancel_requested:
info("Import cancelled")
self.cancelled.emit()
else:
info("Import complete")
loaded_tiles_x = map(lambda t: t.coord()[0], tiles)
loaded_tiles_y = map(lambda t: t.coord()[1], tiles)
if len(loaded_tiles_x) == 0 or len(loaded_tiles_y) == 0:
return None
loaded_extent = {"x_min": int(min(loaded_tiles_x)),
"x_max": int(max(loaded_tiles_x)),
"y_min": int(min(loaded_tiles_y)),
"y_max": int(max(loaded_tiles_y)),
"zoom": int(zoom_level)
}
loaded_extent["width"] = loaded_extent["x_max"] - loaded_extent["x_min"] + 1
loaded_extent["height"] = loaded_extent["y_max"] - loaded_extent["y_min"] + 1
self.loading_finished.emit(zoom_level, loaded_extent)
except Exception as e:
critical("An exception occured: {}, {}", e, traceback.format_exc())
self.cancelled.emit()
def _load_tiles(self,
options,
layers_to_load,
bounds=None,
ignore_limit=False):
if self._debouncer.is_running():
self._debouncer.pause()
merge_tiles = options.merge_tiles_enabled()
apply_styles = options.apply_styles_enabled()
tile_limit = options.tile_number_limit()
load_mask_layer = options.load_mask_layer_enabled()
self._auto_zoom = options.auto_zoom_enabled()
if ignore_limit:
tile_limit = None
manual_zoom = options.manual_zoom()
clip_tiles = options.clip_tiles()
if apply_styles:
self._set_background_color()
reader = self._current_reader
if not reader:
self._is_loading = False
else:
try:
max_zoom = reader.source.max_zoom()
min_zoom = reader.source.min_zoom()
if self._auto_zoom:
zoom = self._get_zoom_for_current_map_scale()
zoom = clamp(zoom, low=min_zoom, high=max_zoom)
else:
zoom = max_zoom
if manual_zoom is not None:
zoom = manual_zoom
self._current_zoom = zoom
source_bounds = reader.source.bounds_tile(zoom)
if source_bounds and not self._extent_overlap_bounds(
bounds, source_bounds):
info(
"The current extent '{}' is not within the bounds of the source '{}'. The extent to load "
"will be set to the bounds of the source", bounds,
source_bounds)
bounds = source_bounds
reader.set_options(layer_filter=layers_to_load,
load_mask_layer=load_mask_layer,
merge_tiles=merge_tiles,
clip_tiles=clip_tiles,
apply_styles=apply_styles,
max_tiles=tile_limit)
self._is_loading = True
reader.load_tiles_async(zoom_level=zoom, bounds=bounds)
except Exception as e:
critical("An exception occured: {}", e)
tb_lines = traceback.format_tb(sys.exc_traceback)
tb_text = ""
for line in tb_lines:
tb_text += line
critical("{}", tb_text)
self.iface.messageBar().pushMessage(
"Something went horribly wrong. Please have a look at the log.",
level=QgsMessageBar.CRITICAL,
duration=5)
if self.progress_dialog:
self.progress_dialog.hide()
self._is_loading = False