class CreatePointsSurveyWizard(QWizard, WIZARD_UI):
WIZARD_NAME = "CreatePointsSurveyWizard"
WIZARD_TOOL_NAME = QCoreApplication.translate(WIZARD_NAME, "Create Point")
def __init__(self, iface, db):
QWizard.__init__(self)
self.setupUi(self)
self.iface = iface
self._db = db
self.logger = Logger()
self.app = AppInterface()
self.names = self._db.names
self.help_strings = HelpStrings()
self._layers = {
self.names.LC_BOUNDARY_POINT_T: None,
self.names.LC_SURVEY_POINT_T: None,
self.names.LC_CONTROL_POINT_T: None
}
self.target_layer = None
# Auxiliary data to set nonlinear next pages
self.pages = [self.wizardPage1, self.wizardPage2, self.wizardPage3]
self.dict_pages_ids = {self.pages[idx] : pid for idx, pid in enumerate(self.pageIds())}
self.mMapLayerComboBox.setFilters(QgsMapLayerProxyModel.PointLayer)
# Set connections
self.btn_browse_file.clicked.connect(
make_file_selector(self.txt_file_path,
file_filter=QCoreApplication.translate("WizardTranslations",'CSV File (*.csv *.txt)')))
self.txt_file_path.textChanged.connect(self.file_path_changed)
self.crsSelector.crsChanged.connect(self.crs_changed)
self.crs = "" # SRS auth id
self.txt_delimiter.textChanged.connect(self.fill_long_lat_combos)
self.mMapLayerComboBox.layerChanged.connect(self.import_layer_changed)
self.known_delimiters = [
{'name': ';', 'value': ';'},
{'name': ',', 'value': ','},
{'name': 'tab', 'value': '\t'},
{'name': 'space', 'value': ' '},
{'name': '|', 'value': '|'},
{'name': '~', 'value': '~'},
{'name': 'Other', 'value': ''}
]
self.cbo_delimiter.addItems([ item['name'] for item in self.known_delimiters ])
self.cbo_delimiter.currentTextChanged.connect(self.separator_changed)
self.restore_settings()
self.txt_file_path.textChanged.emit(self.txt_file_path.text())
self.rad_boundary_point.toggled.connect(self.point_option_changed)
self.rad_control_point.toggled.connect(self.point_option_changed)
self.rad_csv.toggled.connect(self.adjust_page_2_controls)
self.point_option_changed() # Initialize it
self.button(QWizard.FinishButton).clicked.connect(self.finished_dialog)
self.currentIdChanged.connect(self.current_page_changed)
self.txt_help_page_2.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_2_OPTION_CSV)
self.wizardPage2.setButtonText(QWizard.FinishButton,
QCoreApplication.translate("WizardTranslations",
"Import"))
self.txt_help_page_3.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_3_OPTION_CSV)
self.txt_help_page_3.anchorClicked.connect(self.save_template)
self.button(QWizard.HelpButton).clicked.connect(self.show_help)
self.rejected.connect(self.close_wizard)
# Set MessageBar for QWizard
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.setLayout(QGridLayout())
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
def nextId(self):
"""
Set navigation order. Should return an integer. -1 is Finish.
"""
if self.currentId() == self.dict_pages_ids[self.wizardPage1]:
return self.dict_pages_ids[self.wizardPage2]
elif self.currentId() == self.dict_pages_ids[self.wizardPage2]:
if self.rad_csv.isChecked():
return self.dict_pages_ids[self.wizardPage3]
elif self.rad_refactor.isChecked():
return -1
elif self.currentId() == self.dict_pages_ids[self.wizardPage3]:
return -1
else:
return -1
def current_page_changed(self, id):
"""
Reset the Next button. Needed because Next might have been disabled by a
condition in a another SLOT.
"""
enable_next_wizard(self)
if id == self.dict_pages_ids[self.wizardPage2]:
self.adjust_page_2_controls()
elif id == self.dict_pages_ids[self.wizardPage3]:
self.set_buttons_visible(False)
self.set_buttons_enabled(False)
QCoreApplication.processEvents()
self.check_z_in_geometry()
QCoreApplication.processEvents()
self.fill_long_lat_combos("")
QCoreApplication.processEvents()
self.set_buttons_visible(True)
self.set_buttons_enabled(True)
def set_buttons_visible(self, visible):
self.button(self.BackButton).setVisible(visible)
self.button(self.FinishButton).setVisible(visible)
self.button(self.CancelButton).setVisible(visible)
def set_buttons_enabled(self, enabled):
self.wizardPage3.setEnabled(enabled)
self.button(self.BackButton).setEnabled(enabled)
self.button(self.FinishButton).setEnabled(enabled)
self.button(self.CancelButton).setEnabled(enabled)
def check_z_in_geometry(self):
self.target_layer = self.app.core.get_layer(self._db, self.current_point_name(), load=True)
if not self.target_layer:
return
if not QgsWkbTypes().hasZ(self.target_layer.wkbType()):
self.labelZ.setEnabled(False)
self.cbo_elevation.setEnabled(False)
msg = QCoreApplication.translate("WizardTranslations",
"The current model does not support 3D geometries")
self.cbo_elevation.setToolTip(msg)
self.labelZ.setToolTip(msg)
else:
self.labelZ.setEnabled(True)
self.cbo_elevation.setEnabled(True)
self.labelZ.setToolTip("")
self.cbo_elevation.setToolTip("")
def adjust_page_2_controls(self):
self.cbo_mapping.clear()
self.cbo_mapping.addItem("")
self.cbo_mapping.addItems(self.app.core.get_field_mappings_file_names(self.current_point_name()))
if self.rad_refactor.isChecked():
self.lbl_refactor_source.setEnabled(True)
self.mMapLayerComboBox.setEnabled(True)
self.lbl_field_mapping.setEnabled(True)
self.cbo_mapping.setEnabled(True)
self.import_layer_changed(self.mMapLayerComboBox.currentLayer())
disable_next_wizard(self)
self.wizardPage2.setFinalPage(True)
self.txt_help_page_2.setHtml(self.help_strings.get_refactor_help_string(self._db, self._layers[self.current_point_name()]))
elif self.rad_csv.isChecked():
self.lbl_refactor_source.setEnabled(False)
self.mMapLayerComboBox.setEnabled(False)
self.lbl_field_mapping.setEnabled(False)
self.cbo_mapping.setEnabled(False)
self.lbl_refactor_source.setStyleSheet('')
enable_next_wizard(self)
self.wizardPage2.setFinalPage(False)
self.txt_help_page_2.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_2_OPTION_CSV)
def point_option_changed(self):
if self.rad_boundary_point.isChecked():
self.gbx_page_2.setTitle(QCoreApplication.translate("WizardTranslations", "Load data to Boundary Points..."))
self.gbx_page_3.setTitle(QCoreApplication.translate("WizardTranslations", "Configure CSV data source for Boundary Points..."))
self.txt_help_page_1.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_1_OPTION_BP)
elif self.rad_survey_point.isChecked(): # self.rad_survey_point is checked
self.gbx_page_2.setTitle(QCoreApplication.translate("WizardTranslations", "Load data to Survey Points..."))
self.gbx_page_3.setTitle(QCoreApplication.translate("WizardTranslations", "Configure CSV data source for Survey Points..."))
self.txt_help_page_1.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_1_OPTION_SP)
else: # self.rad_control_point is checked
self.gbx_page_2.setTitle(QCoreApplication.translate("WizardTranslations", "Load data to Control Points..."))
self.gbx_page_3.setTitle(QCoreApplication.translate("WizardTranslations", "Configure CSV data source for Control Points..."))
self.txt_help_page_1.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_1_OPTION_CP)
def finished_dialog(self):
self.save_settings()
if self.rad_refactor.isChecked():
output_layer_name = self.current_point_name()
if self.mMapLayerComboBox.currentLayer() is not None:
field_mapping = self.cbo_mapping.currentText()
res_etl_model = self.app.core.show_etl_model(self._db,
self.mMapLayerComboBox.currentLayer(),
output_layer_name,
field_mapping=field_mapping)
if res_etl_model:
self.app.gui.redraw_all_layers() # Redraw all layers to show imported data
# If the result of the etl_model is successful and we used a stored recent mapping, we delete the
# previous mapping used (we give preference to the latest used mapping)
if field_mapping:
self.app.core.delete_old_field_mapping(field_mapping)
self.app.core.save_field_mapping(output_layer_name)
else:
self.logger.warning_msg(__name__, QCoreApplication.translate("WizardTranslations",
"Select a source layer to set the field mapping to '{}'.").format(output_layer_name))
self.close_wizard()
elif self.rad_csv.isChecked():
self.prepare_copy_csv_points_to_db()
def close_wizard(self, message=None, show_message=True):
if message is None:
message = QCoreApplication.translate("WizardTranslations", "'{}' tool has been closed.").format(self.WIZARD_TOOL_NAME)
if show_message:
self.logger.info_msg(__name__, message)
self.close()
def current_point_name(self):
if self.rad_boundary_point.isChecked():
return self.names.LC_BOUNDARY_POINT_T
elif self.rad_survey_point.isChecked():
return self.names.LC_SURVEY_POINT_T
else:
return self.names.LC_CONTROL_POINT_T
def prepare_copy_csv_points_to_db(self):
csv_path = self.txt_file_path.text().strip()
if not csv_path or not os.path.exists(csv_path):
self.logger.warning_msg(__name__, QCoreApplication.translate("WizardTranslations",
"No CSV file given or file doesn't exist."))
return
target_layer_name = self.current_point_name()
with OverrideCursor(Qt.WaitCursor):
csv_layer = self.app.core.csv_to_layer(csv_path,
self.txt_delimiter.text(),
self.cbo_longitude.currentText(),
self.cbo_latitude.currentText(),
self.crs,
self.cbo_elevation.currentText() or None,
self.detect_decimal_point(csv_path))
self.app.core.copy_csv_to_db(csv_layer, self._db, target_layer_name)
def required_layers_are_available(self):
layers_are_available = self.app.core.required_layers_are_available(self._db, self._layers, self.WIZARD_TOOL_NAME)
return layers_are_available
def file_path_changed(self):
self.autodetect_separator()
self.fill_long_lat_combos("")
self.cbo_delimiter.currentTextChanged.connect(self.separator_changed)
def detect_decimal_point(self, csv_path):
if os.path.exists(csv_path):
with open(csv_path) as file:
file.readline() # headers
data = file.readline().strip() # 1st line with data
if data:
fields = self.get_fields_from_csv_file(csv_path)
if self.cbo_latitude.currentText() in fields:
num_col = data.split(self.cbo_delimiter.currentText())[fields.index(self.cbo_latitude.currentText())]
for decimal_point in ['.', ',']:
if decimal_point in num_col:
return decimal_point
return '.' # just use the default one
def autodetect_separator(self):
csv_path = self.txt_file_path.text().strip()
if os.path.exists(csv_path):
with open(csv_path) as file:
first_line = file.readline()
for delimiter in self.known_delimiters:
if delimiter['value'] == '':
continue
# if separator works like a column separator in header
# number of cols is greater than 1
if len(first_line.split(delimiter['value'])) > 1:
self.cbo_delimiter.setCurrentText(delimiter['name'])
return
def update_crs_info(self):
self.crsSelector.setCrs(QgsCoordinateReferenceSystem(self.crs))
def crs_changed(self):
self.crs = get_crs_authid(self.crsSelector.crs())
if self.crs != DEFAULT_SRS_AUTHID:
self.lbl_crs.setStyleSheet('color: orange')
self.lbl_crs.setToolTip(QCoreApplication.translate("WizardTranslations",
"Your CSV data will be reprojected for you to '{}' (Colombian National Origin),<br>before attempting to import it into LADM-COL.").format(DEFAULT_SRS_AUTHID))
else:
self.lbl_crs.setStyleSheet('')
self.lbl_crs.setToolTip(QCoreApplication.translate("WizardTranslations", "Coordinate Reference System"))
def fill_long_lat_combos(self, text):
csv_path = self.txt_file_path.text().strip()
self.cbo_longitude.clear()
self.cbo_latitude.clear()
self.cbo_elevation.clear()
if os.path.exists(csv_path):
self.button(QWizard.FinishButton).setEnabled(True)
fields = self.get_fields_from_csv_file(csv_path)
fields_dict = {field: field.lower() for field in fields}
if not fields:
self.button(QWizard.FinishButton).setEnabled(False)
return
self.cbo_longitude.addItems(fields)
self.cbo_latitude.addItems(fields)
self.cbo_elevation.addItems([""] + fields)
# Heuristics to suggest values for x, y and z
x_potential_names = ['x', 'lon', 'long', 'longitud', 'longitude', 'este', 'east', 'oeste', 'west']
y_potential_names = ['y', 'lat', 'latitud', 'latitude', 'norte', 'north']
z_potential_names = ['z', 'altura', 'elevacion', 'elevation', 'elevación', 'height']
for x_potential_name in x_potential_names:
for k,v in fields_dict.items():
if x_potential_name == v:
self.cbo_longitude.setCurrentText(k)
break
for y_potential_name in y_potential_names:
for k, v in fields_dict.items():
if y_potential_name == v:
self.cbo_latitude.setCurrentText(k)
break
if self.cbo_elevation.isEnabled():
for z_potential_name in z_potential_names:
for k, v in fields_dict.items():
if z_potential_name == v:
self.cbo_elevation.setCurrentText(k)
break
else:
self.button(QWizard.FinishButton).setEnabled(False)
def get_fields_from_csv_file(self, csv_path):
if not self.txt_delimiter.text():
return []
error_reading = False
try:
reader = open(csv_path, "r")
except IOError:
error_reading = True
line = reader.readline().replace("\n", "")
reader.close()
if not line:
error_reading = True
else:
return line.split(self.txt_delimiter.text())
if error_reading:
self.logger.warning_msg(__name__, QCoreApplication.translate("WizardTranslations",
"It was not possible to read field names from the CSV. Check the file and try again."))
return []
def separator_changed(self, text):
# first ocurrence
value = next((x['value'] for x in self.known_delimiters if x['name'] == text), '')
self.txt_delimiter.setText(value)
if value == '':
self.txt_delimiter.setEnabled(True)
else:
self.txt_delimiter.setEnabled(False)
def save_template(self, url):
link = url.url()
if self.rad_boundary_point.isChecked():
if link == '#template':
self.download_csv_file('template_boundary_points.csv')
elif link == '#data':
self.download_csv_file('sample_boundary_points.csv')
elif self.rad_survey_point.isChecked():
if link == '#template':
self.download_csv_file('template_survey_points.csv')
elif link == '#data':
self.download_csv_file('sample_survey_points.csv')
elif self.rad_control_point.isChecked():
if link == '#template':
self.download_csv_file('template_control_points.csv')
elif link == '#data':
self.download_csv_file('sample_control_points.csv')
def download_csv_file(self, filename):
settings = QSettings()
settings.setValue('Asistente-LADM-COL/wizards/points_csv_file_delimiter', self.txt_delimiter.text().strip())
new_filename, filter = QFileDialog.getSaveFileName(self,
QCoreApplication.translate("WizardTranslations",
"Save File"),
os.path.join(settings.value('Asistente-LADM-COL/wizards/points_download_csv_path', '.'), filename),
QCoreApplication.translate("WizardTranslations",
"CSV File (*.csv *.txt)"))
if new_filename:
settings.setValue('Asistente-LADM-COL/wizards/points_download_csv_path', os.path.dirname(new_filename))
template_file = QFile(":/Asistente-LADM-COL/resources/csv/" + filename)
if not template_file.exists():
self.logger.critical(__name__, "CSV doesn't exist! Probably due to a missing 'make' execution to generate resources...")
msg = QCoreApplication.translate("WizardTranslations", "CSV file not found. Update your plugin. For details see log.")
self.show_message(msg, Qgis.Warning)
return
if os.path.isfile(new_filename):
self.logger.info(__name__, 'Removing existing file {}...'.format(new_filename))
os.chmod(new_filename, 0o777)
os.remove(new_filename)
if template_file.copy(new_filename):
os.chmod(new_filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
msg = QCoreApplication.translate("WizardTranslations", """The file <a href="file:///{}">{}</a> was successfully saved!""").format(normalize_local_url(new_filename), os.path.basename(new_filename))
self.show_message(msg, Qgis.Info)
else:
self.logger.warning(__name__, 'There was an error copying the CSV file {}!'.format(new_filename))
msg = QCoreApplication.translate("WizardTranslations", "The file couldn\'t be saved.")
self.show_message(msg, Qgis.Warning)
def import_layer_changed(self, layer):
if layer:
crs = get_crs_authid(layer.crs())
if crs != DEFAULT_SRS_AUTHID:
self.lbl_refactor_source.setStyleSheet('color: orange')
self.lbl_refactor_source.setToolTip(QCoreApplication.translate("WizardTranslations",
"This layer will be reprojected for you to '{}' (Colombian National Origin),<br>before attempting to import it into LADM-COL.").format(
DEFAULT_SRS_AUTHID))
else:
self.lbl_refactor_source.setStyleSheet('')
self.lbl_refactor_source.setToolTip('')
def show_message(self, message, level):
self.bar.clearWidgets() # Remove previous messages before showing a new one
self.bar.pushMessage(message, level, 10)
def save_settings(self):
settings = QSettings()
point_type = None
if self.rad_boundary_point.isChecked():
point_type = 'boundary_point'
elif self.rad_survey_point.isChecked():
point_type = 'survey_point'
else:
point_type = 'control_point'
settings.setValue('Asistente-LADM-COL/wizards/points_add_points_type', point_type)
settings.setValue('Asistente-LADM-COL/wizards/points_load_data_type', 'csv' if self.rad_csv.isChecked() else 'refactor')
settings.setValue('Asistente-LADM-COL/wizards/points_add_points_csv_file', self.txt_file_path.text().strip())
settings.setValue('Asistente-LADM-COL/wizards/points_csv_file_delimiter', self.txt_delimiter.text().strip())
settings.setValue('Asistente-LADM-COL/wizards/points_csv_crs', self.crs)
def restore_settings(self):
settings = QSettings()
point_type = settings.value('Asistente-LADM-COL/wizards/points_add_points_type') or 'boundary_point'
if point_type == 'boundary_point':
self.rad_boundary_point.setChecked(True)
elif point_type == 'survey_point':
self.rad_survey_point.setChecked(True)
else: # 'control_point'
self.rad_control_point.setChecked(True)
load_data_type = settings.value('Asistente-LADM-COL/wizards/points_load_data_type') or 'csv'
if load_data_type == 'refactor':
self.rad_refactor.setChecked(True)
else:
self.rad_csv.setChecked(True)
self.txt_file_path.setText(settings.value('Asistente-LADM-COL/wizards/points_add_points_csv_file'))
self.txt_delimiter.setText(settings.value('Asistente-LADM-COL/wizards/points_csv_file_delimiter'))
self.crs = settings.value('Asistente-LADM-COL/wizards/points_csv_crs', DEFAULT_SRS_AUTHID, str)
self.update_crs_info()
def show_help(self):
show_plugin_help("create_points")
class SymbologyUtils(QObject):
layer_symbology_changed = pyqtSignal(str) # layer id
def __init__(self):
QObject.__init__(self)
self.logger = Logger()
def set_layer_style_from_qml(self,
db,
layer,
is_error_layer=False,
emit=False,
layer_modifiers=dict(),
models=list()): # TODO: Add tests
if db is None:
self.logger.critical(__name__,
"DB connection is none. Style not set.")
return
qml_name = None
if db.is_ladm_layer(layer):
layer_name = db.get_ladm_layer_name(layer)
else:
layer_name = layer.name(
) # we identify some error layer styles using the error table names
if not is_error_layer:
# Check if we should use modifier style group
if LayerConfig.STYLE_GROUP_LAYER_MODIFIERS in layer_modifiers:
style_group_modifiers = layer_modifiers.get(
LayerConfig.STYLE_GROUP_LAYER_MODIFIERS)
if style_group_modifiers:
qml_name = style_group_modifiers.get(layer_name)
if not qml_name: # If None or empty string, we use default styles
qml_name = Symbology().get_default_style_group(
db.names, models).get(layer_name)
else:
style_custom_error_layers = Symbology().get_custom_error_layers()
if layer_name in style_custom_error_layers:
qml_name = style_custom_error_layers.get(layer_name)
else:
qml_name = Symbology().get_default_error_style_layer().get(
layer.geometryType())
if qml_name:
renderer, labeling = self.get_style_from_qml(qml_name)
if renderer:
layer.setRenderer(renderer)
if emit:
self.layer_symbology_changed.emit(layer.id())
if labeling:
layer.setLabeling(labeling)
layer.setLabelsEnabled(True)
def get_style_from_qml(self, qml_name):
renderer = None
labeling = None
style_path = os.path.join(STYLES_DIR, qml_name + '.qml')
file = QFile(style_path)
if not file.open(QIODevice.ReadOnly | QIODevice.Text):
self.logger.warning(
__name__,
"Unable to read style file from {}".format(style_path))
doc = QDomDocument()
doc.setContent(file)
file.close()
doc_elem = doc.documentElement()
nodes = doc_elem.elementsByTagName("renderer-v2")
if nodes.count():
renderer_elem = nodes.at(0).toElement()
renderer = QgsFeatureRenderer.load(renderer_elem,
QgsReadWriteContext())
nodes = doc_elem.elementsByTagName("labeling")
if nodes.count():
labeling_elem = nodes.at(0).toElement()
labeling = QgsAbstractVectorLayerLabeling.create(
labeling_elem, QgsReadWriteContext())
return (renderer, labeling)
class QualityRuleLayerManager(QObject):
"""
Responsible for managing all layers during a Quality Rule execution
session. It goes for LADM-COL layers only once and also manages
intermediate layers (after snapping).
"""
def __init__(self, db, rule_keys, tolerance):
QObject.__init__(self)
self.logger = Logger()
self.app = AppInterface()
self.__db = db
self.__rule_keys = rule_keys
self.__tolerance = tolerance
self.__quality_rule_layers_config = QualityRuleConfig.get_quality_rules_layer_config(
self.__db.names)
# {rule_key: {QUALITY_RULE_LAYERS: {layer_name: layer},
# QUALITY_RULE_LADM_COL_LAYERS: {layer_name: layer}}
self.__layers = dict()
self.__adjusted_layers_cache = dict()
def initialize(self, rule_keys):
"""
Objects of this class are reusable calling initialize()
"""
self.__rule_keys = rule_keys
self.__layers = dict()
def __prepare_layers(self):
"""
Get layers from DB and prepare snapped layers for all rules
"""
self.logger.info(
__name__,
QCoreApplication.translate("QualityRuleLayerManager",
"Preparing layers..."))
# First go for ladm-col layers
ladm_layers = dict()
for rule_key, rule_layers_config in self.__quality_rule_layers_config.items(
):
if rule_key in self.__rule_keys: # Only get selected rules' layers
for layer_name in rule_layers_config[
QUALITY_RULE_LADM_COL_LAYERS]:
ladm_layers[layer_name] = None
self.logger.debug(
__name__,
QCoreApplication.translate("QualityRuleLayerManager",
"Getting {} LADM-COL layers...").format(
len(ladm_layers)))
self.app.core.get_layers(self.__db, ladm_layers, load=True)
if ladm_layers is None: # If there are errors with get_layers, ladm_layers is None
self.logger.critical(
__name__,
QCoreApplication.translate(
"QualityRuleLayerManager",
"Couldn't finish preparing required layers!"))
return False
# If tolerance > 0, prepare adjusted layers
# We create an adjusted_layers dict to override ladm_layers per rule.
# For that, we need to read the config and, if not yet calculated,
# adjust the layers and store them in temporary cache.
# {rule_key: {layer_name: layer}}, because each rule might need
# different adjustments for the same layer, compared to other rules
adjusted_layers = {rule_key: dict() for rule_key in self.__rule_keys}
if self.__tolerance:
self.logger.debug(
__name__,
QCoreApplication.translate(
"QualityRuleLayerManager",
"Tolerance > 0, adjusting layers..."))
self.__adjusted_layers_cache = dict() # adjusted_layers_key: layer
count_rules = 0
total_rules = len([
rk for rk in self.__rule_keys
if rk in self.__quality_rule_layers_config
])
with ProcessWithStatus(
QCoreApplication.translate(
"QualityRuleLayerManager",
"Preparing tolerance on layers...")):
for rule_key, rule_layers_config in self.__quality_rule_layers_config.items(
):
if rule_key in self.__rule_keys: # Only get selected rules' layers
count_rules += 1
self.logger.status(
QCoreApplication.translate(
"QualityRuleLayerManager",
"Preparing tolerance on layers... {}%").format(
int(count_rules / total_rules * 100)))
if QUALITY_RULE_ADJUSTED_LAYERS in rule_layers_config:
for layer_name, snap_config in rule_layers_config[
QUALITY_RULE_ADJUSTED_LAYERS].items():
# Read from config
input_name = snap_config[
ADJUSTED_INPUT_LAYER] # input layer name
reference_name = snap_config[
ADJUSTED_REFERENCE_LAYER] # reference layer name
fix = snap_config[
FIX_ADJUSTED_LAYER] if FIX_ADJUSTED_LAYER in snap_config else False
# Get input and reference layers (note that they could be adjusted layers)
input = self.__adjusted_layers_cache[
input_name] if input_name in self.__adjusted_layers_cache else ladm_layers[
input_name]
reference = self.__adjusted_layers_cache[
reference_name] if reference_name in self.__adjusted_layers_cache else ladm_layers[
reference_name]
# Try to reuse if already calculated!
adjusted_layers_key = get_key_for_quality_rule_adjusted_layer(
input_name, reference_name, fix)
if adjusted_layers_key not in self.__adjusted_layers_cache:
self.__adjusted_layers_cache[
adjusted_layers_key] = self.app.core.adjust_layer(
input, reference, self.__tolerance,
fix)
adjusted_layers[rule_key][
layer_name] = self.__adjusted_layers_cache[
adjusted_layers_key]
self.logger.debug(
__name__,
QCoreApplication.translate("QualityRuleLayerManager",
"Layers adjusted..."))
# Now that we have both ladm_layers and adjusted_layers, use them
# in a single member dict of layers per rule (preserving original LADM-COL layers)
self.__layers = {
rule_key: {
QUALITY_RULE_LAYERS: dict(),
QUALITY_RULE_LADM_COL_LAYERS: dict()
}
for rule_key in self.__rule_keys
}
for rule_key, rule_layers_config in self.__quality_rule_layers_config.items(
):
if rule_key in self.__rule_keys: # Only get selected rules' layers
for layer_name in rule_layers_config[
QUALITY_RULE_LADM_COL_LAYERS]:
# Fill both subdicts
# In LADM-COL layers we send all original layers
self.__layers[rule_key][QUALITY_RULE_LADM_COL_LAYERS][
layer_name] = ladm_layers[
layer_name] if layer_name in ladm_layers else None
# In QR_Layers we store the best layer we have available (preferring adjusted over ladm-col)
if layer_name in adjusted_layers[rule_key]:
self.__layers[rule_key][QUALITY_RULE_LAYERS][
layer_name] = adjusted_layers[rule_key][layer_name]
elif layer_name in ladm_layers:
self.__layers[rule_key][QUALITY_RULE_LAYERS][
layer_name] = ladm_layers[layer_name]
# Let QRs know if they should switch between dicts looking for original geometries
self.__layers[rule_key][HAS_ADJUSTED_LAYERS] = bool(
self.__tolerance)
# Register adjusted layers so that Processing can properly find them
if self.__adjusted_layers_cache:
load_to_registry = [
layer for key, layer in self.__adjusted_layers_cache.items()
if layer is not None
]
self.logger.debug(
__name__,
"{} adjusted layers loaded to QGIS registry...".format(
len(load_to_registry)))
QgsProject.instance().addMapLayers(load_to_registry, False)
return True
def get_layer(self, layer_name, rule_key):
return self.get_layers([layer_name], rule_key)
def get_layers(self, rule_key):
"""
Gets the layers a quality rule requires to run. This is based on the quality rule layer config.
:param rule_key: Key of the quality rule.
:return: Dict of layers for the given rule_key. This dict has both a 'layers' dict which has the best available
layer (which means, if an adjusted layer is required, it will be preferred, and if no adjusted layer is
required, just pass the LADM-COL layer) and a 'ladm-col' dict with the original LADM-COL layers,
because the quality rule might need to refer to the original object (or geometry) to build its result.
"""
# Make sure we only call Prepare layers once for each call to run quality validations.
if not self.__layers:
if not self.__prepare_layers():
return None
return self.__layers[rule_key]
def clean_temporary_layers(self):
# Removes adjusted layers from registry
unload_from_registry = [
layer.id() for key, layer in self.__adjusted_layers_cache.items()
if layer is not None
]
self.logger.debug(
__name__,
"{} adjusted layers removed from QGIS registry...".format(
len(unload_from_registry)))
QgsProject.instance().removeMapLayers(unload_from_registry)
class ImportFromExcelDialog(QDialog, DIALOG_UI):
log_excel_show_message_emitted = pyqtSignal(str)
def __init__(self, iface, db, qgis_utils, parent=None):
QDialog.__init__(self, parent)
self.setupUi(self)
self.iface = iface
self._db = db
self.qgis_utils = qgis_utils
self.logger = Logger()
self.help_strings = HelpStrings()
self.log_dialog_excel_text_content = ""
self.group_parties_exists = False
self.names = self._db.names
self._running_tool = False
self.tool_name = QCoreApplication.translate(
"ImportFromExcelDialog", "Import intermediate structure")
self.fields = {
EXCEL_SHEET_NAME_PLOT: [
EXCEL_SHEET_TITLE_DEPARTMENT, EXCEL_SHEET_TITLE_MUNICIPALITY,
EXCEL_SHEET_TITLE_ZONE, EXCEL_SHEET_TITLE_REGISTRATION_PLOT,
EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_NPV,
EXCEL_SHEET_TITLE_PLOT_NAME, EXCEL_SHEET_TITLE_VALUATION,
EXCEL_SHEET_TITLE_PLOT_CONDITION, EXCEL_SHEET_TITLE_PLOT_TYPE,
EXCEL_SHEET_TITLE_ADDRESS
],
EXCEL_SHEET_NAME_PARTY: [
EXCEL_SHEET_TITLE_FIRST_NAME, EXCEL_SHEET_TITLE_MIDDLE,
EXCEL_SHEET_TITLE_FIRST_SURNAME,
EXCEL_SHEET_TITLE_SECOND_SURNAME,
EXCEL_SHEET_TITLE_BUSINESS_NAME, EXCEL_SHEET_TITLE_SEX,
EXCEL_SHEET_TITLE_DOCUMENT_TYPE,
EXCEL_SHEET_TITLE_DOCUMENT_NUMBER,
EXCEL_SHEET_TITLE_KIND_PERSON,
EXCEL_SHEET_TITLE_ISSUING_ENTITY, EXCEL_SHEET_TITLE_DATE_ISSUE,
EXCEL_SHEET_TITLE_NPN
],
EXCEL_SHEET_NAME_GROUP: [
EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_DOCUMENT_TYPE,
EXCEL_SHEET_TITLE_DOCUMENT_NUMBER, EXCEL_SHEET_TITLE_ID_GROUP
],
EXCEL_SHEET_NAME_RIGHT: [
EXCEL_SHEET_TITLE_TYPE,
EXCEL_SHEET_TITLE_PARTY_DOCUMENT_NUMBER,
EXCEL_SHEET_TITLE_GROUP, EXCEL_SHEET_TITLE_NPN,
EXCEL_SHEET_TITLE_SOURCE_TYPE,
EXCEL_SHEET_TITLE_DESCRIPTION_SOURCE,
EXCEL_SHEET_TITLE_STATE_SOURCE,
EXCEL_SHEET_TITLE_OFFICIALITY_SOURCE,
EXCEL_SHEET_TITLE_STORAGE_PATH
]
}
self.txt_help_page.setHtml(self.help_strings.DLG_IMPORT_FROM_EXCEL)
self.txt_help_page.anchorClicked.connect(self.save_template)
self.buttonBox.accepted.disconnect()
self.buttonBox.accepted.connect(self.accepted)
#self.buttonBox.rejected.connect(self.rejected)
self.buttonBox.helpRequested.connect(self.show_help)
self.btn_browse_file.clicked.connect(
make_file_selector(
self.txt_excel_path,
QCoreApplication.translate(
"ImportFromExcelDialog",
"Select the Excel file with data in the intermediate structure"
),
QCoreApplication.translate("ImportFromExcelDialog",
'Excel File (*.xlsx *.xls)')))
self.buttonBox.button(QDialogButtonBox.Ok).setText(
QCoreApplication.translate("ImportFromExcelDialog", "Import"))
self.initialize_feedback()
self.restore_settings()
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
# self.tabWidget.currentWidget().layout().setContentsMargins(0, 0, 0, 0)
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
def accepted(self):
self.save_settings()
self.import_from_excel()
def import_from_excel(self):
self._running_tool = True
steps = 18
step = 0
self.progress.setVisible(True)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
# Where to store the reports?
excel_path = self.txt_excel_path.text()
if not excel_path:
self.show_message(
QCoreApplication.translate(
"ImportFromExcelDialog",
"You need to select an Excel file before continuing with the import."
), Qgis.Warning)
self.progress.setVisible(False)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
return
if not os.path.exists(excel_path):
self.show_message(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The specified Excel file does not exist!"), Qgis.Warning)
self.progress.setVisible(False)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
return
self.progress.setVisible(True)
self.txt_log.setText(
QCoreApplication.translate(
"ImportFromExcelDialog",
"Loading tables from the Excel file..."))
# Now that we have the Excel file, build vrts to load its sheets appropriately
# Also validate each layer against a number of rules
layer_parcel = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_PLOT)
layer_party = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_PARTY)
layer_group_party = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_GROUP)
layer_right = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_RIGHT)
if layer_parcel is None or layer_party is None or layer_group_party is None or layer_right is None:
# A layer is None if at least an error was found
self.group_parties_exists = False
self.log_excel_show_message_emitted.emit(
self.log_dialog_excel_text_content)
self.done(0)
return
if not layer_group_party.isValid() or not layer_party.isValid(
) or not layer_parcel.isValid() or not layer_right.isValid():
self.show_message(
QCoreApplication.translate(
"ImportFromExcelDialog",
"One of the sheets of the Excel file couldn't be loaded! Check the format again."
), Qgis.Warning)
self.progress.setVisible(False)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
return
QgsProject.instance().addMapLayers(
[layer_group_party, layer_party, layer_parcel, layer_right])
# GET LADM LAYERS
layers = {
self.names.OP_PARTY_T: {
'name': self.names.OP_PARTY_T,
'geometry': None,
LAYER: None
},
self.names.OP_PARCEL_T: {
'name': self.names.OP_PARCEL_T,
'geometry': None,
LAYER: None
},
self.names.OP_RIGHT_T: {
'name': self.names.OP_RIGHT_T,
'geometry': None,
LAYER: None
},
self.names.EXT_ARCHIVE_S: {
'name': self.names.EXT_ARCHIVE_S,
'geometry': None,
LAYER: None
},
self.names.COL_RRR_SOURCE_T: {
'name': self.names.COL_RRR_SOURCE_T,
'geometry': None,
LAYER: None
},
self.names.OP_GROUP_PARTY_T: {
'name': self.names.OP_GROUP_PARTY_T,
'geometry': None,
LAYER: None
},
self.names.MEMBERS_T: {
'name': self.names.MEMBERS_T,
'geometry': None,
LAYER: None
},
self.names.OP_ADMINISTRATIVE_SOURCE_T: {
'name': self.names.OP_ADMINISTRATIVE_SOURCE_T,
'geometry': None,
LAYER: None
}
}
self.qgis_utils.get_layers(self._db, layers, load=True)
if not layers:
return None
# Get feature counts to compare after the ETL and know how many records were imported to each ladm_col table
ladm_tables = [
layers[self.names.OP_PARCEL_T][LAYER],
layers[self.names.OP_PARTY_T][LAYER],
layers[self.names.OP_RIGHT_T][LAYER],
layers[self.names.OP_ADMINISTRATIVE_SOURCE_T][LAYER],
layers[self.names.COL_RRR_SOURCE_T][LAYER],
layers[self.names.OP_GROUP_PARTY_T][LAYER],
layers[self.names.MEMBERS_T][LAYER]
]
ladm_tables_feature_count_before = {
t.name(): t.featureCount()
for t in ladm_tables
}
# Run the ETL
params = {
'agrupacion':
layers[self.names.OP_GROUP_PARTY_T][LAYER],
'colmiembros':
layers[self.names.MEMBERS_T][LAYER],
'colrrrsourcet':
layers[self.names.COL_RRR_SOURCE_T][LAYER],
'extarchivo':
layers[self.names.EXT_ARCHIVE_S][LAYER],
'interesado':
layers[self.names.OP_PARTY_T][LAYER],
'layergroupparty':
layer_group_party,
'layerparcel':
layer_parcel,
'layerparty':
layer_party,
'layerright':
layer_right,
'opderecho':
layers[self.names.OP_RIGHT_T][LAYER],
'opfuenteadministrativatipo':
layers[self.names.OP_ADMINISTRATIVE_SOURCE_T][LAYER],
'parcel':
layers[self.names.OP_PARCEL_T][LAYER]
}
self.qgis_utils.disable_automatic_fields(self._db,
self.names.OP_GROUP_PARTY_T)
self.qgis_utils.disable_automatic_fields(self._db,
self.names.OP_RIGHT_T)
self.qgis_utils.disable_automatic_fields(
self._db, self.names.OP_ADMINISTRATIVE_SOURCE_T)
processing.run("model:ETL_intermediate_structure",
params,
feedback=self.feedback)
if not self.feedback.isCanceled():
self.progress.setValue(100)
self.buttonBox.clear()
self.buttonBox.setEnabled(True)
self.buttonBox.addButton(QDialogButtonBox.Close)
else:
self.initialize_feedback()
# Print summary getting feature count in involved LADM_COL tables...
summary = """<html><head/><body><p>"""
summary += QCoreApplication.translate("ImportFromExcelDialog",
"Import done!!!<br/>")
for table in ladm_tables:
summary += QCoreApplication.translate(
"ImportFromExcelDialog",
"<br/><b>{count}</b> records loaded into table <b>{table}</b>"
).format(count=table.featureCount() -
ladm_tables_feature_count_before[table.name()],
table=table.name())
summary += """</body></html>"""
self.txt_log.setText(summary)
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"QGISUtils",
"Data successfully imported to LADM_COL from intermediate structure (Excel file: '{}')!!!"
).format(excel_path))
self._running_tool = False
def check_layer_from_excel_sheet(self, excel_path, sheetname):
layer = self.get_layer_from_excel_sheet(excel_path, sheetname)
error_counter = 0
if layer is None and sheetname != EXCEL_SHEET_NAME_GROUP: # optional sheet
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The {} sheet has not information or has another name.").
format(sheetname))
error_counter += 1
else:
title_validator = layer.fields().toList()
if sheetname == EXCEL_SHEET_NAME_PLOT and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in the sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"numero predial nuevo" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero predial nuevo has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_field_numeric_layer(layer, 'departamento'):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column departamento has non-numeric values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_field_numeric_layer(layer, 'municipio'):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column municipio has non-numeric values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_field_numeric_layer(layer,
'numero predial nuevo'):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero predial nuevo has non-numeric values in sheet {}."
).format(sheetname))
error_counter += 1
if sheetname == EXCEL_SHEET_NAME_PARTY and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in sheet {}.").
format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"numero de documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_length_attribute_value(
layer, 'numero de documento', 12):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has more characters than expected in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo persona" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo persona has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if sheetname == EXCEL_SHEET_NAME_GROUP and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in the sheet {}."
).format(sheetname))
error_counter += 1
self.group_parties_exists = True
if list(layer.getFeatures('"numero predial nuevo" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero predial nuevo has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"numero de documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"id agrupación" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column id agrupación has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_length_attribute_value(
layer, 'numero de documento', 12):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has more characters of the permitted in sheet {}."
).format(sheetname))
error_counter += 1
if sheetname == EXCEL_SHEET_NAME_RIGHT and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in sheet {}.").
format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo has empty values in sheet {}.").
format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo de fuente" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo de fuente has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(
layer.getFeatures(
'"estado_disponibilidad de la fuente" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column estado_disponibilidad de la fuente has empty values in sheet {}."
).format(sheetname))
error_counter += 1
#if list(layer.getFeatures('"Ruta de Almacenamiento de la fuente" is Null')):
# self.generate_message_excel_error(QCoreApplication.translate("ImportFromExcelDialog",
# "The column Ruta de Almacenamiento de la fuente has empty values in sheet {}.").format(sheetname))
# error_counter += 1
if len(
list(
layer.getFeatures(
'"número documento Interesado" is Null'))) + len(
list(layer.getFeatures('"agrupación" is Null'))
) != layer.featureCount():
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"Number of non-null parties plus number of non-null group parties is not equal to number of records in sheet {}. There might be rights without party or group party associated."
).format(sheetname))
error_counter += 1
if not self.group_parties_exists:
if list(
layer.getFeatures(
'"número documento Interesado" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column número documento Interesado has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if len(list(layer.getFeatures(
'"agrupacion" is Null'))) != layer.featureCount():
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column agrupacion has data but the sheet does not exist in sheet {}."
).format(sheetname))
error_counter += 1
return layer if error_counter == 0 else None
def check_field_numeric_layer(self, layer, name):
id_field_idx = layer.fields().indexFromName(name)
request = QgsFeatureRequest().setSubsetOfAttributes([id_field_idx])
features = layer.getFeatures(request)
is_numeric = True
for feature in features:
try:
int(feature[name])
except:
is_numeric = False
break
return is_numeric
def check_length_attribute_value(self, layer, name, size):
id_field_idx = layer.fields().indexFromName(name)
request = QgsFeatureRequest().setSubsetOfAttributes([id_field_idx])
features = layer.getFeatures(request)
right_length = True
for feature in features:
if len(str(feature[name])) > size:
right_length = False
break
return right_length
def generate_message_excel_error(self, msg):
self.log_dialog_excel_text_content += "{}{}{}{}{}{}".format(
LOG_QUALITY_LIST_CONTAINER_OPEN, LOG_QUALITY_LIST_ITEM_ERROR_OPEN,
msg, LOG_QUALITY_LIST_ITEM_ERROR_CLOSE,
LOG_QUALITY_LIST_CONTAINER_CLOSE, LOG_QUALITY_CONTENT_SEPARATOR)
def get_layer_from_excel_sheet(self, excel_path, sheetname):
basename = os.path.basename(excel_path)
filename = os.path.splitext(basename)[0]
dirname = os.path.dirname(excel_path)
header_in_first_row, count = self.get_excel_info(excel_path, sheetname)
if header_in_first_row is None and count is None:
return None
layer_definition = "<SrcLayer>{sheetname}</SrcLayer>".format(
sheetname=sheetname)
if header_in_first_row:
layer_definition = """<SrcSql dialect="sqlite">SELECT * FROM '{sheetname}' LIMIT {count} OFFSET 1</SrcSql>""".format(
sheetname=sheetname, count=count)
xml_text_group_party = """<?xml version="1.0" encoding="UTF-8"?>
<OGRVRTDataSource>
<OGRVRTLayer name="{filename}-{sheetname}">
<SrcDataSource relativeToVRT="1">{basename}</SrcDataSource>
<!--Header={header}-->
{layer_definition}
{fields}
</OGRVRTLayer>
</OGRVRTDataSource>
""".format(filename=filename,
basename=basename,
header=header_in_first_row,
layer_definition=layer_definition,
sheetname=sheetname,
fields=self.get_vrt_fields(sheetname,
header_in_first_row))
group_party_file_path = os.path.join(
dirname, '{}.{}.vrt'.format(basename, sheetname))
with open(group_party_file_path, 'w') as sheet:
sheet.write(xml_text_group_party)
uri = '{vrtfilepath}|layername={filename}-{sheetname}'.format(
vrtfilepath=group_party_file_path,
sheetname=sheetname,
filename=filename)
self.logger.info(__name__,
"Loading layer from excel with uri='{}'".format(uri))
layer = QgsVectorLayer(uri, '{}-{}'.format('excel', sheetname), 'ogr')
layer.setProviderEncoding('UTF-8')
return layer
def get_excel_info(self, path, sheetname):
data_source = ogr.Open(path, 0)
layer = data_source.GetLayerByName(sheetname)
if layer is None:
# A sheetname couldn't be found
return None, None
feature = layer.GetNextFeature()
# If ogr recognizes the header, the first row will contain data, otherwise it'll contain field names
header_in_first_row = True
for field in self.fields[sheetname]:
if feature.GetField(self.fields[sheetname].index(field)) != field:
header_in_first_row = False
num_rows = layer.GetFeatureCount()
return header_in_first_row, num_rows - 1 if header_in_first_row else num_rows
def get_vrt_fields(self, sheetname, header_in_first_row):
vrt_fields = ""
for index, field in enumerate(self.fields[sheetname]):
vrt_fields += """<Field name="{field}" src="{src}" type="String"/>\n""".format(
field=field,
src='Field{}'.format(index +
1) if header_in_first_row else field)
return vrt_fields.strip()
def save_template(self, url):
link = url.url()
if link == '#template':
self.download_excel_file('plantilla_estructura_excel.xlsx')
elif link == '#data':
self.download_excel_file('datos_estructura_excel.xlsx')
def download_excel_file(self, filename):
settings = QSettings()
new_filename, filter = QFileDialog.getSaveFileName(
self,
QCoreApplication.translate("ImportFromExcelDialog", "Save File"),
os.path.join(
settings.value(
'Asistente-LADM_COL/import_from_excel_dialog/template_save_path',
'.'), filename),
QCoreApplication.translate("ImportFromExcelDialog",
"Excel File (*.xlsx *.xls)"))
if new_filename:
settings.setValue(
'Asistente-LADM_COL/import_from_excel_dialog/template_save_path',
os.path.dirname(new_filename))
template_file = QFile(":/Asistente-LADM_COL/resources/excel/" +
filename)
if not template_file.exists():
self.logger.critical(
__name__,
"Excel doesn't exist! Probably due to a missing 'make' execution to generate resources..."
)
msg = QCoreApplication.translate(
"ImportFromExcelDialog",
"Excel file not found. Update your plugin. For details see log."
)
self.show_message(msg, Qgis.Warning)
return
if os.path.isfile(new_filename):
self.logger.info(
__name__,
'Removing existing file {}...'.format(new_filename))
os.chmod(new_filename, 0o777)
os.remove(new_filename)
if template_file.copy(new_filename):
os.chmod(
new_filename,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
msg = QCoreApplication.translate(
"ImportFromExcelDialog",
"""The file <a href="file:///{}">{}</a> was successfully saved!"""
).format(normalize_local_url(new_filename),
os.path.basename(new_filename))
self.show_message(msg, Qgis.Info)
else:
self.logger.info(
__name__,
'There was an error copying the CSV file {}!'.format(
new_filename))
msg = QCoreApplication.translate(
"ImportFromExcelDialog", "The file couldn\'t be saved.")
self.show_message(msg, Qgis.Warning)
def reject(self):
self.selected_items_dict = dict()
if self._running_tool:
reply = QMessageBox.question(
self, QCoreApplication.translate("import_from_excel",
"Warning"),
QCoreApplication.translate(
"import_from_excel",
"The '{}' tool is still running. Do you want to cancel it? If you cancel, the data might be incomplete in the target database."
).format(self.tool_name), QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.Yes:
self.feedback.cancel()
self._running_tool = False
msg = QCoreApplication.translate(
"import_from_excel",
"The '{}' tool was cancelled.").format(self.tool_name)
self.logger.info(__name__, msg)
self.show_message(msg, Qgis.Info)
else:
self.logger.info(__name__, "Dialog closed.")
self.done(1)
def save_settings(self):
settings = QSettings()
settings.setValue(
'Asistente-LADM_COL/import_from_excel_dialog/excel_path',
self.txt_excel_path.text())
def restore_settings(self):
settings = QSettings()
self.txt_excel_path.setText(
settings.value(
'Asistente-LADM_COL/import_from_excel_dialog/excel_path', ''))
def show_message(self, message, level):
self.bar.clearWidgets(
) # Remove previous messages before showing a new one
self.bar.pushMessage(message, level, 10)
def show_help(self):
self.qgis_utils.show_help("import_from_excel")
def progress_changed(self):
QCoreApplication.processEvents() # Listen to cancel from the user
self.progress.setValue(self.feedback.progress())
def initialize_feedback(self):
self.progress.setValue(0)
self.progress.setVisible(False)
self.feedback = QgsProcessingFeedback()
self.feedback.progressChanged.connect(self.progress_changed)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
class QgisModelBakerUtils(QObject):
def __init__(self):
QObject.__init__(self)
self.logger = Logger()
from asistente_ladm_col.config.config_db_supported import ConfigDBsSupported
self._dbs_supported = ConfigDBsSupported()
self.translatable_config_strings = TranslatableConfigStrings()
def get_generator(self, db):
if 'QgisModelBaker' in qgis.utils.plugins:
tool = self._dbs_supported.get_db_factory(
db.engine).get_model_baker_db_ili_mode()
QgisModelBaker = qgis.utils.plugins["QgisModelBaker"]
generator = QgisModelBaker.get_generator()(
tool, db.uri, "smart2", db.schema, pg_estimated_metadata=False)
return generator
else:
self.logger.critical(
__name__,
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant."
))
return None
def get_model_baker_db_connection(self, db):
generator = self.get_generator(db)
if generator is not None:
return generator._db_connector
return None
def load_layers(self, db, layer_list):
"""
Load a selected list of layers from qgis model baker.
This call should configure relations and bag of enums
between layers being loaded, but not when a layer already
loaded has a relation or is part of a bag of enum. For
that case, we use a cached set of relations and bags of
enums that we get only once per session and configure in
the Asistente LADM-COL.
"""
translated_strings = self.translatable_config_strings.get_translatable_config_strings(
)
if 'QgisModelBaker' in qgis.utils.plugins:
QgisModelBaker = qgis.utils.plugins["QgisModelBaker"]
tool = self._dbs_supported.get_db_factory(
db.engine).get_model_baker_db_ili_mode()
generator = QgisModelBaker.get_generator()(
tool, db.uri, "smart2", db.schema, pg_estimated_metadata=False)
layers = generator.layers(layer_list)
relations, bags_of_enum = generator.relations(layers, layer_list)
legend = generator.legend(
layers,
ignore_node_names=[translated_strings[ERROR_LAYER_GROUP]])
QgisModelBaker.create_project(layers,
relations,
bags_of_enum,
legend,
auto_transaction=False)
else:
self.logger.critical(
__name__,
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant."
))
def get_required_layers_without_load(self, layer_list, db):
"""
Gets a list of layers from a list of layer names using QGIS Model Baker.
Layers are register in QgsProject, but not loaded to the canvas!
:param layer_list: list of layers names (e.g., ['lc_terreno', 'lc_lindero'])
:param db: db connection
:return: list of QgsVectorLayers registered in the project
"""
layers = list()
if 'QgisModelBaker' in qgis.utils.plugins:
QgisModelBaker = qgis.utils.plugins["QgisModelBaker"]
tool = self._dbs_supported.get_db_factory(
db.engine).get_model_baker_db_ili_mode()
generator = QgisModelBaker.get_generator()(
tool, db.uri, "smart2", db.schema, pg_estimated_metadata=False)
model_baker_layers = generator.layers(layer_list)
for model_baker_layer in model_baker_layers:
layer = model_baker_layer.create(
) # Convert Model Baker layer to QGIS layer
QgsProject.instance().addMapLayer(
layer, False) # Do not load it to canvas
layers.append(layer)
else:
self.logger.critical(
__name__,
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant."
))
return layers
def get_layers_and_relations_info(self, db):
"""
Called once per session, this is used to get information
of all relations and bags of enums in the DB and cache it
in the Asistente LADM-COL.
"""
if 'QgisModelBaker' in qgis.utils.plugins:
generator = self.get_generator(db)
layers = generator.get_tables_info_without_ignored_tables()
relations = [
relation for relation in generator.get_relations_info()
]
self.logger.debug(
__name__,
"Relationships before filter: {}".format(len(relations)))
self.filter_relations(relations)
self.logger.debug(
__name__,
"Relationships after filter: {}".format(len(relations)))
return (layers, relations, {})
else:
self.logger.critical(
__name__,
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant."
))
return (list(), list(), dict())
def filter_relations(self, relations):
"""
Modifies the input list of relations, removing elements that meet a condition.
:param relations: List of a dict of relations.
:return: Nothing, changes the input list of relations.
"""
to_delete = list()
for relation in relations:
if relation[QueryNames.REFERENCING_FIELD].startswith(
'uej2_') or relation[
QueryNames.REFERENCING_FIELD].startswith('ue_'):
to_delete.append(relation)
for idx in to_delete:
relations.remove(idx)
def get_tables_info_without_ignored_tables(self, db):
if 'QgisModelBaker' in qgis.utils.plugins:
generator = self.get_generator(db)
return generator.get_tables_info_without_ignored_tables()
else:
self.logger.critical(
__name__,
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant."
))
def get_first_index_for_layer_type(
self, layer_type, group=QgsProject.instance().layerTreeRoot()):
if 'QgisModelBaker' in qgis.utils.plugins:
import QgisModelBaker
return QgisModelBaker.utils.qgis_utils.get_first_index_for_layer_type(
layer_type, group)
return None
@staticmethod
def get_suggested_index_for_layer(layer, group):
if 'QgisModelBaker' in qgis.utils.plugins:
import QgisModelBaker
return QgisModelBaker.utils.qgis_utils.get_suggested_index_for_layer(
layer, group)
return None
class SourceHandler(QObject):
"""
Upload source files from a given field of a layer to a remote server that
is configured in Settings Dialog. The server returns a file URL that is
then stored in the source table.
"""
def __init__(self, qgis_utils):
QObject.__init__(self)
self.qgis_utils = qgis_utils
self.logger = Logger()
def upload_files(self, layer, field_index, features):
"""
Upload given features' source files to remote server and return a dict
formatted as changeAttributeValues expects to update 'datos' attribute
to a remote location.
"""
if not QSettings().value(
'Asistente-LADM_COL/sources/document_repository', False, bool):
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"SourceHandler",
"The source files were not uploaded to the document repository because you have that option unchecked. You can still upload the source files later using the 'Upload Pending Source Files' menu."
), 10)
return dict()
# Test if we have Internet connection and a valid service
res, msg = self.qgis_utils.is_source_service_valid(
) # TODO: Bring this method from qgis_utils
if not res:
msg['text'] = QCoreApplication.translate(
"SourceHandler",
"No file could be uploaded to the document repository. You can do it later from the 'Upload Pending Source Files' menu. Reason: {}"
).format(msg['text'])
self.logger.info_msg(
__name__, msg['text'],
20) # The data is still saved, so always show Info msg
return dict()
file_features = [
feature for feature in features if not feature[field_index] == NULL
and os.path.isfile(feature[field_index])
]
total = len(features)
not_found = total - len(file_features)
upload_dialog = UploadProgressDialog(len(file_features), not_found)
upload_dialog.show()
count = 0
upload_errors = 0
new_values = dict()
for feature in file_features:
data_url = feature[field_index]
file_name = os.path.basename(data_url)
nam = QNetworkAccessManager()
#reply.downloadProgress.connect(upload_dialog.update_current_progress)
multiPart = QHttpMultiPart(QHttpMultiPart.FormDataType)
textPart = QHttpPart()
textPart.setHeader(QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"driver\""))
textPart.setBody(QByteArray().append('Local'))
filePart = QHttpPart()
filePart.setHeader(
QNetworkRequest.ContentDispositionHeader,
QVariant("form-data; name=\"file\"; filename=\"{}\"".format(
file_name)))
file = QFile(data_url)
file.open(QIODevice.ReadOnly)
filePart.setBodyDevice(file)
file.setParent(
multiPart
) # we cannot delete the file now, so delete it with the multiPart
multiPart.append(filePart)
multiPart.append(textPart)
service_url = '/'.join([
QSettings().value(
'Asistente-LADM_COL/sources/service_endpoint',
DEFAULT_ENDPOINT_SOURCE_SERVICE),
SOURCE_SERVICE_UPLOAD_SUFFIX
])
request = QNetworkRequest(QUrl(service_url))
reply = nam.post(request, multiPart)
#reply.uploadProgress.connect(upload_dialog.update_current_progress)
reply.error.connect(self.error_returned)
multiPart.setParent(reply)
# We'll block execution until we get response from the server
loop = QEventLoop()
reply.finished.connect(loop.quit)
loop.exec_()
response = reply.readAll()
data = QTextStream(response, QIODevice.ReadOnly)
content = data.readAll()
if content is None:
self.logger.critical(
__name__,
"There was an error uploading file '{}'".format(data_url))
upload_errors += 1
continue
try:
response = json.loads(content)
except json.decoder.JSONDecodeError:
self.logger.critical(
__name__,
"Couldn't parse JSON response from server for file '{}'!!!"
.format(data_url))
upload_errors += 1
continue
if 'error' in response:
self.logger.critical(
__name__,
"STATUS: {}. ERROR: {} MESSAGE: {} FILE: {}".format(
response['status'], response['error'],
response['message'], data_url))
upload_errors += 1
continue
reply.deleteLater()
if 'url' not in response:
self.logger.critical(
__name__,
"'url' attribute not found in JSON response for file '{}'!"
.format(data_url))
upload_errors += 1
continue
url = self.get_file_url(response['url'])
new_values[feature.id()] = {field_index: url}
count += 1
upload_dialog.update_total_progress(count)
if not_found > 0:
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"SourceHandler",
"{} out of {} records {} not uploaded to the document repository because {} file path is NULL or it couldn't be found in the local disk!"
).format(
not_found, total,
QCoreApplication.translate("SourceHandler", "was")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if not_found == 1 else QCoreApplication.translate(
"SourceHandler", "their")))
if len(new_values):
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files {} uploaded to the document repository and {} remote location stored in the database!"
).format(
len(new_values), total,
QCoreApplication.translate("SourceHandler", "was")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "were"),
QCoreApplication.translate("SourceHandler", "its")
if len(new_values) == 1 else QCoreApplication.translate(
"SourceHandler", "their")))
if upload_errors:
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"SourceHandler",
"{} out of {} files could not be uploaded to the document repository because of upload errors! See log for details."
).format(upload_errors, total))
return new_values
def error_returned(self, error_code):
self.logger.critical(__name__,
"Qt network error code: {}".format(error_code))
def handle_source_upload(self, db, layer, field_name):
layer_name = db.get_ladm_layer_name(layer)
field_index = layer.fields().indexFromName(field_name)
def features_added(layer_id, features):
modified_layer = QgsProject.instance().mapLayer(layer_id)
if modified_layer is None:
return
modified_layer_name = db.get_ladm_layer_name(modified_layer,
validate_is_ladm=True)
if modified_layer_name is None:
return
if modified_layer_name.lower() != layer_name.lower():
return
with OverrideCursor(Qt.WaitCursor):
new_values = self.upload_files(modified_layer, field_index,
features)
if new_values:
modified_layer.dataProvider().changeAttributeValues(new_values)
layer.committedFeaturesAdded.connect(features_added)
def get_file_url(self, part):
endpoint = QSettings().value(
'Asistente-LADM_COL/sources/service_endpoint',
DEFAULT_ENDPOINT_SOURCE_SERVICE)
return '/'.join([endpoint, part[1:] if part.startswith('/') else part])
class ReportGenerator(QObject):
LOG_TAB = 'LADM-COL Reports'
enable_action_requested = pyqtSignal(str, bool)
def __init__(self, ladm_data):
QObject.__init__(self)
self.ladm_data = ladm_data
self.logger = Logger()
self.app = AppInterface()
self.java_dependency = JavaDependency()
self.java_dependency.download_dependency_completed.connect(
self.download_java_complete)
self.report_dependency = ReportDependency()
self.report_dependency.download_dependency_completed.connect(
self.download_report_complete)
self.encoding = locale.getlocale()[1]
# This might be unset
if not self.encoding:
self.encoding = 'UTF8'
self._downloading = False
def stderr_ready(self, proc):
text = bytes(proc.readAllStandardError()).decode(self.encoding)
self.logger.critical(__name__, text, tab=self.LOG_TAB)
def stdout_ready(self, proc):
text = bytes(proc.readAllStandardOutput()).decode(self.encoding)
self.logger.info(__name__, text, tab=self.LOG_TAB)
def update_yaml_config(self, db, config_path):
text = ''
qgs_uri = QgsDataSourceUri(db.uri)
with open(os.path.join(config_path, 'config_template.yaml')) as f:
text = f.read()
text = text.format('{}',
DB_USER=qgs_uri.username(),
DB_PASSWORD=qgs_uri.password(),
DB_HOST=qgs_uri.host(),
DB_PORT=qgs_uri.port(),
DB_NAME=qgs_uri.database())
new_file_path = os.path.join(
config_path, self.get_tmp_filename('yaml_config', 'yaml'))
with open(new_file_path, 'w') as new_yaml:
new_yaml.write(text)
return new_file_path
def get_layer_geojson(self, db, layer_name, plot_id, report_type):
if report_type == ANNEX_17_REPORT:
if layer_name == 'terreno':
return db.get_annex17_plot_data(plot_id, 'only_id')
elif layer_name == 'terrenos':
return db.get_annex17_plot_data(plot_id, 'all_but_id')
elif layer_name == 'terrenos_all':
return db.get_annex17_plot_data(plot_id, 'all')
elif layer_name == 'construcciones':
return db.get_annex17_building_data()
else:
return db.get_annex17_point_data(plot_id)
else: #report_type == ANT_MAP_REPORT:
if layer_name == 'terreno':
return db.get_ant_map_plot_data(plot_id, 'only_id')
elif layer_name == 'terrenos':
return db.get_ant_map_plot_data(plot_id, 'all_but_id')
elif layer_name == 'terrenos_all':
return db.get_annex17_plot_data(plot_id, 'all')
elif layer_name == 'construcciones':
return db.get_annex17_building_data()
elif layer_name == 'puntoLindero':
return db.get_annex17_point_data(plot_id)
else: #layer_name == 'cambio_colindancia':
return db.get_ant_map_neighbouring_change_data(plot_id)
def update_json_data(self, db, json_spec_file, plot_id, tmp_dir,
report_type):
json_data = dict()
with open(json_spec_file) as f:
json_data = json.load(f)
json_data['attributes']['id'] = plot_id
json_data['attributes']['datasetName'] = db.schema
layers = json_data['attributes']['map']['layers']
for layer in layers:
layer['geoJson'] = self.get_layer_geojson(db, layer['name'],
plot_id, report_type)
overview_layers = json_data['attributes']['overviewMap']['layers']
for layer in overview_layers:
layer['geoJson'] = self.get_layer_geojson(db, layer['name'],
plot_id, report_type)
new_json_file_path = os.path.join(
tmp_dir,
self.get_tmp_filename('json_data_{}'.format(plot_id), 'json'))
with open(new_json_file_path, 'w') as new_json:
new_json.write(json.dumps(json_data))
return new_json_file_path
def get_tmp_dir(self, create_random=True):
if create_random:
return tempfile.mkdtemp()
return tempfile.gettempdir()
def get_tmp_filename(self, basename, extension='gpkg'):
return "{}_{}.{}".format(basename,
str(time.time()).replace(".", ""), extension)
def generate_report(self, db, report_type):
# Check if mapfish and Jasper are installed, otherwise show where to
# download them from and return
if not self.report_dependency.check_if_dependency_is_valid():
self.report_dependency.download_dependency(URL_REPORTS_LIBRARIES)
return
java_home_set = self.java_dependency.set_java_home()
if not java_home_set:
self.java_dependency.get_java_on_demand()
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Java is a prerequisite. Since it was not found, it is being configured..."
))
return
plot_layer = self.app.core.get_layer(db, db.names.LC_PLOT_T, load=True)
if not plot_layer:
return
selected_plots = plot_layer.selectedFeatures()
if not selected_plots:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"To generate reports, first select at least a plot!"))
return
# Where to store the reports?
previous_folder = QSettings().value(
"Asistente-LADM-COL/reports/save_into_dir", ".")
save_into_folder = QFileDialog.getExistingDirectory(
None,
QCoreApplication.translate(
"ReportGenerator",
"Select a folder to save the reports to be generated"),
previous_folder)
if not save_into_folder:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You need to select a folder where to save the reports before continuing."
))
return
QSettings().setValue("Asistente-LADM-COL/reports/save_into_dir",
save_into_folder)
config_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, report_type)
json_spec_file = os.path.join(config_path, 'spec_json_file.json')
script_name = ''
if os.name == 'posix':
script_name = 'print'
elif os.name == 'nt':
script_name = 'print.bat'
script_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, 'bin',
script_name)
if not os.path.isfile(script_path):
self.logger.warning(
__name__,
"Script file for reports wasn't found! {}".format(script_path))
return
self.enable_action_requested.emit(report_type, False)
# Update config file
yaml_config_path = self.update_yaml_config(db, config_path)
self.logger.debug(
__name__, "Config file for reports: {}".format(yaml_config_path))
total = len(selected_plots)
step = 0
count = 0
tmp_dir = self.get_tmp_dir()
# Progress bar setup
progress = QProgressBar()
if total == 1:
progress.setRange(0, 0)
else:
progress.setRange(0, 100)
progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.app.gui.create_progress_message_bar(
QCoreApplication.translate("ReportGenerator",
"Generating {} report{}...").format(
total, '' if total == 1 else 's'),
progress)
polygons_with_holes = []
multi_polygons = []
for selected_plot in selected_plots:
plot_id = selected_plot[db.names.T_ID_F]
geometry = selected_plot.geometry()
abstract_geometry = geometry.get()
if abstract_geometry.ringCount() > 1:
polygons_with_holes.append(str(plot_id))
self.logger.warning(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Skipping Annex 17 for plot with {}={} because it has holes. The reporter module does not support such polygons."
).format(db.names.T_ID_F, plot_id))
continue
if abstract_geometry.numGeometries() > 1:
multi_polygons.append(str(plot_id))
self.logger.warning(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Skipping Annex 17 for plot with {}={} because it is a multi-polygon. The reporter module does not support such polygons."
).format(db.names.T_ID_F, plot_id))
continue
# Generate data file
json_file = self.update_json_data(db, json_spec_file, plot_id,
tmp_dir, report_type)
self.logger.debug(__name__,
"JSON file for reports: {}".format(json_file))
# Run sh/bat passing config and data files
proc = QProcess()
proc.readyReadStandardError.connect(
functools.partial(self.stderr_ready, proc=proc))
proc.readyReadStandardOutput.connect(
functools.partial(self.stdout_ready, proc=proc))
parcel_number = self.ladm_data.get_parcels_related_to_plots(
db, [plot_id], db.names.LC_PARCEL_T_PARCEL_NUMBER_F) or ['']
file_name = '{}_{}_{}.pdf'.format(report_type, plot_id,
parcel_number[0])
current_report_path = os.path.join(save_into_folder, file_name)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
# Grant execution permissions
os.chmod(
script_path, stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR
| stat.S_IRUSR | stat.S_IRGRP)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
proc = None
self.logger.warning(
__name__, "Couldn't execute script to generate report...")
else:
loop = QEventLoop()
proc.finished.connect(loop.exit)
loop.exec()
self.logger.debug(__name__,
"{}:{}".format(plot_id, proc.exitCode()))
if proc.exitCode() == 0:
count += 1
step += 1
progress.setValue(step * 100 / total)
os.remove(yaml_config_path)
self.enable_action_requested.emit(report_type, True)
self.logger.clear_message_bar()
if total == count:
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report <a href='file:///{}'>{}</a> was successfully generated!"
).format(normalize_local_url(save_into_folder), file_name)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"All reports were successfully generated in folder <a href='file:///{path}'>{path}</a>!"
).format(path=normalize_local_url(save_into_folder))
self.logger.success_msg(__name__, msg)
else:
details_msg = ''
if polygons_with_holes:
details_msg += QCoreApplication.translate(
"ReportGenerator",
" The following polygons were skipped because they have holes and are not supported: {}."
).format(", ".join(polygons_with_holes))
if multi_polygons:
details_msg += QCoreApplication.translate(
"ReportGenerator",
" The following polygons were skipped because they are multi-polygons and are not supported: {}."
).format(", ".join(multi_polygons))
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report for plot {} couldn't be generated!{} See QGIS log (tab '{}') for details."
).format(plot_id, details_msg, self.LOG_TAB)
else:
if count == 0:
msg = QCoreApplication.translate(
"ReportGenerator",
"No report could be generated!{} See QGIS log (tab '{}') for details."
).format(details_msg, self.LOG_TAB)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"At least one report couldn't be generated!{details_msg} See QGIS log (tab '{log_tab}') for details. Go to <a href='file:///{path}'>{path}</a> to see the reports that were generated."
).format(details_msg=details_msg,
path=normalize_local_url(save_into_folder),
log_tab=self.LOG_TAB)
self.logger.warning_msg(__name__, msg)
def download_java_complete(self):
if self.java_dependency.fetcher_task and not self.java_dependency.fetcher_task.isCanceled(
):
if self.java_dependency.check_if_dependency_is_valid():
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Java was successfully configured!"), 5)
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You have just canceled the Java dependency download."), 5)
def download_report_complete(self):
if self.report_dependency.fetcher_task and not self.report_dependency.fetcher_task.isCanceled(
):
if self.report_dependency.check_if_dependency_is_valid():
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Report dependency was successfully configured!"), 5)
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You have just canceled the report dependency download."),
5)
class XTFModelConverterRegistry(metaclass=Singleton):
"""
Registry of supported model converters.
"""
def __init__(self):
self.logger = Logger()
self.app = AppInterface()
self.__converters = dict() # {converter_key1: LADMColModelConverter1, ...}
# Register default models
self.register_model_converter(Survey10To11Converter())
self.register_model_converter(Survey11To10Converter())
def register_model_converter(self, converter):
"""
:param converter: LADMColModelConverter instance.
:return: True if the converter was registered, False otherwise.
"""
if not isinstance(converter, AbstractLADMColModelConverter):
self.logger.warning(__name__, "The converter '{}' is not a 'LADMColModelConverter' instance!".format(converter.id()))
return False
if not converter.is_valid():
self.logger.warning(__name__, "The converter '{}' is not valid! Check the converter definition!".format(converter.id()))
return False
if converter.id() in self.__converters:
self.logger.warning(__name__, "The converter '{}' is already registered.".format(converter.id()))
return False
self.__converters[converter.id()] = converter
self.logger.info(__name__, "Model converter '{}' has been registered!".format(converter.id()))
return True
def unregister_model_converter(self, converter_key):
"""
Unregisters a model converter.
:param converter_key: Id of the converter to unregister.
:return: True if the converter was unregistered, False otherwise.
"""
if converter_key not in self.__converters:
self.logger.error(__name__, "Converter '{}' was not found in registered model converters, therefore, it cannot be unregistered!".format(converter_key))
return False
self.__converters[converter_key] = None
del self.__converters[converter_key]
self.logger.info(__name__, "Model converter '{}' has been unregistered!".format(converter_key))
return True
def get_converter(self, converter_key):
converter = self.__converters.get(converter_key, None) # To avoid exceptions
if not converter:
self.logger.critical(__name__, "No model converter found with key '{}'".format(converter_key))
return converter
def get_converters_for_models(self, models):
converters = dict() # {converter_key_1: converter_display_name_1, ...]
for converter_key, converter in self.__converters.items():
for model in models:
if converter.supports_source_model(model):
converters[converter_key] = converter.display_name()
return converters
class QualityRuleEngine(QObject):
"""
Engine that executes Quality Rules
:param db: DBConnector object
:param rules: Either a dict {rule_key:rule_name} or a list [rule_key1, rule_key2]
:param tolerance: Tolerance to be used when running the QRs, in millimeters
"""
progress_changed = pyqtSignal(int) # Progress value
def __init__(self, db, rules, tolerance, output_path=''):
QObject.__init__(self)
self.logger = Logger()
self.app = AppInterface()
self.__qr_registry = QualityRuleRegistry()
self.__db = db
self.__db_qr = None
self.__rules = self.__get_dict_rules(rules)
self.__result_layers = list()
self.__with_gui = self.app.settings.with_gui
self.__timestamp = ""
self.__output_path = output_path
self.app.settings.tolerance = tolerance # Tolerance must be given, we don't want anything implicit about it
self.__tolerance = self.app.settings.tolerance # Tolerance input might be altered (e.g., if it comes negative)
self.__layer_manager = QualityRuleLayerManager(db, self.__rules,
self.__tolerance)
self.__layer_manager.progress_changed.connect(
self.__emit_prepare_layers_progress)
self.qr_logger = QualityRuleLogger(self.__db, self.__tolerance)
self.__current_progress = 0
self.__error_db_utils = QualityErrorDBUtils()
self.__error_db_utils.progress_changed.connect(
self.__emit_error_db_progress)
# Clear informality cache before executing QRs.
# Note: between creating an object of this class and calling validate_quality_rules() a lot
# of things could happen (like new caches being built!). It is your responsibility to create
# an instance of this class or initialize() a QREngine object just before calling validate_quality_rules().
self.app.core.clear_cached_informal_spatial_units()
def initialize(self,
db,
rules,
tolerance,
output_path='',
clear_informality_cache=True):
"""
Objects of this class are reusable calling initialize()
"""
self.__result_layers = list()
self.__db = db
self.__db_qr = None
self.__rules = self.__get_dict_rules(rules)
self.__with_gui = self.app.settings.with_gui
self.__timestamp = ""
self.__output_path = output_path
self.app.settings.tolerance = tolerance
self.__tolerance = self.app.settings.tolerance # Tolerance input might be altered (e.g., if it comes negative)
self.__layer_manager.initialize(self.__rules, self.__tolerance)
self.qr_logger.initialize(self.__db, self.__tolerance)
self.__current_progress = 0
# This time, (initializing an existing object) we give you the chance to avoid
# rebuilding the informality cache. It is handy if you're executing validations
# consecutively and you're sure that reusing a previous cache does make sense.
if clear_informality_cache:
self.app.core.clear_cached_informal_spatial_units()
def __get_dict_rules(self, rules):
if isinstance(rules, dict):
return rules # We have everything ready
# If rules is a list, we need to retrieve the quality rule names from the QRRegistry
return {
rule_key: self.__qr_registry.get_quality_rule(rule_key)
for rule_key in rules
}
def validate_quality_rules(self, options=dict()):
"""
:param options: Dict of dicts with of options per rule. {qr1_key: {qr1_opt1: value1, ...}, ...}
"""
res = False
msg = ""
qr_res = dict() # {rule_key: QualityRuleExecutionResult}
if self.__rules:
self.__emit_progress_changed(1)
# First, create the error db and fill its metadata...
self.__timestamp = time.strftime('%Y%m%d_%H%M%S')
res_db, msg_db, self.__db_qr = self.__error_db_utils.get_quality_error_connector(
self.__output_path, self.__timestamp, True)
self.__emit_progress_changed(5)
if not res_db:
msg_db = QCoreApplication.translate(
"QualityRuleEngine",
"There was a problem creating the quality error DB! Details: {}"
).format(msg_db)
self.logger.warning(__name__, msg_db)
return False, msg_db, None
self.qr_logger.set_count_topology_rules(len(self.__rules))
self.logger.info(
__name__,
QCoreApplication.translate(
"QualityRuleEngine",
"Validating {} quality rules (tolerance: {}).").format(
len(self.__rules), self.__tolerance))
first_pass = True
count = 0
for rule_key, rule in self.__rules.items():
count += 1
if rule is not None:
layers = self.__layer_manager.get_layers(
rule_key) # Fist pass might be long if tolerance > 0
if first_pass:
first_pass = False
self.__emit_progress_changed(
25 if self.__tolerance else 15)
if layers:
connect_obj = rule.progress_changed.connect(
partial(self.__emit_qr_progress, count))
qr_res[rule_key] = self.__validate_quality_rule(
rule, layers, options.get(rule_key, dict()))
rule.progress_changed.disconnect(
connect_obj
) # We no longer need the connection, so delete it
else:
qr_msg = QCoreApplication.translate(
"QualityRuleEngine",
"Couldn't execute '{}' quality rule! Required layers are not available. Skipping..."
).format(rule.name())
qr_res[rule_key] = QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL, qr_msg)
self.logger.warning(__name__, qr_msg)
else:
qr_msg = QCoreApplication.translate(
"QualityRuleEngine",
"Quality rule with key '{}' does not exist or is not registered! Skipping..."
).format(rule_key)
qr_res[rule_key] = QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL, qr_msg)
self.logger.warning(__name__, qr_msg)
self.__emit_progress_changed(95)
metadata = {
QR_METADATA_TOOL: QR_METADATA_TOOL_NAME,
QR_METADATA_DATA_SOURCE:
self.__db.get_description_conn_string(),
QR_METADATA_TOLERANCE: self.__tolerance / 1000,
QR_METADATA_TIMESTAMP: self.__timestamp,
QR_METADATA_RULES: list(self.__rules.keys()), # QR keys
QR_METADATA_OPTIONS: self.__normalize_options(options),
QR_METADATA_PERSON: getpass.getuser()
}
QualityErrorDBUtils.save_metadata(self.__db_qr, metadata)
self.export_result_to_pdf()
self.__emit_progress_changed(99)
res = True
msg = "Success!"
self.__layer_manager.clean_temporary_layers()
else:
self.logger.warning(
__name__,
QCoreApplication.translate("QualityRuleEngine",
"No rules to validate!"))
self.__emit_progress_changed(100)
return res, msg, QualityRulesExecutionResult(qr_res)
@_log_quality_rule_validations
def __validate_quality_rule(self, rule, layers, options):
"""
Intermediate function to log quality rule execution.
:param rule: Quality rule instance
:param layers: Layer dict with the layers the quality rule needs (ready to use for tolerance > 0 scenarios)
:param options: Dict of options per rule. {qr1_opt1: value1, ...}
:return: An instance of QualityRuleExecutionResult
"""
return rule.validate(self.__db,
self.__db_qr,
layers,
self.__tolerance,
options=options)
def __emit_progress_changed(self, value, save_value=True):
if value != self.__current_progress: # Avoid emitting the same value twice
if save_value:
self.__current_progress = value
self.progress_changed.emit(value)
def __emit_error_db_progress(self, progress_value):
"""
Add the normalized error db progress value to what we have already in the overall progress
"""
value = self.__current_progress + (progress_value *
TOTAL_PROGRESS_ERROR_DB / 100)
# print("...DB", self.__current_progress, progress_value, value)
self.__emit_progress_changed(
int(value),
progress_value == 100) # Only save when the subprocess is finished
def __emit_prepare_layers_progress(self, progress_value):
"""
Add the normalized prepare layers' progress value to what we have already in the overall progress
"""
range = TOTAL_PROGRESS_PREPARE_LAYERS if self.__tolerance else TOTAL_PROGRESS_PREPARE_LAYERS_NO_TOLERANCE
value = self.__current_progress + (progress_value * range / 100)
# print("...PL", self.__current_progress, progress_value, value)
self.__emit_progress_changed(
int(value),
progress_value == 100) # Only save when the subprocess is finished
def __emit_qr_progress(self, count, qr_progress_value):
"""
Add the normalized current QR progress value to what we have already in the overall progress
"""
num_rules = len(self.__rules)
step = (TOTAL_PROGRESS_QR if self.__tolerance else
TOTAL_PROGRESS_QR_NO_TOLERANCE) / num_rules
value = self.__current_progress + (qr_progress_value * step / 100)
# print("...QR", self.__current_progress, qr_progress_value, value)
self.__emit_progress_changed(
int(value),
qr_progress_value == 100) # Only save when the QR is finished
def get_db_quality(self):
return self.__db_qr
def get_timestamp(self):
# Last timestamp used to validate QRs.
# Note the timestamp is persisted in QR DB's metadata table
return self.__timestamp
def export_result_to_pdf(self):
res, msg, output_path = QualityErrorDBUtils.get_quality_validation_output_path(
self.__output_path, self.__timestamp)
if not res:
self.logger.critical(
__name__,
QCoreApplication.translate(
"QualityRuleEngine",
"PDF report could not be exported, there were problems with the output path '{}'!"
).format(self.__output_path))
return
pdf_path = os.path.join(
output_path, "Reglas_de_Calidad_{}.pdf".format(self.__timestamp))
log = self.qr_logger.get_log_result()
export_title_text_to_pdf(pdf_path, log.title, log.text)
def __normalize_options(self, options):
# Get rid of options that do not correspond to validated QRs
normalized_options = dict()
for k, v in options.items():
for kv, vv in v.items():
if k in list(self.__rules.keys()):
if k in normalized_options:
normalized_options[k][kv] = vv
else:
normalized_options[k] = {kv: vv}
return normalized_options
class QualityRuleController(QObject):
open_report_called = pyqtSignal(QualityRuleResultLog) # log result
quality_rule_layer_removed = pyqtSignal()
refresh_error_layer_symbology = pyqtSignal(QgsVectorLayer)
total_progress_changed = pyqtSignal(int) # Progress value
def __init__(self, db):
QObject.__init__(self)
self.app = AppInterface()
self.logger = Logger()
self.__db = db
self.__tr_dict = TranslatableConfigStrings(
).get_translatable_config_strings()
# Hierarquical dict of qrs and qr groups
self.__qrs_tree_data = dict() # {type: {qr_key1: qr_obj1, ...}, ...}
# Hierarquical dict of qrs and qr groups with general results
self.__general_results_tree_data = dict(
) # {type: {qr_obj1: qr_results1, ...}, ...}
# Hierarchical dict of qrs and their corresponding error instances
# feature1: {uuids, rel_uuids, error_type, nombre_ili_obj, details, values, fixed, exception, geom_fks}
self.__error_results_data = dict() # {qr_key1: {t_id1: feature1}}
self.__qr_results_dir_path = '' # Dir path where results will be stored
self.__selected_qrs = list() # QRs to be validated (at least 1)
self.__selected_qr = None # QR selected by the user to show its corresponding errors (exactly 1)
self.__qr_engine = None # Once set, we can reuse it
self.__qrs_results = None # QualityRulesExecutionResult object
# To cache layers from QR DB
self.__error_layer = None
self.__point_layer = None
self.__line_layer = None
self.__polygon_layer = None
# Cache by t_id (built on demand): {t_id1: 'Error', t_id2: 'Corregido', t_id3: 'Exception'}
self.__error_state_dict = dict()
def get_tr_string(self, key):
return self.__tr_dict.get(key, key)
def validate_qrs(self):
if self.__qr_engine is None:
self.__qr_engine = QualityRuleEngine(self.__db,
self.__selected_qrs,
self.app.settings.tolerance,
self.__qr_results_dir_path)
self.__qr_engine.progress_changed.connect(
self.total_progress_changed)
else:
self.__qr_engine.initialize(self.__db, self.__selected_qrs,
self.app.settings.tolerance,
self.__qr_results_dir_path)
#self.__qr_engine.qr_logger.show_message_emitted.connect(self.show_log_quality_message)
#self.__qr_engine.qr_logger.show_button_emitted.connect(self.show_log_quality_button)
#self.__qr_engine.qr_logger.set_initial_progress_emitted.connect(self.set_log_quality_initial_progress)
#self.__qr_engine.qr_logger.set_final_progress_emitted.connect(self.set_log_quality_final_progress)
use_roads = bool(QSettings().value(
'Asistente-LADM-COL/quality/use_roads', DEFAULT_USE_ROADS_VALUE,
bool))
options = {QR_IGACR3006: {'use_roads': use_roads}}
res, msg, qrs_res = self.__qr_engine.validate_quality_rules(options)
if not res:
return res, msg, None
self.__qrs_results = qrs_res
self.__connect_layer_willbedeleted_signals(
) # Note: Call it after validate_quality_rules!
res_u, msg_u, output_qr_dir = QualityErrorDBUtils.get_quality_validation_output_path(
self.__qr_results_dir_path, self.__qr_engine.get_timestamp())
if len(self.__selected_qrs) == 1:
pre_text = QCoreApplication.translate(
"QualityRules", "The quality rule was checked!")
else:
pre_text = QCoreApplication.translate(
"QualityRules",
"All the {} quality rules were checked!").format(
len(self.__selected_qrs))
post_text = QCoreApplication.translate(
"QualityRules",
"Both a PDF report and a GeoPackage database with errors can be found in <a href='file:///{}'>{}</a>."
).format(normalize_local_url(output_qr_dir), output_qr_dir)
self.logger.success_msg(__name__, "{} {}".format(pre_text, post_text))
self.__emit_refresh_error_layer_symbology()
return res, msg, self.__qrs_results
def __connect_layer_willbedeleted_signals(self):
"""
Iterate QR DB layers from the layer tree and connect their layerwillberemoved signals.
If a QR DB layer is removed, we'll react in the GUI.
"""
group = QualityErrorDBUtils.get_quality_error_group(
self.__qr_engine.get_timestamp())
if group:
for tree_layer in group.findLayers():
try:
tree_layer.layer().willBeDeleted.disconnect(
self.quality_rule_layer_removed)
except:
pass
tree_layer.layer().willBeDeleted.connect(
self.quality_rule_layer_removed)
def disconnect_layer_willberemoved_signals(self):
group = QualityErrorDBUtils.get_quality_error_group(
self.__qr_engine.get_timestamp(), False)
if group:
for tree_layer in group.findLayers():
try:
tree_layer.layer().willBeDeleted.disconnect(
self.quality_rule_layer_removed)
except:
pass
def get_qr_result(self, qr_key):
"""
Return the QRExecutionResult object for the given qr_key.
It first attempts to find it in the __qrs_results dict, but, chances are,
the whole set of QRs hasn't been validated when this method is called,
so, as a last resort, we go for the tree_data, which is updated each time
a QR gets its result.
"""
if self.__qrs_results is not None:
return self.__qrs_results.result(qr_key)
for type, qr_dict in self.__general_results_tree_data.items():
for k, v in qr_dict.items():
if k.id() == qr_key:
return self.__general_results_tree_data[type][k]
return None
def __reset_qrs_results(self):
# To be used when we are returning to select QRs (i.e., to the initial panel)
self.__qrs_results = None
def __get_qrs_per_role_and_models(self):
return QualityRuleRegistry().get_qrs_per_role_and_models(self.__db)
def load_tree_data(self, mode):
"""
Builds a hierarchical dict by qr type: {qr_type1: {qr_key1: qr_obj1, ...}, ...}
Tree data for panel 1.
:params mode: Value from EnumQualityRulePanelMode (either VALIDATE or READ).
For VALIDATE we load QRs from registry (filtered by role and current db models).
For READ we load QRs from the DB itself.
"""
if mode == EnumQualityRulePanelMode.VALIDATE:
qrs = self.__get_qrs_per_role_and_models(
) # Dict of qr key and qr objects.
else:
qrs = dict() # TODO: Read QRs from the QR DB
for qr_key, qr_obj in qrs.items():
type = qr_obj.type()
if type not in self.__qrs_tree_data:
self.__qrs_tree_data[type] = {qr_key: qr_obj}
else:
self.__qrs_tree_data[type][qr_key] = qr_obj
def get_qrs_tree_data(self):
return self.__qrs_tree_data
def set_qr_dir_path(self, path):
self.__qr_results_dir_path = path
def set_selected_qrs(self, selected_qrs):
# We sort them because the engine needs the QRs sorted for the PDF report
for type, qr_dict in self.__qrs_tree_data.items():
for qr_key, qr_obj in qr_dict.items():
if qr_key in selected_qrs:
self.__selected_qrs.append(qr_key)
def get_selected_qrs(self):
return self.__selected_qrs
def __reset_selected_qrs(self):
# To be used when we are returning to select QRs (i.e., to the initial panel)
self.__selected_qrs = list()
def reset_vars_for_general_results_panel(self):
# Initialize variables when we leave the general results panel
self.__reset_general_results_tree_data()
self.__reset_selected_qrs()
self.__reset_qrs_results()
self.__reset_layers()
# Call it before removing QR DB group to avoid triggering parent.layer_removed() slot again.
self.disconnect_layer_willberemoved_signals()
# When we leave the GRP, we remove the QR DB group from layer tree,
# because we won't be working anymore with that QR DB
QualityErrorDBUtils.remove_quality_error_group(
self.__qr_engine.get_timestamp())
def reset_vars_for_error_results_panel(self):
# Initialize variables when we leave the error results panel
self.__reset_error_results_data()
self.__reset_selected_qr()
self.__reset_error_state_dict()
self.__reset_layers()
def load_general_results_tree_data(self):
"""
Builds a hierarchical dict by qr type: {type: {qr_obj1: qr_results1, ...}, ...}
Tree data for panel 2.
"""
for type, qr_dict in self.__qrs_tree_data.items():
for qr_key, qr_obj in qr_dict.items():
if qr_key in self.__selected_qrs:
if type not in self.__general_results_tree_data:
self.__general_results_tree_data[type] = {qr_obj: None}
else:
self.__general_results_tree_data[type][qr_obj] = None
def get_general_results_tree_data(self):
return self.__general_results_tree_data
def __reset_general_results_tree_data(self):
# To be used when we are returning to select QRs (i.e., to the initial panel)
self.__general_results_tree_data = dict()
def set_qr_validation_result(self, qr, qr_result):
"""
When a QR has its validation result after validation,
we can store it in our custom dict by using this method.
"""
for type, qr_dict in self.__general_results_tree_data.items():
for k, v in qr_dict.items():
if k == qr:
self.__general_results_tree_data[type][k] = qr_result
def open_report(self):
if self.__qr_engine:
log_result = self.__qr_engine.qr_logger.get_log_result()
self.open_report_called.emit(log_result)
def set_selected_qr(self, qr_key):
self.__selected_qr = QualityRuleRegistry().get_quality_rule(qr_key)
return self.__selected_qr is not None # We should not be able to continue if we don't find the QR
def get_selected_qr(self):
return self.__selected_qr
def load_error_results_data(self):
"""
Go to table and bring data to the dict.
We should keep this dict updated with changes from the user.
From time to time we reflect this dict changes in the original data source.
"""
db = self.__qr_engine.get_db_quality()
names = db.names
layers = {names.ERR_QUALITY_ERROR_T: None, names.ERR_RULE_TYPE_T: None}
self.app.core.get_layers(db, layers, load=False)
if not layers:
self.logger.critical(
__name__,
"Quality error layers ('{}') not found!".format(",".join(
list(layers.keys()))))
return
# First go for the selected quality error's t_id
features = LADMData.get_features_from_t_ids(
layers[names.ERR_RULE_TYPE_T], names.ERR_RULE_TYPE_T_CODE_F,
[self.__selected_qr.id()])
t_id = features[0][names.T_ID_F] if features else None
if not t_id:
self.logger.critical(
__name__, "Quality error rule ('{}') not found!".format(
self.__selected_qr.id()))
return
# Now go for all features that match the selected quality rule
features = LADMData.get_features_from_t_ids(
layers[names.ERR_QUALITY_ERROR_T],
names.ERR_QUALITY_ERROR_T_RULE_TYPE_F, [t_id])
self.__error_results_data[self.__selected_qr.id()] = {
feature[names.T_ID_F]: feature
for feature in features
}
def get_error_results_data(self):
# Get the subdict {t_id1: feature1, ...} corresponding to selected qr
return self.__error_results_data.get(
self.__selected_qr.id() if self.__selected_qr else '', dict())
def __reset_error_results_data(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__error_results_data = dict()
def error_t_id(self, feature):
return feature[self.__qr_engine.get_db_quality().names.T_ID_F]
def is_fixed_error(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(
state_t_id) == LADMNames.ERR_ERROR_STATE_D_FIXED_V
def is_error(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(
state_t_id) == LADMNames.ERR_ERROR_STATE_D_ERROR_V
def is_exception(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(
state_t_id) == LADMNames.ERR_ERROR_STATE_D_EXCEPTION_V
def uuid_objs(self, feature):
return "\n".join(feature[self.__qr_engine.get_db_quality().names.
ERR_QUALITY_ERROR_T_OBJECT_IDS_F])
def ili_obj_name(self, feature):
ili_name = feature[self.__qr_engine.get_db_quality().names.
ERR_QUALITY_ERROR_T_ILI_NAME_F]
return ili_name.split(".")[-1] if ili_name else ''
def error_type_code_and_display(self, feature):
db = self.__qr_engine.get_db_quality()
names = db.names
layer = self.app.core.get_layer(db, names.ERR_ERROR_TYPE_T, load=False)
features = LADMData.get_features_from_t_ids(
layer, names.T_ID_F,
[feature[db.names.ERR_QUALITY_ERROR_T_ERROR_TYPE_F]]) # tid
return features[0][
names.
ERR_ERROR_TYPE_T_CODE_F] if features else QCoreApplication.translate(
"QualityRules", "No error type found!"
), features[0][
names.
ERR_ERROR_TYPE_T_DESCRIPTION_F] if features else QCoreApplication.translate(
"QualityRules", "No error description found!")
def error_details_and_values(self, feature):
res = ""
db = self.__qr_engine.get_db_quality()
details = feature[db.names.ERR_QUALITY_ERROR_T_DETAILS_F]
values = feature[db.names.ERR_QUALITY_ERROR_T_VALUES_F]
if details:
res = details
if values:
try:
res_values = json.loads(values)
if type(res_values) is dict:
items = ""
for k, v in res_values.items():
items = res + "{}: {}\n".format(k, v)
res_values = items.strip()
else:
res_values = str(res_values)
except json.decoder.JSONDecodeError as e:
res_values = values
res = res_values if not res else "{}\n\n{}".format(res, res_values)
return res
def error_state(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(state_t_id)
def __get_error_state_value(self, state_t_id):
if state_t_id not in self.__error_state_dict:
db = self.__qr_engine.get_db_quality()
self.__error_state_dict[state_t_id] = LADMData(
).get_domain_value_from_code(db, db.names.ERR_ERROR_STATE_D,
state_t_id)
return self.__error_state_dict.get(state_t_id, "")
def __get_error_state_t_id(self, state_value):
# Use __error_state_dict to read cached values, but this time we have the value,
# not the key, so check in dict values and if not found, go for its t_id
if state_value not in self.__error_state_dict.values():
db = self.__qr_engine.get_db_quality()
t_id = LADMData().get_domain_code_from_value(
db, db.names.ERR_ERROR_STATE_D, state_value)
self.__error_state_dict[t_id] = state_value
# Get key by value in a dict:
return next((k for k in self.__error_state_dict
if self.__error_state_dict[k] == state_value), None)
def __get_error_layer(self):
if not self.__error_layer:
db = self.__qr_engine.get_db_quality()
self.__error_layer = self.app.core.get_layer(
db, db.names.ERR_QUALITY_ERROR_T)
return self.__error_layer
def __get_point_error_layer(self):
if not self.__point_layer:
db = self.__qr_engine.get_db_quality()
self.__point_layer = self.app.core.get_layer(
db, db.names.ERR_POINT_T)
return self.__point_layer
def __get_line_error_layer(self):
if not self.__line_layer:
db = self.__qr_engine.get_db_quality()
self.__line_layer = self.app.core.get_layer(
db, db.names.ERR_LINE_T)
return self.__line_layer
def __get_polygon_error_layer(self):
if not self.__polygon_layer:
db = self.__qr_engine.get_db_quality()
self.__polygon_layer = self.app.core.get_layer(
db, db.names.ERR_POLYGON_T)
return self.__polygon_layer
def __reset_layers(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__error_layer = None
self.__point_layer = None
self.__line_layer = None
self.__polygon_layer = None
def __reset_selected_qr(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__selected_qr = None
def __reset_error_state_dict(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__error_state_dict = dict()
def __error_related_geometries(self, error_t_ids):
# Prefered geometry types are polygons, lines, points, in that order
db = self.__qr_engine.get_db_quality()
error_data = self.get_error_results_data()
dict_layer_fids = dict()
for error_t_id in error_t_ids:
feature = error_data.get(error_t_id, None)
if feature:
polygon = feature[db.names.ERR_QUALITY_ERROR_T_POLYGON_F]
line = feature[db.names.ERR_QUALITY_ERROR_T_LINE_F]
point = feature[db.names.ERR_QUALITY_ERROR_T_POINT_F]
if polygon:
if 'polygon' in dict_layer_fids:
dict_layer_fids['polygon']['fids'].append(polygon)
else:
dict_layer_fids['polygon'] = {
'layer': self.__get_polygon_error_layer(),
'fids': [polygon]
}
elif line:
if 'line' in dict_layer_fids:
dict_layer_fids['line']['fids'].append(line)
else:
dict_layer_fids['line'] = {
'layer': self.__get_line_error_layer(),
'fids': [line]
}
elif point:
if 'point' in dict_layer_fids:
dict_layer_fids['point']['fids'].append(point)
else:
dict_layer_fids['point'] = {
'layer': self.__get_point_error_layer(),
'fids': [point]
}
return dict_layer_fids
def highlight_geometries(self, t_ids):
res_geometries = self.__error_related_geometries(t_ids)
if res_geometries:
# First zoom to geometries
if len(res_geometries) == 1: # Only one geometry type related
for geom_type, dict_layer_fids in res_geometries.items(
): # We know this will be called just once
self.app.gui.zoom_to_feature_ids(dict_layer_fids['layer'],
dict_layer_fids['fids'])
else: # Multiple geometry types were found, so combine the extents and then zoom to it
combined_extent = QgsRectangle()
for geom_type, dict_layer_fids in res_geometries.items():
combined_extent.combineExtentWith(
self.app.core.get_extent_from_feature_ids(
dict_layer_fids['layer'], dict_layer_fids['fids']))
self.app.gui.zoom_to_extent(combined_extent)
# Now highlight geometries
for geom_type, dict_layer_fids in res_geometries.items():
self.app.gui.flash_features(dict_layer_fids['layer'],
dict_layer_fids['fids'],
flashes=5)
def get_uuids_display_name(self):
names = self.__qr_engine.get_db_quality().names
res = self.__selected_qr.field_mapping(names).get(
names.ERR_QUALITY_ERROR_T_OBJECT_IDS_F, '')
return res if res else QCoreApplication.translate(
"QualityRules", "UUIDs")
def set_fixed_error(self, error_t_id, fixed):
# Save to the intermediate dict of data and to the underlying data source whether an error is fixed or not
db = self.__qr_engine.get_db_quality()
idx_state = self.__get_error_layer().fields().indexOf(
db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F)
value = LADMNames.ERR_ERROR_STATE_D_FIXED_V if fixed else LADMNames.ERR_ERROR_STATE_D_ERROR_V
fixed_or_error_t_id = self.__get_error_state_t_id(value)
if fixed_or_error_t_id is None:
self.logger.critical(
__name__,
"The error state t_id couldn't be found for value '{}'!".
format(value))
return
# Save to dict
self.get_error_results_data()[error_t_id].setAttribute(
idx_state, fixed_or_error_t_id)
fids = LADMData.get_fids_from_key_values(self.__get_error_layer(),
db.names.T_ID_F, [error_t_id])
# Save to underlying data source
if fids:
res = self.__get_error_layer().dataProvider(
).changeAttributeValues(
{fids[0]: {
idx_state: fixed_or_error_t_id
}})
if not res:
self.logger.critical(__name__,
"Error modifying the error state value!")
else:
self.logger.critical(
__name__, "Error with t_id '' not found!".format(error_t_id))
def __emit_refresh_error_layer_symbology(self):
if self.__get_point_error_layer().featureCount():
self.refresh_error_layer_symbology.emit(
self.__get_point_error_layer())
if self.__get_line_error_layer().featureCount():
self.refresh_error_layer_symbology.emit(
self.__get_line_error_layer())
if self.__get_polygon_error_layer().featureCount():
self.refresh_error_layer_symbology.emit(
self.__get_polygon_error_layer())
class AppProcessingInterface(QObject):
def __init__(self):
QObject.__init__(self)
self.logger = Logger()
self.ladm_col_provider = LADMCOLAlgorithmProvider()
self.__processing_resources_installed = list()
self.__processing_model_dirs_to_register = [PROCESSING_MODELS_DIR]
self.__processing_script_dirs_to_register = [PROCESSING_SCRIPTS_DIR]
def initialize_processing_resources(self):
"""
Add custom provider, models and scripts to QGIS
"""
QgsApplication.processingRegistry().addProvider(self.ladm_col_provider)
connect_provider_added = False
if QgsApplication.processingRegistry().providerById('model'):
self.__add_processing_resources_by_provider('model')
else:
connect_provider_added = True
if QgsApplication.processingRegistry().providerById('script'):
self.__add_processing_resources_by_provider('script')
else:
connect_provider_added = True
if connect_provider_added: # We need to wait until processing is initialized
QgsApplication.processingRegistry().providerAdded.connect(
self.__add_processing_resources_by_provider)
def __add_processing_resources_by_provider(self, provider_id):
if provider_id not in ['model', 'script']:
return
if sorted(self.__processing_resources_installed) == [
"models", "script"
]: # We are done, disconnect.
try:
QgsApplication.processingRegistry().providerAdded.disconnect(
self.__add_processing_resources_by_provider)
except:
pass # Disconnect throws an error if the SLOT is already disconnected
if provider_id == 'model':
for models_dir in self.__processing_model_dirs_to_register:
self.__register_processing_models(models_dir)
self.__processing_resources_installed.append('model')
elif provider_id == 'script':
for scripts_dir in self.__processing_script_dirs_to_register:
self.__register_processing_scripts(scripts_dir)
self.__processing_resources_installed.append('script')
def __register_processing_models(self, models_dir):
# First get model file names from the model root folder
filenames = list()
for filename in glob.glob(os.path.join(models_dir,
'*.model3')): # Non-recursive
filenames.append(filename)
# Now, go for subfolders.
# We store models that depend on QGIS versions in folders like "314" (for QGIS 3.14.x)
# This was initially needed for the FieldMapper input, which was migrated to C++ in QGIS 3.14
qgis_major_version = str(Qgis.QGIS_VERSION_INT)[:3]
qgis_major_version_path = os.path.join(models_dir, qgis_major_version)
if not os.path.isdir(qgis_major_version_path):
# No folder for this version (e.g., unit tests on QGIS-dev), so let's find the most recent version
subfolders = [
sf.name for sf in os.scandir(models_dir) if sf.is_dir()
]
if subfolders:
qgis_major_version_path = os.path.join(models_dir,
max(subfolders))
for filename in glob.glob(
os.path.join(qgis_major_version_path, '*.model3')):
filenames.append(filename)
# Finally, do load the models!
count = 0
registered_models = list()
for filename in filenames:
alg = QgsProcessingModelAlgorithm()
if not alg.fromFile(filename):
self.logger.critical(
__name__,
"Couldn't load model from '{}'!".format(filename))
return
registered_models.append(os.path.basename(filename))
destFilename = os.path.join(ModelerUtils.modelsFolders()[0],
os.path.basename(filename))
shutil.copyfile(filename, destFilename)
count += 1
if count:
QgsApplication.processingRegistry().providerById(
'model').refreshAlgorithms()
if DEFAULT_LOG_MODE == EnumLogMode.DEV:
self.logger.debug(
__name__,
"{} LADM-COL Processing models were installed! {}".format(
count, registered_models))
else:
self.logger.debug(
__name__,
"{} LADM-COL Processing models were installed!".format(
count))
def __register_processing_scripts(self, scripts_dir):
count = 0
registered_scripts = list()
qgis_scripts_dir = ScriptUtils.defaultScriptsFolder()
for filename in glob.glob(os.path.join(scripts_dir, '*.py')):
try:
shutil.copy(filename, qgis_scripts_dir)
count += 1
registered_scripts.append(os.path.basename(filename))
except OSError as e:
self.logger.critical(
__name__,
"Couldn't install LADM-COL script '{}'!".format(filename))
if count:
QgsApplication.processingRegistry().providerById(
"script").refreshAlgorithms()
if DEFAULT_LOG_MODE == EnumLogMode.DEV:
self.logger.debug(
__name__,
"{} LADM-COL Processing scripts were installed! {}".format(
count, registered_scripts))
else:
self.logger.debug(
__name__,
"{} LADM-COL Processing scripts were installed!".format(
count))
def register_add_on_processing_models(self, models_dir):
"""
For add-ons to delegate the registration of their own Processing models.
:param models_dir: Path to the directory containing Processing models
"""
if 'model' in self.__processing_resources_installed:
# The plugin already registered its models, so register right away.
self.__register_processing_models(models_dir)
else:
# The plugin has not yet registered its models (waiting for Processing
# to setup the 'model' provider). Pass the add-on's model dir to a list
# of paths to be registered by the plugin, when Processing is ready.
self.__processing_model_dirs_to_register.append(models_dir)
def register_add_on_processing_scripts(self, scripts_dir):
"""
For add-ons to delegate the registration of their own Processing scripts.
:param models_dir: Path to the directory containing Processing scripts
"""
if 'script' in self.__processing_resources_installed:
# The plugin already registered its scripts, so register right away.
self.__register_processing_scripts(scripts_dir)
else:
# The plugin has not yet registered its scripts (waiting for Processing
# to setup the 'script' provider). Pass the add-on's script dir to a list
# of paths to be registered by the plugin, when Processing is ready.
self.__processing_script_dirs_to_register.append(scripts_dir)
def unload_resources(self):
# TODO: Also unregister models and scripts
QgsApplication.processingRegistry().removeProvider(
self.ladm_col_provider)
class AbstractQualityRule(QObject, metaclass=AbstractQObjectMeta):
"""
Abstract class for LADM-COL quality rules
"""
progress_changed = pyqtSignal(int)
validation_finished = pyqtSignal(QualityRuleExecutionResult)
def __init__(self):
QObject.__init__(self)
self.app = AppInterface()
self.logger = Logger()
self._id = "" # E.g., "IGAC-R1001"
self._name = "" # E.g., "Los puntos de lindero no deben superponerse"
self._type = None # E.g., EnumQualityRuleType.POINT
self._tags = list(
) # List of keywords to search for this QR. Must be lowercase.
self._models = list() # List of model keys required by this rule.
# Dict with error codes (keys) and error messages (values)
self._errors = dict()
self.options = self._initialize_option_definition()
# Optional. Only useful for display purposes.
self._field_mapping = dict(
) # E.g., {'id_objetos': 'ids_punto_lindero', 'valores': 'conteo'}
def id(self):
return self._id
def name(self):
return self._name
def type(self):
return self._type
def tags(self):
return self._tags
def models(self):
return self._models.copy()
def field_mapping(self, names):
# Dictionary to overwrite display of quality error table fields
# For instance, I can rename ERR_QUALITY_ERROR_T_OBJECT_IDS_F by "Plots"
# so that a user is better informed of what is displayed
return self._field_mapping
@staticmethod
def layers_config(names):
# Dictionary of layer configuration. Specifies which layers are needed
# by the rule and how it needs them for a 'tolerance > 0' scenario.
return dict()
def is_valid(self):
return bool(self.id().strip()) and bool(self.name().strip()) and len(
self._errors) and self._type is not None
def validate(self, db, db_qr, layer_dict, tolerance, **kwargs):
"""
Validate the quality rule.
:param db: DBConnector to main DB
:param db_qr: DBConnector to quality rules DB
:param layer_dict: Resolved layers (from LADM and/or after snapping)
:param tolerance: Tolerance in millimeters
:param kwargs: Other parameters needed by the rule. Optional.
:return: An instance of QualityRuleExecutionResult
"""
res_obj = self._validate(db, db_qr, layer_dict, tolerance, **kwargs)
self.validation_finished.emit(res_obj)
return res_obj
@abstractmethod
def _validate(self, db, db_qr, layer_dict, tolerance, **kwargs):
raise NotImplementedError
def validate_features(self, features=None, feature_ids=list()):
return False
def _initialize_option_definition(self):
"""
Overwrite this method if needed.
"""
return QualityRuleOptions(list())
def _read_option_values(self, option_values):
# Create member variables per option to ease value access
for k, v in self.options.get_options().items():
setattr(self.options, k, option_values.get(k, v.default_value()))
def _save_errors(self,
db_qr,
error_code,
error_data,
target_layer=None,
ili_name=None):
"""
Save errors into DB with errores_calidad model structure
:param db_qr: DBConnector of the target database
:param error_code: Exactly as specified in error catalogs
:param error_data: Dict of lists:
{'geometries': [geometries], 'data': [obj_uuids, rel_obj_uuids, values, details, state]}
Note: this dict will always have 2 elements.
Note 2: For geometry errors, this dict will always have the same number of elements in
each of the two lists (and the element order matters!).
Note 3: For geometryless errors, the 'geometries' value won't be even read.
:param target_layer: Useful if a rule of one type needs to write the error in an error layer that doesn't
correspond to its type. For instance, a line QR that needs to write an error as point.
By default None, which means the self._type should be used to know the target layer.
:param ili_name: Interlis name of the class obj_uuids belong to.
:return: Boolean, depending on whether the errors were saved or not.
"""
target_layer = self._type if target_layer is None else target_layer
res, msg = QualityErrorDBUtils.save_errors(db_qr,
self._id,
error_code,
error_data,
target_layer,
ili_name=None)
if not res:
self.logger.critical(__name__, msg)
return res
def _check_prerrequisite_layers(self, layer_dict):
"""
Use it when you don't need the layers themselves,
but just to verify if the layers are valid and have features.
"""
for layer_name, layer in layer_dict[QUALITY_RULE_LAYERS].items():
res, obj = self._check_prerrequisite_layer(layer_name, layer)
if not res:
return res, obj
return True, None
def _check_prerrequisite_layer(self, layer_name, layer):
if not layer:
return False, QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL,
QCoreApplication.translate(
"QualityRules",
"'{}' layer not found!").format(layer_name))
if layer.featureCount() == 0:
return False, QualityRuleExecutionResult(
EnumQualityRuleResult.UNDEFINED,
QCoreApplication.translate(
"QualityRules",
"There are no records in layer '{}' to validate the quality rule!"
).format(layer.name()))
return True, None
def _check_qr_options(self, params):
if self.options.get_num_mandatory_options():
mandatory_option_keys = [
o.id() for o in self.options.get_mandatory_option_list()
]
if 'options' not in params: # No options at all
return False, QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL,
QCoreApplication.translate(
"QualityRules",
"No options were given to the quality rule, but it requires {} mandatory options ({})!"
).format(self.options.get_num_mandatory_options(),
", ".join(mandatory_option_keys)))
# Now check that we've got all mandatory options
not_found = [
k for k in mandatory_option_keys if k not in params['options']
]
if not_found:
return False, QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL,
QCoreApplication.translate(
"QualityRules",
"The following mandatory options were missing: '{}'!").
format("', '".join(not_found)))
return True, None
def _get_layer(self, layer_dict, layer_name=''):
"""
Get layer from layer_dict based on a layer name.
If layer_name is not passed, the first layer in layer_dict will be returned,
so this option is suitable for getting a layer when there is only one in the dict.
"""
if layer_name:
return layer_dict[QUALITY_RULE_LAYERS][layer_name]
else:
for layer_name, layer in layer_dict[QUALITY_RULE_LAYERS].items():
return layer # Get the first one
return None
