class CreatePointsSurveyWizard(QWizard, WIZARD_UI): WIZARD_NAME = "CreatePointsSurveyWizard" WIZARD_TOOL_NAME = QCoreApplication.translate(WIZARD_NAME, "Create Point") def __init__(self, iface, db): QWizard.__init__(self) self.setupUi(self) self.iface = iface self._db = db self.logger = Logger() self.app = AppInterface() self.names = self._db.names self.help_strings = HelpStrings() self._layers = { self.names.LC_BOUNDARY_POINT_T: None, self.names.LC_SURVEY_POINT_T: None, self.names.LC_CONTROL_POINT_T: None } self.target_layer = None # Auxiliary data to set nonlinear next pages self.pages = [self.wizardPage1, self.wizardPage2, self.wizardPage3] self.dict_pages_ids = {self.pages[idx] : pid for idx, pid in enumerate(self.pageIds())} self.mMapLayerComboBox.setFilters(QgsMapLayerProxyModel.PointLayer) # Set connections self.btn_browse_file.clicked.connect( make_file_selector(self.txt_file_path, file_filter=QCoreApplication.translate("WizardTranslations",'CSV File (*.csv *.txt)'))) self.txt_file_path.textChanged.connect(self.file_path_changed) self.crsSelector.crsChanged.connect(self.crs_changed) self.crs = "" # SRS auth id self.txt_delimiter.textChanged.connect(self.fill_long_lat_combos) self.mMapLayerComboBox.layerChanged.connect(self.import_layer_changed) self.known_delimiters = [ {'name': ';', 'value': ';'}, {'name': ',', 'value': ','}, {'name': 'tab', 'value': '\t'}, {'name': 'space', 'value': ' '}, {'name': '|', 'value': '|'}, {'name': '~', 'value': '~'}, {'name': 'Other', 'value': ''} ] self.cbo_delimiter.addItems([ item['name'] for item in self.known_delimiters ]) self.cbo_delimiter.currentTextChanged.connect(self.separator_changed) self.restore_settings() self.txt_file_path.textChanged.emit(self.txt_file_path.text()) self.rad_boundary_point.toggled.connect(self.point_option_changed) self.rad_control_point.toggled.connect(self.point_option_changed) self.rad_csv.toggled.connect(self.adjust_page_2_controls) self.point_option_changed() # Initialize it self.button(QWizard.FinishButton).clicked.connect(self.finished_dialog) self.currentIdChanged.connect(self.current_page_changed) self.txt_help_page_2.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_2_OPTION_CSV) self.wizardPage2.setButtonText(QWizard.FinishButton, QCoreApplication.translate("WizardTranslations", "Import")) self.txt_help_page_3.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_3_OPTION_CSV) self.txt_help_page_3.anchorClicked.connect(self.save_template) self.button(QWizard.HelpButton).clicked.connect(self.show_help) self.rejected.connect(self.close_wizard) # Set MessageBar for QWizard self.bar = QgsMessageBar() self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed) self.setLayout(QGridLayout()) self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop) def nextId(self): """ Set navigation order. Should return an integer. -1 is Finish. """ if self.currentId() == self.dict_pages_ids[self.wizardPage1]: return self.dict_pages_ids[self.wizardPage2] elif self.currentId() == self.dict_pages_ids[self.wizardPage2]: if self.rad_csv.isChecked(): return self.dict_pages_ids[self.wizardPage3] elif self.rad_refactor.isChecked(): return -1 elif self.currentId() == self.dict_pages_ids[self.wizardPage3]: return -1 else: return -1 def current_page_changed(self, id): """ Reset the Next button. Needed because Next might have been disabled by a condition in a another SLOT. """ enable_next_wizard(self) if id == self.dict_pages_ids[self.wizardPage2]: self.adjust_page_2_controls() elif id == self.dict_pages_ids[self.wizardPage3]: self.set_buttons_visible(False) self.set_buttons_enabled(False) QCoreApplication.processEvents() self.check_z_in_geometry() QCoreApplication.processEvents() self.fill_long_lat_combos("") QCoreApplication.processEvents() self.set_buttons_visible(True) self.set_buttons_enabled(True) def set_buttons_visible(self, visible): self.button(self.BackButton).setVisible(visible) self.button(self.FinishButton).setVisible(visible) self.button(self.CancelButton).setVisible(visible) def set_buttons_enabled(self, enabled): self.wizardPage3.setEnabled(enabled) self.button(self.BackButton).setEnabled(enabled) self.button(self.FinishButton).setEnabled(enabled) self.button(self.CancelButton).setEnabled(enabled) def check_z_in_geometry(self): self.target_layer = self.app.core.get_layer(self._db, self.current_point_name(), load=True) if not self.target_layer: return if not QgsWkbTypes().hasZ(self.target_layer.wkbType()): self.labelZ.setEnabled(False) self.cbo_elevation.setEnabled(False) msg = QCoreApplication.translate("WizardTranslations", "The current model does not support 3D geometries") self.cbo_elevation.setToolTip(msg) self.labelZ.setToolTip(msg) else: self.labelZ.setEnabled(True) self.cbo_elevation.setEnabled(True) self.labelZ.setToolTip("") self.cbo_elevation.setToolTip("") def adjust_page_2_controls(self): self.cbo_mapping.clear() self.cbo_mapping.addItem("") self.cbo_mapping.addItems(self.app.core.get_field_mappings_file_names(self.current_point_name())) if self.rad_refactor.isChecked(): self.lbl_refactor_source.setEnabled(True) self.mMapLayerComboBox.setEnabled(True) self.lbl_field_mapping.setEnabled(True) self.cbo_mapping.setEnabled(True) self.import_layer_changed(self.mMapLayerComboBox.currentLayer()) disable_next_wizard(self) self.wizardPage2.setFinalPage(True) self.txt_help_page_2.setHtml(self.help_strings.get_refactor_help_string(self._db, self._layers[self.current_point_name()])) elif self.rad_csv.isChecked(): self.lbl_refactor_source.setEnabled(False) self.mMapLayerComboBox.setEnabled(False) self.lbl_field_mapping.setEnabled(False) self.cbo_mapping.setEnabled(False) self.lbl_refactor_source.setStyleSheet('') enable_next_wizard(self) self.wizardPage2.setFinalPage(False) self.txt_help_page_2.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_2_OPTION_CSV) def point_option_changed(self): if self.rad_boundary_point.isChecked(): self.gbx_page_2.setTitle(QCoreApplication.translate("WizardTranslations", "Load data to Boundary Points...")) self.gbx_page_3.setTitle(QCoreApplication.translate("WizardTranslations", "Configure CSV data source for Boundary Points...")) self.txt_help_page_1.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_1_OPTION_BP) elif self.rad_survey_point.isChecked(): # self.rad_survey_point is checked self.gbx_page_2.setTitle(QCoreApplication.translate("WizardTranslations", "Load data to Survey Points...")) self.gbx_page_3.setTitle(QCoreApplication.translate("WizardTranslations", "Configure CSV data source for Survey Points...")) self.txt_help_page_1.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_1_OPTION_SP) else: # self.rad_control_point is checked self.gbx_page_2.setTitle(QCoreApplication.translate("WizardTranslations", "Load data to Control Points...")) self.gbx_page_3.setTitle(QCoreApplication.translate("WizardTranslations", "Configure CSV data source for Control Points...")) self.txt_help_page_1.setHtml(self.help_strings.WIZ_ADD_POINTS_SURVEY_PAGE_1_OPTION_CP) def finished_dialog(self): self.save_settings() if self.rad_refactor.isChecked(): output_layer_name = self.current_point_name() if self.mMapLayerComboBox.currentLayer() is not None: field_mapping = self.cbo_mapping.currentText() res_etl_model = self.app.core.show_etl_model(self._db, self.mMapLayerComboBox.currentLayer(), output_layer_name, field_mapping=field_mapping) if res_etl_model: self.app.gui.redraw_all_layers() # Redraw all layers to show imported data # If the result of the etl_model is successful and we used a stored recent mapping, we delete the # previous mapping used (we give preference to the latest used mapping) if field_mapping: self.app.core.delete_old_field_mapping(field_mapping) self.app.core.save_field_mapping(output_layer_name) else: self.logger.warning_msg(__name__, QCoreApplication.translate("WizardTranslations", "Select a source layer to set the field mapping to '{}'.").format(output_layer_name)) self.close_wizard() elif self.rad_csv.isChecked(): self.prepare_copy_csv_points_to_db() def close_wizard(self, message=None, show_message=True): if message is None: message = QCoreApplication.translate("WizardTranslations", "'{}' tool has been closed.").format(self.WIZARD_TOOL_NAME) if show_message: self.logger.info_msg(__name__, message) self.close() def current_point_name(self): if self.rad_boundary_point.isChecked(): return self.names.LC_BOUNDARY_POINT_T elif self.rad_survey_point.isChecked(): return self.names.LC_SURVEY_POINT_T else: return self.names.LC_CONTROL_POINT_T def prepare_copy_csv_points_to_db(self): csv_path = self.txt_file_path.text().strip() if not csv_path or not os.path.exists(csv_path): self.logger.warning_msg(__name__, QCoreApplication.translate("WizardTranslations", "No CSV file given or file doesn't exist.")) return target_layer_name = self.current_point_name() with OverrideCursor(Qt.WaitCursor): csv_layer = self.app.core.csv_to_layer(csv_path, self.txt_delimiter.text(), self.cbo_longitude.currentText(), self.cbo_latitude.currentText(), self.crs, self.cbo_elevation.currentText() or None, self.detect_decimal_point(csv_path)) self.app.core.copy_csv_to_db(csv_layer, self._db, target_layer_name) def required_layers_are_available(self): layers_are_available = self.app.core.required_layers_are_available(self._db, self._layers, self.WIZARD_TOOL_NAME) return layers_are_available def file_path_changed(self): self.autodetect_separator() self.fill_long_lat_combos("") self.cbo_delimiter.currentTextChanged.connect(self.separator_changed) def detect_decimal_point(self, csv_path): if os.path.exists(csv_path): with open(csv_path) as file: file.readline() # headers data = file.readline().strip() # 1st line with data if data: fields = self.get_fields_from_csv_file(csv_path) if self.cbo_latitude.currentText() in fields: num_col = data.split(self.cbo_delimiter.currentText())[fields.index(self.cbo_latitude.currentText())] for decimal_point in ['.', ',']: if decimal_point in num_col: return decimal_point return '.' # just use the default one def autodetect_separator(self): csv_path = self.txt_file_path.text().strip() if os.path.exists(csv_path): with open(csv_path) as file: first_line = file.readline() for delimiter in self.known_delimiters: if delimiter['value'] == '': continue # if separator works like a column separator in header # number of cols is greater than 1 if len(first_line.split(delimiter['value'])) > 1: self.cbo_delimiter.setCurrentText(delimiter['name']) return def update_crs_info(self): self.crsSelector.setCrs(QgsCoordinateReferenceSystem(self.crs)) def crs_changed(self): self.crs = get_crs_authid(self.crsSelector.crs()) if self.crs != DEFAULT_SRS_AUTHID: self.lbl_crs.setStyleSheet('color: orange') self.lbl_crs.setToolTip(QCoreApplication.translate("WizardTranslations", "Your CSV data will be reprojected for you to '{}' (Colombian National Origin),<br>before attempting to import it into LADM-COL.").format(DEFAULT_SRS_AUTHID)) else: self.lbl_crs.setStyleSheet('') self.lbl_crs.setToolTip(QCoreApplication.translate("WizardTranslations", "Coordinate Reference System")) def fill_long_lat_combos(self, text): csv_path = self.txt_file_path.text().strip() self.cbo_longitude.clear() self.cbo_latitude.clear() self.cbo_elevation.clear() if os.path.exists(csv_path): self.button(QWizard.FinishButton).setEnabled(True) fields = self.get_fields_from_csv_file(csv_path) fields_dict = {field: field.lower() for field in fields} if not fields: self.button(QWizard.FinishButton).setEnabled(False) return self.cbo_longitude.addItems(fields) self.cbo_latitude.addItems(fields) self.cbo_elevation.addItems([""] + fields) # Heuristics to suggest values for x, y and z x_potential_names = ['x', 'lon', 'long', 'longitud', 'longitude', 'este', 'east', 'oeste', 'west'] y_potential_names = ['y', 'lat', 'latitud', 'latitude', 'norte', 'north'] z_potential_names = ['z', 'altura', 'elevacion', 'elevation', 'elevación', 'height'] for x_potential_name in x_potential_names: for k,v in fields_dict.items(): if x_potential_name == v: self.cbo_longitude.setCurrentText(k) break for y_potential_name in y_potential_names: for k, v in fields_dict.items(): if y_potential_name == v: self.cbo_latitude.setCurrentText(k) break if self.cbo_elevation.isEnabled(): for z_potential_name in z_potential_names: for k, v in fields_dict.items(): if z_potential_name == v: self.cbo_elevation.setCurrentText(k) break else: self.button(QWizard.FinishButton).setEnabled(False) def get_fields_from_csv_file(self, csv_path): if not self.txt_delimiter.text(): return [] error_reading = False try: reader = open(csv_path, "r") except IOError: error_reading = True line = reader.readline().replace("\n", "") reader.close() if not line: error_reading = True else: return line.split(self.txt_delimiter.text()) if error_reading: self.logger.warning_msg(__name__, QCoreApplication.translate("WizardTranslations", "It was not possible to read field names from the CSV. Check the file and try again.")) return [] def separator_changed(self, text): # first ocurrence value = next((x['value'] for x in self.known_delimiters if x['name'] == text), '') self.txt_delimiter.setText(value) if value == '': self.txt_delimiter.setEnabled(True) else: self.txt_delimiter.setEnabled(False) def save_template(self, url): link = url.url() if self.rad_boundary_point.isChecked(): if link == '#template': self.download_csv_file('template_boundary_points.csv') elif link == '#data': self.download_csv_file('sample_boundary_points.csv') elif self.rad_survey_point.isChecked(): if link == '#template': self.download_csv_file('template_survey_points.csv') elif link == '#data': self.download_csv_file('sample_survey_points.csv') elif self.rad_control_point.isChecked(): if link == '#template': self.download_csv_file('template_control_points.csv') elif link == '#data': self.download_csv_file('sample_control_points.csv') def download_csv_file(self, filename): settings = QSettings() settings.setValue('Asistente-LADM-COL/wizards/points_csv_file_delimiter', self.txt_delimiter.text().strip()) new_filename, filter = QFileDialog.getSaveFileName(self, QCoreApplication.translate("WizardTranslations", "Save File"), os.path.join(settings.value('Asistente-LADM-COL/wizards/points_download_csv_path', '.'), filename), QCoreApplication.translate("WizardTranslations", "CSV File (*.csv *.txt)")) if new_filename: settings.setValue('Asistente-LADM-COL/wizards/points_download_csv_path', os.path.dirname(new_filename)) template_file = QFile(":/Asistente-LADM-COL/resources/csv/" + filename) if not template_file.exists(): self.logger.critical(__name__, "CSV doesn't exist! Probably due to a missing 'make' execution to generate resources...") msg = QCoreApplication.translate("WizardTranslations", "CSV file not found. Update your plugin. For details see log.") self.show_message(msg, Qgis.Warning) return if os.path.isfile(new_filename): self.logger.info(__name__, 'Removing existing file {}...'.format(new_filename)) os.chmod(new_filename, 0o777) os.remove(new_filename) if template_file.copy(new_filename): os.chmod(new_filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) msg = QCoreApplication.translate("WizardTranslations", """The file <a href="file:///{}">{}</a> was successfully saved!""").format(normalize_local_url(new_filename), os.path.basename(new_filename)) self.show_message(msg, Qgis.Info) else: self.logger.warning(__name__, 'There was an error copying the CSV file {}!'.format(new_filename)) msg = QCoreApplication.translate("WizardTranslations", "The file couldn\'t be saved.") self.show_message(msg, Qgis.Warning) def import_layer_changed(self, layer): if layer: crs = get_crs_authid(layer.crs()) if crs != DEFAULT_SRS_AUTHID: self.lbl_refactor_source.setStyleSheet('color: orange') self.lbl_refactor_source.setToolTip(QCoreApplication.translate("WizardTranslations", "This layer will be reprojected for you to '{}' (Colombian National Origin),<br>before attempting to import it into LADM-COL.").format( DEFAULT_SRS_AUTHID)) else: self.lbl_refactor_source.setStyleSheet('') self.lbl_refactor_source.setToolTip('') def show_message(self, message, level): self.bar.clearWidgets() # Remove previous messages before showing a new one self.bar.pushMessage(message, level, 10) def save_settings(self): settings = QSettings() point_type = None if self.rad_boundary_point.isChecked(): point_type = 'boundary_point' elif self.rad_survey_point.isChecked(): point_type = 'survey_point' else: point_type = 'control_point' settings.setValue('Asistente-LADM-COL/wizards/points_add_points_type', point_type) settings.setValue('Asistente-LADM-COL/wizards/points_load_data_type', 'csv' if self.rad_csv.isChecked() else 'refactor') settings.setValue('Asistente-LADM-COL/wizards/points_add_points_csv_file', self.txt_file_path.text().strip()) settings.setValue('Asistente-LADM-COL/wizards/points_csv_file_delimiter', self.txt_delimiter.text().strip()) settings.setValue('Asistente-LADM-COL/wizards/points_csv_crs', self.crs) def restore_settings(self): settings = QSettings() point_type = settings.value('Asistente-LADM-COL/wizards/points_add_points_type') or 'boundary_point' if point_type == 'boundary_point': self.rad_boundary_point.setChecked(True) elif point_type == 'survey_point': self.rad_survey_point.setChecked(True) else: # 'control_point' self.rad_control_point.setChecked(True) load_data_type = settings.value('Asistente-LADM-COL/wizards/points_load_data_type') or 'csv' if load_data_type == 'refactor': self.rad_refactor.setChecked(True) else: self.rad_csv.setChecked(True) self.txt_file_path.setText(settings.value('Asistente-LADM-COL/wizards/points_add_points_csv_file')) self.txt_delimiter.setText(settings.value('Asistente-LADM-COL/wizards/points_csv_file_delimiter')) self.crs = settings.value('Asistente-LADM-COL/wizards/points_csv_crs', DEFAULT_SRS_AUTHID, str) self.update_crs_info() def show_help(self): show_plugin_help("create_points")
class SymbologyUtils(QObject): layer_symbology_changed = pyqtSignal(str) # layer id def __init__(self): QObject.__init__(self) self.logger = Logger() def set_layer_style_from_qml(self, db, layer, is_error_layer=False, emit=False, layer_modifiers=dict(), models=list()): # TODO: Add tests if db is None: self.logger.critical(__name__, "DB connection is none. Style not set.") return qml_name = None if db.is_ladm_layer(layer): layer_name = db.get_ladm_layer_name(layer) else: layer_name = layer.name( ) # we identify some error layer styles using the error table names if not is_error_layer: # Check if we should use modifier style group if LayerConfig.STYLE_GROUP_LAYER_MODIFIERS in layer_modifiers: style_group_modifiers = layer_modifiers.get( LayerConfig.STYLE_GROUP_LAYER_MODIFIERS) if style_group_modifiers: qml_name = style_group_modifiers.get(layer_name) if not qml_name: # If None or empty string, we use default styles qml_name = Symbology().get_default_style_group( db.names, models).get(layer_name) else: style_custom_error_layers = Symbology().get_custom_error_layers() if layer_name in style_custom_error_layers: qml_name = style_custom_error_layers.get(layer_name) else: qml_name = Symbology().get_default_error_style_layer().get( layer.geometryType()) if qml_name: renderer, labeling = self.get_style_from_qml(qml_name) if renderer: layer.setRenderer(renderer) if emit: self.layer_symbology_changed.emit(layer.id()) if labeling: layer.setLabeling(labeling) layer.setLabelsEnabled(True) def get_style_from_qml(self, qml_name): renderer = None labeling = None style_path = os.path.join(STYLES_DIR, qml_name + '.qml') file = QFile(style_path) if not file.open(QIODevice.ReadOnly | QIODevice.Text): self.logger.warning( __name__, "Unable to read style file from {}".format(style_path)) doc = QDomDocument() doc.setContent(file) file.close() doc_elem = doc.documentElement() nodes = doc_elem.elementsByTagName("renderer-v2") if nodes.count(): renderer_elem = nodes.at(0).toElement() renderer = QgsFeatureRenderer.load(renderer_elem, QgsReadWriteContext()) nodes = doc_elem.elementsByTagName("labeling") if nodes.count(): labeling_elem = nodes.at(0).toElement() labeling = QgsAbstractVectorLayerLabeling.create( labeling_elem, QgsReadWriteContext()) return (renderer, labeling)
class QualityRuleLayerManager(QObject): """ Responsible for managing all layers during a Quality Rule execution session. It goes for LADM-COL layers only once and also manages intermediate layers (after snapping). """ def __init__(self, db, rule_keys, tolerance): QObject.__init__(self) self.logger = Logger() self.app = AppInterface() self.__db = db self.__rule_keys = rule_keys self.__tolerance = tolerance self.__quality_rule_layers_config = QualityRuleConfig.get_quality_rules_layer_config( self.__db.names) # {rule_key: {QUALITY_RULE_LAYERS: {layer_name: layer}, # QUALITY_RULE_LADM_COL_LAYERS: {layer_name: layer}} self.__layers = dict() self.__adjusted_layers_cache = dict() def initialize(self, rule_keys): """ Objects of this class are reusable calling initialize() """ self.__rule_keys = rule_keys self.__layers = dict() def __prepare_layers(self): """ Get layers from DB and prepare snapped layers for all rules """ self.logger.info( __name__, QCoreApplication.translate("QualityRuleLayerManager", "Preparing layers...")) # First go for ladm-col layers ladm_layers = dict() for rule_key, rule_layers_config in self.__quality_rule_layers_config.items( ): if rule_key in self.__rule_keys: # Only get selected rules' layers for layer_name in rule_layers_config[ QUALITY_RULE_LADM_COL_LAYERS]: ladm_layers[layer_name] = None self.logger.debug( __name__, QCoreApplication.translate("QualityRuleLayerManager", "Getting {} LADM-COL layers...").format( len(ladm_layers))) self.app.core.get_layers(self.__db, ladm_layers, load=True) if ladm_layers is None: # If there are errors with get_layers, ladm_layers is None self.logger.critical( __name__, QCoreApplication.translate( "QualityRuleLayerManager", "Couldn't finish preparing required layers!")) return False # If tolerance > 0, prepare adjusted layers # We create an adjusted_layers dict to override ladm_layers per rule. # For that, we need to read the config and, if not yet calculated, # adjust the layers and store them in temporary cache. # {rule_key: {layer_name: layer}}, because each rule might need # different adjustments for the same layer, compared to other rules adjusted_layers = {rule_key: dict() for rule_key in self.__rule_keys} if self.__tolerance: self.logger.debug( __name__, QCoreApplication.translate( "QualityRuleLayerManager", "Tolerance > 0, adjusting layers...")) self.__adjusted_layers_cache = dict() # adjusted_layers_key: layer count_rules = 0 total_rules = len([ rk for rk in self.__rule_keys if rk in self.__quality_rule_layers_config ]) with ProcessWithStatus( QCoreApplication.translate( "QualityRuleLayerManager", "Preparing tolerance on layers...")): for rule_key, rule_layers_config in self.__quality_rule_layers_config.items( ): if rule_key in self.__rule_keys: # Only get selected rules' layers count_rules += 1 self.logger.status( QCoreApplication.translate( "QualityRuleLayerManager", "Preparing tolerance on layers... {}%").format( int(count_rules / total_rules * 100))) if QUALITY_RULE_ADJUSTED_LAYERS in rule_layers_config: for layer_name, snap_config in rule_layers_config[ QUALITY_RULE_ADJUSTED_LAYERS].items(): # Read from config input_name = snap_config[ ADJUSTED_INPUT_LAYER] # input layer name reference_name = snap_config[ ADJUSTED_REFERENCE_LAYER] # reference layer name fix = snap_config[ FIX_ADJUSTED_LAYER] if FIX_ADJUSTED_LAYER in snap_config else False # Get input and reference layers (note that they could be adjusted layers) input = self.__adjusted_layers_cache[ input_name] if input_name in self.__adjusted_layers_cache else ladm_layers[ input_name] reference = self.__adjusted_layers_cache[ reference_name] if reference_name in self.__adjusted_layers_cache else ladm_layers[ reference_name] # Try to reuse if already calculated! adjusted_layers_key = get_key_for_quality_rule_adjusted_layer( input_name, reference_name, fix) if adjusted_layers_key not in self.__adjusted_layers_cache: self.__adjusted_layers_cache[ adjusted_layers_key] = self.app.core.adjust_layer( input, reference, self.__tolerance, fix) adjusted_layers[rule_key][ layer_name] = self.__adjusted_layers_cache[ adjusted_layers_key] self.logger.debug( __name__, QCoreApplication.translate("QualityRuleLayerManager", "Layers adjusted...")) # Now that we have both ladm_layers and adjusted_layers, use them # in a single member dict of layers per rule (preserving original LADM-COL layers) self.__layers = { rule_key: { QUALITY_RULE_LAYERS: dict(), QUALITY_RULE_LADM_COL_LAYERS: dict() } for rule_key in self.__rule_keys } for rule_key, rule_layers_config in self.__quality_rule_layers_config.items( ): if rule_key in self.__rule_keys: # Only get selected rules' layers for layer_name in rule_layers_config[ QUALITY_RULE_LADM_COL_LAYERS]: # Fill both subdicts # In LADM-COL layers we send all original layers self.__layers[rule_key][QUALITY_RULE_LADM_COL_LAYERS][ layer_name] = ladm_layers[ layer_name] if layer_name in ladm_layers else None # In QR_Layers we store the best layer we have available (preferring adjusted over ladm-col) if layer_name in adjusted_layers[rule_key]: self.__layers[rule_key][QUALITY_RULE_LAYERS][ layer_name] = adjusted_layers[rule_key][layer_name] elif layer_name in ladm_layers: self.__layers[rule_key][QUALITY_RULE_LAYERS][ layer_name] = ladm_layers[layer_name] # Let QRs know if they should switch between dicts looking for original geometries self.__layers[rule_key][HAS_ADJUSTED_LAYERS] = bool( self.__tolerance) # Register adjusted layers so that Processing can properly find them if self.__adjusted_layers_cache: load_to_registry = [ layer for key, layer in self.__adjusted_layers_cache.items() if layer is not None ] self.logger.debug( __name__, "{} adjusted layers loaded to QGIS registry...".format( len(load_to_registry))) QgsProject.instance().addMapLayers(load_to_registry, False) return True def get_layer(self, layer_name, rule_key): return self.get_layers([layer_name], rule_key) def get_layers(self, rule_key): """ Gets the layers a quality rule requires to run. This is based on the quality rule layer config. :param rule_key: Key of the quality rule. :return: Dict of layers for the given rule_key. This dict has both a 'layers' dict which has the best available layer (which means, if an adjusted layer is required, it will be preferred, and if no adjusted layer is required, just pass the LADM-COL layer) and a 'ladm-col' dict with the original LADM-COL layers, because the quality rule might need to refer to the original object (or geometry) to build its result. """ # Make sure we only call Prepare layers once for each call to run quality validations. if not self.__layers: if not self.__prepare_layers(): return None return self.__layers[rule_key] def clean_temporary_layers(self): # Removes adjusted layers from registry unload_from_registry = [ layer.id() for key, layer in self.__adjusted_layers_cache.items() if layer is not None ] self.logger.debug( __name__, "{} adjusted layers removed from QGIS registry...".format( len(unload_from_registry))) QgsProject.instance().removeMapLayers(unload_from_registry)
class ImportFromExcelDialog(QDialog, DIALOG_UI): log_excel_show_message_emitted = pyqtSignal(str) def __init__(self, iface, db, qgis_utils, parent=None): QDialog.__init__(self, parent) self.setupUi(self) self.iface = iface self._db = db self.qgis_utils = qgis_utils self.logger = Logger() self.help_strings = HelpStrings() self.log_dialog_excel_text_content = "" self.group_parties_exists = False self.names = self._db.names self._running_tool = False self.tool_name = QCoreApplication.translate( "ImportFromExcelDialog", "Import intermediate structure") self.fields = { EXCEL_SHEET_NAME_PLOT: [ EXCEL_SHEET_TITLE_DEPARTMENT, EXCEL_SHEET_TITLE_MUNICIPALITY, EXCEL_SHEET_TITLE_ZONE, EXCEL_SHEET_TITLE_REGISTRATION_PLOT, EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_NPV, EXCEL_SHEET_TITLE_PLOT_NAME, EXCEL_SHEET_TITLE_VALUATION, EXCEL_SHEET_TITLE_PLOT_CONDITION, EXCEL_SHEET_TITLE_PLOT_TYPE, EXCEL_SHEET_TITLE_ADDRESS ], EXCEL_SHEET_NAME_PARTY: [ EXCEL_SHEET_TITLE_FIRST_NAME, EXCEL_SHEET_TITLE_MIDDLE, EXCEL_SHEET_TITLE_FIRST_SURNAME, EXCEL_SHEET_TITLE_SECOND_SURNAME, EXCEL_SHEET_TITLE_BUSINESS_NAME, EXCEL_SHEET_TITLE_SEX, EXCEL_SHEET_TITLE_DOCUMENT_TYPE, EXCEL_SHEET_TITLE_DOCUMENT_NUMBER, EXCEL_SHEET_TITLE_KIND_PERSON, EXCEL_SHEET_TITLE_ISSUING_ENTITY, EXCEL_SHEET_TITLE_DATE_ISSUE, EXCEL_SHEET_TITLE_NPN ], EXCEL_SHEET_NAME_GROUP: [ EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_DOCUMENT_TYPE, EXCEL_SHEET_TITLE_DOCUMENT_NUMBER, EXCEL_SHEET_TITLE_ID_GROUP ], EXCEL_SHEET_NAME_RIGHT: [ EXCEL_SHEET_TITLE_TYPE, EXCEL_SHEET_TITLE_PARTY_DOCUMENT_NUMBER, EXCEL_SHEET_TITLE_GROUP, EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_SOURCE_TYPE, EXCEL_SHEET_TITLE_DESCRIPTION_SOURCE, EXCEL_SHEET_TITLE_STATE_SOURCE, EXCEL_SHEET_TITLE_OFFICIALITY_SOURCE, EXCEL_SHEET_TITLE_STORAGE_PATH ] } self.txt_help_page.setHtml(self.help_strings.DLG_IMPORT_FROM_EXCEL) self.txt_help_page.anchorClicked.connect(self.save_template) self.buttonBox.accepted.disconnect() self.buttonBox.accepted.connect(self.accepted) #self.buttonBox.rejected.connect(self.rejected) self.buttonBox.helpRequested.connect(self.show_help) self.btn_browse_file.clicked.connect( make_file_selector( self.txt_excel_path, QCoreApplication.translate( "ImportFromExcelDialog", "Select the Excel file with data in the intermediate structure" ), QCoreApplication.translate("ImportFromExcelDialog", 'Excel File (*.xlsx *.xls)'))) self.buttonBox.button(QDialogButtonBox.Ok).setText( QCoreApplication.translate("ImportFromExcelDialog", "Import")) self.initialize_feedback() self.restore_settings() self.bar = QgsMessageBar() self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed) # self.tabWidget.currentWidget().layout().setContentsMargins(0, 0, 0, 0) self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop) def accepted(self): self.save_settings() self.import_from_excel() def import_from_excel(self): self._running_tool = True steps = 18 step = 0 self.progress.setVisible(True) self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False) # Where to store the reports? excel_path = self.txt_excel_path.text() if not excel_path: self.show_message( QCoreApplication.translate( "ImportFromExcelDialog", "You need to select an Excel file before continuing with the import." ), Qgis.Warning) self.progress.setVisible(False) self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True) return if not os.path.exists(excel_path): self.show_message( QCoreApplication.translate( "ImportFromExcelDialog", "The specified Excel file does not exist!"), Qgis.Warning) self.progress.setVisible(False) self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True) return self.progress.setVisible(True) self.txt_log.setText( QCoreApplication.translate( "ImportFromExcelDialog", "Loading tables from the Excel file...")) # Now that we have the Excel file, build vrts to load its sheets appropriately # Also validate each layer against a number of rules layer_parcel = self.check_layer_from_excel_sheet( excel_path, EXCEL_SHEET_NAME_PLOT) layer_party = self.check_layer_from_excel_sheet( excel_path, EXCEL_SHEET_NAME_PARTY) layer_group_party = self.check_layer_from_excel_sheet( excel_path, EXCEL_SHEET_NAME_GROUP) layer_right = self.check_layer_from_excel_sheet( excel_path, EXCEL_SHEET_NAME_RIGHT) if layer_parcel is None or layer_party is None or layer_group_party is None or layer_right is None: # A layer is None if at least an error was found self.group_parties_exists = False self.log_excel_show_message_emitted.emit( self.log_dialog_excel_text_content) self.done(0) return if not layer_group_party.isValid() or not layer_party.isValid( ) or not layer_parcel.isValid() or not layer_right.isValid(): self.show_message( QCoreApplication.translate( "ImportFromExcelDialog", "One of the sheets of the Excel file couldn't be loaded! Check the format again." ), Qgis.Warning) self.progress.setVisible(False) self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True) return QgsProject.instance().addMapLayers( [layer_group_party, layer_party, layer_parcel, layer_right]) # GET LADM LAYERS layers = { self.names.OP_PARTY_T: { 'name': self.names.OP_PARTY_T, 'geometry': None, LAYER: None }, self.names.OP_PARCEL_T: { 'name': self.names.OP_PARCEL_T, 'geometry': None, LAYER: None }, self.names.OP_RIGHT_T: { 'name': self.names.OP_RIGHT_T, 'geometry': None, LAYER: None }, self.names.EXT_ARCHIVE_S: { 'name': self.names.EXT_ARCHIVE_S, 'geometry': None, LAYER: None }, self.names.COL_RRR_SOURCE_T: { 'name': self.names.COL_RRR_SOURCE_T, 'geometry': None, LAYER: None }, self.names.OP_GROUP_PARTY_T: { 'name': self.names.OP_GROUP_PARTY_T, 'geometry': None, LAYER: None }, self.names.MEMBERS_T: { 'name': self.names.MEMBERS_T, 'geometry': None, LAYER: None }, self.names.OP_ADMINISTRATIVE_SOURCE_T: { 'name': self.names.OP_ADMINISTRATIVE_SOURCE_T, 'geometry': None, LAYER: None } } self.qgis_utils.get_layers(self._db, layers, load=True) if not layers: return None # Get feature counts to compare after the ETL and know how many records were imported to each ladm_col table ladm_tables = [ layers[self.names.OP_PARCEL_T][LAYER], layers[self.names.OP_PARTY_T][LAYER], layers[self.names.OP_RIGHT_T][LAYER], layers[self.names.OP_ADMINISTRATIVE_SOURCE_T][LAYER], layers[self.names.COL_RRR_SOURCE_T][LAYER], layers[self.names.OP_GROUP_PARTY_T][LAYER], layers[self.names.MEMBERS_T][LAYER] ] ladm_tables_feature_count_before = { t.name(): t.featureCount() for t in ladm_tables } # Run the ETL params = { 'agrupacion': layers[self.names.OP_GROUP_PARTY_T][LAYER], 'colmiembros': layers[self.names.MEMBERS_T][LAYER], 'colrrrsourcet': layers[self.names.COL_RRR_SOURCE_T][LAYER], 'extarchivo': layers[self.names.EXT_ARCHIVE_S][LAYER], 'interesado': layers[self.names.OP_PARTY_T][LAYER], 'layergroupparty': layer_group_party, 'layerparcel': layer_parcel, 'layerparty': layer_party, 'layerright': layer_right, 'opderecho': layers[self.names.OP_RIGHT_T][LAYER], 'opfuenteadministrativatipo': layers[self.names.OP_ADMINISTRATIVE_SOURCE_T][LAYER], 'parcel': layers[self.names.OP_PARCEL_T][LAYER] } self.qgis_utils.disable_automatic_fields(self._db, self.names.OP_GROUP_PARTY_T) self.qgis_utils.disable_automatic_fields(self._db, self.names.OP_RIGHT_T) self.qgis_utils.disable_automatic_fields( self._db, self.names.OP_ADMINISTRATIVE_SOURCE_T) processing.run("model:ETL_intermediate_structure", params, feedback=self.feedback) if not self.feedback.isCanceled(): self.progress.setValue(100) self.buttonBox.clear() self.buttonBox.setEnabled(True) self.buttonBox.addButton(QDialogButtonBox.Close) else: self.initialize_feedback() # Print summary getting feature count in involved LADM_COL tables... summary = """<html><head/><body><p>""" summary += QCoreApplication.translate("ImportFromExcelDialog", "Import done!!!<br/>") for table in ladm_tables: summary += QCoreApplication.translate( "ImportFromExcelDialog", "<br/><b>{count}</b> records loaded into table <b>{table}</b>" ).format(count=table.featureCount() - ladm_tables_feature_count_before[table.name()], table=table.name()) summary += """</body></html>""" self.txt_log.setText(summary) self.logger.success_msg( __name__, QCoreApplication.translate( "QGISUtils", "Data successfully imported to LADM_COL from intermediate structure (Excel file: '{}')!!!" ).format(excel_path)) self._running_tool = False def check_layer_from_excel_sheet(self, excel_path, sheetname): layer = self.get_layer_from_excel_sheet(excel_path, sheetname) error_counter = 0 if layer is None and sheetname != EXCEL_SHEET_NAME_GROUP: # optional sheet self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The {} sheet has not information or has another name."). format(sheetname)) error_counter += 1 else: title_validator = layer.fields().toList() if sheetname == EXCEL_SHEET_NAME_PLOT and layer is not None: if not title_validator: self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The title does not match the format in the sheet {}." ).format(sheetname)) error_counter += 1 if list(layer.getFeatures('"numero predial nuevo" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero predial nuevo has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if not self.check_field_numeric_layer(layer, 'departamento'): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column departamento has non-numeric values in sheet {}." ).format(sheetname)) error_counter += 1 if not self.check_field_numeric_layer(layer, 'municipio'): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column municipio has non-numeric values in sheet {}." ).format(sheetname)) error_counter += 1 if not self.check_field_numeric_layer(layer, 'numero predial nuevo'): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero predial nuevo has non-numeric values in sheet {}." ).format(sheetname)) error_counter += 1 if sheetname == EXCEL_SHEET_NAME_PARTY and layer is not None: if not title_validator: self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The title does not match the format in sheet {}."). format(sheetname)) error_counter += 1 if list(layer.getFeatures('"tipo documento" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column tipo documento has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if list(layer.getFeatures('"numero de documento" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero de documento has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if not self.check_length_attribute_value( layer, 'numero de documento', 12): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero de documento has more characters than expected in sheet {}." ).format(sheetname)) error_counter += 1 if list(layer.getFeatures('"tipo persona" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column tipo persona has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if sheetname == EXCEL_SHEET_NAME_GROUP and layer is not None: if not title_validator: self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The title does not match the format in the sheet {}." ).format(sheetname)) error_counter += 1 self.group_parties_exists = True if list(layer.getFeatures('"numero predial nuevo" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero predial nuevo has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if list(layer.getFeatures('"tipo documento" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column tipo documento has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if list(layer.getFeatures('"numero de documento" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero de documento has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if list(layer.getFeatures('"id agrupación" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column id agrupación has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if not self.check_length_attribute_value( layer, 'numero de documento', 12): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column numero de documento has more characters of the permitted in sheet {}." ).format(sheetname)) error_counter += 1 if sheetname == EXCEL_SHEET_NAME_RIGHT and layer is not None: if not title_validator: self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The title does not match the format in sheet {}."). format(sheetname)) error_counter += 1 if list(layer.getFeatures('"tipo" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column tipo has empty values in sheet {}."). format(sheetname)) error_counter += 1 if list(layer.getFeatures('"tipo de fuente" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column tipo de fuente has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if list( layer.getFeatures( '"estado_disponibilidad de la fuente" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column estado_disponibilidad de la fuente has empty values in sheet {}." ).format(sheetname)) error_counter += 1 #if list(layer.getFeatures('"Ruta de Almacenamiento de la fuente" is Null')): # self.generate_message_excel_error(QCoreApplication.translate("ImportFromExcelDialog", # "The column Ruta de Almacenamiento de la fuente has empty values in sheet {}.").format(sheetname)) # error_counter += 1 if len( list( layer.getFeatures( '"número documento Interesado" is Null'))) + len( list(layer.getFeatures('"agrupación" is Null')) ) != layer.featureCount(): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "Number of non-null parties plus number of non-null group parties is not equal to number of records in sheet {}. There might be rights without party or group party associated." ).format(sheetname)) error_counter += 1 if not self.group_parties_exists: if list( layer.getFeatures( '"número documento Interesado" is Null')): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column número documento Interesado has empty values in sheet {}." ).format(sheetname)) error_counter += 1 if len(list(layer.getFeatures( '"agrupacion" is Null'))) != layer.featureCount(): self.generate_message_excel_error( QCoreApplication.translate( "ImportFromExcelDialog", "The column agrupacion has data but the sheet does not exist in sheet {}." ).format(sheetname)) error_counter += 1 return layer if error_counter == 0 else None def check_field_numeric_layer(self, layer, name): id_field_idx = layer.fields().indexFromName(name) request = QgsFeatureRequest().setSubsetOfAttributes([id_field_idx]) features = layer.getFeatures(request) is_numeric = True for feature in features: try: int(feature[name]) except: is_numeric = False break return is_numeric def check_length_attribute_value(self, layer, name, size): id_field_idx = layer.fields().indexFromName(name) request = QgsFeatureRequest().setSubsetOfAttributes([id_field_idx]) features = layer.getFeatures(request) right_length = True for feature in features: if len(str(feature[name])) > size: right_length = False break return right_length def generate_message_excel_error(self, msg): self.log_dialog_excel_text_content += "{}{}{}{}{}{}".format( LOG_QUALITY_LIST_CONTAINER_OPEN, LOG_QUALITY_LIST_ITEM_ERROR_OPEN, msg, LOG_QUALITY_LIST_ITEM_ERROR_CLOSE, LOG_QUALITY_LIST_CONTAINER_CLOSE, LOG_QUALITY_CONTENT_SEPARATOR) def get_layer_from_excel_sheet(self, excel_path, sheetname): basename = os.path.basename(excel_path) filename = os.path.splitext(basename)[0] dirname = os.path.dirname(excel_path) header_in_first_row, count = self.get_excel_info(excel_path, sheetname) if header_in_first_row is None and count is None: return None layer_definition = "<SrcLayer>{sheetname}</SrcLayer>".format( sheetname=sheetname) if header_in_first_row: layer_definition = """<SrcSql dialect="sqlite">SELECT * FROM '{sheetname}' LIMIT {count} OFFSET 1</SrcSql>""".format( sheetname=sheetname, count=count) xml_text_group_party = """<?xml version="1.0" encoding="UTF-8"?> <OGRVRTDataSource> <OGRVRTLayer name="{filename}-{sheetname}"> <SrcDataSource relativeToVRT="1">{basename}</SrcDataSource> <!--Header={header}--> {layer_definition} {fields} </OGRVRTLayer> </OGRVRTDataSource> """.format(filename=filename, basename=basename, header=header_in_first_row, layer_definition=layer_definition, sheetname=sheetname, fields=self.get_vrt_fields(sheetname, header_in_first_row)) group_party_file_path = os.path.join( dirname, '{}.{}.vrt'.format(basename, sheetname)) with open(group_party_file_path, 'w') as sheet: sheet.write(xml_text_group_party) uri = '{vrtfilepath}|layername={filename}-{sheetname}'.format( vrtfilepath=group_party_file_path, sheetname=sheetname, filename=filename) self.logger.info(__name__, "Loading layer from excel with uri='{}'".format(uri)) layer = QgsVectorLayer(uri, '{}-{}'.format('excel', sheetname), 'ogr') layer.setProviderEncoding('UTF-8') return layer def get_excel_info(self, path, sheetname): data_source = ogr.Open(path, 0) layer = data_source.GetLayerByName(sheetname) if layer is None: # A sheetname couldn't be found return None, None feature = layer.GetNextFeature() # If ogr recognizes the header, the first row will contain data, otherwise it'll contain field names header_in_first_row = True for field in self.fields[sheetname]: if feature.GetField(self.fields[sheetname].index(field)) != field: header_in_first_row = False num_rows = layer.GetFeatureCount() return header_in_first_row, num_rows - 1 if header_in_first_row else num_rows def get_vrt_fields(self, sheetname, header_in_first_row): vrt_fields = "" for index, field in enumerate(self.fields[sheetname]): vrt_fields += """<Field name="{field}" src="{src}" type="String"/>\n""".format( field=field, src='Field{}'.format(index + 1) if header_in_first_row else field) return vrt_fields.strip() def save_template(self, url): link = url.url() if link == '#template': self.download_excel_file('plantilla_estructura_excel.xlsx') elif link == '#data': self.download_excel_file('datos_estructura_excel.xlsx') def download_excel_file(self, filename): settings = QSettings() new_filename, filter = QFileDialog.getSaveFileName( self, QCoreApplication.translate("ImportFromExcelDialog", "Save File"), os.path.join( settings.value( 'Asistente-LADM_COL/import_from_excel_dialog/template_save_path', '.'), filename), QCoreApplication.translate("ImportFromExcelDialog", "Excel File (*.xlsx *.xls)")) if new_filename: settings.setValue( 'Asistente-LADM_COL/import_from_excel_dialog/template_save_path', os.path.dirname(new_filename)) template_file = QFile(":/Asistente-LADM_COL/resources/excel/" + filename) if not template_file.exists(): self.logger.critical( __name__, "Excel doesn't exist! Probably due to a missing 'make' execution to generate resources..." ) msg = QCoreApplication.translate( "ImportFromExcelDialog", "Excel file not found. Update your plugin. For details see log." ) self.show_message(msg, Qgis.Warning) return if os.path.isfile(new_filename): self.logger.info( __name__, 'Removing existing file {}...'.format(new_filename)) os.chmod(new_filename, 0o777) os.remove(new_filename) if template_file.copy(new_filename): os.chmod( new_filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) msg = QCoreApplication.translate( "ImportFromExcelDialog", """The file <a href="file:///{}">{}</a> was successfully saved!""" ).format(normalize_local_url(new_filename), os.path.basename(new_filename)) self.show_message(msg, Qgis.Info) else: self.logger.info( __name__, 'There was an error copying the CSV file {}!'.format( new_filename)) msg = QCoreApplication.translate( "ImportFromExcelDialog", "The file couldn\'t be saved.") self.show_message(msg, Qgis.Warning) def reject(self): self.selected_items_dict = dict() if self._running_tool: reply = QMessageBox.question( self, QCoreApplication.translate("import_from_excel", "Warning"), QCoreApplication.translate( "import_from_excel", "The '{}' tool is still running. Do you want to cancel it? If you cancel, the data might be incomplete in the target database." ).format(self.tool_name), QMessageBox.Yes, QMessageBox.No) if reply == QMessageBox.Yes: self.feedback.cancel() self._running_tool = False msg = QCoreApplication.translate( "import_from_excel", "The '{}' tool was cancelled.").format(self.tool_name) self.logger.info(__name__, msg) self.show_message(msg, Qgis.Info) else: self.logger.info(__name__, "Dialog closed.") self.done(1) def save_settings(self): settings = QSettings() settings.setValue( 'Asistente-LADM_COL/import_from_excel_dialog/excel_path', self.txt_excel_path.text()) def restore_settings(self): settings = QSettings() self.txt_excel_path.setText( settings.value( 'Asistente-LADM_COL/import_from_excel_dialog/excel_path', '')) def show_message(self, message, level): self.bar.clearWidgets( ) # Remove previous messages before showing a new one self.bar.pushMessage(message, level, 10) def show_help(self): self.qgis_utils.show_help("import_from_excel") def progress_changed(self): QCoreApplication.processEvents() # Listen to cancel from the user self.progress.setValue(self.feedback.progress()) def initialize_feedback(self): self.progress.setValue(0) self.progress.setVisible(False) self.feedback = QgsProcessingFeedback() self.feedback.progressChanged.connect(self.progress_changed) self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
class QgisModelBakerUtils(QObject): def __init__(self): QObject.__init__(self) self.logger = Logger() from asistente_ladm_col.config.config_db_supported import ConfigDBsSupported self._dbs_supported = ConfigDBsSupported() self.translatable_config_strings = TranslatableConfigStrings() def get_generator(self, db): if 'QgisModelBaker' in qgis.utils.plugins: tool = self._dbs_supported.get_db_factory( db.engine).get_model_baker_db_ili_mode() QgisModelBaker = qgis.utils.plugins["QgisModelBaker"] generator = QgisModelBaker.get_generator()( tool, db.uri, "smart2", db.schema, pg_estimated_metadata=False) return generator else: self.logger.critical( __name__, QCoreApplication.translate( "AsistenteLADMCOLPlugin", "The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant." )) return None def get_model_baker_db_connection(self, db): generator = self.get_generator(db) if generator is not None: return generator._db_connector return None def load_layers(self, db, layer_list): """ Load a selected list of layers from qgis model baker. This call should configure relations and bag of enums between layers being loaded, but not when a layer already loaded has a relation or is part of a bag of enum. For that case, we use a cached set of relations and bags of enums that we get only once per session and configure in the Asistente LADM-COL. """ translated_strings = self.translatable_config_strings.get_translatable_config_strings( ) if 'QgisModelBaker' in qgis.utils.plugins: QgisModelBaker = qgis.utils.plugins["QgisModelBaker"] tool = self._dbs_supported.get_db_factory( db.engine).get_model_baker_db_ili_mode() generator = QgisModelBaker.get_generator()( tool, db.uri, "smart2", db.schema, pg_estimated_metadata=False) layers = generator.layers(layer_list) relations, bags_of_enum = generator.relations(layers, layer_list) legend = generator.legend( layers, ignore_node_names=[translated_strings[ERROR_LAYER_GROUP]]) QgisModelBaker.create_project(layers, relations, bags_of_enum, legend, auto_transaction=False) else: self.logger.critical( __name__, QCoreApplication.translate( "AsistenteLADMCOLPlugin", "The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant." )) def get_required_layers_without_load(self, layer_list, db): """ Gets a list of layers from a list of layer names using QGIS Model Baker. Layers are register in QgsProject, but not loaded to the canvas! :param layer_list: list of layers names (e.g., ['lc_terreno', 'lc_lindero']) :param db: db connection :return: list of QgsVectorLayers registered in the project """ layers = list() if 'QgisModelBaker' in qgis.utils.plugins: QgisModelBaker = qgis.utils.plugins["QgisModelBaker"] tool = self._dbs_supported.get_db_factory( db.engine).get_model_baker_db_ili_mode() generator = QgisModelBaker.get_generator()( tool, db.uri, "smart2", db.schema, pg_estimated_metadata=False) model_baker_layers = generator.layers(layer_list) for model_baker_layer in model_baker_layers: layer = model_baker_layer.create( ) # Convert Model Baker layer to QGIS layer QgsProject.instance().addMapLayer( layer, False) # Do not load it to canvas layers.append(layer) else: self.logger.critical( __name__, QCoreApplication.translate( "AsistenteLADMCOLPlugin", "The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant." )) return layers def get_layers_and_relations_info(self, db): """ Called once per session, this is used to get information of all relations and bags of enums in the DB and cache it in the Asistente LADM-COL. """ if 'QgisModelBaker' in qgis.utils.plugins: generator = self.get_generator(db) layers = generator.get_tables_info_without_ignored_tables() relations = [ relation for relation in generator.get_relations_info() ] self.logger.debug( __name__, "Relationships before filter: {}".format(len(relations))) self.filter_relations(relations) self.logger.debug( __name__, "Relationships after filter: {}".format(len(relations))) return (layers, relations, {}) else: self.logger.critical( __name__, QCoreApplication.translate( "AsistenteLADMCOLPlugin", "The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant." )) return (list(), list(), dict()) def filter_relations(self, relations): """ Modifies the input list of relations, removing elements that meet a condition. :param relations: List of a dict of relations. :return: Nothing, changes the input list of relations. """ to_delete = list() for relation in relations: if relation[QueryNames.REFERENCING_FIELD].startswith( 'uej2_') or relation[ QueryNames.REFERENCING_FIELD].startswith('ue_'): to_delete.append(relation) for idx in to_delete: relations.remove(idx) def get_tables_info_without_ignored_tables(self, db): if 'QgisModelBaker' in qgis.utils.plugins: generator = self.get_generator(db) return generator.get_tables_info_without_ignored_tables() else: self.logger.critical( __name__, QCoreApplication.translate( "AsistenteLADMCOLPlugin", "The QGIS Model Baker plugin is a prerequisite, install it before using LADM-COL Assistant." )) def get_first_index_for_layer_type( self, layer_type, group=QgsProject.instance().layerTreeRoot()): if 'QgisModelBaker' in qgis.utils.plugins: import QgisModelBaker return QgisModelBaker.utils.qgis_utils.get_first_index_for_layer_type( layer_type, group) return None @staticmethod def get_suggested_index_for_layer(layer, group): if 'QgisModelBaker' in qgis.utils.plugins: import QgisModelBaker return QgisModelBaker.utils.qgis_utils.get_suggested_index_for_layer( layer, group) return None
class SourceHandler(QObject): """ Upload source files from a given field of a layer to a remote server that is configured in Settings Dialog. The server returns a file URL that is then stored in the source table. """ def __init__(self, qgis_utils): QObject.__init__(self) self.qgis_utils = qgis_utils self.logger = Logger() def upload_files(self, layer, field_index, features): """ Upload given features' source files to remote server and return a dict formatted as changeAttributeValues expects to update 'datos' attribute to a remote location. """ if not QSettings().value( 'Asistente-LADM_COL/sources/document_repository', False, bool): self.logger.info_msg( __name__, QCoreApplication.translate( "SourceHandler", "The source files were not uploaded to the document repository because you have that option unchecked. You can still upload the source files later using the 'Upload Pending Source Files' menu." ), 10) return dict() # Test if we have Internet connection and a valid service res, msg = self.qgis_utils.is_source_service_valid( ) # TODO: Bring this method from qgis_utils if not res: msg['text'] = QCoreApplication.translate( "SourceHandler", "No file could be uploaded to the document repository. You can do it later from the 'Upload Pending Source Files' menu. Reason: {}" ).format(msg['text']) self.logger.info_msg( __name__, msg['text'], 20) # The data is still saved, so always show Info msg return dict() file_features = [ feature for feature in features if not feature[field_index] == NULL and os.path.isfile(feature[field_index]) ] total = len(features) not_found = total - len(file_features) upload_dialog = UploadProgressDialog(len(file_features), not_found) upload_dialog.show() count = 0 upload_errors = 0 new_values = dict() for feature in file_features: data_url = feature[field_index] file_name = os.path.basename(data_url) nam = QNetworkAccessManager() #reply.downloadProgress.connect(upload_dialog.update_current_progress) multiPart = QHttpMultiPart(QHttpMultiPart.FormDataType) textPart = QHttpPart() textPart.setHeader(QNetworkRequest.ContentDispositionHeader, QVariant("form-data; name=\"driver\"")) textPart.setBody(QByteArray().append('Local')) filePart = QHttpPart() filePart.setHeader( QNetworkRequest.ContentDispositionHeader, QVariant("form-data; name=\"file\"; filename=\"{}\"".format( file_name))) file = QFile(data_url) file.open(QIODevice.ReadOnly) filePart.setBodyDevice(file) file.setParent( multiPart ) # we cannot delete the file now, so delete it with the multiPart multiPart.append(filePart) multiPart.append(textPart) service_url = '/'.join([ QSettings().value( 'Asistente-LADM_COL/sources/service_endpoint', DEFAULT_ENDPOINT_SOURCE_SERVICE), SOURCE_SERVICE_UPLOAD_SUFFIX ]) request = QNetworkRequest(QUrl(service_url)) reply = nam.post(request, multiPart) #reply.uploadProgress.connect(upload_dialog.update_current_progress) reply.error.connect(self.error_returned) multiPart.setParent(reply) # We'll block execution until we get response from the server loop = QEventLoop() reply.finished.connect(loop.quit) loop.exec_() response = reply.readAll() data = QTextStream(response, QIODevice.ReadOnly) content = data.readAll() if content is None: self.logger.critical( __name__, "There was an error uploading file '{}'".format(data_url)) upload_errors += 1 continue try: response = json.loads(content) except json.decoder.JSONDecodeError: self.logger.critical( __name__, "Couldn't parse JSON response from server for file '{}'!!!" .format(data_url)) upload_errors += 1 continue if 'error' in response: self.logger.critical( __name__, "STATUS: {}. ERROR: {} MESSAGE: {} FILE: {}".format( response['status'], response['error'], response['message'], data_url)) upload_errors += 1 continue reply.deleteLater() if 'url' not in response: self.logger.critical( __name__, "'url' attribute not found in JSON response for file '{}'!" .format(data_url)) upload_errors += 1 continue url = self.get_file_url(response['url']) new_values[feature.id()] = {field_index: url} count += 1 upload_dialog.update_total_progress(count) if not_found > 0: self.logger.info_msg( __name__, QCoreApplication.translate( "SourceHandler", "{} out of {} records {} not uploaded to the document repository because {} file path is NULL or it couldn't be found in the local disk!" ).format( not_found, total, QCoreApplication.translate("SourceHandler", "was") if not_found == 1 else QCoreApplication.translate( "SourceHandler", "were"), QCoreApplication.translate("SourceHandler", "its") if not_found == 1 else QCoreApplication.translate( "SourceHandler", "their"))) if len(new_values): self.logger.info_msg( __name__, QCoreApplication.translate( "SourceHandler", "{} out of {} files {} uploaded to the document repository and {} remote location stored in the database!" ).format( len(new_values), total, QCoreApplication.translate("SourceHandler", "was") if len(new_values) == 1 else QCoreApplication.translate( "SourceHandler", "were"), QCoreApplication.translate("SourceHandler", "its") if len(new_values) == 1 else QCoreApplication.translate( "SourceHandler", "their"))) if upload_errors: self.logger.info_msg( __name__, QCoreApplication.translate( "SourceHandler", "{} out of {} files could not be uploaded to the document repository because of upload errors! See log for details." ).format(upload_errors, total)) return new_values def error_returned(self, error_code): self.logger.critical(__name__, "Qt network error code: {}".format(error_code)) def handle_source_upload(self, db, layer, field_name): layer_name = db.get_ladm_layer_name(layer) field_index = layer.fields().indexFromName(field_name) def features_added(layer_id, features): modified_layer = QgsProject.instance().mapLayer(layer_id) if modified_layer is None: return modified_layer_name = db.get_ladm_layer_name(modified_layer, validate_is_ladm=True) if modified_layer_name is None: return if modified_layer_name.lower() != layer_name.lower(): return with OverrideCursor(Qt.WaitCursor): new_values = self.upload_files(modified_layer, field_index, features) if new_values: modified_layer.dataProvider().changeAttributeValues(new_values) layer.committedFeaturesAdded.connect(features_added) def get_file_url(self, part): endpoint = QSettings().value( 'Asistente-LADM_COL/sources/service_endpoint', DEFAULT_ENDPOINT_SOURCE_SERVICE) return '/'.join([endpoint, part[1:] if part.startswith('/') else part])
class ReportGenerator(QObject): LOG_TAB = 'LADM-COL Reports' enable_action_requested = pyqtSignal(str, bool) def __init__(self, ladm_data): QObject.__init__(self) self.ladm_data = ladm_data self.logger = Logger() self.app = AppInterface() self.java_dependency = JavaDependency() self.java_dependency.download_dependency_completed.connect( self.download_java_complete) self.report_dependency = ReportDependency() self.report_dependency.download_dependency_completed.connect( self.download_report_complete) self.encoding = locale.getlocale()[1] # This might be unset if not self.encoding: self.encoding = 'UTF8' self._downloading = False def stderr_ready(self, proc): text = bytes(proc.readAllStandardError()).decode(self.encoding) self.logger.critical(__name__, text, tab=self.LOG_TAB) def stdout_ready(self, proc): text = bytes(proc.readAllStandardOutput()).decode(self.encoding) self.logger.info(__name__, text, tab=self.LOG_TAB) def update_yaml_config(self, db, config_path): text = '' qgs_uri = QgsDataSourceUri(db.uri) with open(os.path.join(config_path, 'config_template.yaml')) as f: text = f.read() text = text.format('{}', DB_USER=qgs_uri.username(), DB_PASSWORD=qgs_uri.password(), DB_HOST=qgs_uri.host(), DB_PORT=qgs_uri.port(), DB_NAME=qgs_uri.database()) new_file_path = os.path.join( config_path, self.get_tmp_filename('yaml_config', 'yaml')) with open(new_file_path, 'w') as new_yaml: new_yaml.write(text) return new_file_path def get_layer_geojson(self, db, layer_name, plot_id, report_type): if report_type == ANNEX_17_REPORT: if layer_name == 'terreno': return db.get_annex17_plot_data(plot_id, 'only_id') elif layer_name == 'terrenos': return db.get_annex17_plot_data(plot_id, 'all_but_id') elif layer_name == 'terrenos_all': return db.get_annex17_plot_data(plot_id, 'all') elif layer_name == 'construcciones': return db.get_annex17_building_data() else: return db.get_annex17_point_data(plot_id) else: #report_type == ANT_MAP_REPORT: if layer_name == 'terreno': return db.get_ant_map_plot_data(plot_id, 'only_id') elif layer_name == 'terrenos': return db.get_ant_map_plot_data(plot_id, 'all_but_id') elif layer_name == 'terrenos_all': return db.get_annex17_plot_data(plot_id, 'all') elif layer_name == 'construcciones': return db.get_annex17_building_data() elif layer_name == 'puntoLindero': return db.get_annex17_point_data(plot_id) else: #layer_name == 'cambio_colindancia': return db.get_ant_map_neighbouring_change_data(plot_id) def update_json_data(self, db, json_spec_file, plot_id, tmp_dir, report_type): json_data = dict() with open(json_spec_file) as f: json_data = json.load(f) json_data['attributes']['id'] = plot_id json_data['attributes']['datasetName'] = db.schema layers = json_data['attributes']['map']['layers'] for layer in layers: layer['geoJson'] = self.get_layer_geojson(db, layer['name'], plot_id, report_type) overview_layers = json_data['attributes']['overviewMap']['layers'] for layer in overview_layers: layer['geoJson'] = self.get_layer_geojson(db, layer['name'], plot_id, report_type) new_json_file_path = os.path.join( tmp_dir, self.get_tmp_filename('json_data_{}'.format(plot_id), 'json')) with open(new_json_file_path, 'w') as new_json: new_json.write(json.dumps(json_data)) return new_json_file_path def get_tmp_dir(self, create_random=True): if create_random: return tempfile.mkdtemp() return tempfile.gettempdir() def get_tmp_filename(self, basename, extension='gpkg'): return "{}_{}.{}".format(basename, str(time.time()).replace(".", ""), extension) def generate_report(self, db, report_type): # Check if mapfish and Jasper are installed, otherwise show where to # download them from and return if not self.report_dependency.check_if_dependency_is_valid(): self.report_dependency.download_dependency(URL_REPORTS_LIBRARIES) return java_home_set = self.java_dependency.set_java_home() if not java_home_set: self.java_dependency.get_java_on_demand() self.logger.info_msg( __name__, QCoreApplication.translate( "ReportGenerator", "Java is a prerequisite. Since it was not found, it is being configured..." )) return plot_layer = self.app.core.get_layer(db, db.names.LC_PLOT_T, load=True) if not plot_layer: return selected_plots = plot_layer.selectedFeatures() if not selected_plots: self.logger.warning_msg( __name__, QCoreApplication.translate( "ReportGenerator", "To generate reports, first select at least a plot!")) return # Where to store the reports? previous_folder = QSettings().value( "Asistente-LADM-COL/reports/save_into_dir", ".") save_into_folder = QFileDialog.getExistingDirectory( None, QCoreApplication.translate( "ReportGenerator", "Select a folder to save the reports to be generated"), previous_folder) if not save_into_folder: self.logger.warning_msg( __name__, QCoreApplication.translate( "ReportGenerator", "You need to select a folder where to save the reports before continuing." )) return QSettings().setValue("Asistente-LADM-COL/reports/save_into_dir", save_into_folder) config_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, report_type) json_spec_file = os.path.join(config_path, 'spec_json_file.json') script_name = '' if os.name == 'posix': script_name = 'print' elif os.name == 'nt': script_name = 'print.bat' script_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, 'bin', script_name) if not os.path.isfile(script_path): self.logger.warning( __name__, "Script file for reports wasn't found! {}".format(script_path)) return self.enable_action_requested.emit(report_type, False) # Update config file yaml_config_path = self.update_yaml_config(db, config_path) self.logger.debug( __name__, "Config file for reports: {}".format(yaml_config_path)) total = len(selected_plots) step = 0 count = 0 tmp_dir = self.get_tmp_dir() # Progress bar setup progress = QProgressBar() if total == 1: progress.setRange(0, 0) else: progress.setRange(0, 100) progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) self.app.gui.create_progress_message_bar( QCoreApplication.translate("ReportGenerator", "Generating {} report{}...").format( total, '' if total == 1 else 's'), progress) polygons_with_holes = [] multi_polygons = [] for selected_plot in selected_plots: plot_id = selected_plot[db.names.T_ID_F] geometry = selected_plot.geometry() abstract_geometry = geometry.get() if abstract_geometry.ringCount() > 1: polygons_with_holes.append(str(plot_id)) self.logger.warning( __name__, QCoreApplication.translate( "ReportGenerator", "Skipping Annex 17 for plot with {}={} because it has holes. The reporter module does not support such polygons." ).format(db.names.T_ID_F, plot_id)) continue if abstract_geometry.numGeometries() > 1: multi_polygons.append(str(plot_id)) self.logger.warning( __name__, QCoreApplication.translate( "ReportGenerator", "Skipping Annex 17 for plot with {}={} because it is a multi-polygon. The reporter module does not support such polygons." ).format(db.names.T_ID_F, plot_id)) continue # Generate data file json_file = self.update_json_data(db, json_spec_file, plot_id, tmp_dir, report_type) self.logger.debug(__name__, "JSON file for reports: {}".format(json_file)) # Run sh/bat passing config and data files proc = QProcess() proc.readyReadStandardError.connect( functools.partial(self.stderr_ready, proc=proc)) proc.readyReadStandardOutput.connect( functools.partial(self.stdout_ready, proc=proc)) parcel_number = self.ladm_data.get_parcels_related_to_plots( db, [plot_id], db.names.LC_PARCEL_T_PARCEL_NUMBER_F) or [''] file_name = '{}_{}_{}.pdf'.format(report_type, plot_id, parcel_number[0]) current_report_path = os.path.join(save_into_folder, file_name) proc.start(script_path, [ '-config', yaml_config_path, '-spec', json_file, '-output', current_report_path ]) if not proc.waitForStarted(): # Grant execution permissions os.chmod( script_path, stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR | stat.S_IRUSR | stat.S_IRGRP) proc.start(script_path, [ '-config', yaml_config_path, '-spec', json_file, '-output', current_report_path ]) if not proc.waitForStarted(): proc = None self.logger.warning( __name__, "Couldn't execute script to generate report...") else: loop = QEventLoop() proc.finished.connect(loop.exit) loop.exec() self.logger.debug(__name__, "{}:{}".format(plot_id, proc.exitCode())) if proc.exitCode() == 0: count += 1 step += 1 progress.setValue(step * 100 / total) os.remove(yaml_config_path) self.enable_action_requested.emit(report_type, True) self.logger.clear_message_bar() if total == count: if total == 1: msg = QCoreApplication.translate( "ReportGenerator", "The report <a href='file:///{}'>{}</a> was successfully generated!" ).format(normalize_local_url(save_into_folder), file_name) else: msg = QCoreApplication.translate( "ReportGenerator", "All reports were successfully generated in folder <a href='file:///{path}'>{path}</a>!" ).format(path=normalize_local_url(save_into_folder)) self.logger.success_msg(__name__, msg) else: details_msg = '' if polygons_with_holes: details_msg += QCoreApplication.translate( "ReportGenerator", " The following polygons were skipped because they have holes and are not supported: {}." ).format(", ".join(polygons_with_holes)) if multi_polygons: details_msg += QCoreApplication.translate( "ReportGenerator", " The following polygons were skipped because they are multi-polygons and are not supported: {}." ).format(", ".join(multi_polygons)) if total == 1: msg = QCoreApplication.translate( "ReportGenerator", "The report for plot {} couldn't be generated!{} See QGIS log (tab '{}') for details." ).format(plot_id, details_msg, self.LOG_TAB) else: if count == 0: msg = QCoreApplication.translate( "ReportGenerator", "No report could be generated!{} See QGIS log (tab '{}') for details." ).format(details_msg, self.LOG_TAB) else: msg = QCoreApplication.translate( "ReportGenerator", "At least one report couldn't be generated!{details_msg} See QGIS log (tab '{log_tab}') for details. Go to <a href='file:///{path}'>{path}</a> to see the reports that were generated." ).format(details_msg=details_msg, path=normalize_local_url(save_into_folder), log_tab=self.LOG_TAB) self.logger.warning_msg(__name__, msg) def download_java_complete(self): if self.java_dependency.fetcher_task and not self.java_dependency.fetcher_task.isCanceled( ): if self.java_dependency.check_if_dependency_is_valid(): self.logger.info_msg( __name__, QCoreApplication.translate( "ReportGenerator", "Java was successfully configured!"), 5) else: self.logger.warning_msg( __name__, QCoreApplication.translate( "ReportGenerator", "You have just canceled the Java dependency download."), 5) def download_report_complete(self): if self.report_dependency.fetcher_task and not self.report_dependency.fetcher_task.isCanceled( ): if self.report_dependency.check_if_dependency_is_valid(): self.logger.info_msg( __name__, QCoreApplication.translate( "ReportGenerator", "Report dependency was successfully configured!"), 5) else: self.logger.warning_msg( __name__, QCoreApplication.translate( "ReportGenerator", "You have just canceled the report dependency download."), 5)
class XTFModelConverterRegistry(metaclass=Singleton): """ Registry of supported model converters. """ def __init__(self): self.logger = Logger() self.app = AppInterface() self.__converters = dict() # {converter_key1: LADMColModelConverter1, ...} # Register default models self.register_model_converter(Survey10To11Converter()) self.register_model_converter(Survey11To10Converter()) def register_model_converter(self, converter): """ :param converter: LADMColModelConverter instance. :return: True if the converter was registered, False otherwise. """ if not isinstance(converter, AbstractLADMColModelConverter): self.logger.warning(__name__, "The converter '{}' is not a 'LADMColModelConverter' instance!".format(converter.id())) return False if not converter.is_valid(): self.logger.warning(__name__, "The converter '{}' is not valid! Check the converter definition!".format(converter.id())) return False if converter.id() in self.__converters: self.logger.warning(__name__, "The converter '{}' is already registered.".format(converter.id())) return False self.__converters[converter.id()] = converter self.logger.info(__name__, "Model converter '{}' has been registered!".format(converter.id())) return True def unregister_model_converter(self, converter_key): """ Unregisters a model converter. :param converter_key: Id of the converter to unregister. :return: True if the converter was unregistered, False otherwise. """ if converter_key not in self.__converters: self.logger.error(__name__, "Converter '{}' was not found in registered model converters, therefore, it cannot be unregistered!".format(converter_key)) return False self.__converters[converter_key] = None del self.__converters[converter_key] self.logger.info(__name__, "Model converter '{}' has been unregistered!".format(converter_key)) return True def get_converter(self, converter_key): converter = self.__converters.get(converter_key, None) # To avoid exceptions if not converter: self.logger.critical(__name__, "No model converter found with key '{}'".format(converter_key)) return converter def get_converters_for_models(self, models): converters = dict() # {converter_key_1: converter_display_name_1, ...] for converter_key, converter in self.__converters.items(): for model in models: if converter.supports_source_model(model): converters[converter_key] = converter.display_name() return converters
class QualityRuleEngine(QObject): """ Engine that executes Quality Rules :param db: DBConnector object :param rules: Either a dict {rule_key:rule_name} or a list [rule_key1, rule_key2] :param tolerance: Tolerance to be used when running the QRs, in millimeters """ progress_changed = pyqtSignal(int) # Progress value def __init__(self, db, rules, tolerance, output_path=''): QObject.__init__(self) self.logger = Logger() self.app = AppInterface() self.__qr_registry = QualityRuleRegistry() self.__db = db self.__db_qr = None self.__rules = self.__get_dict_rules(rules) self.__result_layers = list() self.__with_gui = self.app.settings.with_gui self.__timestamp = "" self.__output_path = output_path self.app.settings.tolerance = tolerance # Tolerance must be given, we don't want anything implicit about it self.__tolerance = self.app.settings.tolerance # Tolerance input might be altered (e.g., if it comes negative) self.__layer_manager = QualityRuleLayerManager(db, self.__rules, self.__tolerance) self.__layer_manager.progress_changed.connect( self.__emit_prepare_layers_progress) self.qr_logger = QualityRuleLogger(self.__db, self.__tolerance) self.__current_progress = 0 self.__error_db_utils = QualityErrorDBUtils() self.__error_db_utils.progress_changed.connect( self.__emit_error_db_progress) # Clear informality cache before executing QRs. # Note: between creating an object of this class and calling validate_quality_rules() a lot # of things could happen (like new caches being built!). It is your responsibility to create # an instance of this class or initialize() a QREngine object just before calling validate_quality_rules(). self.app.core.clear_cached_informal_spatial_units() def initialize(self, db, rules, tolerance, output_path='', clear_informality_cache=True): """ Objects of this class are reusable calling initialize() """ self.__result_layers = list() self.__db = db self.__db_qr = None self.__rules = self.__get_dict_rules(rules) self.__with_gui = self.app.settings.with_gui self.__timestamp = "" self.__output_path = output_path self.app.settings.tolerance = tolerance self.__tolerance = self.app.settings.tolerance # Tolerance input might be altered (e.g., if it comes negative) self.__layer_manager.initialize(self.__rules, self.__tolerance) self.qr_logger.initialize(self.__db, self.__tolerance) self.__current_progress = 0 # This time, (initializing an existing object) we give you the chance to avoid # rebuilding the informality cache. It is handy if you're executing validations # consecutively and you're sure that reusing a previous cache does make sense. if clear_informality_cache: self.app.core.clear_cached_informal_spatial_units() def __get_dict_rules(self, rules): if isinstance(rules, dict): return rules # We have everything ready # If rules is a list, we need to retrieve the quality rule names from the QRRegistry return { rule_key: self.__qr_registry.get_quality_rule(rule_key) for rule_key in rules } def validate_quality_rules(self, options=dict()): """ :param options: Dict of dicts with of options per rule. {qr1_key: {qr1_opt1: value1, ...}, ...} """ res = False msg = "" qr_res = dict() # {rule_key: QualityRuleExecutionResult} if self.__rules: self.__emit_progress_changed(1) # First, create the error db and fill its metadata... self.__timestamp = time.strftime('%Y%m%d_%H%M%S') res_db, msg_db, self.__db_qr = self.__error_db_utils.get_quality_error_connector( self.__output_path, self.__timestamp, True) self.__emit_progress_changed(5) if not res_db: msg_db = QCoreApplication.translate( "QualityRuleEngine", "There was a problem creating the quality error DB! Details: {}" ).format(msg_db) self.logger.warning(__name__, msg_db) return False, msg_db, None self.qr_logger.set_count_topology_rules(len(self.__rules)) self.logger.info( __name__, QCoreApplication.translate( "QualityRuleEngine", "Validating {} quality rules (tolerance: {}).").format( len(self.__rules), self.__tolerance)) first_pass = True count = 0 for rule_key, rule in self.__rules.items(): count += 1 if rule is not None: layers = self.__layer_manager.get_layers( rule_key) # Fist pass might be long if tolerance > 0 if first_pass: first_pass = False self.__emit_progress_changed( 25 if self.__tolerance else 15) if layers: connect_obj = rule.progress_changed.connect( partial(self.__emit_qr_progress, count)) qr_res[rule_key] = self.__validate_quality_rule( rule, layers, options.get(rule_key, dict())) rule.progress_changed.disconnect( connect_obj ) # We no longer need the connection, so delete it else: qr_msg = QCoreApplication.translate( "QualityRuleEngine", "Couldn't execute '{}' quality rule! Required layers are not available. Skipping..." ).format(rule.name()) qr_res[rule_key] = QualityRuleExecutionResult( EnumQualityRuleResult.CRITICAL, qr_msg) self.logger.warning(__name__, qr_msg) else: qr_msg = QCoreApplication.translate( "QualityRuleEngine", "Quality rule with key '{}' does not exist or is not registered! Skipping..." ).format(rule_key) qr_res[rule_key] = QualityRuleExecutionResult( EnumQualityRuleResult.CRITICAL, qr_msg) self.logger.warning(__name__, qr_msg) self.__emit_progress_changed(95) metadata = { QR_METADATA_TOOL: QR_METADATA_TOOL_NAME, QR_METADATA_DATA_SOURCE: self.__db.get_description_conn_string(), QR_METADATA_TOLERANCE: self.__tolerance / 1000, QR_METADATA_TIMESTAMP: self.__timestamp, QR_METADATA_RULES: list(self.__rules.keys()), # QR keys QR_METADATA_OPTIONS: self.__normalize_options(options), QR_METADATA_PERSON: getpass.getuser() } QualityErrorDBUtils.save_metadata(self.__db_qr, metadata) self.export_result_to_pdf() self.__emit_progress_changed(99) res = True msg = "Success!" self.__layer_manager.clean_temporary_layers() else: self.logger.warning( __name__, QCoreApplication.translate("QualityRuleEngine", "No rules to validate!")) self.__emit_progress_changed(100) return res, msg, QualityRulesExecutionResult(qr_res) @_log_quality_rule_validations def __validate_quality_rule(self, rule, layers, options): """ Intermediate function to log quality rule execution. :param rule: Quality rule instance :param layers: Layer dict with the layers the quality rule needs (ready to use for tolerance > 0 scenarios) :param options: Dict of options per rule. {qr1_opt1: value1, ...} :return: An instance of QualityRuleExecutionResult """ return rule.validate(self.__db, self.__db_qr, layers, self.__tolerance, options=options) def __emit_progress_changed(self, value, save_value=True): if value != self.__current_progress: # Avoid emitting the same value twice if save_value: self.__current_progress = value self.progress_changed.emit(value) def __emit_error_db_progress(self, progress_value): """ Add the normalized error db progress value to what we have already in the overall progress """ value = self.__current_progress + (progress_value * TOTAL_PROGRESS_ERROR_DB / 100) # print("...DB", self.__current_progress, progress_value, value) self.__emit_progress_changed( int(value), progress_value == 100) # Only save when the subprocess is finished def __emit_prepare_layers_progress(self, progress_value): """ Add the normalized prepare layers' progress value to what we have already in the overall progress """ range = TOTAL_PROGRESS_PREPARE_LAYERS if self.__tolerance else TOTAL_PROGRESS_PREPARE_LAYERS_NO_TOLERANCE value = self.__current_progress + (progress_value * range / 100) # print("...PL", self.__current_progress, progress_value, value) self.__emit_progress_changed( int(value), progress_value == 100) # Only save when the subprocess is finished def __emit_qr_progress(self, count, qr_progress_value): """ Add the normalized current QR progress value to what we have already in the overall progress """ num_rules = len(self.__rules) step = (TOTAL_PROGRESS_QR if self.__tolerance else TOTAL_PROGRESS_QR_NO_TOLERANCE) / num_rules value = self.__current_progress + (qr_progress_value * step / 100) # print("...QR", self.__current_progress, qr_progress_value, value) self.__emit_progress_changed( int(value), qr_progress_value == 100) # Only save when the QR is finished def get_db_quality(self): return self.__db_qr def get_timestamp(self): # Last timestamp used to validate QRs. # Note the timestamp is persisted in QR DB's metadata table return self.__timestamp def export_result_to_pdf(self): res, msg, output_path = QualityErrorDBUtils.get_quality_validation_output_path( self.__output_path, self.__timestamp) if not res: self.logger.critical( __name__, QCoreApplication.translate( "QualityRuleEngine", "PDF report could not be exported, there were problems with the output path '{}'!" ).format(self.__output_path)) return pdf_path = os.path.join( output_path, "Reglas_de_Calidad_{}.pdf".format(self.__timestamp)) log = self.qr_logger.get_log_result() export_title_text_to_pdf(pdf_path, log.title, log.text) def __normalize_options(self, options): # Get rid of options that do not correspond to validated QRs normalized_options = dict() for k, v in options.items(): for kv, vv in v.items(): if k in list(self.__rules.keys()): if k in normalized_options: normalized_options[k][kv] = vv else: normalized_options[k] = {kv: vv} return normalized_options
class QualityRuleController(QObject): open_report_called = pyqtSignal(QualityRuleResultLog) # log result quality_rule_layer_removed = pyqtSignal() refresh_error_layer_symbology = pyqtSignal(QgsVectorLayer) total_progress_changed = pyqtSignal(int) # Progress value def __init__(self, db): QObject.__init__(self) self.app = AppInterface() self.logger = Logger() self.__db = db self.__tr_dict = TranslatableConfigStrings( ).get_translatable_config_strings() # Hierarquical dict of qrs and qr groups self.__qrs_tree_data = dict() # {type: {qr_key1: qr_obj1, ...}, ...} # Hierarquical dict of qrs and qr groups with general results self.__general_results_tree_data = dict( ) # {type: {qr_obj1: qr_results1, ...}, ...} # Hierarchical dict of qrs and their corresponding error instances # feature1: {uuids, rel_uuids, error_type, nombre_ili_obj, details, values, fixed, exception, geom_fks} self.__error_results_data = dict() # {qr_key1: {t_id1: feature1}} self.__qr_results_dir_path = '' # Dir path where results will be stored self.__selected_qrs = list() # QRs to be validated (at least 1) self.__selected_qr = None # QR selected by the user to show its corresponding errors (exactly 1) self.__qr_engine = None # Once set, we can reuse it self.__qrs_results = None # QualityRulesExecutionResult object # To cache layers from QR DB self.__error_layer = None self.__point_layer = None self.__line_layer = None self.__polygon_layer = None # Cache by t_id (built on demand): {t_id1: 'Error', t_id2: 'Corregido', t_id3: 'Exception'} self.__error_state_dict = dict() def get_tr_string(self, key): return self.__tr_dict.get(key, key) def validate_qrs(self): if self.__qr_engine is None: self.__qr_engine = QualityRuleEngine(self.__db, self.__selected_qrs, self.app.settings.tolerance, self.__qr_results_dir_path) self.__qr_engine.progress_changed.connect( self.total_progress_changed) else: self.__qr_engine.initialize(self.__db, self.__selected_qrs, self.app.settings.tolerance, self.__qr_results_dir_path) #self.__qr_engine.qr_logger.show_message_emitted.connect(self.show_log_quality_message) #self.__qr_engine.qr_logger.show_button_emitted.connect(self.show_log_quality_button) #self.__qr_engine.qr_logger.set_initial_progress_emitted.connect(self.set_log_quality_initial_progress) #self.__qr_engine.qr_logger.set_final_progress_emitted.connect(self.set_log_quality_final_progress) use_roads = bool(QSettings().value( 'Asistente-LADM-COL/quality/use_roads', DEFAULT_USE_ROADS_VALUE, bool)) options = {QR_IGACR3006: {'use_roads': use_roads}} res, msg, qrs_res = self.__qr_engine.validate_quality_rules(options) if not res: return res, msg, None self.__qrs_results = qrs_res self.__connect_layer_willbedeleted_signals( ) # Note: Call it after validate_quality_rules! res_u, msg_u, output_qr_dir = QualityErrorDBUtils.get_quality_validation_output_path( self.__qr_results_dir_path, self.__qr_engine.get_timestamp()) if len(self.__selected_qrs) == 1: pre_text = QCoreApplication.translate( "QualityRules", "The quality rule was checked!") else: pre_text = QCoreApplication.translate( "QualityRules", "All the {} quality rules were checked!").format( len(self.__selected_qrs)) post_text = QCoreApplication.translate( "QualityRules", "Both a PDF report and a GeoPackage database with errors can be found in <a href='file:///{}'>{}</a>." ).format(normalize_local_url(output_qr_dir), output_qr_dir) self.logger.success_msg(__name__, "{} {}".format(pre_text, post_text)) self.__emit_refresh_error_layer_symbology() return res, msg, self.__qrs_results def __connect_layer_willbedeleted_signals(self): """ Iterate QR DB layers from the layer tree and connect their layerwillberemoved signals. If a QR DB layer is removed, we'll react in the GUI. """ group = QualityErrorDBUtils.get_quality_error_group( self.__qr_engine.get_timestamp()) if group: for tree_layer in group.findLayers(): try: tree_layer.layer().willBeDeleted.disconnect( self.quality_rule_layer_removed) except: pass tree_layer.layer().willBeDeleted.connect( self.quality_rule_layer_removed) def disconnect_layer_willberemoved_signals(self): group = QualityErrorDBUtils.get_quality_error_group( self.__qr_engine.get_timestamp(), False) if group: for tree_layer in group.findLayers(): try: tree_layer.layer().willBeDeleted.disconnect( self.quality_rule_layer_removed) except: pass def get_qr_result(self, qr_key): """ Return the QRExecutionResult object for the given qr_key. It first attempts to find it in the __qrs_results dict, but, chances are, the whole set of QRs hasn't been validated when this method is called, so, as a last resort, we go for the tree_data, which is updated each time a QR gets its result. """ if self.__qrs_results is not None: return self.__qrs_results.result(qr_key) for type, qr_dict in self.__general_results_tree_data.items(): for k, v in qr_dict.items(): if k.id() == qr_key: return self.__general_results_tree_data[type][k] return None def __reset_qrs_results(self): # To be used when we are returning to select QRs (i.e., to the initial panel) self.__qrs_results = None def __get_qrs_per_role_and_models(self): return QualityRuleRegistry().get_qrs_per_role_and_models(self.__db) def load_tree_data(self, mode): """ Builds a hierarchical dict by qr type: {qr_type1: {qr_key1: qr_obj1, ...}, ...} Tree data for panel 1. :params mode: Value from EnumQualityRulePanelMode (either VALIDATE or READ). For VALIDATE we load QRs from registry (filtered by role and current db models). For READ we load QRs from the DB itself. """ if mode == EnumQualityRulePanelMode.VALIDATE: qrs = self.__get_qrs_per_role_and_models( ) # Dict of qr key and qr objects. else: qrs = dict() # TODO: Read QRs from the QR DB for qr_key, qr_obj in qrs.items(): type = qr_obj.type() if type not in self.__qrs_tree_data: self.__qrs_tree_data[type] = {qr_key: qr_obj} else: self.__qrs_tree_data[type][qr_key] = qr_obj def get_qrs_tree_data(self): return self.__qrs_tree_data def set_qr_dir_path(self, path): self.__qr_results_dir_path = path def set_selected_qrs(self, selected_qrs): # We sort them because the engine needs the QRs sorted for the PDF report for type, qr_dict in self.__qrs_tree_data.items(): for qr_key, qr_obj in qr_dict.items(): if qr_key in selected_qrs: self.__selected_qrs.append(qr_key) def get_selected_qrs(self): return self.__selected_qrs def __reset_selected_qrs(self): # To be used when we are returning to select QRs (i.e., to the initial panel) self.__selected_qrs = list() def reset_vars_for_general_results_panel(self): # Initialize variables when we leave the general results panel self.__reset_general_results_tree_data() self.__reset_selected_qrs() self.__reset_qrs_results() self.__reset_layers() # Call it before removing QR DB group to avoid triggering parent.layer_removed() slot again. self.disconnect_layer_willberemoved_signals() # When we leave the GRP, we remove the QR DB group from layer tree, # because we won't be working anymore with that QR DB QualityErrorDBUtils.remove_quality_error_group( self.__qr_engine.get_timestamp()) def reset_vars_for_error_results_panel(self): # Initialize variables when we leave the error results panel self.__reset_error_results_data() self.__reset_selected_qr() self.__reset_error_state_dict() self.__reset_layers() def load_general_results_tree_data(self): """ Builds a hierarchical dict by qr type: {type: {qr_obj1: qr_results1, ...}, ...} Tree data for panel 2. """ for type, qr_dict in self.__qrs_tree_data.items(): for qr_key, qr_obj in qr_dict.items(): if qr_key in self.__selected_qrs: if type not in self.__general_results_tree_data: self.__general_results_tree_data[type] = {qr_obj: None} else: self.__general_results_tree_data[type][qr_obj] = None def get_general_results_tree_data(self): return self.__general_results_tree_data def __reset_general_results_tree_data(self): # To be used when we are returning to select QRs (i.e., to the initial panel) self.__general_results_tree_data = dict() def set_qr_validation_result(self, qr, qr_result): """ When a QR has its validation result after validation, we can store it in our custom dict by using this method. """ for type, qr_dict in self.__general_results_tree_data.items(): for k, v in qr_dict.items(): if k == qr: self.__general_results_tree_data[type][k] = qr_result def open_report(self): if self.__qr_engine: log_result = self.__qr_engine.qr_logger.get_log_result() self.open_report_called.emit(log_result) def set_selected_qr(self, qr_key): self.__selected_qr = QualityRuleRegistry().get_quality_rule(qr_key) return self.__selected_qr is not None # We should not be able to continue if we don't find the QR def get_selected_qr(self): return self.__selected_qr def load_error_results_data(self): """ Go to table and bring data to the dict. We should keep this dict updated with changes from the user. From time to time we reflect this dict changes in the original data source. """ db = self.__qr_engine.get_db_quality() names = db.names layers = {names.ERR_QUALITY_ERROR_T: None, names.ERR_RULE_TYPE_T: None} self.app.core.get_layers(db, layers, load=False) if not layers: self.logger.critical( __name__, "Quality error layers ('{}') not found!".format(",".join( list(layers.keys())))) return # First go for the selected quality error's t_id features = LADMData.get_features_from_t_ids( layers[names.ERR_RULE_TYPE_T], names.ERR_RULE_TYPE_T_CODE_F, [self.__selected_qr.id()]) t_id = features[0][names.T_ID_F] if features else None if not t_id: self.logger.critical( __name__, "Quality error rule ('{}') not found!".format( self.__selected_qr.id())) return # Now go for all features that match the selected quality rule features = LADMData.get_features_from_t_ids( layers[names.ERR_QUALITY_ERROR_T], names.ERR_QUALITY_ERROR_T_RULE_TYPE_F, [t_id]) self.__error_results_data[self.__selected_qr.id()] = { feature[names.T_ID_F]: feature for feature in features } def get_error_results_data(self): # Get the subdict {t_id1: feature1, ...} corresponding to selected qr return self.__error_results_data.get( self.__selected_qr.id() if self.__selected_qr else '', dict()) def __reset_error_results_data(self): # To be used when we are returning to select QR results (i.e., to the general results panel) self.__error_results_data = dict() def error_t_id(self, feature): return feature[self.__qr_engine.get_db_quality().names.T_ID_F] def is_fixed_error(self, feature): db = self.__qr_engine.get_db_quality() state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F] return self.__get_error_state_value( state_t_id) == LADMNames.ERR_ERROR_STATE_D_FIXED_V def is_error(self, feature): db = self.__qr_engine.get_db_quality() state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F] return self.__get_error_state_value( state_t_id) == LADMNames.ERR_ERROR_STATE_D_ERROR_V def is_exception(self, feature): db = self.__qr_engine.get_db_quality() state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F] return self.__get_error_state_value( state_t_id) == LADMNames.ERR_ERROR_STATE_D_EXCEPTION_V def uuid_objs(self, feature): return "\n".join(feature[self.__qr_engine.get_db_quality().names. ERR_QUALITY_ERROR_T_OBJECT_IDS_F]) def ili_obj_name(self, feature): ili_name = feature[self.__qr_engine.get_db_quality().names. ERR_QUALITY_ERROR_T_ILI_NAME_F] return ili_name.split(".")[-1] if ili_name else '' def error_type_code_and_display(self, feature): db = self.__qr_engine.get_db_quality() names = db.names layer = self.app.core.get_layer(db, names.ERR_ERROR_TYPE_T, load=False) features = LADMData.get_features_from_t_ids( layer, names.T_ID_F, [feature[db.names.ERR_QUALITY_ERROR_T_ERROR_TYPE_F]]) # tid return features[0][ names. ERR_ERROR_TYPE_T_CODE_F] if features else QCoreApplication.translate( "QualityRules", "No error type found!" ), features[0][ names. ERR_ERROR_TYPE_T_DESCRIPTION_F] if features else QCoreApplication.translate( "QualityRules", "No error description found!") def error_details_and_values(self, feature): res = "" db = self.__qr_engine.get_db_quality() details = feature[db.names.ERR_QUALITY_ERROR_T_DETAILS_F] values = feature[db.names.ERR_QUALITY_ERROR_T_VALUES_F] if details: res = details if values: try: res_values = json.loads(values) if type(res_values) is dict: items = "" for k, v in res_values.items(): items = res + "{}: {}\n".format(k, v) res_values = items.strip() else: res_values = str(res_values) except json.decoder.JSONDecodeError as e: res_values = values res = res_values if not res else "{}\n\n{}".format(res, res_values) return res def error_state(self, feature): db = self.__qr_engine.get_db_quality() state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F] return self.__get_error_state_value(state_t_id) def __get_error_state_value(self, state_t_id): if state_t_id not in self.__error_state_dict: db = self.__qr_engine.get_db_quality() self.__error_state_dict[state_t_id] = LADMData( ).get_domain_value_from_code(db, db.names.ERR_ERROR_STATE_D, state_t_id) return self.__error_state_dict.get(state_t_id, "") def __get_error_state_t_id(self, state_value): # Use __error_state_dict to read cached values, but this time we have the value, # not the key, so check in dict values and if not found, go for its t_id if state_value not in self.__error_state_dict.values(): db = self.__qr_engine.get_db_quality() t_id = LADMData().get_domain_code_from_value( db, db.names.ERR_ERROR_STATE_D, state_value) self.__error_state_dict[t_id] = state_value # Get key by value in a dict: return next((k for k in self.__error_state_dict if self.__error_state_dict[k] == state_value), None) def __get_error_layer(self): if not self.__error_layer: db = self.__qr_engine.get_db_quality() self.__error_layer = self.app.core.get_layer( db, db.names.ERR_QUALITY_ERROR_T) return self.__error_layer def __get_point_error_layer(self): if not self.__point_layer: db = self.__qr_engine.get_db_quality() self.__point_layer = self.app.core.get_layer( db, db.names.ERR_POINT_T) return self.__point_layer def __get_line_error_layer(self): if not self.__line_layer: db = self.__qr_engine.get_db_quality() self.__line_layer = self.app.core.get_layer( db, db.names.ERR_LINE_T) return self.__line_layer def __get_polygon_error_layer(self): if not self.__polygon_layer: db = self.__qr_engine.get_db_quality() self.__polygon_layer = self.app.core.get_layer( db, db.names.ERR_POLYGON_T) return self.__polygon_layer def __reset_layers(self): # To be used when we are returning to select QR results (i.e., to the general results panel) self.__error_layer = None self.__point_layer = None self.__line_layer = None self.__polygon_layer = None def __reset_selected_qr(self): # To be used when we are returning to select QR results (i.e., to the general results panel) self.__selected_qr = None def __reset_error_state_dict(self): # To be used when we are returning to select QR results (i.e., to the general results panel) self.__error_state_dict = dict() def __error_related_geometries(self, error_t_ids): # Prefered geometry types are polygons, lines, points, in that order db = self.__qr_engine.get_db_quality() error_data = self.get_error_results_data() dict_layer_fids = dict() for error_t_id in error_t_ids: feature = error_data.get(error_t_id, None) if feature: polygon = feature[db.names.ERR_QUALITY_ERROR_T_POLYGON_F] line = feature[db.names.ERR_QUALITY_ERROR_T_LINE_F] point = feature[db.names.ERR_QUALITY_ERROR_T_POINT_F] if polygon: if 'polygon' in dict_layer_fids: dict_layer_fids['polygon']['fids'].append(polygon) else: dict_layer_fids['polygon'] = { 'layer': self.__get_polygon_error_layer(), 'fids': [polygon] } elif line: if 'line' in dict_layer_fids: dict_layer_fids['line']['fids'].append(line) else: dict_layer_fids['line'] = { 'layer': self.__get_line_error_layer(), 'fids': [line] } elif point: if 'point' in dict_layer_fids: dict_layer_fids['point']['fids'].append(point) else: dict_layer_fids['point'] = { 'layer': self.__get_point_error_layer(), 'fids': [point] } return dict_layer_fids def highlight_geometries(self, t_ids): res_geometries = self.__error_related_geometries(t_ids) if res_geometries: # First zoom to geometries if len(res_geometries) == 1: # Only one geometry type related for geom_type, dict_layer_fids in res_geometries.items( ): # We know this will be called just once self.app.gui.zoom_to_feature_ids(dict_layer_fids['layer'], dict_layer_fids['fids']) else: # Multiple geometry types were found, so combine the extents and then zoom to it combined_extent = QgsRectangle() for geom_type, dict_layer_fids in res_geometries.items(): combined_extent.combineExtentWith( self.app.core.get_extent_from_feature_ids( dict_layer_fids['layer'], dict_layer_fids['fids'])) self.app.gui.zoom_to_extent(combined_extent) # Now highlight geometries for geom_type, dict_layer_fids in res_geometries.items(): self.app.gui.flash_features(dict_layer_fids['layer'], dict_layer_fids['fids'], flashes=5) def get_uuids_display_name(self): names = self.__qr_engine.get_db_quality().names res = self.__selected_qr.field_mapping(names).get( names.ERR_QUALITY_ERROR_T_OBJECT_IDS_F, '') return res if res else QCoreApplication.translate( "QualityRules", "UUIDs") def set_fixed_error(self, error_t_id, fixed): # Save to the intermediate dict of data and to the underlying data source whether an error is fixed or not db = self.__qr_engine.get_db_quality() idx_state = self.__get_error_layer().fields().indexOf( db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F) value = LADMNames.ERR_ERROR_STATE_D_FIXED_V if fixed else LADMNames.ERR_ERROR_STATE_D_ERROR_V fixed_or_error_t_id = self.__get_error_state_t_id(value) if fixed_or_error_t_id is None: self.logger.critical( __name__, "The error state t_id couldn't be found for value '{}'!". format(value)) return # Save to dict self.get_error_results_data()[error_t_id].setAttribute( idx_state, fixed_or_error_t_id) fids = LADMData.get_fids_from_key_values(self.__get_error_layer(), db.names.T_ID_F, [error_t_id]) # Save to underlying data source if fids: res = self.__get_error_layer().dataProvider( ).changeAttributeValues( {fids[0]: { idx_state: fixed_or_error_t_id }}) if not res: self.logger.critical(__name__, "Error modifying the error state value!") else: self.logger.critical( __name__, "Error with t_id '' not found!".format(error_t_id)) def __emit_refresh_error_layer_symbology(self): if self.__get_point_error_layer().featureCount(): self.refresh_error_layer_symbology.emit( self.__get_point_error_layer()) if self.__get_line_error_layer().featureCount(): self.refresh_error_layer_symbology.emit( self.__get_line_error_layer()) if self.__get_polygon_error_layer().featureCount(): self.refresh_error_layer_symbology.emit( self.__get_polygon_error_layer())
class AppProcessingInterface(QObject): def __init__(self): QObject.__init__(self) self.logger = Logger() self.ladm_col_provider = LADMCOLAlgorithmProvider() self.__processing_resources_installed = list() self.__processing_model_dirs_to_register = [PROCESSING_MODELS_DIR] self.__processing_script_dirs_to_register = [PROCESSING_SCRIPTS_DIR] def initialize_processing_resources(self): """ Add custom provider, models and scripts to QGIS """ QgsApplication.processingRegistry().addProvider(self.ladm_col_provider) connect_provider_added = False if QgsApplication.processingRegistry().providerById('model'): self.__add_processing_resources_by_provider('model') else: connect_provider_added = True if QgsApplication.processingRegistry().providerById('script'): self.__add_processing_resources_by_provider('script') else: connect_provider_added = True if connect_provider_added: # We need to wait until processing is initialized QgsApplication.processingRegistry().providerAdded.connect( self.__add_processing_resources_by_provider) def __add_processing_resources_by_provider(self, provider_id): if provider_id not in ['model', 'script']: return if sorted(self.__processing_resources_installed) == [ "models", "script" ]: # We are done, disconnect. try: QgsApplication.processingRegistry().providerAdded.disconnect( self.__add_processing_resources_by_provider) except: pass # Disconnect throws an error if the SLOT is already disconnected if provider_id == 'model': for models_dir in self.__processing_model_dirs_to_register: self.__register_processing_models(models_dir) self.__processing_resources_installed.append('model') elif provider_id == 'script': for scripts_dir in self.__processing_script_dirs_to_register: self.__register_processing_scripts(scripts_dir) self.__processing_resources_installed.append('script') def __register_processing_models(self, models_dir): # First get model file names from the model root folder filenames = list() for filename in glob.glob(os.path.join(models_dir, '*.model3')): # Non-recursive filenames.append(filename) # Now, go for subfolders. # We store models that depend on QGIS versions in folders like "314" (for QGIS 3.14.x) # This was initially needed for the FieldMapper input, which was migrated to C++ in QGIS 3.14 qgis_major_version = str(Qgis.QGIS_VERSION_INT)[:3] qgis_major_version_path = os.path.join(models_dir, qgis_major_version) if not os.path.isdir(qgis_major_version_path): # No folder for this version (e.g., unit tests on QGIS-dev), so let's find the most recent version subfolders = [ sf.name for sf in os.scandir(models_dir) if sf.is_dir() ] if subfolders: qgis_major_version_path = os.path.join(models_dir, max(subfolders)) for filename in glob.glob( os.path.join(qgis_major_version_path, '*.model3')): filenames.append(filename) # Finally, do load the models! count = 0 registered_models = list() for filename in filenames: alg = QgsProcessingModelAlgorithm() if not alg.fromFile(filename): self.logger.critical( __name__, "Couldn't load model from '{}'!".format(filename)) return registered_models.append(os.path.basename(filename)) destFilename = os.path.join(ModelerUtils.modelsFolders()[0], os.path.basename(filename)) shutil.copyfile(filename, destFilename) count += 1 if count: QgsApplication.processingRegistry().providerById( 'model').refreshAlgorithms() if DEFAULT_LOG_MODE == EnumLogMode.DEV: self.logger.debug( __name__, "{} LADM-COL Processing models were installed! {}".format( count, registered_models)) else: self.logger.debug( __name__, "{} LADM-COL Processing models were installed!".format( count)) def __register_processing_scripts(self, scripts_dir): count = 0 registered_scripts = list() qgis_scripts_dir = ScriptUtils.defaultScriptsFolder() for filename in glob.glob(os.path.join(scripts_dir, '*.py')): try: shutil.copy(filename, qgis_scripts_dir) count += 1 registered_scripts.append(os.path.basename(filename)) except OSError as e: self.logger.critical( __name__, "Couldn't install LADM-COL script '{}'!".format(filename)) if count: QgsApplication.processingRegistry().providerById( "script").refreshAlgorithms() if DEFAULT_LOG_MODE == EnumLogMode.DEV: self.logger.debug( __name__, "{} LADM-COL Processing scripts were installed! {}".format( count, registered_scripts)) else: self.logger.debug( __name__, "{} LADM-COL Processing scripts were installed!".format( count)) def register_add_on_processing_models(self, models_dir): """ For add-ons to delegate the registration of their own Processing models. :param models_dir: Path to the directory containing Processing models """ if 'model' in self.__processing_resources_installed: # The plugin already registered its models, so register right away. self.__register_processing_models(models_dir) else: # The plugin has not yet registered its models (waiting for Processing # to setup the 'model' provider). Pass the add-on's model dir to a list # of paths to be registered by the plugin, when Processing is ready. self.__processing_model_dirs_to_register.append(models_dir) def register_add_on_processing_scripts(self, scripts_dir): """ For add-ons to delegate the registration of their own Processing scripts. :param models_dir: Path to the directory containing Processing scripts """ if 'script' in self.__processing_resources_installed: # The plugin already registered its scripts, so register right away. self.__register_processing_scripts(scripts_dir) else: # The plugin has not yet registered its scripts (waiting for Processing # to setup the 'script' provider). Pass the add-on's script dir to a list # of paths to be registered by the plugin, when Processing is ready. self.__processing_script_dirs_to_register.append(scripts_dir) def unload_resources(self): # TODO: Also unregister models and scripts QgsApplication.processingRegistry().removeProvider( self.ladm_col_provider)
class AbstractQualityRule(QObject, metaclass=AbstractQObjectMeta): """ Abstract class for LADM-COL quality rules """ progress_changed = pyqtSignal(int) validation_finished = pyqtSignal(QualityRuleExecutionResult) def __init__(self): QObject.__init__(self) self.app = AppInterface() self.logger = Logger() self._id = "" # E.g., "IGAC-R1001" self._name = "" # E.g., "Los puntos de lindero no deben superponerse" self._type = None # E.g., EnumQualityRuleType.POINT self._tags = list( ) # List of keywords to search for this QR. Must be lowercase. self._models = list() # List of model keys required by this rule. # Dict with error codes (keys) and error messages (values) self._errors = dict() self.options = self._initialize_option_definition() # Optional. Only useful for display purposes. self._field_mapping = dict( ) # E.g., {'id_objetos': 'ids_punto_lindero', 'valores': 'conteo'} def id(self): return self._id def name(self): return self._name def type(self): return self._type def tags(self): return self._tags def models(self): return self._models.copy() def field_mapping(self, names): # Dictionary to overwrite display of quality error table fields # For instance, I can rename ERR_QUALITY_ERROR_T_OBJECT_IDS_F by "Plots" # so that a user is better informed of what is displayed return self._field_mapping @staticmethod def layers_config(names): # Dictionary of layer configuration. Specifies which layers are needed # by the rule and how it needs them for a 'tolerance > 0' scenario. return dict() def is_valid(self): return bool(self.id().strip()) and bool(self.name().strip()) and len( self._errors) and self._type is not None def validate(self, db, db_qr, layer_dict, tolerance, **kwargs): """ Validate the quality rule. :param db: DBConnector to main DB :param db_qr: DBConnector to quality rules DB :param layer_dict: Resolved layers (from LADM and/or after snapping) :param tolerance: Tolerance in millimeters :param kwargs: Other parameters needed by the rule. Optional. :return: An instance of QualityRuleExecutionResult """ res_obj = self._validate(db, db_qr, layer_dict, tolerance, **kwargs) self.validation_finished.emit(res_obj) return res_obj @abstractmethod def _validate(self, db, db_qr, layer_dict, tolerance, **kwargs): raise NotImplementedError def validate_features(self, features=None, feature_ids=list()): return False def _initialize_option_definition(self): """ Overwrite this method if needed. """ return QualityRuleOptions(list()) def _read_option_values(self, option_values): # Create member variables per option to ease value access for k, v in self.options.get_options().items(): setattr(self.options, k, option_values.get(k, v.default_value())) def _save_errors(self, db_qr, error_code, error_data, target_layer=None, ili_name=None): """ Save errors into DB with errores_calidad model structure :param db_qr: DBConnector of the target database :param error_code: Exactly as specified in error catalogs :param error_data: Dict of lists: {'geometries': [geometries], 'data': [obj_uuids, rel_obj_uuids, values, details, state]} Note: this dict will always have 2 elements. Note 2: For geometry errors, this dict will always have the same number of elements in each of the two lists (and the element order matters!). Note 3: For geometryless errors, the 'geometries' value won't be even read. :param target_layer: Useful if a rule of one type needs to write the error in an error layer that doesn't correspond to its type. For instance, a line QR that needs to write an error as point. By default None, which means the self._type should be used to know the target layer. :param ili_name: Interlis name of the class obj_uuids belong to. :return: Boolean, depending on whether the errors were saved or not. """ target_layer = self._type if target_layer is None else target_layer res, msg = QualityErrorDBUtils.save_errors(db_qr, self._id, error_code, error_data, target_layer, ili_name=None) if not res: self.logger.critical(__name__, msg) return res def _check_prerrequisite_layers(self, layer_dict): """ Use it when you don't need the layers themselves, but just to verify if the layers are valid and have features. """ for layer_name, layer in layer_dict[QUALITY_RULE_LAYERS].items(): res, obj = self._check_prerrequisite_layer(layer_name, layer) if not res: return res, obj return True, None def _check_prerrequisite_layer(self, layer_name, layer): if not layer: return False, QualityRuleExecutionResult( EnumQualityRuleResult.CRITICAL, QCoreApplication.translate( "QualityRules", "'{}' layer not found!").format(layer_name)) if layer.featureCount() == 0: return False, QualityRuleExecutionResult( EnumQualityRuleResult.UNDEFINED, QCoreApplication.translate( "QualityRules", "There are no records in layer '{}' to validate the quality rule!" ).format(layer.name())) return True, None def _check_qr_options(self, params): if self.options.get_num_mandatory_options(): mandatory_option_keys = [ o.id() for o in self.options.get_mandatory_option_list() ] if 'options' not in params: # No options at all return False, QualityRuleExecutionResult( EnumQualityRuleResult.CRITICAL, QCoreApplication.translate( "QualityRules", "No options were given to the quality rule, but it requires {} mandatory options ({})!" ).format(self.options.get_num_mandatory_options(), ", ".join(mandatory_option_keys))) # Now check that we've got all mandatory options not_found = [ k for k in mandatory_option_keys if k not in params['options'] ] if not_found: return False, QualityRuleExecutionResult( EnumQualityRuleResult.CRITICAL, QCoreApplication.translate( "QualityRules", "The following mandatory options were missing: '{}'!"). format("', '".join(not_found))) return True, None def _get_layer(self, layer_dict, layer_name=''): """ Get layer from layer_dict based on a layer name. If layer_name is not passed, the first layer in layer_dict will be returned, so this option is suitable for getting a layer when there is only one in the dict. """ if layer_name: return layer_dict[QUALITY_RULE_LAYERS][layer_name] else: for layer_name, layer in layer_dict[QUALITY_RULE_LAYERS].items(): return layer # Get the first one return None