def at_least_one_ladm_col_model_exists(self):
"""
Check that at least one hidden_and_supported model (hidden models are supposed to be the building blocks
of extended ones) is also supported in the DB and that there is at least one non-hidden_and_supported
model that is supported in the DB.
Note: Both hidden/non hidden and supported models depend on the active role,
which is already taken into account by LADMColModelRegistry().
"""
hidden_model_ids = [
model.id()
for model in LADMColModelRegistry().hidden_and_supported_models()
]
non_hidden_model_ids = [
model.id() for model in
LADMColModelRegistry().non_hidden_and_supported_models()
]
hidden_models_supported = list()
non_hidden_models_supported = list()
for model_id, is_supported in self.model_version_is_supported.items():
if model_id in hidden_model_ids:
hidden_models_supported.append(is_supported)
elif model_id in non_hidden_model_ids:
non_hidden_models_supported.append(is_supported)
return any(hidden_models_supported) and any(
non_hidden_models_supported)
def __refresh_mapping_for_role(self, db_models):
model_registry = LADMColModelRegistry()
for model in model_registry.supported_models():
if model.full_name() in db_models:
self.__register_db_mapping(
model.id(), model_registry.get_model_mapping(model.id()))
def __init__(self, uri, conn_dict=dict()):
QObject.__init__(self)
self.logger = Logger()
self.engine = ''
self.provider = '' # QGIS provider name. e.g., postgres
self._uri = None
self.schema = None
self.conn = None
self._dict_conn_params = None
self.names = DBMappingRegistry()
self.__db_mapping = None # To cache query response from the DB getting table and field names.
# Flag to control whether the DB connector should update the name registry. It should be True in two scenarios:
# 1) when the DB connector is created, and 2) when the plugin's active role has changed.
self._should_update_db_mapping_values = True
# Model parser instance. If it's None, it will be recreated.
# The ModelParser compares DB models vs. role supported models.
# It should be set to None in two scenarios:
# 1) when the DB connector is created, and 2) when the plugin's active role has changed.
self._model_parser = None
if uri is not None:
self.uri = uri
else:
self.dict_conn_params = conn_dict
self.__ladmcol_models = LADMColModelRegistry()
def decorated_function(*args, **kwargs):
inst = args[0] if type(
args[0]).__name__ == 'AsistenteLADMCOLPlugin' else args[0].ladmcol
context = args[1]
for db_source in context.get_db_sources():
db = inst.conn_manager.get_db_connector_from_source(
db_source=db_source)
db.test_connection()
if not db.cadastral_cartography_model_exists():
widget = inst.iface.messageBar().createMessage(
"Asistente LADM-COL",
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"Check your {} database connection. The '{} {}' model is required for this functionality, but could not be found in your current database. Click the button to go to Settings."
).format(
Tr.tr_db_source(db_source),
LADMColModelRegistry().model(
LADMNames.CADASTRAL_CARTOGRAPHY_MODEL_KEY).alias(),
LADMColModelRegistry().model(
LADMNames.CADASTRAL_CARTOGRAPHY_MODEL_KEY).alias())
)
button = QPushButton(widget)
button.setText(
QCoreApplication.translate("AsistenteLADMCOLPlugin",
"Settings"))
settings_context = SettingsContext(db_source)
settings_context.required_models = [
LADMNames.CADASTRAL_CARTOGRAPHY_MODEL_KEY
]
settings_context.tab_pages_list = [
SETTINGS_CONNECTION_TAB_INDEX
]
settings_context.title = QCoreApplication.translate(
"SettingsDialog", "{} Connection Settings").format(
Tr.tr_db_source(db_source))
settings_context.tip = QCoreApplication.translate(
"SettingsDialog",
"Set a DB connection with the '{}' model.").format(
LADMColModelRegistry().model(
LADMNames.CADASTRAL_CARTOGRAPHY_MODEL_KEY).alias())
button.pressed.connect(
partial(inst.show_settings_clear_message_bar,
settings_context))
widget.layout().addWidget(button)
inst.iface.messageBar().pushWidget(widget, Qgis.Warning, 15)
inst.logger.warning(
__name__,
QCoreApplication.translate(
"AsistenteLADMCOLPlugin",
"A dialog/tool couldn't be opened/executed, connection to DB was not valid."
))
return
func_to_decorate(*args, **kwargs)
def test_xtf_with_empty_element_tag(self):
found_models = get_models_from_xtf(
get_test_path("xtf/empty_element_tag.xtf"))
expected_models = [
LADMColModelRegistry().model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
LADMColModelRegistry().model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
LADMColModelRegistry().model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name()
]
self.assertEqual(expected_models.sort(), found_models.sort())
def _validate(self, db, db_qr, layer_dict, tolerance, **kwargs):
self.progress_changed.emit(5)
# First, run an ili2db --validate on the data
model = LADMColModelRegistry().model(LADMNames.SURVEY_MODEL_KEY)
res, msg = Ili2DB().validate(db, [model.full_name()], self._xtf_log)
if not res:
return QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL,
QCoreApplication.translate(
"QualityRules",
"There was an error running the quality rule '{}'! Details: '{}'."
).format(self._id, msg))
self.progress_changed.emit(85)
error_layer = self.app.core.get_layer(db_qr,
db_qr.names.ERR_QUALITY_ERROR_T,
load=True)
count_before = error_layer.featureCount()
# Write errors to QR DB
res, msg = IliVErrorsToErroresCalidad01Converter().convert(
self._xtf_log, db_qr, params=dict())
self.progress_changed.emit(100)
if not res:
return QualityRuleExecutionResult(
EnumQualityRuleResult.CRITICAL,
QCoreApplication.translate(
"QualityRules",
"There was an error running the quality rule '{}'! Details: '{}'."
)).format(self._id, msg)
else:
count = error_layer.featureCount() - count_before
if count:
return QualityRuleExecutionResult(
EnumQualityRuleResult.ERRORS,
QCoreApplication.translate(
"QualityRules",
"There were {} errors validating the data against their model!"
).format(count), count)
else:
return QualityRuleExecutionResult(
EnumQualityRuleResult.SUCCESS,
QCoreApplication.translate(
"QualityRules", "The data comply with their model."))
def __load_available_layers(self):
# Call qgis model baker tables_info and fill the tree
tables_info = self.app.core.get_cached_layers()
self.models_tree = dict()
ladmcol_models = {
model.full_name(): model.alias()
for model in LADMColModelRegistry().supported_models()
}
for record in tables_info:
if record[QueryNames.MODEL] is not None:
model = ladmcol_models.get(record[QueryNames.MODEL],
record[QueryNames.MODEL])
if model not in self.models_tree:
self.models_tree[model] = {
record[QueryNames.TABLE_ALIAS] or record[QueryNames.TABLE_NAME_MODEL_BAKER]:
record
}
else:
self.models_tree[model][
record[QueryNames.TABLE_ALIAS]
or record[QueryNames.TABLE_NAME_MODEL_BAKER]] = record
self.__update_available_layers()
def open_schema_import_dialog_model2():
open_schema_import_dialog({
'selected_models': [
LADMColModelRegistry().model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name()
]
})
def get_basket_id_for_new_receiver(self):
# 1. Make sure we've got the FDC dataset
dataset_t_id, msg = self._get_fdc_dataset()
if dataset_t_id is None:
return None, QCoreApplication.translate(
"FieldDataCaptureAdminController",
"The Field Data Capture dataset does not exist and couldn't be created! No coordinator can be created."
)
# 2. Get a new basket for such dataset
fdc_model = LADMColModelRegistry().model(
LADMNames.FIELD_DATA_CAPTURE_MODEL_KEY).full_name()
topic_name = "{}.{}".format(fdc_model, LADMNames.FDC_TOPIC_NAME)
basket_feature, msg = self._ladm_data.create_ili2db_basket(
self._db, dataset_t_id, topic_name)
if basket_feature is None:
return None, QCoreApplication.translate(
"FieldDataCaptureAdminController",
"The new coordinator could not be created. Details: Basket could not be created!"
)
basket_t_id = basket_feature[self._db.names.T_ID_F]
return basket_t_id, "Success!"
def get_ili_models(self):
ili_models = list()
model_names = self._db.get_models()
if model_names:
for model in LADMColModelRegistry().supported_models():
if not model.hidden() and model.full_name() in model_names:
ili_models.append(model.full_name())
return ili_models
def get_field_admin_role_models():
"""
Function to delay configuration for admin role models, since it
needs to overwrite ili2db params from the main model config
"""
from asistente_ladm_col.lib.model_registry import LADMColModelRegistry
field_admin_role_models = COMMON_ROLE_MODELS.copy()
field_admin_role_models[ROLE_SUPPORTED_MODELS] = COMMON_SUPPORTED_MODELS + [LADMNames.FIELD_DATA_CAPTURE_MODEL_KEY]
field_admin_role_models[ROLE_CHECKED_MODELS] = [LADMNames.FIELD_DATA_CAPTURE_MODEL_KEY]
fdc_model = LADMColModelRegistry().model(LADMNames.FIELD_DATA_CAPTURE_MODEL_KEY)
params = fdc_model.get_ili2db_params()
if ILI2DB_UPDATE in params:
params[ILI2DB_UPDATE].append((ILI2DB_DATASET_KEY, FDC_DATASET_NAME))
else:
params[ILI2DB_UPDATE] = [(ILI2DB_DATASET_KEY, FDC_DATASET_NAME)]
field_admin_role_models[ROLE_MODEL_ILI2DB_PARAMETERS] = {LADMNames.FIELD_DATA_CAPTURE_MODEL_KEY: params}
return field_admin_role_models
def setUpClass(cls):
print(
"\nINFO: Setting up copy layer With different Geometries to DB validation..."
)
print("INFO: Restoring databases to be used")
cls.app = AppInterface()
cls.base_test_path = tempfile.mkdtemp()
cls.ladmcol_models = LADMColModelRegistry()
def test_xtf_with_start_and_closing_tag(self):
found_models = get_models_from_xtf(
get_test_path("xtf/start_and_closing_tag.xtf"))
# print(get_models_from_xtf("/docs/borrar/xtf/empty_element_tag.xtf"))
expected_models = [
LADMColModelRegistry().model(
LADMNames.SUPPLIES_MODEL_KEY).full_name()
]
self.assertEqual(expected_models, found_models)
def get_qrs_per_role_and_models(self, db, as_dict=True):
"""
:param as_dict: Boolean. If False, the result is returned as a list or rule keys
"""
qrs = dict()
role_registry = RoleRegistry()
role_qrs = role_registry.get_role_quality_rules(role_registry.get_active_role())
if role_qrs == ALL_QUALITY_RULES:
role_qrs = self.__quality_rules
if role_qrs:
db_models = db.get_models()
model_registry = LADMColModelRegistry()
for qr in role_qrs:
# First check if the role QR is registered
if qr in self.__quality_rules:
# Then check if the models required by the QR are in the DB
req_models = self.__quality_rules[qr].models()
num_models = len(req_models)
all_models_found = True
if num_models: # We don't check models if a QR has no required models (e.g., iliValidator)
for req_model in req_models:
model = model_registry.model(req_model)
model_key = model.full_name()
if model_key and model_key not in db_models:
all_models_found = False
self.logger.debug(__name__,
"Model '{}' not found in the DB. QR '{}' cannot be listed.".format(
model_key, qr
))
break
if all_models_found:
qrs[qr] = self.__quality_rules[qr]
return qrs if as_dict else list(qrs.keys())
def __init__(self, db):
QObject.__init__(self)
self.logger = Logger()
ladmcol_models = LADMColModelRegistry()
self.current_model_version = {
model_id: None
for model_id in ladmcol_models.model_keys()
}
self.model_version_is_supported = {
model_id: False
for model_id in ladmcol_models.model_keys()
}
self._db = db
# Fill versions for each model found
for current_model_name in self._get_models():
for model_key, v in self.current_model_version.items():
if current_model_name.startswith(model_key):
parts = current_model_name.split(model_key)
if len(parts) > 1:
current_version = self.parse_version(parts[1])
current_version_valid = is_version_valid(
current_version,
ladmcol_models.model(
model_key).supported_version(),
True, # Exact version required
QCoreApplication.translate("ModelParser",
model_key))
self.current_model_version[model_key] = current_version
self.model_version_is_supported[
model_key] = current_version_valid
self.logger.debug(
__name__, "Model '{}' found! Valid: {}".format(
model_key, current_version_valid))
break
def update_model_names(self):
self.export_models_qmodel = QStandardItemModel()
model_names = self.db.get_models()
if model_names:
for model in LADMColModelRegistry().supported_models():
if not model.hidden() and model.full_name() in model_names:
item = QStandardItem(model.full_alias())
item.setData(model.full_name(), Qt.UserRole)
item.setCheckable(False)
item.setEditable(False)
self.export_models_qmodel.appendRow(item)
self.export_models_list_view.setModel(self.export_models_qmodel)
def __init__(self,
iface,
conn_manager,
context,
selected_models=list(),
link_to_import_data=True,
parent=None):
QDialog.__init__(self, parent)
self.iface = iface
self.conn_manager = conn_manager
self.selected_models = selected_models
self.link_to_import_data = link_to_import_data
self.logger = Logger()
self.app = AppInterface()
self.__ladmcol_models = LADMColModelRegistry()
self.java_dependency = JavaDependency()
self.java_dependency.download_dependency_completed.connect(
self.download_java_complete)
self.java_dependency.download_dependency_progress_changed.connect(
self.download_java_progress_change)
self.db_source = context.get_db_sources()[0]
self.db = self.conn_manager.get_db_connector_from_source(
self.db_source)
self._dbs_supported = ConfigDBsSupported()
self._running_tool = False
# There may be two cases where we need to emit a db_connection_changed from the Schema Import dialog:
# 1) Connection Settings was opened and the DB conn was changed.
# 2) Connection Settings was never opened but the Schema Import ran successfully, in a way that new models may
# convert a db/schema LADM-COL compliant.
self._db_was_changed = False # To postpone calling refresh gui until we close this dialog instead of settings
# Similarly, we could call a refresh on layers and relations cache in two cases:
# 1) If the SI dialog was called for the COLLECTED source: opening Connection Settings and changing the DB
# connection.
# 2) Not opening the Connection Settings, but running a successful Schema Import on the COLLECTED DB, which
# invalidates the cache as models change.
self._schedule_layers_and_relations_refresh = False
self.setupUi(self)
self.validators = Validators()
self.update_import_models()
self.previous_item = QListWidgetItem()
self.connection_setting_button.clicked.connect(self.show_settings)
self.connection_setting_button.setText(
QCoreApplication.translate("DialogImportSchema",
"Connection Settings"))
# CRS Setting
self.srs_auth = DEFAULT_SRS_AUTH
self.srs_code = DEFAULT_SRS_CODE
self.crsSelector.crsChanged.connect(self.crs_changed)
# LOG
self.log_config.setTitle(
QCoreApplication.translate("DialogImportSchema", "Show log"))
self.buttonBox.accepted.disconnect()
self.buttonBox.clicked.connect(self.accepted_import_schema)
self.buttonBox.clear()
self.buttonBox.addButton(QDialogButtonBox.Cancel)
self._accept_button = self.buttonBox.addButton(
self.BUTTON_NAME_CREATE_STRUCTURE, QDialogButtonBox.AcceptRole)
self.buttonBox.addButton(QDialogButtonBox.Help)
self.buttonBox.helpRequested.connect(self.show_help)
self.import_models_list_widget.setDisabled(bool(
selected_models)) # If we got models from params, disable panel
self.update_connection_info()
self.restore_configuration()
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
class DBConnector(QObject):
"""
Superclass for all DB connectors.
"""
_DEFAULT_VALUES = dict(
) # You should set it, so that testing empty parameters can be handled easily.
def __init__(self, uri, conn_dict=dict()):
QObject.__init__(self)
self.logger = Logger()
self.engine = ''
self.provider = '' # QGIS provider name. e.g., postgres
self._uri = None
self.schema = None
self.conn = None
self._dict_conn_params = None
self.names = DBMappingRegistry()
self.__db_mapping = None # To cache query response from the DB getting table and field names.
# Flag to control whether the DB connector should update the name registry. It should be True in two scenarios:
# 1) when the DB connector is created, and 2) when the plugin's active role has changed.
self._should_update_db_mapping_values = True
# Model parser instance. If it's None, it will be recreated.
# The ModelParser compares DB models vs. role supported models.
# It should be set to None in two scenarios:
# 1) when the DB connector is created, and 2) when the plugin's active role has changed.
self._model_parser = None
if uri is not None:
self.uri = uri
else:
self.dict_conn_params = conn_dict
self.__ladmcol_models = LADMColModelRegistry()
@property
def dict_conn_params(self):
return self._dict_conn_params.copy()
@dict_conn_params.setter
def dict_conn_params(self, dict_values):
dict_values = {k: v
for k, v in dict_values.items() if v
} # To avoid empty values to overwrite default values
self._dict_conn_params = self._DEFAULT_VALUES.copy()
self._dict_conn_params.update(dict_values)
self._uri = self.get_connection_uri(self._dict_conn_params, level=1)
@property
def uri(self):
return self._uri
@uri.setter
def uri(self, value):
raise NotImplementedError
def equals(self, db):
return self.dict_conn_params == db.dict_conn_params
def _table_exists(self, table_name):
raise NotImplementedError
def _metadata_exists(self):
raise NotImplementedError
def has_basket_col(self):
raise NotImplementedError
def close_connection(self):
raise NotImplementedError
def get_description(self):
return "Current connection details: '{}' -> {} {}".format(
self.engine, self._uri,
'schema:{}'.format(self.schema) if self.schema else '')
def get_ladm_units(self):
raise NotImplementedError
def get_models(self, schema=None):
raise NotImplementedError
def get_display_conn_string(self):
# Do not use to connect to a DB, only for display purposes
tmp_dict_conn_params = self._dict_conn_params.copy()
if 'password' in tmp_dict_conn_params:
del tmp_dict_conn_params['password']
return ' '.join(
["{}={}".format(k, v) for k, v in tmp_dict_conn_params.items()])
def get_description_conn_string(self):
raise NotImplementedError
def get_connection_uri(self, dict_conn, level=1):
"""
:param dict_conn: (dict) dictionary with the parameters to establish a connection
:param level: (int) At what level the connection will be established
0: server level
1: database level
:return: (str) string uri to establish a connection
"""
raise NotImplementedError
def reset_db_model_parser(self):
"""
Call it to let the connector know it has to update the DB model parser (e.g.,
when the active role has changed, since the new one could support other models).
"""
self._model_parser = None
def read_model_parser(self):
if self._model_parser is None:
try:
self._model_parser = ModelParser(self)
except psycopg2.ProgrammingError as e:
# if it is not possible to access the schema due to lack of privileges
return False
return True
def is_ladm_layer(self, layer):
"""
Whether the layer corresponds to the DB connector source or not.
"""
raise NotImplementedError
def get_ladm_layer_name(self, layer, validate_is_ladm=False):
raise NotImplementedError
def get_ili2db_version(self):
raise NotImplementedError
def get_db_mapping(self):
"""
Cache the get_db_mapping call, which should be called only once per db connector.
:return: A dict with table ilinames as keys and dict as values. See __get_db_mapping for details.
"""
if not self.__db_mapping:
self.__db_mapping = self.__get_db_mapping()
return self.__db_mapping
def __get_db_mapping(self):
"""
Get table and field names from the DB. Should be called only once for a single connection.
:return: dict with table ilinames as keys and dict as values. The dicts found in the value contain field
ilinames as keys and sqlnames as values. The table name itself is added with the key 'table_name'.
Example:
"LADM_COL.LADM_Nucleo.col_masCcl": {
'table_name': 'col_masccl',
'LADM_COL.LADM_Nucleo.col_masCcl.ccl_mas..Levantamiento_Catastral.Levantamiento_Catastral.LC_Lindero': 'ccl_mas',
'LADM_COL.LADM_Nucleo.col_masCcl.ue_mas..Levantamiento_Catastral.Levantamiento_Catastral.LC_Construccion': 'ue_mas_lc_construccion',
'LADM_COL.LADM_Nucleo.col_masCcl.ue_mas..Levantamiento_Catastral.Levantamiento_Catastral.LC_ServidumbreTransito': 'ue_mas_lc_servidumbretransito',
'LADM_COL.LADM_Nucleo.col_masCcl.another_ili_attr': 'corresponding_sql_name'
}
"""
# Get both table and field names. Only include field names that are not FKs, they will be added in a second step
records = self._get_table_and_field_names()
dict_names = dict()
for record in records:
if record[QueryNames.TABLE_ILINAME] is None:
# Any t_ili2db_* tables (INTERLIS meta-attrs)
continue
table_iliname = normalize_iliname(record[QueryNames.TABLE_ILINAME])
if not table_iliname in dict_names:
dict_names[table_iliname] = dict()
dict_names[table_iliname][QueryNames.TABLE_NAME] = record[
QueryNames.TABLE_NAME]
if record[QueryNames.FIELD_ILINAME] is None:
# Fields for domains, like 'description' (we map it in a custom way later in this class method)
continue
field_iliname = normalize_iliname(record[QueryNames.FIELD_ILINAME])
dict_names[table_iliname][field_iliname] = record[
QueryNames.FIELD_NAME]
# Map FK ilinames (i.e., those whose t_ili2db_attrname target column is not NULL)
# Spatial_Unit-->Ext_Address_ID (Ext_Address)
# Key: "LADM_COL_V1_2.LADM_Nucleo.COL_UnidadEspacial.Ext_Direccion_ID"
# Values: lc_construccion_ext_direccion_id and lc_terreno_ext_direccion_id
records = self._get_fk_fields()
for record in records:
composed_key = "{}{}{}".format(
normalize_iliname(record['iliname']), COMPOSED_KEY_SEPARATOR,
normalize_iliname(record['iliname2']))
table_iliname = normalize_iliname(record[QueryNames.TABLE_ILINAME])
if table_iliname in dict_names:
dict_names[table_iliname][composed_key] = record['sqlname']
else:
colowner = normalize_iliname(record['colowner'])
if colowner in dict_names:
dict_names[colowner][composed_key] = record['sqlname']
dict_names.update(self._get_ili2db_names(
)) # Now add ili2db names like t_id, t_ili_id, t_basket, etc.
return dict_names
def _get_table_and_field_names(self):
"""Gets both table and field names from DB. Only includes field names that are not FKs.
Execute below Sql statement (pseudo-SQL):
SELECT
IliName AS {QueryNames.TABLE_ILINAME}, -- (1)
table_name AS {QueryNames.TABLE_NAME}, -- (2)
IliName AS {QueryNames.FIELD_ILINAME}, -- (3)
SqlName AS {QueryNames.FIELD_NAME} -- (4)
FROM Dbms_tbl_metadata INNER JOIN T_ILI2DB_CLASSNAME LEFT JOIN T_ILI2DB_ATTRNAME
WHERE ilicol.Target IS NULL (because it does not include FKs)
*Dbms_tbl_metadata="Metadata table of specific DBMS"
+-----------------+ +------------------+ +-----------------+
|Dbms_tbl_metadata| |T_ILI2DB_CLASSNAME| |T_ILI2DB_ATTRNAME|
|-----------------| |------------------| |-----------------|
|table_name (2)+ +---+ |IliName (1) | |IliName (3) |
+--------------|--+ +---+SqlName | |SqlName (4) |
| +------------------+ +---+ColOwner |
| | |Target |
+---------------------------------+ +-----------------+
:return: dict
"""
raise NotImplementedError
def _get_fk_fields(self):
"""Maps FK ilinames (i.e., those whose t_ili2db_attrname target column is not NULL)
i.e.
Spatial_Unit-->Ext_Address_ID (Ext_Address)
Key: "LADM_COL_V1_2.LADM_Nucleo.COL_UnidadEspacial.Ext_Direccion_ID"
Values: lc_construccion_ext_direccion_id and lc_terreno_ext_direccion_id
Execute below Sql statement (seudo-SQL):
SELECT "iliname before the last point" as QueryNames.TABLE_ILINAME,
iliname, -- (2)
sqlname, -- (3)
iliname as iliname2, (4)
iliname as colowner
FROM T_ILI2DB_CLASSNAME AS main_class INNER_JOIN T_ILI2DB_ATTRNAME INNER JOIN T_ILI2DB_CLASSNAME as target class
+------------------+ +-----------------+ +------------------+
|T_ILI2DB_CLASSNAME| |T_ILI2DB_ATTRNAME| |T_ILI2DB_CLASSNAME|
| (main class) | |-----------------| | (target class) |
|------------------| |IliName (1,2) | |------------------|
|IliName (5) | |SqlName (3) | |IliName |
|SqlName <------------+ColOwner | +-->SqlName (4) |
+------------------+ |Target +------+ +------------------+
+-----------------+
:return: dict
"""
raise NotImplementedError
def _get_ili2db_names(self):
"""Returns field common names of databases, e.g., T_Id, T_Ili_Tid, dispName, t_basket, etc.
:return: Dictionary with ili2db keys:
T_ID_KEY, T_ILI_TID_KEY, etc., from db_mapping_registry
"""
raise NotImplementedError
def _initialize_names(self):
"""
Gets table and field names from the DB and initializes the db mapping values in the registry.
Could be called more than once per connector, namely, when the role changes, the db mapping registry values
should be updated.
"""
self.logger.info(__name__,
"Resetting db mapping registry values from the DB!")
self.names.initialize_table_and_field_names(self.get_db_mapping(),
self.get_models())
self._should_update_db_mapping_values = False # Since we just updated, avoid it for the next test_connection.
# self.logger.debug(__name__, "DEBUG DICT: {}".format(dict_names["Operacion.Operacion.OP_Derecho"]))
def reset_db_mapping_values(self):
"""
Call it to let the connector know it has to update the DB mapping values in the DB Mapping Registry (e.g., when
the active role has changed, since the new one could support other models).
"""
self._should_update_db_mapping_values = True
def check_db_models(self, models):
code = EnumTestConnectionMsg.DB_MODELS_ARE_CORRECT
if models.get(REQUIRED_MODELS):
res, msg = self.check_required_models(models[REQUIRED_MODELS])
if not res:
code = EnumTestConnectionMsg.REQUIRED_LADM_MODELS_NOT_FOUND
else:
# 2 options here: if models has info, we test against such info, but if not,
# we test against the info from model parser. In both scenarios, we test that
# all hidden (base) models are in the DB and that at least one of the non-hidden
# models is present in the DB.
res, msg = self.at_least_one_ladm_col_model_exists(models)
if not res:
code = EnumTestConnectionMsg.NO_LADM_MODELS_FOUND_IN_SUPPORTED_VERSION
return res, code, msg
def check_required_models(self, models):
msg = QCoreApplication.translate("DBConnector",
"All required models are in the DB!")
not_found = [
model for model in models if not self.ladm_col_model_exists(model)
]
if not_found:
msg = QCoreApplication.translate(
"DBConnector",
"The following required model(s) could not be found in the DB: {}."
).format(', '.join(not_found))
return not bool(not_found), msg
def at_least_one_ladm_col_model_exists(self, models):
if self.read_model_parser():
if models.get(ROLE_SUPPORTED_MODELS) and models.get(
ROLE_HIDDEN_MODELS):
# We won't test against model registry (active role), but only against info in models dict
# All hidden models should exist in the DB
not_found = [
model for model in models[ROLE_HIDDEN_MODELS]
if not self.ladm_col_model_exists(model)
]
if not_found:
return False, QCoreApplication.translate(
"DBConnector",
"The following required model(s) could not be found in the DB: '{}'."
).format(', '.join(not_found))
# At least one non-hidden model should exist in the DB
non_hidden_models = [
model for model in models[ROLE_SUPPORTED_MODELS]
if model not in models[ROLE_HIDDEN_MODELS]
]
found = [
model for model in non_hidden_models
if self.ladm_col_model_exists(model)
]
if not found:
return False, QCoreApplication.translate(
"DBConnector",
"At least one of the following required model(s) should exist in the DB: '{}'. Your DB has '{}'."
).format(', '.join(non_hidden_models),
', '.join(self.get_models()))
else:
# Go to the model_parser, which deals with model_registry and will use info from the active role.
msg = ''
res = self._model_parser.at_least_one_ladm_col_model_exists()
if not res:
msg = QCoreApplication.translate(
"DBConnector",
"At least one LADM-COL model should exist in the required version (besides the basic ones: '{}')! Supported models are: '{}', but your DB has '{}'"
).format(
', '.join([
m.full_name() for m in self.__ladmcol_models.
hidden_and_supported_models()
]), ', '.join([
m.full_name()
for m in self.__ladmcol_models.supported_models()
]), ', '.join(self.get_models()))
return False, msg
return True, ""
return False, "Error getting the model parser!"
def ladm_col_model_exists(self, model_prefix):
if self.read_model_parser():
return self._model_parser.ladm_col_model_exists(model_prefix)
return False
def survey_model_exists(self):
if self.read_model_parser():
return self._model_parser.survey_model_exists()
return False
def valuation_model_exists(self):
if self.read_model_parser():
return self._model_parser.valuation_model_exists()
return False
def ladm_model_exists(self):
if self.read_model_parser():
return self._model_parser.ladm_model_exists()
return False
def cadastral_cartography_model_exists(self):
if self.read_model_parser():
return self._model_parser.cadastral_cartography_model_exists()
return False
def snr_data_model_exists(self):
if self.read_model_parser():
return self._model_parser.snr_data_model_exists()
return False
def supplies_integration_model_exists(self):
if self.read_model_parser():
return self._model_parser.supplies_integration_model_exists()
return False
def supplies_model_exists(self):
if self.read_model_parser():
return self._model_parser.supplies_model_exists()
return False
def open_connection(self):
"""
:return: Whether the connection is opened after calling this method or not
"""
raise NotImplementedError
def test_connection(self,
test_level=EnumTestLevel.LADM,
user_level=EnumUserLevel.CONNECT,
models={}):
"""
'Template method' subclasses should overwrite it, proposing their own way to test a connection.
"""
raise NotImplementedError
def _test_connection_to_db(self):
raise NotImplementedError
def _test_connection_to_ladm(self, models):
raise NotImplementedError
def _db_should_have_basket_support(self):
"""
:return: Tuple: Whether the current DB should have baskets or not, name of the 1st model that requires baskets!
"""
# Get models in the DB that are supported and not hidden
model_names_in_db = self.get_models()
if model_names_in_db:
for model in self.__ladmcol_models.supported_models():
if not model.hidden() and model.full_name(
) in model_names_in_db:
params = model.get_ili2db_params(
) # Note: params depend on the model and on the active role
if ILI2DB_SCHEMAIMPORT in params:
for param in params[
ILI2DB_SCHEMAIMPORT]: # List of tuples
if param[
0] == ILI2DB_CREATE_BASKET_COL_KEY: # param: (option, value)
self.logger.debug(
__name__,
"Model '{}' requires baskets...".format(
model.alias()))
return True, model.alias()
return False, ''
@staticmethod
def _parse_models_from_db_meta_attrs(lst_models):
"""
Reads a list of models as saved by ili2db and returns a dict of model dependencies.
E.g.:
INPUT-> ["D_G_C_V2_9_6{ LADM_COL_V1_2 ISO19107_PLANAS_V1} D_SNR_V2_9_6{ LADM_COL_V1_2} D_I_I_V2_9_6{ D_SNR_V2_9_6 D_G_C_V2_9_6}", "LADM_COL_V1_2"]
OUTPUT-> {'D_G_C_V2_9_6': ['LADM_COL_V1_2', 'ISO19107_PLANAS_V1'], 'D_SNR_V2_9_6': ['LADM_COL_V1_2'], 'D_I_I_V2_9_6': ['D_SNR_V2_9_6', 'D_G_C_V2_9_6'], 'LADM_COL_V1_2': []}
:param lst_models: The list of values stored in the DB meta attrs model table (column 'modelname').
:return: Dict of model dependencies.
"""
model_hierarchy = dict()
for str_model in lst_models:
parts = str_model.split("}")
if len(parts) > 1: # With dependencies
for part in parts:
if part: # The last element of parts is ''
model, dependencies = part.split("{")
model_hierarchy[
model.strip()] = dependencies.strip().split(" ")
elif len(parts) == 1: # No dependencies
model_hierarchy[parts[0].strip()] = list()
return model_hierarchy
def get_quality_error_connector(self,
output_path,
timestamp,
load_layers=False):
# TODO: should we support both new and existent gpkgs in this method? I guess so!
self.progress_changed.emit(0)
res, msg, output_path = QualityErrorDBUtils.get_quality_validation_output_path(
output_path, timestamp)
if not res:
return False, msg, None
db_file = os.path.join(output_path,
"Reglas_de_Calidad_{}.gpkg".format(timestamp))
db = GPKGConnector(db_file)
ili2db = Ili2DB()
error_model = LADMColModelRegistry().model(
LADMNames.QUALITY_ERROR_MODEL_KEY)
configuration = ili2db.get_import_schema_configuration(
db, [error_model.full_name()])
res, msg = ili2db.import_schema(db, configuration)
self.progress_changed.emit(50)
if res:
for catalog_key, catalog_xtf_path in error_model.get_catalogs(
).items():
logger.info(
__name__,
"Importing catalog '{}' to quality error database...".
format(catalog_key))
configuration = ili2db.get_import_data_configuration(
db, catalog_xtf_path)
res_xtf, msg_xtf = ili2db.import_data(db, configuration)
if not res_xtf:
logger.warning(
__name__,
"There was a problem importing catalog '{}'! Skipping..."
.format(catalog_key))
else:
return False, "There were errors creating the 'Errores Calidad' structure!", None
self.progress_changed.emit(80)
if getattr(db.names, "T_ID_F",
None) is None or db.names.T_ID_F is None:
db.test_connection() # Just to build the names object
if load_layers:
names = db.names
layers = {
names.ERR_QUALITY_ERROR_T: None,
names.ERR_RULE_TYPE_T: None,
names.ERR_ERROR_TYPE_T: None,
names.ERR_POINT_T: None,
names.ERR_LINE_T: None,
names.ERR_POLYGON_T: None,
names.ERR_METADATA_T: None,
names.ERR_ERROR_STATE_D: None
}
app.core.get_layers(
db,
layers,
load=True,
group=QualityErrorDBUtils.get_quality_error_group(timestamp))
self.progress_changed.emit(100)
return res, msg, None if not res else db
def __init__(self, iface, conn_manager, context, link_to_import_schema=True, parent=None):
QDialog.__init__(self, parent)
self.setupUi(self)
QgsGui.instance().enableAutoGeometryRestore(self)
self.iface = iface
self.conn_manager = conn_manager
self.db_source = context.get_db_sources()[0]
self.link_to_import_schema = link_to_import_schema
self.db = self.conn_manager.get_db_connector_from_source(self.db_source)
self.logger = Logger()
self.app = AppInterface()
self.__ladmcol_models = LADMColModelRegistry()
self.java_dependency = JavaDependency()
self.java_dependency.download_dependency_completed.connect(self.download_java_complete)
self.java_dependency.download_dependency_progress_changed.connect(self.download_java_progress_change)
self._dbs_supported = ConfigDBsSupported()
self._running_tool = False
# There may be 1 case where we need to emit a db_connection_changed from the Import Data dialog:
# 1) Connection Settings was opened and the DB conn was changed.
self._db_was_changed = False # To postpone calling refresh gui until we close this dialog instead of settings
# Similarly, we could call a refresh on layers and relations cache in 1 case:
# 1) If the ID dialog was called for the COLLECTED source: opening Connection Settings and changing the DB
# connection.
self._schedule_layers_and_relations_refresh = False
# We need bar definition above calling clear_messages
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
self.xtf_file_browse_button.clicked.connect(
make_file_selector(self.xtf_file_line_edit, title=QCoreApplication.translate("DialogImportData", "Open Transfer or Catalog File"),
file_filter=QCoreApplication.translate("DialogImportData",'Transfer File (*.xtf *.itf);;Catalog File (*.xml *.xls *.xlsx)')))
self.validators = Validators()
self.xtf_file_line_edit.setPlaceholderText(QCoreApplication.translate("DialogImportData", "[Name of the XTF to be imported]"))
fileValidator = FileValidator(pattern=['*.xtf', '*.itf', '*.xml'])
self.xtf_file_line_edit.setValidator(fileValidator)
self.xtf_file_line_edit.textChanged.connect(self.update_import_models)
self.xtf_file_line_edit.textChanged.emit(self.xtf_file_line_edit.text())
# db
self.connection_setting_button.clicked.connect(self.show_settings)
self.connection_setting_button.setText(QCoreApplication.translate("DialogImportData", "Connection Settings"))
# LOG
self.log_config.setTitle(QCoreApplication.translate("DialogImportData", "Show log"))
self.buttonBox.accepted.disconnect()
self.buttonBox.clicked.connect(self.accepted_import_data)
self.buttonBox.clear()
self.buttonBox.addButton(QDialogButtonBox.Cancel)
self._accept_button = self.buttonBox.addButton(self.BUTTON_NAME_IMPORT_DATA, QDialogButtonBox.AcceptRole)
self.buttonBox.addButton(QDialogButtonBox.Help)
self.buttonBox.helpRequested.connect(self.show_help)
self.update_connection_info()
self.restore_configuration()
class DialogImportData(QDialog, DIALOG_UI):
open_dlg_import_schema = pyqtSignal(dict) # dict with key-value params
BUTTON_NAME_IMPORT_DATA = QCoreApplication.translate("DialogImportData", "Import data")
BUTTON_NAME_GO_TO_CREATE_STRUCTURE = QCoreApplication.translate("DialogImportData", "Go to Create Structure...")
def __init__(self, iface, conn_manager, context, link_to_import_schema=True, parent=None):
QDialog.__init__(self, parent)
self.setupUi(self)
QgsGui.instance().enableAutoGeometryRestore(self)
self.iface = iface
self.conn_manager = conn_manager
self.db_source = context.get_db_sources()[0]
self.link_to_import_schema = link_to_import_schema
self.db = self.conn_manager.get_db_connector_from_source(self.db_source)
self.logger = Logger()
self.app = AppInterface()
self.__ladmcol_models = LADMColModelRegistry()
self.java_dependency = JavaDependency()
self.java_dependency.download_dependency_completed.connect(self.download_java_complete)
self.java_dependency.download_dependency_progress_changed.connect(self.download_java_progress_change)
self._dbs_supported = ConfigDBsSupported()
self._running_tool = False
# There may be 1 case where we need to emit a db_connection_changed from the Import Data dialog:
# 1) Connection Settings was opened and the DB conn was changed.
self._db_was_changed = False # To postpone calling refresh gui until we close this dialog instead of settings
# Similarly, we could call a refresh on layers and relations cache in 1 case:
# 1) If the ID dialog was called for the COLLECTED source: opening Connection Settings and changing the DB
# connection.
self._schedule_layers_and_relations_refresh = False
# We need bar definition above calling clear_messages
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
self.xtf_file_browse_button.clicked.connect(
make_file_selector(self.xtf_file_line_edit, title=QCoreApplication.translate("DialogImportData", "Open Transfer or Catalog File"),
file_filter=QCoreApplication.translate("DialogImportData",'Transfer File (*.xtf *.itf);;Catalog File (*.xml *.xls *.xlsx)')))
self.validators = Validators()
self.xtf_file_line_edit.setPlaceholderText(QCoreApplication.translate("DialogImportData", "[Name of the XTF to be imported]"))
fileValidator = FileValidator(pattern=['*.xtf', '*.itf', '*.xml'])
self.xtf_file_line_edit.setValidator(fileValidator)
self.xtf_file_line_edit.textChanged.connect(self.update_import_models)
self.xtf_file_line_edit.textChanged.emit(self.xtf_file_line_edit.text())
# db
self.connection_setting_button.clicked.connect(self.show_settings)
self.connection_setting_button.setText(QCoreApplication.translate("DialogImportData", "Connection Settings"))
# LOG
self.log_config.setTitle(QCoreApplication.translate("DialogImportData", "Show log"))
self.buttonBox.accepted.disconnect()
self.buttonBox.clicked.connect(self.accepted_import_data)
self.buttonBox.clear()
self.buttonBox.addButton(QDialogButtonBox.Cancel)
self._accept_button = self.buttonBox.addButton(self.BUTTON_NAME_IMPORT_DATA, QDialogButtonBox.AcceptRole)
self.buttonBox.addButton(QDialogButtonBox.Help)
self.buttonBox.helpRequested.connect(self.show_help)
self.update_connection_info()
self.restore_configuration()
def accepted_import_data(self, button):
if self.buttonBox.buttonRole(button) == QDialogButtonBox.AcceptRole:
if button.text() == self.BUTTON_NAME_IMPORT_DATA:
self.accepted()
elif button.text() == self.BUTTON_NAME_GO_TO_CREATE_STRUCTURE:
self.close() # Close import data dialog
self.open_dlg_import_schema.emit({'selected_models': self.get_ili_models()}) # Emit signal to open import schema dialog
def reject(self):
if self._running_tool:
QMessageBox.information(self,
QCoreApplication.translate("DialogImportData", "Warning"),
QCoreApplication.translate("DialogImportData", "The Import Data tool is still running. Please wait until it finishes."))
else:
self.close_dialog()
def close_dialog(self):
"""
We use this method to be safe when emitting the db_connection_changed, otherwise we could trigger slots that
unload the plugin, destroying dialogs and thus, leading to crashes.
"""
if self._schedule_layers_and_relations_refresh:
self.conn_manager.db_connection_changed.connect(self.app.core.cache_layers_and_relations)
if self._db_was_changed:
# If the db was changed, it implies it complies with ladm_col, hence the second parameter
self.conn_manager.db_connection_changed.emit(self.db, True, self.db_source)
if self._schedule_layers_and_relations_refresh:
self.conn_manager.db_connection_changed.disconnect(self.app.core.cache_layers_and_relations)
self.logger.info(__name__, "Dialog closed.")
self.done(QDialog.Accepted)
def update_connection_info(self):
db_description = self.db.get_description_conn_string()
if db_description:
self.db_connect_label.setText(db_description)
self.db_connect_label.setToolTip(self.db.get_display_conn_string())
self._accept_button.setEnabled(True)
else:
self.db_connect_label.setText(QCoreApplication.translate("DialogImportData", "The database is not defined!"))
self.db_connect_label.setToolTip('')
self._accept_button.setEnabled(False)
def update_import_models(self):
self.clear_messages()
error_msg = None
if not self.xtf_file_line_edit.text().strip():
color = '#ffd356' # Light orange
self.import_models_qmodel = QStandardItemModel()
self.import_models_list_view.setModel(self.import_models_qmodel)
else:
if os.path.isfile(self.xtf_file_line_edit.text().strip()):
color = '#fff' # White
self.import_models_qmodel = QStandardItemModel()
model_names = get_models_from_xtf(self.xtf_file_line_edit.text().strip())
for model in self.__ladmcol_models.non_hidden_and_supported_models():
if model.full_name() in model_names:
item = QStandardItem(model.full_alias())
item.setData(model.full_name(), Qt.UserRole)
item.setCheckable(False)
item.setEditable(False)
self.import_models_qmodel.appendRow(item)
if self.import_models_qmodel.rowCount() > 0:
self.import_models_list_view.setModel(self.import_models_qmodel)
else:
error_msg = QCoreApplication.translate("DialogImportData",
"No models were found in the XTF. Is it a valid file?")
color = '#ffd356' # Light orange
self.import_models_qmodel = QStandardItemModel()
self.import_models_list_view.setModel(self.import_models_qmodel)
else:
error_msg = QCoreApplication.translate("DialogImportData", "Please set a valid XTF file")
color = '#ffd356' # Light orange
self.import_models_qmodel = QStandardItemModel()
self.import_models_list_view.setModel(self.import_models_qmodel)
self.xtf_file_line_edit.setStyleSheet('QLineEdit {{ background-color: {} }}'.format(color))
if error_msg:
self.txtStdout.setText(error_msg)
self.show_message(error_msg, Qgis.Warning)
self.import_models_list_view.setFocus()
return
def get_ili_models(self):
ili_models = list()
for index in range(self.import_models_qmodel.rowCount()):
item = self.import_models_qmodel.item(index)
ili_models.append(item.data(Qt.UserRole))
return ili_models
def show_settings(self):
# We only need those tabs related to Model Baker/ili2db operations
dlg = SettingsDialog(self.conn_manager, parent=self)
dlg.setWindowTitle(QCoreApplication.translate("DialogImportData", "Target DB Connection Settings"))
dlg.show_tip(QCoreApplication.translate("DialogImportData", "Configure where do you want the XTF data to be imported."))
dlg.set_db_source(self.db_source)
dlg.set_tab_pages_list([SETTINGS_CONNECTION_TAB_INDEX, SETTINGS_MODELS_TAB_INDEX])
# Connect signals (DBUtils, Core)
dlg.db_connection_changed.connect(self.db_connection_changed)
if self.db_source == COLLECTED_DB_SOURCE:
self._schedule_layers_and_relations_refresh = True
dlg.set_action_type(EnumDbActionType.IMPORT)
if dlg.exec_():
self.db = dlg.get_db_connection()
self.update_connection_info()
def db_connection_changed(self, db, ladm_col_db, db_source):
self._db_was_changed = True
self.clear_messages()
def accepted(self):
self._running_tool = True
self.txtStdout.clear()
self.progress_bar.setValue(0)
self.bar.clearWidgets()
if not os.path.isfile(self.xtf_file_line_edit.text().strip()):
self._running_tool = False
error_msg = QCoreApplication.translate("DialogImportData", "Please set a valid XTF file before importing data. XTF file does not exist.")
self.txtStdout.setText(error_msg)
self.show_message(error_msg, Qgis.Warning)
self.xtf_file_line_edit.setFocus()
return
java_home_set = self.java_dependency.set_java_home()
if not java_home_set:
message_java = QCoreApplication.translate("DialogImportData", """Configuring Java {}...""").format(JAVA_REQUIRED_VERSION)
self.txtStdout.setTextColor(QColor('#000000'))
self.txtStdout.clear()
self.txtStdout.setText(message_java)
self.java_dependency.get_java_on_demand()
self.disable()
return
ili2db = Ili2DB()
configuration = self.update_configuration(ili2db)
configuration = self.apply_role_model_configuration(configuration)
if configuration.disable_validation: # If data validation at import is disabled, we ask for confirmation
self.msg = QMessageBox()
self.msg.setIcon(QMessageBox.Question)
self.msg.setText(QCoreApplication.translate("DialogImportData",
"Are you sure you want to import your data without validation?"))
self.msg.setWindowTitle(QCoreApplication.translate("DialogImportData", "Import XTF without validation?"))
self.msg.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
res = self.msg.exec_()
if res == QMessageBox.No:
self._running_tool = False
return
if not self.xtf_file_line_edit.validator().validate(configuration.xtffile, 0)[0] == QValidator.Acceptable:
self._running_tool = False
error_msg = QCoreApplication.translate("DialogImportData", "Please set a valid XTF before importing data.")
self.txtStdout.setText(error_msg)
self.show_message(error_msg, Qgis.Warning)
self.xtf_file_line_edit.setFocus()
return
if not self.get_ili_models():
self._running_tool = False
error_msg = QCoreApplication.translate("DialogImportData", "The selected XTF file does not have information according to the LADM-COL model to import.")
self.txtStdout.setText(error_msg)
self.show_message(error_msg, Qgis.Warning)
self.import_models_list_view.setFocus()
return
# Get list of models present in the XTF file, in the DB and in the list of required models (by the plugin)
ili_models = set([ili_model for ili_model in self.get_ili_models()])
supported_models_in_ili = set([m.full_name() for m in self.__ladmcol_models.supported_models()]).intersection(ili_models)
if not supported_models_in_ili:
self._running_tool = False
error_msg = QCoreApplication.translate("DialogImportData",
"The selected XTF file does not have data from any LADM-COL model supported by the LADM-COL Assistant. " \
"Therefore, you cannot import it! These are the models supported:\n\n * {}").format(" \n * ".join([m.full_alias() for m in self.__ladmcol_models.supported_models()]))
self.txtStdout.setText(error_msg)
self.show_message(error_msg, Qgis.Warning)
self.import_models_list_view.setFocus()
return
db_models = set(self.db.get_models())
suggested_models = sorted(ili_models.difference(db_models))
if not ili_models.issubset(db_models):
self._running_tool = False
error_msg = QCoreApplication.translate("DialogImportData",
"IMPORT ERROR: The XTF file to import does not have the same models as the target database schema. " \
"Please create a schema that also includes the following missing modules:\n\n * {}").format(
" \n * ".join(suggested_models))
self.txtStdout.clear()
self.txtStdout.setTextColor(QColor('#000000'))
self.txtStdout.setText(error_msg)
self.show_message(error_msg, Qgis.Warning)
self.xtf_file_line_edit.setFocus()
# button is removed to define order in GUI
for button in self.buttonBox.buttons():
if button.text() == self.BUTTON_NAME_IMPORT_DATA:
self.buttonBox.removeButton(button)
# Check if button was previously added
self.remove_create_structure_button()
if self.link_to_import_schema:
self.buttonBox.addButton(self.BUTTON_NAME_GO_TO_CREATE_STRUCTURE,
QDialogButtonBox.AcceptRole).setStyleSheet("color: #aa2222;")
self.buttonBox.addButton(self.BUTTON_NAME_IMPORT_DATA,
QDialogButtonBox.AcceptRole)
return
self.progress_bar.show()
self.disable()
self.txtStdout.setTextColor(QColor('#000000'))
self.txtStdout.clear()
self._connect_ili2db_signals(ili2db)
self.save_configuration(configuration)
res, msg = ili2db.import_data(self.db, configuration)
self._disconnect_ili2db_signals(ili2db)
self._running_tool = False
self.progress_bar.setValue(25)
if res:
self.buttonBox.clear()
self.buttonBox.setEnabled(True)
self.buttonBox.addButton(QDialogButtonBox.Close)
self.progress_bar.setValue(100)
message_type = Qgis.Success if res else Qgis.Warning
self.show_message(msg, message_type)
def download_java_complete(self):
self.accepted()
def download_java_progress_change(self, progress):
self.progress_bar.setValue(progress/2)
if (progress % 20) == 0:
self.txtStdout.append('...')
def remove_create_structure_button(self):
for button in self.buttonBox.buttons():
if button.text() == self.BUTTON_NAME_GO_TO_CREATE_STRUCTURE:
self.buttonBox.removeButton(button)
def save_configuration(self, configuration):
settings = QSettings()
settings.setValue('Asistente-LADM-COL/QgisModelBaker/ili2pg/xtffile_import', configuration.xtffile)
settings.setValue('Asistente-LADM-COL/QgisModelBaker/show_log', not self.log_config.isCollapsed())
def restore_configuration(self):
settings = QSettings()
self.xtf_file_line_edit.setText(settings.value('Asistente-LADM-COL/QgisModelBaker/ili2pg/xtffile_import'))
# Show log
value_show_log = settings.value('Asistente-LADM-COL/QgisModelBaker/show_log', False, type=bool)
self.log_config.setCollapsed(not value_show_log)
# set model repository
# if there is no option by default use online model repository
self.use_local_models = self.app.settings.custom_models
if self.use_local_models:
self.custom_model_directories = self.app.settings.custom_model_dirs
def update_configuration(self, ili2db: Ili2DB):
"""
Get the configuration that is updated with the user configuration changes on the dialog.
:return: Configuration
"""
disable_validation = \
not QSettings().value('Asistente-LADM-COL/models/validate_data_importing_exporting', True, bool)
configuration = ili2db.get_import_data_configuration(self.db, self.xtf_file_line_edit.text().strip(),
disable_validation=disable_validation)
configuration.delete_data = False
# TODO this is different to ili2db.py
if self.get_ili_models():
configuration.ilimodels = ';'.join(self.get_ili_models())
return configuration
def apply_role_model_configuration(self, configuration):
"""
Applies the configuration that the active role has set over models that are in both the DB and the XTF.
Important:
Note that this works better if the checked models are limited to one (e.g. Field Data Capture) or limited to
a group of related models (e.g., the 3 supplies models). When the checked models are heterogeneous, results
start to be unpredictable, as the configuration for a single model may affect the others.
:param configuration: SchemaImportConfiguration object
:return: configuration object updated
"""
model_names = get_models_from_xtf(self.xtf_file_line_edit.text().strip())
for model in self.__ladmcol_models.non_hidden_and_supported_models():
if model.full_name() in model_names: # We'll check models in the DB that are also in the XTF
params = model.get_ili2db_params()
if ILI2DB_IMPORT in params:
for param in params[ILI2DB_IMPORT]: # List of tuples
if param[0] == ILI2DB_DATASET: # param: (option, value)
configuration.dataset = param[1]
self.logger.debug(__name__, "Import XTF data into dataset '{}'! (taken from role config)".format(param[1]))
return configuration
def print_info(self, text, text_color='#000000', clear=False):
self.txtStdout.setTextColor(QColor(text_color))
self.txtStdout.append(text)
QCoreApplication.processEvents()
def on_stderr(self, text):
color_log_text(text, self.txtStdout)
self.advance_progress_bar_by_text(text)
def on_process_started(self, command):
self.disable()
self.txtStdout.setTextColor(QColor('#000000'))
self.txtStdout.clear()
self.txtStdout.setText(command)
QCoreApplication.processEvents()
def on_process_finished(self, exit_code, result):
color = '#004905' if exit_code == 0 else '#aa2222'
self.txtStdout.setTextColor(QColor(color))
self.txtStdout.append('Finished ({})'.format(exit_code))
if result == iliimporter.Importer.SUCCESS:
self.buttonBox.clear()
self.buttonBox.setEnabled(True)
self.buttonBox.addButton(QDialogButtonBox.Close)
else:
self.show_message(QCoreApplication.translate("DialogImportData", "Error when importing data"), Qgis.Warning)
# Open log
if self.log_config.isCollapsed():
self.log_config.setCollapsed(False)
def advance_progress_bar_by_text(self, text):
if text.strip() == 'Info: compile models...':
self.progress_bar.setValue(50)
QCoreApplication.processEvents()
elif text.strip() == 'Info: create table structure...':
self.progress_bar.setValue(55)
QCoreApplication.processEvents()
elif text.strip() == 'Info: first validation pass...':
self.progress_bar.setValue(60)
QCoreApplication.processEvents()
elif text.strip() == 'Info: second validation pass...':
self.progress_bar.setValue(80)
QCoreApplication.processEvents()
def clear_messages(self):
self.bar.clearWidgets() # Remove previous messages before showing a new one
self.txtStdout.clear() # Clear previous log messages
self.progress_bar.setValue(0) # Initialize progress bar
def show_help(self):
show_plugin_help("import_data")
def disable(self):
self.source_config.setEnabled(False)
self.target_config.setEnabled(False)
self.buttonBox.setEnabled(False)
def enable(self):
self.source_config.setEnabled(True)
self.target_config.setEnabled(True)
self.buttonBox.setEnabled(True)
def show_message(self, message, level):
if level == Qgis.Warning:
self.enable()
self.bar.clearWidgets() # Remove previous messages before showing a new one
self.bar.pushMessage("Asistente LADM-COL", message, level, duration=0)
def _connect_ili2db_signals(self, ili2db):
ili2db.process_started.connect(self.on_process_started)
ili2db.stderr.connect(self.on_stderr)
ili2db.stdout.connect(self.print_info)
ili2db.process_finished.connect(self.on_process_finished)
def _disconnect_ili2db_signals(self, ili2db):
ili2db.process_started.disconnect(self.on_process_started)
ili2db.stderr.disconnect(self.on_stderr)
ili2db.stdout.disconnect(self.print_info)
ili2db.process_finished.disconnect(self.on_process_finished)
class DialogImportSchema(QDialog, DIALOG_UI):
open_dlg_import_data = pyqtSignal(dict) # dict with key-value params
on_result = pyqtSignal(
bool) # whether the tool was run successfully or not
BUTTON_NAME_CREATE_STRUCTURE = QCoreApplication.translate(
"DialogImportSchema", "Create LADM-COL structure")
BUTTON_NAME_GO_TO_IMPORT_DATA = QCoreApplication.translate(
"DialogImportData", "Go to Import Data...")
def __init__(self,
iface,
conn_manager,
context,
selected_models=list(),
link_to_import_data=True,
parent=None):
QDialog.__init__(self, parent)
self.iface = iface
self.conn_manager = conn_manager
self.selected_models = selected_models
self.link_to_import_data = link_to_import_data
self.logger = Logger()
self.app = AppInterface()
self.__ladmcol_models = LADMColModelRegistry()
self.java_dependency = JavaDependency()
self.java_dependency.download_dependency_completed.connect(
self.download_java_complete)
self.java_dependency.download_dependency_progress_changed.connect(
self.download_java_progress_change)
self.db_source = context.get_db_sources()[0]
self.db = self.conn_manager.get_db_connector_from_source(
self.db_source)
self._dbs_supported = ConfigDBsSupported()
self._running_tool = False
# There may be two cases where we need to emit a db_connection_changed from the Schema Import dialog:
# 1) Connection Settings was opened and the DB conn was changed.
# 2) Connection Settings was never opened but the Schema Import ran successfully, in a way that new models may
# convert a db/schema LADM-COL compliant.
self._db_was_changed = False # To postpone calling refresh gui until we close this dialog instead of settings
# Similarly, we could call a refresh on layers and relations cache in two cases:
# 1) If the SI dialog was called for the COLLECTED source: opening Connection Settings and changing the DB
# connection.
# 2) Not opening the Connection Settings, but running a successful Schema Import on the COLLECTED DB, which
# invalidates the cache as models change.
self._schedule_layers_and_relations_refresh = False
self.setupUi(self)
self.validators = Validators()
self.update_import_models()
self.previous_item = QListWidgetItem()
self.connection_setting_button.clicked.connect(self.show_settings)
self.connection_setting_button.setText(
QCoreApplication.translate("DialogImportSchema",
"Connection Settings"))
# CRS Setting
self.srs_auth = DEFAULT_SRS_AUTH
self.srs_code = DEFAULT_SRS_CODE
self.crsSelector.crsChanged.connect(self.crs_changed)
# LOG
self.log_config.setTitle(
QCoreApplication.translate("DialogImportSchema", "Show log"))
self.buttonBox.accepted.disconnect()
self.buttonBox.clicked.connect(self.accepted_import_schema)
self.buttonBox.clear()
self.buttonBox.addButton(QDialogButtonBox.Cancel)
self._accept_button = self.buttonBox.addButton(
self.BUTTON_NAME_CREATE_STRUCTURE, QDialogButtonBox.AcceptRole)
self.buttonBox.addButton(QDialogButtonBox.Help)
self.buttonBox.helpRequested.connect(self.show_help)
self.import_models_list_widget.setDisabled(bool(
selected_models)) # If we got models from params, disable panel
self.update_connection_info()
self.restore_configuration()
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
def accepted_import_schema(self, button):
if self.buttonBox.buttonRole(button) == QDialogButtonBox.AcceptRole:
if button.text() == self.BUTTON_NAME_CREATE_STRUCTURE:
self.accepted()
elif button.text() == self.BUTTON_NAME_GO_TO_IMPORT_DATA:
self.close(
) # Close import schema dialog and open import open dialog
self.open_dlg_import_data.emit({"db_source": self.db_source})
def reject(self):
if self._running_tool:
QMessageBox.information(
self,
QCoreApplication.translate("DialogImportSchema", "Warning"),
QCoreApplication.translate(
"DialogImportSchema",
"The Import Schema tool is still running. Please wait until it finishes."
))
else:
self.close_dialog()
def close_dialog(self):
"""
We use this method to be safe when emitting the db_connection_changed, otherwise we could trigger slots that
unload the plugin, destroying dialogs and thus, leading to crashes.
"""
if self._schedule_layers_and_relations_refresh:
self.conn_manager.db_connection_changed.connect(
self.app.core.cache_layers_and_relations)
if self._db_was_changed:
self.conn_manager.db_connection_changed.emit(
self.db,
self.db.test_connection()[0], self.db_source)
if self._schedule_layers_and_relations_refresh:
self.conn_manager.db_connection_changed.disconnect(
self.app.core.cache_layers_and_relations)
self.logger.info(__name__, "Dialog closed.")
self.done(QDialog.Accepted)
def update_connection_info(self):
db_description = self.db.get_description_conn_string()
if db_description:
self.db_connect_label.setText(db_description)
self.db_connect_label.setToolTip(self.db.get_display_conn_string())
self._accept_button.setEnabled(True)
else:
self.db_connect_label.setText(
QCoreApplication.translate("DialogImportSchema",
"The database is not defined!"))
self.db_connect_label.setToolTip('')
self._accept_button.setEnabled(False)
def update_import_models(self):
for model in self.__ladmcol_models.supported_models():
if not model.hidden():
item = QListWidgetItem(model.full_alias())
item.setData(Qt.UserRole, model.full_name())
item.setFlags(item.flags() | Qt.ItemIsUserCheckable)
if self.selected_models: # From parameters
item.setCheckState(Qt.Checked if model.full_name() in
self.selected_models else Qt.Unchecked)
else: # By default
item.setCheckState(
Qt.Checked if model.checked() else Qt.Unchecked)
self.import_models_list_widget.addItem(item)
self.import_models_list_widget.itemClicked.connect(
self.on_item_clicked_import_model)
self.import_models_list_widget.itemChanged.connect(
self.on_itemchanged_import_model)
def on_item_clicked_import_model(self, item):
# disconnect signal to do changes in the items
self.import_models_list_widget.itemChanged.disconnect(
self.on_itemchanged_import_model)
if self.previous_item.text() != item.text():
item.setCheckState(Qt.Unchecked if item.checkState() ==
Qt.Checked else Qt.Checked)
# connect signal to check when the items change
self.import_models_list_widget.itemChanged.connect(
self.on_itemchanged_import_model)
self.previous_item = item
def on_itemchanged_import_model(self, item):
if self.previous_item.text() != item.text():
item.setSelected(True)
self.previous_item = item
def get_checked_models(self):
checked_models = list()
for index in range(self.import_models_list_widget.count()):
item = self.import_models_list_widget.item(index)
if item.checkState() == Qt.Checked:
checked_models.append(item.data(Qt.UserRole))
return checked_models
def show_settings(self):
# We only need those tabs related to Model Baker/ili2db operations
dlg = SettingsDialog(self.conn_manager, parent=self)
dlg.setWindowTitle(
QCoreApplication.translate("DialogImportSchema",
"Target DB Connection Settings"))
dlg.show_tip(
QCoreApplication.translate(
"DialogImportSchema",
"Configure where do you want the LADM-COL structure to be created."
))
dlg.set_db_source(self.db_source)
dlg.set_tab_pages_list(
[SETTINGS_CONNECTION_TAB_INDEX, SETTINGS_MODELS_TAB_INDEX])
# Connect signals (DBUtils, Core)
dlg.db_connection_changed.connect(self.db_connection_changed)
if self.db_source == COLLECTED_DB_SOURCE:
self._schedule_layers_and_relations_refresh = True
dlg.set_action_type(EnumDbActionType.SCHEMA_IMPORT)
if dlg.exec_():
self.db = dlg.get_db_connection()
self.update_connection_info()
def db_connection_changed(self, db, ladm_col_db, db_source):
# We dismiss parameters here, after all, we already have the db, and the ladm_col_db may change from this moment
# until we close the import schema dialog
self._db_was_changed = True
self.clear_messages() # Clean GUI messages if db connection changed
def accepted(self):
self._running_tool = True
self.txtStdout.clear()
self.progress_bar.setValue(0)
self.bar.clearWidgets()
java_home_set = self.java_dependency.set_java_home()
if not java_home_set:
message_java = QCoreApplication.translate(
"DialogImportSchema",
"""Configuring Java {}...""").format(JAVA_REQUIRED_VERSION)
self.txtStdout.setTextColor(QColor('#000000'))
self.txtStdout.clear()
self.txtStdout.setText(message_java)
self.java_dependency.get_java_on_demand()
self.disable()
return
if not self.get_checked_models():
self._running_tool = False
message_error = QCoreApplication.translate(
"DialogImportSchema",
"You should select a valid model(s) before creating the LADM-COL structure."
)
self.txtStdout.setText(message_error)
self.show_message(message_error, Qgis.Warning)
self.import_models_list_widget.setFocus()
return
self.save_configuration()
self.progress_bar.show()
self.disable()
self.txtStdout.setTextColor(QColor('#000000'))
self.txtStdout.clear()
ili2db = Ili2DB()
self._connect_ili2db_signals(ili2db)
models = self.get_checked_models()
configuration = ili2db.get_import_schema_configuration(self.db, models)
configuration = self.apply_role_model_configuration(configuration)
res, msg = ili2db.import_schema(self.db, configuration)
self._disconnect_ili2db_signals(ili2db)
self._running_tool = False
if res:
self.buttonBox.clear()
if self.link_to_import_data:
self.buttonBox.addButton(
self.BUTTON_NAME_GO_TO_IMPORT_DATA,
QDialogButtonBox.AcceptRole).setStyleSheet(
"color: #007208;")
self.buttonBox.setEnabled(True)
self.buttonBox.addButton(QDialogButtonBox.Close)
self.progress_bar.setValue(100)
self.print_info(
QCoreApplication.translate("DialogImportSchema", "\nDone!"),
'#004905')
self._db_was_changed = True # Schema could become LADM compliant after a schema import
if self.db_source == COLLECTED_DB_SOURCE:
self.logger.info(
__name__,
"Schedule a call to refresh db relations cache since a Schema Import was run on the current 'collected' DB."
)
self._schedule_layers_and_relations_refresh = True
message_type = Qgis.Success if res else Qgis.Warning
self.show_message(msg, message_type)
# Inform other classes whether the execution was successful or not
self.on_result.emit(res)
def download_java_complete(self):
self.accepted()
def download_java_progress_change(self, progress):
self.progress_bar.setValue(progress / 2)
if (progress % 20) == 0:
self.txtStdout.append('...')
def save_configuration(self):
settings = QSettings()
settings.setValue('Asistente-LADM-COL/QgisModelBaker/show_log',
not self.log_config.isCollapsed())
settings.setValue('Asistente-LADM-COL/QgisModelBaker/srs_auth',
self.srs_auth)
settings.setValue('Asistente-LADM-COL/QgisModelBaker/srs_code',
self.srs_code)
def restore_configuration(self):
settings = QSettings()
# CRS
srs_auth = settings.value('Asistente-LADM-COL/QgisModelBaker/srs_auth',
DEFAULT_SRS_AUTH, str)
srs_code = settings.value('Asistente-LADM-COL/QgisModelBaker/srs_code',
int(DEFAULT_SRS_CODE), int)
self.crsSelector.setCrs(get_crs_from_auth_and_code(srs_auth, srs_code))
self.crs_changed()
# Show log
value_show_log = settings.value(
'Asistente-LADM-COL/QgisModelBaker/show_log', False, type=bool)
self.log_config.setCollapsed(not value_show_log)
# set model repository
# if there is no option by default use online model repository
self.use_local_models = self.app.settings.custom_models
if self.use_local_models:
self.custom_model_directories = self.app.settings.custom_model_dirs
def crs_changed(self):
self.srs_auth, self.srs_code = get_crs_authid(
self.crsSelector.crs()).split(":")
if self.srs_code != DEFAULT_SRS_CODE or self.srs_auth != DEFAULT_SRS_AUTH:
self.crs_label.setStyleSheet('color: orange')
self.crs_label.setToolTip(
QCoreApplication.translate(
"DialogImportSchema",
"The {} (Colombian National Origin) is recommended,<br>since official models were created for that projection."
).format(DEFAULT_SRS_AUTHID))
else:
self.crs_label.setStyleSheet('')
self.crs_label.setToolTip(
QCoreApplication.translate("DialogImportSchema",
"Coordinate Reference System"))
def apply_role_model_configuration(self, configuration):
"""
Applies the configuration that the active role has set over checked models.
Important:
Note that this works better if the checked models are limited to one (e.g. Field Data Capture) or limited to
a group of related models (e.g., the 3 supplies models). When the checked models are heterogeneous, results
start to be unpredictable, as the configuration for a single model may affect the others.
:param configuration: SchemaImportConfiguration object
:return: configuration object updated
"""
for checked_model in self.get_checked_models():
model = self.__ladmcol_models.model_by_full_name(checked_model)
params = model.get_ili2db_params()
if ILI2DB_SCHEMAIMPORT in params:
for param in params[ILI2DB_SCHEMAIMPORT]: # List of tuples
if param[
0] == ILI2DB_CREATE_BASKET_COL_KEY: # param: (option, value)
configuration.create_basket_col = True
self.logger.debug(
__name__,
"Schema Import createBasketCol enabled (model '{}')! (taken from role config)"
.format(model.id()))
return configuration
def print_info(self, text, text_color='#000000', clear=False):
self.txtStdout.setTextColor(QColor(text_color))
self.txtStdout.append(text)
QCoreApplication.processEvents()
def on_stderr(self, text):
color_log_text(text, self.txtStdout)
self.advance_progress_bar_by_text(text)
def on_process_started(self, command):
self.txtStdout.append(command)
QCoreApplication.processEvents()
def on_process_finished(self, exit_code, result):
if exit_code == 0:
color = '#004905'
message = QCoreApplication.translate(
"DialogImportSchema",
"Model(s) successfully imported into the database!")
else:
color = '#aa2222'
message = QCoreApplication.translate("DialogImportSchema",
"Finished with errors!")
# Open log
if self.log_config.isCollapsed():
self.log_config.setCollapsed(False)
self.txtStdout.setTextColor(QColor(color))
self.txtStdout.append(message)
def advance_progress_bar_by_text(self, text):
if text.strip() == 'Info: compile models…':
self.progress_bar.setValue(20)
QCoreApplication.processEvents()
elif text.strip() == 'Info: create table structure…':
self.progress_bar.setValue(70)
QCoreApplication.processEvents()
def clear_messages(self):
self.bar.clearWidgets(
) # Remove previous messages before showing a new one
self.txtStdout.clear() # Clear previous log messages
self.progress_bar.setValue(0) # Initialize progress bar
def show_help(self):
show_plugin_help("import_schema")
def disable(self):
self.db_config.setEnabled(False)
self.ili_config.setEnabled(False)
self.buttonBox.setEnabled(False)
def enable(self):
self.db_config.setEnabled(True)
self.ili_config.setEnabled(True)
self.buttonBox.setEnabled(True)
def show_message(self, message, level):
if level == Qgis.Warning:
self.enable()
self.bar.clearWidgets(
) # Remove previous messages before showing a new one
self.bar.pushMessage("Asistente LADM-COL", message, level, duration=0)
def _connect_ili2db_signals(self, ili2db):
ili2db.process_started.connect(self.on_process_started)
ili2db.stderr.connect(self.on_stderr)
ili2db.stdout.connect(self.print_info)
ili2db.process_finished.connect(self.on_process_finished)
def _disconnect_ili2db_signals(self, ili2db):
ili2db.process_started.disconnect(self.on_process_started)
ili2db.stderr.disconnect(self.on_stderr)
ili2db.stdout.disconnect(self.print_info)
ili2db.process_finished.disconnect(self.on_process_finished)
def setUpClass(cls):
cls.ladmcol_models = LADMColModelRegistry()