def _create_config(self) -> Ili2DbCommandConfiguration:
if self.__data_import:
configuration = ImportDataConfiguration()
else:
configuration = SchemaImportConfiguration()
return configuration
def __init__(self, dataImport=False, parent=None):
QObject.__init__(self, parent)
self.filename = None
self.tool = None
self.dataImport = dataImport
if dataImport:
self.configuration = ImportDataConfiguration()
else:
self.configuration = SchemaImportConfiguration()
self.encoding = locale.getlocale()[1]
# This might be unset
# (https://stackoverflow.com/questions/1629699/locale-getlocale-problems-on-osx)
if not self.encoding:
self.encoding = 'UTF8'
def ili2db_command_reload(self):
config = None
if self.ili2db_action_combobox.currentData() == 'schemaimport':
config = SchemaImportConfiguration()
elif self.ili2db_action_combobox.currentData() == 'import':
config = ImportDataConfiguration()
elif self.ili2db_action_combobox.currentData() == 'export':
config = ExportConfiguration()
executable = 'java -jar {}.jar'.format(
self.ili2db_tool_combobox.currentData().name)
command = '\n '.join([executable] + config.to_ili2db_args())
self.ili2db_options_textedit.setText(command)
def __init__(self, iface, base_config, parent):
QWizard.__init__(self, parent)
self.setWindowTitle(self.tr("QGIS Model Baker Wizard"))
self.setWizardStyle(QWizard.ModernStyle)
self.setOption(QWizard.NoCancelButtonOnLastPage)
self.current_id = 0
self.iface = iface
self.log_panel = parent.log_panel
# configuration objects are keeped on top level to be able to access them from individual pages
self.base_config = base_config
self.import_schema_configuration = SchemaImportConfiguration()
self.import_data_configuration = ImportDataConfiguration()
self.update_data_configuration = UpdateDataConfiguration()
self.export_data_configuration = ExportConfiguration()
self.import_schema_configuration.base_configuration = self.base_config
self.import_data_configuration.base_configuration = self.base_config
self.update_data_configuration.base_configuration = self.base_config
self.update_data_configuration.with_importbid = True
self.export_data_configuration.base_configuration = self.base_config
# data models are keeped on top level because sometimes they need to be accessed to evaluate the wizard workflow
# the source_model keeps all the sources (files or repositories) used and the dataset property
self.source_model = SourceModel()
self.source_model.print_info.connect(self.log_panel.print_info)
# the import_models_model keeps every single model as entry and a checked state
self.import_models_model = ImportModelsModel()
self.import_models_model.print_info.connect(self.log_panel.print_info)
# the import_data_file_model keeps the filtered out transfer files (from source model) and functions to get ordered import sessions
self.import_data_file_model = ImportDataModel()
self.import_data_file_model.print_info.connect(
self.log_panel.print_info)
self.import_data_file_model.setSourceModel(self.source_model)
self.import_data_file_model.setFilterRole(int(SourceModel.Roles.TYPE))
self.import_data_file_model.setFilterRegExp(
"|".join(TransferExtensions))
self.ilireferencedatacache = IliDataCache(
self.import_schema_configuration.base_configuration,
"referenceData",
)
self.ilireferencedatacache.new_message.connect(
self.log_panel.show_message)
# the current_models_model keeps every single model found in the current database and keeps the selected models
self.current_models_model = SchemaModelsModel()
# the current_datasets_model keeps every dataset found in the current database and keeps the selected dataset
self.current_datasets_model = SchemaDatasetsModel()
# the current_baskets_model keeps every baskets found in the current database and keeps the selected baskets
self.current_baskets_model = SchemaBasketsModel()
# the current export target is the current set target file for the export. It's keeped top level to have a consequent behavior of those information.
self.current_export_target = ""
self.current_filter_mode = SchemaDataFilterMode.NO_FILTER
# pages setup
self.intro_page = IntroPage(self,
self._current_page_title(PageIds.Intro))
self.source_selection_page = ImportSourceSelectionPage(
self, self._current_page_title(PageIds.ImportSourceSelection))
self.import_database_selection_page = DatabaseSelectionPage(
self,
self._current_page_title(PageIds.ImportDatabaseSelection),
DbActionType.IMPORT_DATA,
)
self.schema_configuration_page = ImportSchemaConfigurationPage(
self, self._current_page_title(PageIds.ImportSchemaConfiguration))
self.import_schema_execution_page = ExecutionPage(
self,
self._current_page_title(PageIds.ImportSchemaExecution),
DbActionType.GENERATE,
)
self.data_configuration_page = ImportDataConfigurationPage(
self, self._current_page_title(PageIds.ImportDataConfiguration))
self.import_data_execution_page = ExecutionPage(
self,
self._current_page_title(PageIds.ImportDataExecution),
DbActionType.IMPORT_DATA,
)
self.project_creation_page = ProjectCreationPage(
self, self._current_page_title(PageIds.ProjectCreation))
self.generate_database_selection_page = DatabaseSelectionPage(
self,
self._current_page_title(PageIds.GenerateDatabaseSelection),
DbActionType.GENERATE,
)
self.export_database_selection_page = DatabaseSelectionPage(
self,
self._current_page_title(PageIds.ExportDatabaseSelection),
DbActionType.EXPORT,
)
self.export_data_configuration_page = ExportDataConfigurationPage(
self, self._current_page_title(PageIds.ExportDataConfiguration))
self.export_data_execution_page = ExecutionPage(
self,
self._current_page_title(PageIds.ExportDataExecution),
DbActionType.EXPORT,
)
self.setPage(PageIds.Intro, self.intro_page)
self.setPage(PageIds.ImportSourceSelection, self.source_selection_page)
self.setPage(PageIds.ImportDatabaseSelection,
self.import_database_selection_page)
self.setPage(PageIds.ImportSchemaConfiguration,
self.schema_configuration_page)
self.setPage(PageIds.ImportSchemaExecution,
self.import_schema_execution_page)
self.setPage(PageIds.ImportDataConfiguration,
self.data_configuration_page)
self.setPage(PageIds.ImportDataExecution,
self.import_data_execution_page)
self.setPage(PageIds.ProjectCreation, self.project_creation_page)
self.setPage(PageIds.GenerateDatabaseSelection,
self.generate_database_selection_page)
self.setPage(PageIds.ExportDatabaseSelection,
self.export_database_selection_page)
self.setPage(PageIds.ExportDataConfiguration,
self.export_data_configuration_page)
self.setPage(PageIds.ExportDataExecution,
self.export_data_execution_page)
self.currentIdChanged.connect(self.id_changed)
def test_import_data_in_mssql(self):
print("\nINFO: Validate Import Data in MS SQL Server...")
schema = 'test_ladm_col'
reset_db_mssql(schema)
restore_schema_mssql(schema)
db_conn = get_mssql_conn(schema)
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
model_list = [
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
]
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'mssql'
configuration.dbusr = '******'
configuration.dbpwd = '<YourStrong!Passw0rd>'
configuration.dbport = '1433'
configuration.database = schema # use schema because delete schemas in mssql is difficult
configuration.dbschema = schema
configuration.db_odbc_driver = 'ODBC Driver 17 for SQL Server'
configuration.delete_data = True
configuration.ilimodels = ';'.join(model_list)
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2mssql
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v1_0.xtf')
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2mssql,
'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},1433;DATABASE={db};UID={user};PWD={pwd}'
.format(db=configuration.database,
user=configuration.dbusr,
pwd=configuration.dbpwd,
host=configuration.dbhost,
port=configuration.dbport), configuration.inheritance,
importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 156)
res, code, msg = db_conn.test_connection()
self.assertTrue(res, msg)
test_layer = self.app.core.get_layer(db_conn,
db_conn.names.LC_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_conn.conn.close()
def test_import_data_in_gpkg(self):
print("\nINFO: Validate Import Data in GPKG...")
gpkg_path = get_test_copy_path(
'geopackage/test_import_data_ladm_v1_0.gpkg')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.tool = DbIliMode.ili2gpkg
configuration.dbfile = gpkg_path
configuration.srs_auth = DEFAULT_SRS_AUTH
configuration.srs_code = DEFAULT_SRS_CODE
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.delete_data = True
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2gpkg
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v1_0.xtf')
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
config_manager = GpkgCommandConfigManager(importer.configuration)
generator = Generator(DbIliMode.ili2gpkg, config_manager.get_uri(),
configuration.inheritance)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 156)
db_gpkg = get_gpkg_conn_from_path(config_manager.get_uri())
res, code, msg = db_gpkg.test_connection()
self.assertTrue(res, msg)
test_layer = self.app.core.get_layer(db_gpkg,
db_gpkg.names.LC_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_gpkg.conn.close()
def test_import_data_in_pg(self):
print("\nINFO: Validate Import Data in PG...")
restore_schema('test_import_data')
db_pg = get_pg_conn('test_import_data')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'postgres'
configuration.dbusr = '******'
configuration.dbpwd = 'clave_ladm_col'
configuration.database = 'ladm_col'
configuration.srs_auth = DEFAULT_SRS_AUTH
configuration.srs_code = DEFAULT_SRS_CODE
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.dbschema = 'test_import_data'
configuration.delete_data = True
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2pg
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v1_0.xtf')
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2pg, 'dbname={} user={} password={} host={}'.format(
configuration.database, configuration.dbusr,
configuration.dbpwd, configuration.dbhost),
configuration.inheritance, importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 156)
res, code, msg = db_pg.test_connection()
self.assertTrue(res, msg)
test_layer = self.app.core.get_layer(db_pg,
db_pg.names.LC_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_pg.conn.close()
def update_configuration(self):
"""
Get the configuration that is updated with the user configuration changes on the dialog.
:return: Configuration
"""
db_factory = self._dbs_supported.get_db_factory(self.db.engine)
configuration = ImportDataConfiguration()
db_factory.set_ili2db_configuration_params(self.db.dict_conn_params,
configuration)
configuration.xtffile = self.xtf_file_line_edit.text().strip()
configuration.delete_data = False
configuration.srs_auth = QSettings().value(
'Asistente-LADM-COL/QgisModelBaker/srs_auth', DEFAULT_SRS_AUTH,
str)
configuration.srs_code = QSettings().value(
'Asistente-LADM-COL/QgisModelBaker/srs_code',
int(DEFAULT_SRS_CODE), int)
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.with_importtid = True
full_java_exe_path = JavaDependency.get_full_java_exe_path()
if full_java_exe_path:
self.base_configuration.java_path = full_java_exe_path
# User could have changed the default values
self.use_local_models = QSettings().value(
'Asistente-LADM-COL/models/custom_model_directories_is_checked',
DEFAULT_USE_CUSTOM_MODELS,
type=bool)
self.custom_model_directories = QSettings().value(
'Asistente-LADM-COL/models/custom_models', DEFAULT_MODELS_DIR)
# Check custom model directories
if self.use_local_models:
if not self.custom_model_directories:
self.base_configuration.custom_model_directories_enabled = False
else:
self.base_configuration.custom_model_directories = self.custom_model_directories
self.base_configuration.custom_model_directories_enabled = True
configuration.base_configuration = self.base_configuration
if self.get_ili_models():
configuration.ilimodels = ';'.join(self.get_ili_models())
configuration.disable_validation = not QSettings().value(
'Asistente-LADM-COL/models/validate_data_importing_exporting',
True, bool)
return configuration
def update_configuration(self):
"""
Get the configuration that is updated with the user configuration changes on the dialog.
:return: Configuration
"""
item_db = self._conf_db.get_db_items()[self.db.mode]
configuration = ImportDataConfiguration()
item_db.set_db_configuration_params(self.db.dict_conn_params,
configuration)
configuration.xtffile = self.xtf_file_line_edit.text().strip()
configuration.delete_data = False
configuration.epsg = DEFAULT_EPSG
configuration.inheritance = DEFAULT_INHERITANCE
configuration.create_basket_col = CREATE_BASKET_COL
configuration.create_import_tid = CREATE_IMPORT_TID
configuration.stroke_arcs = STROKE_ARCS
java_path = get_java_path_from_qgis_model_baker()
if java_path:
self.base_configuration.java_path = java_path
# Check custom model directories
if self.use_local_models:
if self.custom_model_directories is None:
self.base_configuration.custom_model_directories_enabled = False
else:
self.base_configuration.custom_model_directories = self.custom_model_directories
self.base_configuration.custom_model_directories_enabled = True
configuration.base_configuration = self.base_configuration
if self.get_ili_models():
configuration.ilimodels = ';'.join(self.get_ili_models())
configuration.disable_validation = not QSettings().value(
'Asistente-LADM_COL/advanced_settings/validate_data_importing_exporting',
True, bool)
return configuration
def update_configuration(self):
"""
Get the configuration that is updated with the user configuration changes on the dialog.
:return: Configuration
"""
configuration = ImportDataConfiguration()
if self.type_combo_box.currentData() == 'pg':
# PostgreSQL specific options
configuration.dbhost = self.db.dict_conn_params['host']
configuration.dbport = self.db.dict_conn_params['port']
configuration.dbusr = self.db.dict_conn_params['username']
configuration.database = self.db.dict_conn_params['database']
configuration.dbschema = self.get_checked_schema()
configuration.dbpwd = self.db.dict_conn_params['password']
elif self.type_combo_box.currentData() == 'gpkg':
configuration.dbfile = self.db.dict_conn_params['dbfile']
configuration.xtffile = self.xtf_file_line_edit.text().strip()
configuration.delete_data = False
configuration.epsg = DEFAULT_EPSG
configuration.inheritance = DEFAULT_INHERITANCE
configuration.create_basket_col = CREATE_BASKET_COL
configuration.create_import_tid = CREATE_IMPORT_TID
configuration.stroke_arcs = STROKE_ARCS
java_path = get_java_path_from_qgis_model_baker()
if java_path:
self.base_configuration.java_path = java_path
# Check custom model directories
if self.use_local_models:
if self.custom_model_directories is None:
self.base_configuration.custom_model_directories_enabled = False
else:
self.base_configuration.custom_model_directories = self.custom_model_directories
self.base_configuration.custom_model_directories_enabled = True
configuration.base_configuration = self.base_configuration
if self.get_ili_models():
configuration.ilimodels = ';'.join(self.get_ili_models())
return configuration
def test_import_data_in_gpkg(self):
print("\nINFO: Validate Import Data in GPKG...")
gpkg_path = get_test_copy_path('geopackage/test_import_data.gpkg')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path('models/LADM_COL')
base_config.custom_model_directories_enabled = True
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.tool = DbIliMode.ili2gpkg
configuration.dbfile = gpkg_path
configuration.epsg = DEFAULT_EPSG
configuration.inheritance = LADMNames.DEFAULT_INHERITANCE
configuration.create_basket_col = LADMNames.CREATE_BASKET_COL
configuration.create_import_tid = LADMNames.CREATE_IMPORT_TID
configuration.stroke_arcs = LADMNames.STROKE_ARCS
configuration.delete_data = True
configuration.ilimodels = ';'.join([
LADMNames.SUPPORTED_LADM_MODEL, LADMNames.SUPPORTED_SNR_DATA_MODEL,
LADMNames.SUPPORTED_SUPPLIES_MODEL,
LADMNames.SUPPORTED_SUPPLIES_INTEGRATION_MODEL,
LADMNames.SUPPORTED_OPERATION_MODEL, LADMNames.SUPPORTED_ANT_MODEL,
LADMNames.SUPPORTED_CADASTRAL_FORM_MODEL,
LADMNames.SUPPORTED_VALUATION_MODEL
])
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2gpkg
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v2.9.6.xtf')
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
config_manager = GpkgCommandConfigManager(importer.configuration)
generator = Generator(DbIliMode.ili2gpkg, config_manager.get_uri(),
configuration.inheritance)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 189)
db_gpkg = get_gpkg_conn_from_path(config_manager.get_uri())
res, code, msg = db_gpkg.test_connection()
self.assertTrue(res, msg)
test_layer = self.qgis_utils.get_layer(
db_gpkg, db_gpkg.names.OP_BOUNDARY_POINT_T, load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_gpkg.conn.close()
def test_import_data_in_pg(self):
print("\nINFO: Validate Import Data in PG...")
restore_schema('test_import_data')
db_pg = get_pg_conn('test_import_data')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path('models/LADM_COL')
base_config.custom_model_directories_enabled = True
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'postgres'
configuration.dbusr = '******'
configuration.dbpwd = 'clave_ladm_col'
configuration.database = 'ladm_col'
configuration.epsg = DEFAULT_EPSG
configuration.inheritance = LADMNames.DEFAULT_INHERITANCE
configuration.create_basket_col = LADMNames.CREATE_BASKET_COL
configuration.create_import_tid = LADMNames.CREATE_IMPORT_TID
configuration.stroke_arcs = LADMNames.STROKE_ARCS
configuration.dbschema = 'test_import_data'
configuration.delete_data = True
configuration.ilimodels = ';'.join([
LADMNames.SUPPORTED_LADM_MODEL, LADMNames.SUPPORTED_SNR_DATA_MODEL,
LADMNames.SUPPORTED_SUPPLIES_MODEL,
LADMNames.SUPPORTED_SUPPLIES_INTEGRATION_MODEL,
LADMNames.SUPPORTED_OPERATION_MODEL, LADMNames.SUPPORTED_ANT_MODEL,
LADMNames.SUPPORTED_CADASTRAL_FORM_MODEL,
LADMNames.SUPPORTED_VALUATION_MODEL
])
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2pg
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v2.9.6.xtf')
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2pg, 'dbname={} user={} password={} host={}'.format(
configuration.database, configuration.dbusr,
configuration.dbpwd, configuration.dbhost),
configuration.inheritance, importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 189)
res, code, msg = db_pg.test_connection()
self.assertTrue(res, msg)
test_layer = self.qgis_utils.get_layer(db_pg,
db_pg.names.OP_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_pg.conn.close()