def test_export_data_in_gpkg(self):
print("\nINFO: Validate Export Data in GPKG...")
gpkg_path = get_test_copy_path(
'db/ladm/gpkg/test_export_data_ladm_v1_1.gpkg')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = ExportConfiguration()
configuration.base_configuration = base_config
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
configuration.dbfile = gpkg_path
exporter = iliexporter.Exporter()
exporter.tool = DbIliMode.ili2gpkg
exporter.configuration = configuration
exporter.configuration.xtffile = os.path.join(self.base_test_path,
'test_export_data.xtf')
# exporter.stderr.connect(self.on_stderr)
self.assertEqual(exporter.run(), iliexporter.Exporter.SUCCESS)
self.check_export_xtf(exporter.configuration.xtffile)
def test_import_schema_in_mssql(self):
schema = 'test_import_schema'
reset_db_mssql(schema)
print("\nINFO: Validate Import Schema in MS SQL Server")
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(
'xtf') + ';' + testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = SchemaImportConfiguration()
configuration.base_configuration = base_config
configuration.tool = DbIliMode.ili2mssql
configuration.dbhost = 'mssql'
configuration.dbusr = '******'
configuration.dbpwd = '<YourStrong!Passw0rd>'
configuration.dbport = '1433'
configuration.database = schema
configuration.dbschema = schema
configuration.db_odbc_driver = 'ODBC Driver 17 for SQL Server'
configuration.tomlfile = TOML_FILE_DIR
configuration.epsg = 9377
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
importer = iliimporter.Importer()
importer.tool = DbIliMode.ili2mssql
importer.configuration = configuration
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2mssql,
'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},1433;DATABASE={db};UID={user};PWD={pwd}'
.format(db=configuration.database,
user=configuration.dbusr,
pwd=configuration.dbpwd,
host=configuration.dbhost,
port=configuration.dbport), configuration.inheritance,
importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 160)
def test_import_data_in_gpkg(self):
print("\nINFO: Validate Import Data in GPKG...")
gpkg_path = get_test_copy_path(
'db/ladm/gpkg/test_import_data_ladm_v1_1.gpkg')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.tool = DbIliMode.ili2gpkg
configuration.dbfile = gpkg_path
configuration.srs_auth = DEFAULT_SRS_AUTH
configuration.srs_code = DEFAULT_SRS_CODE
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.delete_data = True
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2gpkg
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v1_1.xtf')
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
config_manager = GpkgCommandConfigManager(importer.configuration)
generator = Generator(DbIliMode.ili2gpkg, config_manager.get_uri(),
configuration.inheritance)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 160)
db_gpkg = get_gpkg_conn_from_path(config_manager.get_uri())
res, code, msg = db_gpkg.test_connection()
self.assertTrue(res, msg)
test_layer = self.app.core.get_layer(db_gpkg,
db_gpkg.names.LC_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_gpkg.conn.close()
def test_import_schema_in_pg(self):
print("\nINFO: Validate Import Schema in PG...")
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(
'xtf') + ';' + testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = SchemaImportConfiguration()
configuration.base_configuration = base_config
configuration.tool = DbIliMode.ili2pg
configuration.dbhost = 'postgres'
configuration.dbusr = '******'
configuration.dbpwd = 'clave_ladm_col'
configuration.database = 'ladm_col'
configuration.dbschema = 'test_import_schema'
configuration.tomlfile = TOML_FILE_DIR
configuration.srs_code = 3116
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.CADASTRAL_CARTOGRAPHY_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.VALUATION_MODEL_KEY).full_name()
])
importer = iliimporter.Importer()
importer.tool = DbIliMode.ili2pg
importer.configuration = configuration
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2pg, 'dbname={} user={} password={} host={}'.format(
configuration.database, configuration.dbusr,
configuration.dbpwd, configuration.dbhost),
configuration.inheritance, importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 188)
def test_export_data_in_mssql(self):
print("\nINFO: Validate Export Data in MS SQL Server...")
schema = 'test_export_data'
reset_db_mssql(schema)
restore_schema_mssql(schema)
db_conn = get_mssql_conn(schema)
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
model_list = [
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
]
configuration = ExportConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'mssql'
configuration.dbusr = '******'
configuration.dbpwd = '<YourStrong!Passw0rd>'
configuration.dbport = '1433'
configuration.database = schema # use schema because delete schemas in mssql is difficult
configuration.dbschema = schema
configuration.db_odbc_driver = 'ODBC Driver 17 for SQL Server'
configuration.delete_data = True
configuration.ilimodels = ';'.join(model_list)
exporter = iliexporter.Exporter()
exporter.tool = DbIliMode.ili2mssql
exporter.configuration = configuration
exporter.configuration.xtffile = os.path.join(tempfile.mkdtemp(),
'test_export_data.xtf')
# exporter.stderr.connect(self.on_stderr)
self.assertEqual(exporter.run(), iliexporter.Exporter.SUCCESS)
self.check_export_xtf(exporter.configuration.xtffile)
db_conn.conn.close()
def test_import_schema_in_gpkg(self):
print("\nINFO: Validate Import Schema in GPKG...")
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = SchemaImportConfiguration()
configuration.base_configuration = base_config
configuration.tool = DbIliMode.ili2gpkg
configuration.dbfile = os.path.join(self.base_test_path,
'tmp_import_schema.gpkg')
configuration.tomlfile = TOML_FILE_DIR
configuration.srs_code = 3116
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.CADASTRAL_CARTOGRAPHY_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.VALUATION_MODEL_KEY).full_name()
])
importer = iliimporter.Importer()
importer.tool = DbIliMode.ili2gpkg
importer.configuration = configuration
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
config_manager = GpkgCommandConfigManager(importer.configuration)
generator = Generator(DbIliMode.ili2gpkg, config_manager.get_uri(),
configuration.inheritance)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 188)
def test_export_data_in_pg(self):
print("\nINFO: Validate Export Data in PG...")
restore_schema('test_export_data')
db_pg = get_pg_conn('test_export_data')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = ExportConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'postgres'
configuration.dbusr = '******'
configuration.dbpwd = 'clave_ladm_col'
configuration.database = 'ladm_col'
configuration.dbschema = 'test_export_data'
configuration.delete_data = True
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
exporter = iliexporter.Exporter()
exporter.tool = DbIliMode.ili2pg
exporter.configuration = configuration
exporter.configuration.xtffile = os.path.join(tempfile.mkdtemp(),
'test_export_data.xtf')
# exporter.stderr.connect(self.on_stderr)
self.assertEqual(exporter.run(), iliexporter.Exporter.SUCCESS)
self.check_export_xtf(exporter.configuration.xtffile)
db_pg.conn.close()
def _get_base_configuration(self):
"""
:return: BaseConfiguration object. If it's already configured, it returns the existing object, so that it can
be shared among chained operations (e.g., export DB1-->schema import DB2-->import DB2).
"""
if not self._base_configuration:
self._base_configuration = BaseConfiguration()
self._ilicache = IliCache(self._base_configuration)
self._base_configuration.java_path = self.get_full_java_exe_path(
) # It is well configured at this point!
# Check custom model directories
if QSettings().value(
'Asistente-LADM-COL/models/custom_model_directories_is_checked',
DEFAULT_USE_CUSTOM_MODELS,
type=bool):
custom_model_directories = QSettings().value(
'Asistente-LADM-COL/models/custom_models',
DEFAULT_MODELS_DIR)
if not custom_model_directories:
self._base_configuration.custom_model_directories_enabled = False
else:
self._base_configuration.custom_model_directories = custom_model_directories
self._base_configuration.custom_model_directories_enabled = True
# Debug mode
self._base_configuration.debugging_enabled = \
QSettings().value('Asistente-LADM-COL/models/debug', DEFAULT_ILI2DB_DEBUG_MODE, type=bool)
self._base_configuration.logfile_path = QSettings().value(
'Asistente-LADM-COL/models/log_file_path', '')
self._ilicache.refresh(
) # Always call it after setting custom_model_directories
return self._base_configuration
def export_baskets(self):
java_home_set = self.java_dependency.set_java_home()
if not java_home_set:
message_java = QCoreApplication.translate("BasketExporter", """Configuring Java {}...""").format(
JAVA_REQUIRED_VERSION)
self.logger.status(message_java)
self.java_dependency.get_java_on_demand()
return
self.base_configuration = BaseConfiguration()
self.ilicache = IliCache(self.base_configuration)
db_factory = self._dbs_supported.get_db_factory(self._db.engine)
self.configuration = ExportConfiguration()
db_factory.set_ili2db_configuration_params(self._db.dict_conn_params, self.configuration)
self.configuration.with_exporttid = True
full_java_exe_path = JavaDependency.get_full_java_exe_path()
if full_java_exe_path:
self.base_configuration.java_path = full_java_exe_path
# Check custom model directories
if QSettings().value('Asistente-LADM-COL/models/custom_model_directories_is_checked', DEFAULT_USE_CUSTOM_MODELS, type=bool):
custom_model_directories = QSettings().value('Asistente-LADM-COL/models/custom_models', DEFAULT_MODELS_DIR)
if not custom_model_directories:
self.base_configuration.custom_model_directories_enabled = False
else:
self.base_configuration.custom_model_directories = custom_model_directories
self.base_configuration.custom_model_directories_enabled = True
self.ilicache.refresh() # Always call it after setting custom_model_directories
self.configuration.base_configuration = self.base_configuration
if self.get_ili_models():
self.configuration.ilimodels = ';'.join(self.get_ili_models())
self.exporter = iliexporter.Exporter()
self.exporter.tool = db_factory.get_model_baker_db_ili_mode()
self.exporter.process_started.connect(self.on_process_started)
self.exporter.stderr.connect(self.on_stderr)
#self.exporter.process_finished.connect(self.on_process_finished)
res = dict()
count = 0
total = len(self._basket_dict)
with OverrideCursor(Qt.WaitCursor):
for basket,name in self._basket_dict.items():
self.log = ''
self.configuration.xtffile = os.path.join(self._export_dir, "{}.xtf".format(name))
self.configuration.baskets = basket
self.exporter.configuration = self.configuration
try:
if self.exporter.run() != iliexporter.Exporter.SUCCESS:
msg = QCoreApplication.translate("BasketExporter", "An error occurred when exporting the data for '{}' (check the QGIS log panel).").format(name)
res[basket] = (False, msg)
QgsMessageLog.logMessage(self.log, QCoreApplication.translate("BasketExporter", "Allocate to coordinators"), Qgis.Critical)
else:
res[basket] = (True, QCoreApplication.translate("BasketExporter", "XTF export for '{}' successful!").format(name) )
except JavaNotFoundError:
msg = QCoreApplication.translate("BasketExporter", "Java {} could not be found. You can configure the JAVA_HOME environment variable manually, restart QGIS and try again.").format(JAVA_REQUIRED_VERSION)
res[basket] = (False, msg)
count += 1
self.total_progress_updated.emit(count/total*100)
return res
def test_import_data_in_mssql(self):
print("\nINFO: Validate Import Data in MS SQL Server...")
schema = 'test_ladm_col'
reset_db_mssql(schema)
restore_schema_mssql(schema)
db_conn = get_mssql_conn(schema)
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
model_list = [
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
]
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'mssql'
configuration.dbusr = '******'
configuration.dbpwd = '<YourStrong!Passw0rd>'
configuration.dbport = '1433'
configuration.database = schema # use schema because delete schemas in mssql is difficult
configuration.dbschema = schema
configuration.db_odbc_driver = 'ODBC Driver 17 for SQL Server'
configuration.delete_data = True
configuration.ilimodels = ';'.join(model_list)
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2mssql
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v1_1.xtf')
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2mssql,
'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},1433;DATABASE={db};UID={user};PWD={pwd}'
.format(db=configuration.database,
user=configuration.dbusr,
pwd=configuration.dbpwd,
host=configuration.dbhost,
port=configuration.dbport), configuration.inheritance,
importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 160)
res, code, msg = db_conn.test_connection()
self.assertTrue(res, msg)
test_layer = self.app.core.get_layer(db_conn,
db_conn.names.LC_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_conn.conn.close()
def test_import_data_in_pg(self):
print("\nINFO: Validate Import Data in PG...")
restore_schema('test_import_data')
db_pg = get_pg_conn('test_import_data')
base_config = BaseConfiguration()
base_config.custom_model_directories = testdata_path(MODELS_PATH)
base_config.custom_model_directories_enabled = True
configuration = ImportDataConfiguration()
configuration.base_configuration = base_config
configuration.dbhost = 'postgres'
configuration.dbusr = '******'
configuration.dbpwd = 'clave_ladm_col'
configuration.database = 'ladm_col'
configuration.srs_auth = DEFAULT_SRS_AUTH
configuration.srs_code = DEFAULT_SRS_CODE
configuration.inheritance = ILI2DBNames.DEFAULT_INHERITANCE
configuration.create_basket_col = ILI2DBNames.CREATE_BASKET_COL
configuration.create_import_tid = ILI2DBNames.CREATE_IMPORT_TID
configuration.stroke_arcs = ILI2DBNames.STROKE_ARCS
configuration.dbschema = 'test_import_data'
configuration.delete_data = True
configuration.ilimodels = ';'.join([
self.ladmcol_models.model(
LADMNames.LADM_COL_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SNR_DATA_SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_MODEL_KEY).full_name(),
self.ladmcol_models.model(
LADMNames.SUPPLIES_INTEGRATION_MODEL_KEY).full_name(),
self.ladmcol_models.model(LADMNames.SURVEY_MODEL_KEY).full_name()
])
importer = iliimporter.Importer(dataImport=True)
importer.tool = DbIliMode.ili2pg
importer.configuration = configuration
importer.configuration.xtffile = testdata_path(
'xtf/test_ladm_col_queries_v1_1.xtf')
# importer.stderr.connect(self.on_stderr)
self.assertEqual(importer.run(), iliimporter.Importer.SUCCESS)
generator = Generator(
DbIliMode.ili2pg, 'dbname={} user={} password={} host={}'.format(
configuration.database, configuration.dbusr,
configuration.dbpwd, configuration.dbhost),
configuration.inheritance, importer.configuration.dbschema)
available_layers = generator.layers()
self.assertEqual(len(available_layers), 160)
res, code, msg = db_pg.test_connection()
self.assertTrue(res, msg)
test_layer = self.app.core.get_layer(db_pg,
db_pg.names.LC_BOUNDARY_POINT_T,
load=True)
self.assertEqual(test_layer.featureCount(), 390)
db_pg.conn.close()