def delete_data_in_database(self):
"""Deletes the project (= database schema) in the database.
Returns:
False: If something went wrong with deleting in the database.
Otherwise True.
"""
try:
if self.settings.value("options/general/use_pg_projects_database",
False,
type=bool):
db = get_default_db()
else:
db = open_psql_db(self.db_host, self.db_name, self.db_port,
self.db_admin, self.db_admin_pwd)
sql = "BEGIN;"
sql += "DROP SCHEMA IF EXISTS %s CASCADE;" % self.db_schema
sql += "COMMIT;"
query = db.exec_(sql)
if not query.isActive():
message = "Error occured while deleting project in database."
raise VerisoError(
message, long_message=QSqlQuery.lastError(query).text())
db.close()
del db
return True
except Exception as e:
message = "Something went wrong while deleting the project."
raise VerisoError(message, e)
def delete_data_in_database(self):
"""Deletes the project (= database schema) in the database.
Returns:
False: If something went wrong with deleting in the database.
Otherwise True.
"""
try:
db = open_psql_db(self.db_host, self.db_name, self.db_port,
self.db_admin, self.db_admin_pwd)
sql = "BEGIN;"
sql += "DROP SCHEMA IF EXISTS %s CASCADE;" % self.db_schema
sql += "DELETE FROM geometry_columns WHERE f_table_schema='%s' ;" % self.db_schema
sql += "COMMIT;"
query = db.exec_(sql)
if not query.isActive():
message = "Error occured while deleting project in database."
raise VerisoError(
message, long_message=QSqlQuery.lastError(query).text())
db.close()
del db
return True
except Exception as e:
message = "Something went wrong while deleting the project."
raise VerisoError(message, e)
def update_project_database_sqlite(self):
"""Deletes the deleted project from the sqlite project database.
Returns:
False: If something went wrong. Otherwise True.
"""
try:
db = get_projects_db()
sql = "DELETE FROM projects WHERE dbschema = '%s';" % (
self.db_schema)
query = db.exec_(sql)
if not query.isActive():
message = "Error while reading from projects database."
raise VerisoError(
message, long_message=QSqlQuery.lastError(query).text())
db.close()
del db
return True
except Exception as e:
message = "Something went wrong while updating projects database."
raise VerisoError(message, e)
def update_projects_database_sqlite(self):
"""Updates the sqlite projects database.
Returns:
False: When there an error occured. Otherswise True.
"""
error_message = ("Something went wrong while updating projects "
"database. You need to delete the database schema "
"manually.")
try:
# Create a new projects database if there is none (copy one from
# the templates).
if self.projects_database == "":
template = get_absolute_path("templates/template_projects.db")
self.projects_database = QDir.toNativeSeparators(
QDir.cleanPath(self.projects_root_directory +
"/projects.db"))
shutil.copyfile(template, self.projects_database)
self.settings.setValue("options/general/projects_database",
self.projects_database)
db = get_projects_db()
project_root_directory = QDir.toNativeSeparators(
QDir.cleanPath(self.projects_root_directory + "/" +
str(self.db_schema)))
values = (self.db_schema, self.db_schema, self.db_host,
self.db_name, self.db_port, self.db_schema, self.db_user,
self.db_pwd, self.db_admin, self.db_admin_pwd, self.epsg,
self.ili, self.app_module, self.app_module_name,
self.projects_root_directory, project_root_directory,
self.data_date, self.notes, self.itf, self.max_scale)
values = """VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s',
'%s', '%s', 'postgres', '%s', '%s', '%s', '%s', '%s',
'%s', '%s', '%s', '%s', '%s')""" % values
sql = "INSERT INTO projects (id, displayname, dbhost, dbname, " \
"dbport, dbschema, dbuser, dbpwd, dbadmin, dbadminpwd, " \
"provider, epsg, ilimodelname, appmodule, appmodulename, " \
"projectrootdir, projectdir, datadate, notes, itf, " \
"max_scale)" + values
query = db.exec_(sql)
if not query.isActive():
message = "Error while updating projects database."
raise VerisoError(
message, long_message=QSqlQuery.lastError(query).text())
db.close()
self.projectsDatabaseHasChanged.emit()
return True
except Exception as e:
raise VerisoError(error_message, e)
def open_psql_db(db_host, db_name, db_port, db_admin, db_admin_pwd):
"""
Open PostGIS db connection taking care that the connection exists only once
:param db_host: the db host
:param db_name: the db name
:param db_port: the db port
:param db_admin: the db administrator username
:param db_admin_pwd: the db administrator username password
:return: a db object
:rtype: QSqlDatabase
"""
connection_name = "%s@%s:%s/%s" % (db_admin, db_host, db_port, db_name)
try:
if connection_name in QSqlDatabase.connectionNames():
db = QSqlDatabase.database(connection_name)
else:
db = QSqlDatabase.addDatabase("QPSQL", connection_name)
db.setHostName(db_host)
db.setPort(int(db_port))
db.setDatabaseName(db_name)
db.setUserName(db_admin)
db.setPassword(db_admin_pwd)
if not db.open():
raise Exception()
except Exception as e:
message = "Could not open psql database: %s" % connection_name
message = tr(message)
raise VerisoError(message, e, db.lastError().text())
return db
def read_import_output(self):
"""
Reads the output of the ili2pg conversion
:return:
"""
output = self.textEditImportOutput.toPlainText()
if output.find("Info: ...import done") < 0 or output.find(
"compiler failed") >= 0:
message = "Import process not successfully finished."
raise VerisoError(message)
def get_info_from_yaml(path):
try:
data = yaml_load_file(path)
name = data['name']
except Exception as e:
raise VerisoError('error parsing %s' % path, e)
try:
shortcut = data['shortcut']
except KeyError:
shortcut = ''
return name, shortcut
def yaml_load_file(file_path):
"""
High level method to parse YAML safely into an OrderedDict
:param file_path: file path of the yml
:return: dict
"""
try:
with open(file_path) as f:
parsed_dict = yaml_ordered_load(f, Loader=yaml.SafeLoader)
return parsed_dict
except Exception as e:
raise VerisoError("Something went wrong when parsing %s" % file_path,
e)
def open_psql_db(db_host, db_name, db_port, db_admin, db_admin_pwd):
"""
Open PostGIS db connection taking care that the connection exists only once
:param db_host: the db host
:param db_name: the db name
:param db_port: the db port
:param db_admin: the db administrator username
:param db_admin_pwd: the db administrator username password
:return: a db object
:rtype: QSqlDatabase
"""
connection_name = "%s@%s:%s/%s" % (db_admin, db_host, db_port, db_name)
try:
if connection_name in QSqlDatabase.connectionNames():
db = QSqlDatabase.database(connection_name)
else:
if not QSqlDatabase.isDriverAvailable("QPSQL"):
raise VerisoError('Please install the PSQL Qt driver\n'
'(libqt4-sql-psql in ubuntu).\n')
db = QSqlDatabase.addDatabase("QPSQL", connection_name)
db.setHostName(db_host)
db.setPort(int(db_port))
db.setDatabaseName(db_name)
db.setUserName(db_admin)
db.setPassword(db_admin_pwd)
if not db.open():
raise Exception()
except VerisoError:
raise
except Exception as e:
message = "Could not open psql database: %s see log for more details"\
% connection_name
message = tr(message)
#
raise VerisoError(message, e, db.lastError().text())
return db
def get_absolute_path(path):
path = "/python/plugins/veriso/%s" % path
filename = QDir.toNativeSeparators(QDir.cleanPath(
QgsApplication.qgisSettingsDirPath() + path))
if not os.path.isfile(filename):
filename = QDir.toNativeSeparators(QDir.cleanPath(
QgsApplication.pkgDataPath() + path))
# the plugin is not in the .qgis2 folder
# lets try in the qgis installation folder (for central installation
# on servers)
if not os.path.isfile(filename):
raise VerisoError('File not found at %s' % filename)
return filename
def dynamic_import(module_name):
"""
import a module from the given path in a safe manner
:param module_name: string path to the module to load
:return: the loaded module
"""
try:
module = importlib.import_module(module_name)
QgsMessageLog.logMessage('Successfully loaded: %s ' % module_name,
"VeriSO", QgsMessageLog.INFO)
return module
except Exception as e:
message = tr(
"Error while loading application module: %s") % module_name
raise VerisoError(message, e)
def create_project_directory(self):
"""Creates a directory with the same name as the project (db schema)
in the project root directory. This will be for exports, maps etc.
It emits a projects database changed signal.
Returns:
False: If the directory could not be created. Otherwise True.
"""
try:
os.makedirs(
os.path.join(str(self.projects_root_directory),
str(self.db_schema)))
return True
except Exception as e:
message = "Something went wrong while creating project directory."
raise VerisoError(message, e)
def btnTestConnection_clicked(self):
try:
db = open_psql_db(self.lineEditDbHost.text(),
self.lineEditDbDatabase.text(),
self.lineEditDbPort.text(),
self.lineEditDbAdmin.text(),
self.lineEditDbAdminPwd.text())
sql = "select postgis_full_version();"
query = db.exec_(sql)
count = query.size()
db.close()
if count < 1:
raise VerisoError(
"No PostGIS found on the DB %s" % db.connectionName())
except Exception as e:
self.test_connection_failed(e)
else:
self.test_connection_succes()
def delete_project_directory(self):
"""Deletes the directory with the same name as the project (db
schema) in the project root directory.
It emits a projects database changed signal.
Returns:
False: If the directory could not be deleted. Otherwise True.
"""
try:
projects_root_directory = QSettings(
"CatAIS",
"VeriSO").value("options/general/projects_root_directory")
path = os.path.join(str(projects_root_directory),
str(self.db_schema))
shutil.rmtree(path)
return True
except Exception as e:
message = "Something went wrong while deleting the project folder."
raise VerisoError(message, e)
def postprocess_data(self, queries):
"""Does the postprocessing in the postgresql/postgis database.
Returns:
-1: If the process fails (e.g. no db connection etc.). Otherwise
number of errors occured while postprocessing.
"""
try:
db = open_psql_db(self.db_host, self.db_name, self.db_port,
self.db_admin, self.db_admin_pwd)
errors = 0
self.report_progress("\n\nInfo: Starting postprocessing...")
for sql in queries:
self.report_progress("\n\n%s" % sql)
query = db.exec_(str(sql))
if not query.isActive():
errors += 1
message = "Error while postprocessing data:"
QgsMessageLog.logMessage(tr(message), "VeriSO",
Qgis.Critical)
QgsMessageLog.logMessage(
str(QSqlQuery.lastError(query).text()) + str(sql),
"VeriSO", Qgis.Critical)
self.report_progress("--> error, see log", 'orange')
if errors > 0:
self.report_progress(
"Error: ...postprocessing completed with errors", "red")
if not self.ignore_postprocessing_errors:
raise Exception()
self.report_progress("Info: ...postprocessing completed")
db.close
del db
except Exception as e:
message = "Something went wrong while postprocessing data. You " \
"need to delete the database schema manually."
raise VerisoError(message, e)
def open_sqlite_db(file_path, connection_name):
"""
Opens SQLite db connection taking care that the connection exists only once
:param file_path: the oprional path of the SQLite file
:return: a db object
:rtype: QSqlDatabase
"""
try:
if connection_name in QSqlDatabase.connectionNames():
db = QSqlDatabase.database(connection_name)
else:
db = QSqlDatabase.addDatabase("QSQLITE", connection_name)
db.setDatabaseName(file_path)
if not db.open():
raise Exception()
except Exception as e:
message = "Could not open sqlite database: %s" % connection_name
message = tr(message)
raise VerisoError(message, e, db.lastError().text())
return db
def get_postprocessing_queries(self):
"""Gets the SQL queries that are stored in the sqlite database for
the postprocessing process which is done in postgis.
Language support: Everything that is not french or italian will be
german.
Returns:
False: If the queries could not be fetched from the sqlite
database. Otherwise a list with the SQL queries.
"""
# originial
""" filename = QDir.convertSeparators(QDir.cleanPath(
QgsApplication.qgisSettingsDirPath() +
"/python/plugins/veriso/modules/" + self.app_module +
"/postprocessing/postprocessing.db"))"""
# Hack
filename = QDir.convertSeparators(
QDir.cleanPath((os.path.realpath(__file__)).split("python")[0] +
"/python/plugins/veriso/modules/" +
self.app_module +
"/postprocessing/postprocessing.db"))
self.report_progress("Info: getting postprocessing queries...")
try:
# This is NOT the project db
connection_name = 'postprocessing_' + self.app_module
db = open_sqlite_db(filename, connection_name)
locale = QSettings().value('locale/userLocale')[0:2]
if locale == "fr":
lang = locale
elif locale == "it":
lang = locale
else:
lang = "de"
sql = "SELECT * FROM postprocessing " \
"WHERE (lang = '%s' " \
"OR lang IS NULL) AND apply = 1 " \
"ORDER BY 'order', ogc_fid;" % lang
query = db.exec_(sql)
if not query.isActive():
message = "Database query not active."
raise VerisoError(
message, long_message=QSqlQuery.lastError(query).text())
queries = []
record = query.record()
while next(query):
sql_query = str(query.value(record.indexOf("sql_query")))
sql_query = sql_query.replace("$$DBSCHEMA", self.db_schema)
sql_query = sql_query.replace("$$USER", self.db_user)
sql_query = sql_query.replace("$$EPSG", self.epsg)
queries.append(sql_query)
db.close()
del db
return queries
except Exception as e:
message = "Something went wrong while catching postprocessing " \
"queries from sqlite database. You need to delete the " \
"database schema manually."
raise VerisoError(message, e)
def get_check_topics(module_name):
"""Get all check topics (aka groups).
Different languages are support. See the yaml file how to deal with it.
Returns:
A ordered dictionary with the topic name and corresponding check
files (python). False if something went wrong.
"""
topics_dir = os.path.join(get_modules_dir(), module_name, 'checks')
checks = []
for topic_dir in get_subdirs(topics_dir):
topic_file = os.path.join(topics_dir, topic_dir, 'topic.yml')
if os.path.isfile(topic_file):
try:
topic = yaml_load_file(topic_file)
topic['topic_dir'] = topic_dir
checks.append(topic)
except VerisoError:
raise
try:
locale = QSettings().value('locale/userLocale')[0:2]
except TypeError:
locale = 'de'
try:
topics = OrderedDict()
checks = sorted(checks, key=lambda k: k['topic'][locale])
for check in checks:
topic = check["topic"]
topic_dir = check["topic_dir"]
# Check, if yaml file is multilingual.
try:
if topic in topics:
continue
topics[topic] = check
# TODO control this whe using checks_from_files
# yaml is *not* multilingual.
except:
# yaml is multilingual.
# If the language set in QGIS is not available in the
# yaml file, the first language will be chosen
# dinamically get the checks based on the available files
checks_from_files = get_checks_from_files(
module_name, topic_dir)
try:
my_topic = topic[locale]
my_check = OrderedDict()
my_check["topic"] = my_topic
# my_check["checks"] = check["checks"]
my_check["checks"] = checks_from_files
my_check["topic_dir"] = topic_dir
topics[my_topic] = my_check
# language found
except:
# language *not* found
my_check = OrderedDict()
my_check["topic"] = list(topic.values())[0]
# my_check["checks"] = check["checks"]
my_check["checks"] = checks_from_files
my_check["topic_dir"] = topic_dir
topics[my_check["topic"]] = my_check
return topics
except Exception as e:
print(str(e))
raise VerisoError(str(e), e, tag=module_name)
def update_projects_database_pg(self):
"""Updates the postgres projects database.
Returns:
False: When there an error occured. Otherswise True.
"""
error_message = ("Something went wrong while updating projects "
"database. You need to delete the database schema "
"manually.")
try:
# Create a new projects database (schema and table on pg) if
# there is none
table_exists = False
schema_exists = False
db = get_default_db()
sql = "SELECT 1 FROM pg_namespace " \
"WHERE nspname = 'veriso_conf'"
query = db.exec_(sql)
if query.size() > 0:
schema_exists = True
sql = """SELECT 1 FROM pg_tables
WHERE schemaname = 'veriso_conf'
AND tablename = 'project'"""
query = db.exec_(sql)
if query.size() > 0:
table_exists = True
if not schema_exists:
sql = "CREATE SCHEMA veriso_conf"
query = db.exec_(sql)
if not table_exists:
sql = "CREATE TABLE veriso_conf.project (" \
"ogc_fid serial primary key, " \
"id character varying, " \
"displayname character varying," \
"provider character varying, " \
"epsg integer, " \
"ilimodelname character varying, " \
"appmodule character varying, " \
"appmodulename character varying, " \
"datadate timestamp, " \
"notes character varying, " \
"itf character varying, " \
"max_scale integer default 0 " \
")"
query = db.exec_(sql)
values = (self.db_schema, self.db_schema, self.epsg, self.ili,
self.app_module, self.app_module_name, self.data_date,
self.notes, self.itf, self.max_scale)
values = "VALUES ( "\
"'%s', '%s', 'postgres', '%s', '%s', '%s', '%s', '%s', " \
"'%s', '%s', '%s')" % values
sql = "INSERT INTO veriso_conf.project (id, displayname, " \
"provider, epsg, ilimodelname, appmodule, appmodulename, " \
"datadate, notes, itf, " \
"max_scale)" + values
query = db.exec_(sql)
if not query.isActive():
message = "Error while updating projects database."
raise VerisoError(
message, long_message=QSqlQuery.lastError(query).text())
db.close()
self.projectsDatabaseHasChanged.emit()
return True
except Exception as e:
raise VerisoError(error_message, e)
