class STTaskManager(QObject):
"""
Retrieve tasks for a user from the Transitional System's Task Service and store them during the session.
"""
task_started = pyqtSignal(int) # task_id
task_canceled = pyqtSignal(int) # task_id
task_closed = pyqtSignal(int) # task_id
def __init__(self):
QObject.__init__(self)
self.logger = Logger()
self.__registered_tasks = dict()
self.st_config = TransitionalSystemConfig()
@_with_override_cursor
def __retrieve_tasks(self, st_user, task_type=None, task_status=None):
headers = {
'Authorization': "Bearer {}".format(st_user.get_token()),
# 'User-Agent': "PostmanRuntime/7.20.1",
'Accept': "*/*",
'Cache-Control': "no-cache",
# 'Postman-Token': "987c7fbf-af4d-42e8-adee-687f35f4a4a0,0547120a-6f8e-42a8-b97f-f052602cc7ff",
# 'Host': "st.local:8090",
'Accept-Encoding': "gzip, deflate",
'Connection': "keep-alive",
'cache-control': "no-cache"
}
try:
self.logger.debug(__name__, "Retrieving tasks from server...")
response = requests.request("GET", self.st_config.ST_GET_TASKS_SERVICE_URL, headers=headers)
except requests.ConnectionError as e:
msg = self.st_config.ST_CONNECTION_ERROR_MSG.format(e)
self.logger.warning(__name__, msg)
return False, msg
status_OK = response.status_code == 200
if status_OK:
# Parse, create and register tasks
response_data = json.loads(response.text)
for task_data in response_data:
task = STTask(task_data)
if task.is_valid():
self.__register_task(task)
else:
if response.status_code == 500:
self.logger.warning(__name__, self.st_config.ST_STATUS_500_MSG)
elif response.status_code > 500 and response.status_code < 600:
self.logger.warning(__name__, self.st_config.ST_STATUS_GT_500_MSG)
elif response.status_code == 401:
self.logger.warning(__name__, self.st_config.ST_STATUS_401_MSG)
def get_tasks(self, st_user, task_type=None, task_status=None):
"""
Go to server for current tasks per user
:param st_user:
:param task_type: To filter task types. Still unused.
:param task_status: To filter task statuses. Still unused.
:return: dict of task ids with the corresponding task object
"""
# Each call refreshes the registered tasks.
self.unregister_tasks()
self.__retrieve_tasks(st_user, task_type, task_status)
return self.__registered_tasks
def get_task(self, task_id):
task = self.__registered_tasks[task_id] if task_id in self.__registered_tasks else None
if task is None:
self.logger.warning(__name__, "Task {} not found!!!".format(task_id))
else:
self.logger.info(__name__, "Task {} found!!!".format(task_id))
return task
def __register_task(self, task):
self.logger.debug(__name__, "Task {} registered!".format(task.get_id()))
self.__registered_tasks[task.get_id()] = task
def __unregister_task(self, task_id):
self.logger.debug(__name__, "Task {} unregistered!".format(task_id))
self.__registered_tasks[task_id] = None
del self.__registered_tasks[task_id]
def unregister_tasks(self):
for k,v in self.__registered_tasks.items():
self.__registered_tasks[k] = None
self.__registered_tasks = dict()
self.logger.info(__name__, "All tasks have been unregistered!")
@_with_override_cursor
def start_task(self, st_user, task_id):
payload = {}
headers = {
'Authorization': "Bearer {}".format(st_user.get_token()),
}
try:
self.logger.debug(__name__, "Telling the server to start a task...")
response = requests.request("PUT", self.st_config.ST_START_TASK_SERVICE_URL.format(task_id), headers=headers, data=payload)
except requests.ConnectionError as e:
msg = self.st_config.ST_CONNECTION_ERROR_MSG.format(e)
self.logger.warning(__name__, msg)
return False, msg
status_OK = response.status_code == 200
if status_OK:
# Parse response
response_data = json.loads(response.text)
self.logger.info(__name__, "Task id '{}' started in server!...".format(task_id))
self.logger.info_msg(__name__, QCoreApplication.translate("TaskManager",
"The task '{}' was successfully started!".format(
self.get_task(task_id).get_name())))
self.update_task_info(task_id, response_data)
self.task_started.emit(task_id)
else:
if response.status_code == 500:
self.logger.warning(__name__, self.st_config.ST_STATUS_500_MSG)
elif response.status_code > 500 and response.status_code < 600:
self.logger.warning(__name__, self.st_config.ST_STATUS_GT_500_MSG)
elif response.status_code == 401:
self.logger.warning(__name__, self.st_config.ST_STATUS_401_MSG)
else:
self.logger.warning(__name__, "Status code not handled: {}".format(response.status_code))
@_with_override_cursor
def cancel_task(self, st_user, task_id, reason):
payload = json.dumps({"reason": reason})
headers = {
'Authorization': "Bearer {}".format(st_user.get_token()),
'Content-Type': 'application/json'
}
try:
self.logger.debug(__name__, "Telling the server to cancel a task...")
response = requests.request("PUT", self.st_config.ST_CANCEL_TASK_SERVICE_URL.format(task_id), headers=headers, data=payload)
except requests.ConnectionError as e:
msg = self.st_config.ST_CONNECTION_ERROR_MSG.format(e)
self.logger.warning(__name__, msg)
return False, msg
status_OK = response.status_code == 200
if status_OK:
# No need to parse response this time, we'll ask tasks from server again anyways
self.logger.info(__name__, "Task id '{}' canceled in server!".format(task_id))
self.logger.info_msg(__name__, QCoreApplication.translate("TaskManager", "The task '{}' was successfully canceled!".format(self.get_task(task_id).get_name())))
self.task_canceled.emit(task_id)
else:
if response.status_code == 500:
self.logger.warning(__name__, self.st_config.ST_STATUS_500_MSG)
elif response.status_code > 500 and response.status_code < 600:
self.logger.warning(__name__, self.st_config.ST_STATUS_GT_500_MSG)
elif response.status_code == 401:
self.logger.warning(__name__, self.st_config.ST_STATUS_401_MSG)
else:
self.logger.warning(__name__, "Status code not handled: {}, payload: {}".format(response.status_code, payload))
@_with_override_cursor
def close_task(self, st_user, task_id):
payload = {}
headers = {
'Authorization': "Bearer {}".format(st_user.get_token()),
}
try:
self.logger.debug(__name__, "Telling the server to close a task...")
response = requests.request("PUT", self.st_config.ST_CLOSE_TASK_SERVICE_URL.format(task_id), headers=headers, data=payload)
except requests.ConnectionError as e:
msg = self.st_config.ST_CONNECTION_ERROR_MSG.format(e)
self.logger.warning(__name__, msg)
return False, msg
status_OK = response.status_code == 200
if status_OK:
# No need to parse response this time, we'll ask tasks from server again anyways
self.logger.success(__name__, "Task id '{}' closed in server!".format(task_id))
self.logger.success_msg(__name__, QCoreApplication.translate("TaskManager",
"The task '{}' was successfully closed!".format(
self.get_task(task_id).get_name())))
self.task_closed.emit(task_id)
else:
if response.status_code == 500:
self.logger.warning(__name__, self.st_config.ST_STATUS_500_MSG)
elif response.status_code > 500 and response.status_code < 600:
self.logger.warning(__name__, self.st_config.ST_STATUS_GT_500_MSG)
elif response.status_code == 401:
self.logger.warning(__name__, self.st_config.ST_STATUS_401_MSG)
elif response.status_code == 422:
response_data = json.loads(response.text)
msg = QCoreApplication.translate("STSession", QCoreApplication.translate("TaskManager",
"Task not closed! Details: {}").format(response_data['message'] if 'message' in response_data else "Unreadable response from server."))
self.logger.warning_msg(__name__, msg)
else:
self.logger.warning(__name__, "Status code not handled: {}".format(response.status_code))
def update_task_info(self, task_id, task_data):
task = STTask(task_data)
if task.is_valid():
self.__unregister_task(task_id)
self.__register_task(task)
class BaseConfigureReceiversPanelWidget(QgsPanelWidget, WIDGET_UI):
clear_message_bar_requested = pyqtSignal()
def __init__(self, parent, controller):
QgsPanelWidget.__init__(self, parent)
self.setupUi(self)
self.parent = parent
self.logger = Logger()
self._controller = controller
self.setDockMode(True)
self.setPanelTitle(
QCoreApplication.translate("BaseConfigureReceiversPanelWidget",
"Configure receivers"))
self.panelAccepted.connect(self.panel_accepted)
self.tbl_receivers.itemSelectionChanged.connect(self.selection_changed)
self.btn_save.clicked.connect(self.save_receiver)
self.btn_delete.clicked.connect(self.delete_receiver)
self.btn_delete.setEnabled(False)
self.fill_data()
self.tbl_receivers.resizeColumnsToContents()
def panel_accepted(self):
self.clear_message_bar_requested.emit()
def fill_data(self):
self.tbl_receivers.clearContents()
receivers_data = self._controller.get_receivers_data()
self.tbl_receivers.setRowCount(len(receivers_data))
self.tbl_receivers.setSortingEnabled(False)
for row, data in enumerate(receivers_data.items()):
receiver_name, receiver_docid = data[1]
self.fill_row(data[0], receiver_name, receiver_docid, row)
self.tbl_receivers.setSortingEnabled(True)
self.tbl_receivers.resizeColumnsToContents()
def fill_row(self, receiver_t_id, receiver_name, receiver_docid, row):
item = QTableWidgetItem(receiver_name)
item.setData(Qt.UserRole, receiver_t_id)
self.tbl_receivers.setItem(row, 0, item)
item2 = QTableWidgetItem(receiver_docid)
item2.setData(Qt.UserRole, receiver_t_id)
self.tbl_receivers.setItem(row, 1, item2)
def selection_changed(self):
self.btn_delete.setEnabled(bool(self.tbl_receivers.selectedItems()))
def save_receiver(self):
if self.txt_first_name.text().strip(
) and self.txt_first_last_name.text().strip(
) and self.txt_document_id.text().strip():
try:
int(self.txt_document_id.text().strip())
except ValueError as e:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseConfigureReceiversPanelWidget",
"Invalid value for document id. Only digits are accepted."
))
return
basket_t_id, msg = self._controller.get_basket_id_for_new_receiver(
)
if basket_t_id is None:
self.logger.warning_msg(__name__, msg)
return
names = self._controller.db().names
receiver_data = {
names.FDC_USER_T_DOCUMENT_TYPE_F:
self._controller.receiver_type,
names.FDC_USER_T_DOCUMENT_ID_F:
self.txt_document_id.text().strip(),
names.FDC_USER_T_FIRST_NAME_F:
self.txt_first_name.text().strip(),
names.FDC_USER_T_SECOND_NAME_F:
self.txt_second_name.text().strip(),
names.FDC_USER_T_FIRST_LAST_NAME_F:
self.txt_first_last_name.text().strip(),
names.FDC_USER_T_SECOND_LAST_NAME_F:
self.txt_second_last_name.text().strip(),
names.T_ILI_TID_F:
str(uuid.uuid4()),
names.T_BASKET_F:
basket_t_id
}
res = self._controller.save_receiver(receiver_data)
if res:
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"BaseConfigureReceiversPanelWidget",
"Receiver saved!"))
self.fill_data()
self.initialize_input_controls()
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseConfigureReceiversPanelWidget",
"There was an error saving the receiver."))
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseConfigureReceiversPanelWidget",
"First name, last name and document id are mandatory."))
def delete_receiver(self):
selected_receiver_id = [
item.data(Qt.UserRole)
for item in self.tbl_receivers.selectedItems()
]
if selected_receiver_id:
res, msg = self._controller.delete_receiver(
selected_receiver_id[0])
if res:
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"BaseConfigureReceiversPanelWidget",
"Receiver deleted!"))
self.fill_data()
else:
if not msg:
msg = QCoreApplication.translate(
"BaseConfigureReceiversPanelWidget",
"There was an error deleting the receiver.")
self.logger.warning_msg(__name__, msg)
def initialize_input_controls(self):
self.txt_document_id.setText('')
self.txt_first_name.setText('')
self.txt_second_name.setText('')
self.txt_first_last_name.setText('')
self.txt_second_last_name.setText('')
class BaseAllocateParcelsToReceiverPanelWidget(QgsPanelWidget, WIDGET_UI):
refresh_parcel_data_requested = pyqtSignal()
def __init__(self, parent, controller, parcels_to_be_allocated):
QgsPanelWidget.__init__(self, parent)
self.setupUi(self)
self.parent = parent
self._controller = controller
self.logger = Logger()
# Main dicts to store parcels that are not yet allocated but were selected
# from the previous panel and parcels that are already allocated in the DB
self.__parcels_to_be_allocated = parcels_to_be_allocated # {parcel_fid: parcel_number}
self.__parcels_already_allocated = dict(
) # {parcel_fid: parcel_number}
self.setDockMode(True)
self.setPanelTitle(
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"Allocate parcels to receiver"))
self.panelAccepted.connect(self.panel_accepted)
self.btn_save_allocation.clicked.connect(self.save_allocation)
self.btn_discard_parcels.clicked.connect(self.discard_parcels)
self.cbo_receiver.currentIndexChanged.connect(self.receiver_changed)
self.__txt_already_allocated = QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget", "{} already allocated")
self.__txt_to_be_allocated = QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget", "{} to be allocated")
self.fill_table()
self.fill_receivers()
self.tbl_parcels.resizeColumnsToContents()
def panel_accepted(self):
self.refresh_parcel_data_requested.emit()
def fill_table(self, refresh_allocated_parcels=True):
self.tbl_parcels.clearContents()
if refresh_allocated_parcels:
self.__parcels_already_allocated = self._controller.get_already_allocated_parcels_for_receiver(
self.cbo_receiver.currentData())
number_of_rows = len(self.__parcels_to_be_allocated) + len(
self.__parcels_already_allocated)
self.tbl_parcels.setRowCount(number_of_rows)
self.tbl_parcels.setSortingEnabled(False)
# Fill parcels to be allocated
for row, (parcel_fid, parcel_number) in enumerate(
self.__parcels_to_be_allocated.items()):
self.fill_row(parcel_fid, parcel_number, False, row)
# Fill already allocated parcels
for row, (parcel_fid, parcel_number) in enumerate(
self.__parcels_already_allocated.items()):
self.fill_row(parcel_fid, parcel_number, True,
row + len(self.__parcels_to_be_allocated))
self.tbl_parcels.setSortingEnabled(True)
self.update_count_labels()
def fill_row(self, parcel_fid, parcel_number, allocated, row):
item = QTableWidgetItem(parcel_number)
item.setData(Qt.UserRole, parcel_fid)
self.tbl_parcels.setItem(row, 0, item)
text = QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"Already allocated") if allocated else QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget", "To be allocated")
item2 = QTableWidgetItem(text)
item2.setData(Qt.UserRole, allocated)
if not allocated:
item2.setBackground(QBrush(NOT_ALLOCATED_PARCEL_COLOR))
self.tbl_parcels.setItem(row, 1, item2)
def fill_receivers(self):
self.cbo_receiver.clear()
for receiver_id, receiver_data in sorted(
self._controller.get_receivers_data().items(),
key=lambda x: locale.strxfrm(str(x[1][0]))):
# receiver_id: either t_id (surveyor) or t_basket (coordinator)
# receiver_data: (name, doc id)
self.cbo_receiver.addItem(receiver_data[0], receiver_id)
def receiver_changed(self, index):
self.fill_table()
def update_count_labels(self):
self.lbl_already_allocated.setText(
self.__txt_already_allocated.format(
len(self.__parcels_already_allocated)))
self.lbl_to_be_allocated.setText(
self.__txt_to_be_allocated.format(
len(self.__parcels_to_be_allocated)))
def discard_parcels(self):
# Take 2 cases into account:
# 1) an already allocated parcel is being discarded --> fill_table()
# 2) a 'to be allocated' parcel is being discarded --> fill_table(refresh_allocated_parcels=False)
already_allocated = list()
to_be_allocated = list()
selected_gui_items = [
item.data(Qt.UserRole)
for item in self.tbl_parcels.selectedItems()
]
for row in range(self.tbl_parcels.rowCount()):
item = self.tbl_parcels.item(row, 0)
fid = item.data(Qt.UserRole)
if fid in selected_gui_items:
item2 = self.tbl_parcels.item(row, 1)
if item2.data(Qt.UserRole): # Allocated?
already_allocated.append(fid)
else:
to_be_allocated.append(fid)
if not already_allocated and not to_be_allocated:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"First select some parcels in the list."))
return
reply = QMessageBox.question(
self,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget", "Do you confirm?"),
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"Are you sure you want to remove the allocation of selected parcels in the list?"
), QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.Yes:
res = False
# 1)
if already_allocated:
res = self._controller.discard_parcel_allocation(
already_allocated)
# 2)
for parcel_fid in to_be_allocated:
del self.__parcels_to_be_allocated[parcel_fid]
res = True
if res:
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"Selected parcels were successfully discarded!"))
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"There were troubles discarding parcels!"))
# Finally, reload the table, refreshing from data source only when already-allocated parcels were discarded
# For safety, we reload even if not res, just to make sure our data is totally in sync
self.fill_table(bool(already_allocated))
def save_allocation(self):
if not self.__parcels_to_be_allocated:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"There are no parcels to be allocated! Go back and select some parcels first."
))
return
parcel_ids_to_allocate = list(self.__parcels_to_be_allocated.keys())
res = self._controller.save_allocation_for_receiver(
parcel_ids_to_allocate, self.cbo_receiver.currentData())
if res:
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"{} parcels were allocated to user {}!").format(
len(parcel_ids_to_allocate),
self.cbo_receiver.currentText()))
self.__parcels_to_be_allocated = dict()
self.fill_table()
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"BaseAllocateParcelsToReceiverPanelWidget",
"There was an error allocating the parcels!"))
class ImportFromExcelDialog(QDialog, DIALOG_UI):
log_excel_show_message_emitted = pyqtSignal(str)
def __init__(self, iface, db, qgis_utils, parent=None):
QDialog.__init__(self, parent)
self.setupUi(self)
self.iface = iface
self._db = db
self.qgis_utils = qgis_utils
self.logger = Logger()
self.help_strings = HelpStrings()
self.log_dialog_excel_text_content = ""
self.group_parties_exists = False
self.names = self._db.names
self._running_tool = False
self.tool_name = QCoreApplication.translate(
"ImportFromExcelDialog", "Import intermediate structure")
self.fields = {
EXCEL_SHEET_NAME_PLOT: [
EXCEL_SHEET_TITLE_DEPARTMENT, EXCEL_SHEET_TITLE_MUNICIPALITY,
EXCEL_SHEET_TITLE_ZONE, EXCEL_SHEET_TITLE_REGISTRATION_PLOT,
EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_NPV,
EXCEL_SHEET_TITLE_PLOT_NAME, EXCEL_SHEET_TITLE_VALUATION,
EXCEL_SHEET_TITLE_PLOT_CONDITION, EXCEL_SHEET_TITLE_PLOT_TYPE,
EXCEL_SHEET_TITLE_ADDRESS
],
EXCEL_SHEET_NAME_PARTY: [
EXCEL_SHEET_TITLE_FIRST_NAME, EXCEL_SHEET_TITLE_MIDDLE,
EXCEL_SHEET_TITLE_FIRST_SURNAME,
EXCEL_SHEET_TITLE_SECOND_SURNAME,
EXCEL_SHEET_TITLE_BUSINESS_NAME, EXCEL_SHEET_TITLE_SEX,
EXCEL_SHEET_TITLE_DOCUMENT_TYPE,
EXCEL_SHEET_TITLE_DOCUMENT_NUMBER,
EXCEL_SHEET_TITLE_KIND_PERSON,
EXCEL_SHEET_TITLE_ISSUING_ENTITY, EXCEL_SHEET_TITLE_DATE_ISSUE,
EXCEL_SHEET_TITLE_NPN
],
EXCEL_SHEET_NAME_GROUP: [
EXCEL_SHEET_TITLE_NPN, EXCEL_SHEET_TITLE_DOCUMENT_TYPE,
EXCEL_SHEET_TITLE_DOCUMENT_NUMBER, EXCEL_SHEET_TITLE_ID_GROUP
],
EXCEL_SHEET_NAME_RIGHT: [
EXCEL_SHEET_TITLE_TYPE,
EXCEL_SHEET_TITLE_PARTY_DOCUMENT_NUMBER,
EXCEL_SHEET_TITLE_GROUP, EXCEL_SHEET_TITLE_NPN,
EXCEL_SHEET_TITLE_SOURCE_TYPE,
EXCEL_SHEET_TITLE_DESCRIPTION_SOURCE,
EXCEL_SHEET_TITLE_STATE_SOURCE,
EXCEL_SHEET_TITLE_OFFICIALITY_SOURCE,
EXCEL_SHEET_TITLE_STORAGE_PATH
]
}
self.txt_help_page.setHtml(self.help_strings.DLG_IMPORT_FROM_EXCEL)
self.txt_help_page.anchorClicked.connect(self.save_template)
self.buttonBox.accepted.disconnect()
self.buttonBox.accepted.connect(self.accepted)
#self.buttonBox.rejected.connect(self.rejected)
self.buttonBox.helpRequested.connect(self.show_help)
self.btn_browse_file.clicked.connect(
make_file_selector(
self.txt_excel_path,
QCoreApplication.translate(
"ImportFromExcelDialog",
"Select the Excel file with data in the intermediate structure"
),
QCoreApplication.translate("ImportFromExcelDialog",
'Excel File (*.xlsx *.xls)')))
self.buttonBox.button(QDialogButtonBox.Ok).setText(
QCoreApplication.translate("ImportFromExcelDialog", "Import"))
self.initialize_feedback()
self.restore_settings()
self.bar = QgsMessageBar()
self.bar.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Fixed)
# self.tabWidget.currentWidget().layout().setContentsMargins(0, 0, 0, 0)
self.layout().addWidget(self.bar, 0, 0, Qt.AlignTop)
def accepted(self):
self.save_settings()
self.import_from_excel()
def import_from_excel(self):
self._running_tool = True
steps = 18
step = 0
self.progress.setVisible(True)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
# Where to store the reports?
excel_path = self.txt_excel_path.text()
if not excel_path:
self.show_message(
QCoreApplication.translate(
"ImportFromExcelDialog",
"You need to select an Excel file before continuing with the import."
), Qgis.Warning)
self.progress.setVisible(False)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
return
if not os.path.exists(excel_path):
self.show_message(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The specified Excel file does not exist!"), Qgis.Warning)
self.progress.setVisible(False)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
return
self.progress.setVisible(True)
self.txt_log.setText(
QCoreApplication.translate(
"ImportFromExcelDialog",
"Loading tables from the Excel file..."))
# Now that we have the Excel file, build vrts to load its sheets appropriately
# Also validate each layer against a number of rules
layer_parcel = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_PLOT)
layer_party = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_PARTY)
layer_group_party = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_GROUP)
layer_right = self.check_layer_from_excel_sheet(
excel_path, EXCEL_SHEET_NAME_RIGHT)
if layer_parcel is None or layer_party is None or layer_group_party is None or layer_right is None:
# A layer is None if at least an error was found
self.group_parties_exists = False
self.log_excel_show_message_emitted.emit(
self.log_dialog_excel_text_content)
self.done(0)
return
if not layer_group_party.isValid() or not layer_party.isValid(
) or not layer_parcel.isValid() or not layer_right.isValid():
self.show_message(
QCoreApplication.translate(
"ImportFromExcelDialog",
"One of the sheets of the Excel file couldn't be loaded! Check the format again."
), Qgis.Warning)
self.progress.setVisible(False)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
return
QgsProject.instance().addMapLayers(
[layer_group_party, layer_party, layer_parcel, layer_right])
# GET LADM LAYERS
layers = {
self.names.OP_PARTY_T: {
'name': self.names.OP_PARTY_T,
'geometry': None,
LAYER: None
},
self.names.OP_PARCEL_T: {
'name': self.names.OP_PARCEL_T,
'geometry': None,
LAYER: None
},
self.names.OP_RIGHT_T: {
'name': self.names.OP_RIGHT_T,
'geometry': None,
LAYER: None
},
self.names.EXT_ARCHIVE_S: {
'name': self.names.EXT_ARCHIVE_S,
'geometry': None,
LAYER: None
},
self.names.COL_RRR_SOURCE_T: {
'name': self.names.COL_RRR_SOURCE_T,
'geometry': None,
LAYER: None
},
self.names.OP_GROUP_PARTY_T: {
'name': self.names.OP_GROUP_PARTY_T,
'geometry': None,
LAYER: None
},
self.names.MEMBERS_T: {
'name': self.names.MEMBERS_T,
'geometry': None,
LAYER: None
},
self.names.OP_ADMINISTRATIVE_SOURCE_T: {
'name': self.names.OP_ADMINISTRATIVE_SOURCE_T,
'geometry': None,
LAYER: None
}
}
self.qgis_utils.get_layers(self._db, layers, load=True)
if not layers:
return None
# Get feature counts to compare after the ETL and know how many records were imported to each ladm_col table
ladm_tables = [
layers[self.names.OP_PARCEL_T][LAYER],
layers[self.names.OP_PARTY_T][LAYER],
layers[self.names.OP_RIGHT_T][LAYER],
layers[self.names.OP_ADMINISTRATIVE_SOURCE_T][LAYER],
layers[self.names.COL_RRR_SOURCE_T][LAYER],
layers[self.names.OP_GROUP_PARTY_T][LAYER],
layers[self.names.MEMBERS_T][LAYER]
]
ladm_tables_feature_count_before = {
t.name(): t.featureCount()
for t in ladm_tables
}
# Run the ETL
params = {
'agrupacion':
layers[self.names.OP_GROUP_PARTY_T][LAYER],
'colmiembros':
layers[self.names.MEMBERS_T][LAYER],
'colrrrsourcet':
layers[self.names.COL_RRR_SOURCE_T][LAYER],
'extarchivo':
layers[self.names.EXT_ARCHIVE_S][LAYER],
'interesado':
layers[self.names.OP_PARTY_T][LAYER],
'layergroupparty':
layer_group_party,
'layerparcel':
layer_parcel,
'layerparty':
layer_party,
'layerright':
layer_right,
'opderecho':
layers[self.names.OP_RIGHT_T][LAYER],
'opfuenteadministrativatipo':
layers[self.names.OP_ADMINISTRATIVE_SOURCE_T][LAYER],
'parcel':
layers[self.names.OP_PARCEL_T][LAYER]
}
self.qgis_utils.disable_automatic_fields(self._db,
self.names.OP_GROUP_PARTY_T)
self.qgis_utils.disable_automatic_fields(self._db,
self.names.OP_RIGHT_T)
self.qgis_utils.disable_automatic_fields(
self._db, self.names.OP_ADMINISTRATIVE_SOURCE_T)
processing.run("model:ETL_intermediate_structure",
params,
feedback=self.feedback)
if not self.feedback.isCanceled():
self.progress.setValue(100)
self.buttonBox.clear()
self.buttonBox.setEnabled(True)
self.buttonBox.addButton(QDialogButtonBox.Close)
else:
self.initialize_feedback()
# Print summary getting feature count in involved LADM_COL tables...
summary = """<html><head/><body><p>"""
summary += QCoreApplication.translate("ImportFromExcelDialog",
"Import done!!!<br/>")
for table in ladm_tables:
summary += QCoreApplication.translate(
"ImportFromExcelDialog",
"<br/><b>{count}</b> records loaded into table <b>{table}</b>"
).format(count=table.featureCount() -
ladm_tables_feature_count_before[table.name()],
table=table.name())
summary += """</body></html>"""
self.txt_log.setText(summary)
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"QGISUtils",
"Data successfully imported to LADM_COL from intermediate structure (Excel file: '{}')!!!"
).format(excel_path))
self._running_tool = False
def check_layer_from_excel_sheet(self, excel_path, sheetname):
layer = self.get_layer_from_excel_sheet(excel_path, sheetname)
error_counter = 0
if layer is None and sheetname != EXCEL_SHEET_NAME_GROUP: # optional sheet
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The {} sheet has not information or has another name.").
format(sheetname))
error_counter += 1
else:
title_validator = layer.fields().toList()
if sheetname == EXCEL_SHEET_NAME_PLOT and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in the sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"numero predial nuevo" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero predial nuevo has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_field_numeric_layer(layer, 'departamento'):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column departamento has non-numeric values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_field_numeric_layer(layer, 'municipio'):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column municipio has non-numeric values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_field_numeric_layer(layer,
'numero predial nuevo'):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero predial nuevo has non-numeric values in sheet {}."
).format(sheetname))
error_counter += 1
if sheetname == EXCEL_SHEET_NAME_PARTY and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in sheet {}.").
format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"numero de documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_length_attribute_value(
layer, 'numero de documento', 12):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has more characters than expected in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo persona" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo persona has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if sheetname == EXCEL_SHEET_NAME_GROUP and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in the sheet {}."
).format(sheetname))
error_counter += 1
self.group_parties_exists = True
if list(layer.getFeatures('"numero predial nuevo" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero predial nuevo has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"numero de documento" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(layer.getFeatures('"id agrupación" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column id agrupación has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if not self.check_length_attribute_value(
layer, 'numero de documento', 12):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column numero de documento has more characters of the permitted in sheet {}."
).format(sheetname))
error_counter += 1
if sheetname == EXCEL_SHEET_NAME_RIGHT and layer is not None:
if not title_validator:
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The title does not match the format in sheet {}.").
format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo has empty values in sheet {}.").
format(sheetname))
error_counter += 1
if list(layer.getFeatures('"tipo de fuente" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column tipo de fuente has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if list(
layer.getFeatures(
'"estado_disponibilidad de la fuente" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column estado_disponibilidad de la fuente has empty values in sheet {}."
).format(sheetname))
error_counter += 1
#if list(layer.getFeatures('"Ruta de Almacenamiento de la fuente" is Null')):
# self.generate_message_excel_error(QCoreApplication.translate("ImportFromExcelDialog",
# "The column Ruta de Almacenamiento de la fuente has empty values in sheet {}.").format(sheetname))
# error_counter += 1
if len(
list(
layer.getFeatures(
'"número documento Interesado" is Null'))) + len(
list(layer.getFeatures('"agrupación" is Null'))
) != layer.featureCount():
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"Number of non-null parties plus number of non-null group parties is not equal to number of records in sheet {}. There might be rights without party or group party associated."
).format(sheetname))
error_counter += 1
if not self.group_parties_exists:
if list(
layer.getFeatures(
'"número documento Interesado" is Null')):
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column número documento Interesado has empty values in sheet {}."
).format(sheetname))
error_counter += 1
if len(list(layer.getFeatures(
'"agrupacion" is Null'))) != layer.featureCount():
self.generate_message_excel_error(
QCoreApplication.translate(
"ImportFromExcelDialog",
"The column agrupacion has data but the sheet does not exist in sheet {}."
).format(sheetname))
error_counter += 1
return layer if error_counter == 0 else None
def check_field_numeric_layer(self, layer, name):
id_field_idx = layer.fields().indexFromName(name)
request = QgsFeatureRequest().setSubsetOfAttributes([id_field_idx])
features = layer.getFeatures(request)
is_numeric = True
for feature in features:
try:
int(feature[name])
except:
is_numeric = False
break
return is_numeric
def check_length_attribute_value(self, layer, name, size):
id_field_idx = layer.fields().indexFromName(name)
request = QgsFeatureRequest().setSubsetOfAttributes([id_field_idx])
features = layer.getFeatures(request)
right_length = True
for feature in features:
if len(str(feature[name])) > size:
right_length = False
break
return right_length
def generate_message_excel_error(self, msg):
self.log_dialog_excel_text_content += "{}{}{}{}{}{}".format(
LOG_QUALITY_LIST_CONTAINER_OPEN, LOG_QUALITY_LIST_ITEM_ERROR_OPEN,
msg, LOG_QUALITY_LIST_ITEM_ERROR_CLOSE,
LOG_QUALITY_LIST_CONTAINER_CLOSE, LOG_QUALITY_CONTENT_SEPARATOR)
def get_layer_from_excel_sheet(self, excel_path, sheetname):
basename = os.path.basename(excel_path)
filename = os.path.splitext(basename)[0]
dirname = os.path.dirname(excel_path)
header_in_first_row, count = self.get_excel_info(excel_path, sheetname)
if header_in_first_row is None and count is None:
return None
layer_definition = "<SrcLayer>{sheetname}</SrcLayer>".format(
sheetname=sheetname)
if header_in_first_row:
layer_definition = """<SrcSql dialect="sqlite">SELECT * FROM '{sheetname}' LIMIT {count} OFFSET 1</SrcSql>""".format(
sheetname=sheetname, count=count)
xml_text_group_party = """<?xml version="1.0" encoding="UTF-8"?>
<OGRVRTDataSource>
<OGRVRTLayer name="{filename}-{sheetname}">
<SrcDataSource relativeToVRT="1">{basename}</SrcDataSource>
<!--Header={header}-->
{layer_definition}
{fields}
</OGRVRTLayer>
</OGRVRTDataSource>
""".format(filename=filename,
basename=basename,
header=header_in_first_row,
layer_definition=layer_definition,
sheetname=sheetname,
fields=self.get_vrt_fields(sheetname,
header_in_first_row))
group_party_file_path = os.path.join(
dirname, '{}.{}.vrt'.format(basename, sheetname))
with open(group_party_file_path, 'w') as sheet:
sheet.write(xml_text_group_party)
uri = '{vrtfilepath}|layername={filename}-{sheetname}'.format(
vrtfilepath=group_party_file_path,
sheetname=sheetname,
filename=filename)
self.logger.info(__name__,
"Loading layer from excel with uri='{}'".format(uri))
layer = QgsVectorLayer(uri, '{}-{}'.format('excel', sheetname), 'ogr')
layer.setProviderEncoding('UTF-8')
return layer
def get_excel_info(self, path, sheetname):
data_source = ogr.Open(path, 0)
layer = data_source.GetLayerByName(sheetname)
if layer is None:
# A sheetname couldn't be found
return None, None
feature = layer.GetNextFeature()
# If ogr recognizes the header, the first row will contain data, otherwise it'll contain field names
header_in_first_row = True
for field in self.fields[sheetname]:
if feature.GetField(self.fields[sheetname].index(field)) != field:
header_in_first_row = False
num_rows = layer.GetFeatureCount()
return header_in_first_row, num_rows - 1 if header_in_first_row else num_rows
def get_vrt_fields(self, sheetname, header_in_first_row):
vrt_fields = ""
for index, field in enumerate(self.fields[sheetname]):
vrt_fields += """<Field name="{field}" src="{src}" type="String"/>\n""".format(
field=field,
src='Field{}'.format(index +
1) if header_in_first_row else field)
return vrt_fields.strip()
def save_template(self, url):
link = url.url()
if link == '#template':
self.download_excel_file('plantilla_estructura_excel.xlsx')
elif link == '#data':
self.download_excel_file('datos_estructura_excel.xlsx')
def download_excel_file(self, filename):
settings = QSettings()
new_filename, filter = QFileDialog.getSaveFileName(
self,
QCoreApplication.translate("ImportFromExcelDialog", "Save File"),
os.path.join(
settings.value(
'Asistente-LADM_COL/import_from_excel_dialog/template_save_path',
'.'), filename),
QCoreApplication.translate("ImportFromExcelDialog",
"Excel File (*.xlsx *.xls)"))
if new_filename:
settings.setValue(
'Asistente-LADM_COL/import_from_excel_dialog/template_save_path',
os.path.dirname(new_filename))
template_file = QFile(":/Asistente-LADM_COL/resources/excel/" +
filename)
if not template_file.exists():
self.logger.critical(
__name__,
"Excel doesn't exist! Probably due to a missing 'make' execution to generate resources..."
)
msg = QCoreApplication.translate(
"ImportFromExcelDialog",
"Excel file not found. Update your plugin. For details see log."
)
self.show_message(msg, Qgis.Warning)
return
if os.path.isfile(new_filename):
self.logger.info(
__name__,
'Removing existing file {}...'.format(new_filename))
os.chmod(new_filename, 0o777)
os.remove(new_filename)
if template_file.copy(new_filename):
os.chmod(
new_filename,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
msg = QCoreApplication.translate(
"ImportFromExcelDialog",
"""The file <a href="file:///{}">{}</a> was successfully saved!"""
).format(normalize_local_url(new_filename),
os.path.basename(new_filename))
self.show_message(msg, Qgis.Info)
else:
self.logger.info(
__name__,
'There was an error copying the CSV file {}!'.format(
new_filename))
msg = QCoreApplication.translate(
"ImportFromExcelDialog", "The file couldn\'t be saved.")
self.show_message(msg, Qgis.Warning)
def reject(self):
self.selected_items_dict = dict()
if self._running_tool:
reply = QMessageBox.question(
self, QCoreApplication.translate("import_from_excel",
"Warning"),
QCoreApplication.translate(
"import_from_excel",
"The '{}' tool is still running. Do you want to cancel it? If you cancel, the data might be incomplete in the target database."
).format(self.tool_name), QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.Yes:
self.feedback.cancel()
self._running_tool = False
msg = QCoreApplication.translate(
"import_from_excel",
"The '{}' tool was cancelled.").format(self.tool_name)
self.logger.info(__name__, msg)
self.show_message(msg, Qgis.Info)
else:
self.logger.info(__name__, "Dialog closed.")
self.done(1)
def save_settings(self):
settings = QSettings()
settings.setValue(
'Asistente-LADM_COL/import_from_excel_dialog/excel_path',
self.txt_excel_path.text())
def restore_settings(self):
settings = QSettings()
self.txt_excel_path.setText(
settings.value(
'Asistente-LADM_COL/import_from_excel_dialog/excel_path', ''))
def show_message(self, message, level):
self.bar.clearWidgets(
) # Remove previous messages before showing a new one
self.bar.pushMessage(message, level, 10)
def show_help(self):
self.qgis_utils.show_help("import_from_excel")
def progress_changed(self):
QCoreApplication.processEvents() # Listen to cancel from the user
self.progress.setValue(self.feedback.progress())
def initialize_feedback(self):
self.progress.setValue(0)
self.progress.setVisible(False)
self.feedback = QgsProcessingFeedback()
self.feedback.progressChanged.connect(self.progress_changed)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
class ReportGenerator(QObject):
LOG_TAB = 'LADM-COL Reports'
enable_action_requested = pyqtSignal(str, bool)
def __init__(self, ladm_data):
QObject.__init__(self)
self.ladm_data = ladm_data
self.logger = Logger()
self.app = AppInterface()
self.java_dependency = JavaDependency()
self.java_dependency.download_dependency_completed.connect(
self.download_java_complete)
self.report_dependency = ReportDependency()
self.report_dependency.download_dependency_completed.connect(
self.download_report_complete)
self.encoding = locale.getlocale()[1]
# This might be unset
if not self.encoding:
self.encoding = 'UTF8'
self._downloading = False
def stderr_ready(self, proc):
text = bytes(proc.readAllStandardError()).decode(self.encoding)
self.logger.critical(__name__, text, tab=self.LOG_TAB)
def stdout_ready(self, proc):
text = bytes(proc.readAllStandardOutput()).decode(self.encoding)
self.logger.info(__name__, text, tab=self.LOG_TAB)
def update_yaml_config(self, db, config_path):
text = ''
qgs_uri = QgsDataSourceUri(db.uri)
with open(os.path.join(config_path, 'config_template.yaml')) as f:
text = f.read()
text = text.format('{}',
DB_USER=qgs_uri.username(),
DB_PASSWORD=qgs_uri.password(),
DB_HOST=qgs_uri.host(),
DB_PORT=qgs_uri.port(),
DB_NAME=qgs_uri.database())
new_file_path = os.path.join(
config_path, self.get_tmp_filename('yaml_config', 'yaml'))
with open(new_file_path, 'w') as new_yaml:
new_yaml.write(text)
return new_file_path
def get_layer_geojson(self, db, layer_name, plot_id, report_type):
if report_type == ANNEX_17_REPORT:
if layer_name == 'terreno':
return db.get_annex17_plot_data(plot_id, 'only_id')
elif layer_name == 'terrenos':
return db.get_annex17_plot_data(plot_id, 'all_but_id')
elif layer_name == 'terrenos_all':
return db.get_annex17_plot_data(plot_id, 'all')
elif layer_name == 'construcciones':
return db.get_annex17_building_data()
else:
return db.get_annex17_point_data(plot_id)
else: #report_type == ANT_MAP_REPORT:
if layer_name == 'terreno':
return db.get_ant_map_plot_data(plot_id, 'only_id')
elif layer_name == 'terrenos':
return db.get_ant_map_plot_data(plot_id, 'all_but_id')
elif layer_name == 'terrenos_all':
return db.get_annex17_plot_data(plot_id, 'all')
elif layer_name == 'construcciones':
return db.get_annex17_building_data()
elif layer_name == 'puntoLindero':
return db.get_annex17_point_data(plot_id)
else: #layer_name == 'cambio_colindancia':
return db.get_ant_map_neighbouring_change_data(plot_id)
def update_json_data(self, db, json_spec_file, plot_id, tmp_dir,
report_type):
json_data = dict()
with open(json_spec_file) as f:
json_data = json.load(f)
json_data['attributes']['id'] = plot_id
json_data['attributes']['datasetName'] = db.schema
layers = json_data['attributes']['map']['layers']
for layer in layers:
layer['geoJson'] = self.get_layer_geojson(db, layer['name'],
plot_id, report_type)
overview_layers = json_data['attributes']['overviewMap']['layers']
for layer in overview_layers:
layer['geoJson'] = self.get_layer_geojson(db, layer['name'],
plot_id, report_type)
new_json_file_path = os.path.join(
tmp_dir,
self.get_tmp_filename('json_data_{}'.format(plot_id), 'json'))
with open(new_json_file_path, 'w') as new_json:
new_json.write(json.dumps(json_data))
return new_json_file_path
def get_tmp_dir(self, create_random=True):
if create_random:
return tempfile.mkdtemp()
return tempfile.gettempdir()
def get_tmp_filename(self, basename, extension='gpkg'):
return "{}_{}.{}".format(basename,
str(time.time()).replace(".", ""), extension)
def generate_report(self, db, report_type):
# Check if mapfish and Jasper are installed, otherwise show where to
# download them from and return
if not self.report_dependency.check_if_dependency_is_valid():
self.report_dependency.download_dependency(URL_REPORTS_LIBRARIES)
return
java_home_set = self.java_dependency.set_java_home()
if not java_home_set:
self.java_dependency.get_java_on_demand()
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Java is a prerequisite. Since it was not found, it is being configured..."
))
return
plot_layer = self.app.core.get_layer(db, db.names.LC_PLOT_T, load=True)
if not plot_layer:
return
selected_plots = plot_layer.selectedFeatures()
if not selected_plots:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"To generate reports, first select at least a plot!"))
return
# Where to store the reports?
previous_folder = QSettings().value(
"Asistente-LADM-COL/reports/save_into_dir", ".")
save_into_folder = QFileDialog.getExistingDirectory(
None,
QCoreApplication.translate(
"ReportGenerator",
"Select a folder to save the reports to be generated"),
previous_folder)
if not save_into_folder:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You need to select a folder where to save the reports before continuing."
))
return
QSettings().setValue("Asistente-LADM-COL/reports/save_into_dir",
save_into_folder)
config_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, report_type)
json_spec_file = os.path.join(config_path, 'spec_json_file.json')
script_name = ''
if os.name == 'posix':
script_name = 'print'
elif os.name == 'nt':
script_name = 'print.bat'
script_path = os.path.join(DEPENDENCY_REPORTS_DIR_NAME, 'bin',
script_name)
if not os.path.isfile(script_path):
self.logger.warning(
__name__,
"Script file for reports wasn't found! {}".format(script_path))
return
self.enable_action_requested.emit(report_type, False)
# Update config file
yaml_config_path = self.update_yaml_config(db, config_path)
self.logger.debug(
__name__, "Config file for reports: {}".format(yaml_config_path))
total = len(selected_plots)
step = 0
count = 0
tmp_dir = self.get_tmp_dir()
# Progress bar setup
progress = QProgressBar()
if total == 1:
progress.setRange(0, 0)
else:
progress.setRange(0, 100)
progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.app.gui.create_progress_message_bar(
QCoreApplication.translate("ReportGenerator",
"Generating {} report{}...").format(
total, '' if total == 1 else 's'),
progress)
polygons_with_holes = []
multi_polygons = []
for selected_plot in selected_plots:
plot_id = selected_plot[db.names.T_ID_F]
geometry = selected_plot.geometry()
abstract_geometry = geometry.get()
if abstract_geometry.ringCount() > 1:
polygons_with_holes.append(str(plot_id))
self.logger.warning(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Skipping Annex 17 for plot with {}={} because it has holes. The reporter module does not support such polygons."
).format(db.names.T_ID_F, plot_id))
continue
if abstract_geometry.numGeometries() > 1:
multi_polygons.append(str(plot_id))
self.logger.warning(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Skipping Annex 17 for plot with {}={} because it is a multi-polygon. The reporter module does not support such polygons."
).format(db.names.T_ID_F, plot_id))
continue
# Generate data file
json_file = self.update_json_data(db, json_spec_file, plot_id,
tmp_dir, report_type)
self.logger.debug(__name__,
"JSON file for reports: {}".format(json_file))
# Run sh/bat passing config and data files
proc = QProcess()
proc.readyReadStandardError.connect(
functools.partial(self.stderr_ready, proc=proc))
proc.readyReadStandardOutput.connect(
functools.partial(self.stdout_ready, proc=proc))
parcel_number = self.ladm_data.get_parcels_related_to_plots(
db, [plot_id], db.names.LC_PARCEL_T_PARCEL_NUMBER_F) or ['']
file_name = '{}_{}_{}.pdf'.format(report_type, plot_id,
parcel_number[0])
current_report_path = os.path.join(save_into_folder, file_name)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
# Grant execution permissions
os.chmod(
script_path, stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR
| stat.S_IRUSR | stat.S_IRGRP)
proc.start(script_path, [
'-config', yaml_config_path, '-spec', json_file, '-output',
current_report_path
])
if not proc.waitForStarted():
proc = None
self.logger.warning(
__name__, "Couldn't execute script to generate report...")
else:
loop = QEventLoop()
proc.finished.connect(loop.exit)
loop.exec()
self.logger.debug(__name__,
"{}:{}".format(plot_id, proc.exitCode()))
if proc.exitCode() == 0:
count += 1
step += 1
progress.setValue(step * 100 / total)
os.remove(yaml_config_path)
self.enable_action_requested.emit(report_type, True)
self.logger.clear_message_bar()
if total == count:
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report <a href='file:///{}'>{}</a> was successfully generated!"
).format(normalize_local_url(save_into_folder), file_name)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"All reports were successfully generated in folder <a href='file:///{path}'>{path}</a>!"
).format(path=normalize_local_url(save_into_folder))
self.logger.success_msg(__name__, msg)
else:
details_msg = ''
if polygons_with_holes:
details_msg += QCoreApplication.translate(
"ReportGenerator",
" The following polygons were skipped because they have holes and are not supported: {}."
).format(", ".join(polygons_with_holes))
if multi_polygons:
details_msg += QCoreApplication.translate(
"ReportGenerator",
" The following polygons were skipped because they are multi-polygons and are not supported: {}."
).format(", ".join(multi_polygons))
if total == 1:
msg = QCoreApplication.translate(
"ReportGenerator",
"The report for plot {} couldn't be generated!{} See QGIS log (tab '{}') for details."
).format(plot_id, details_msg, self.LOG_TAB)
else:
if count == 0:
msg = QCoreApplication.translate(
"ReportGenerator",
"No report could be generated!{} See QGIS log (tab '{}') for details."
).format(details_msg, self.LOG_TAB)
else:
msg = QCoreApplication.translate(
"ReportGenerator",
"At least one report couldn't be generated!{details_msg} See QGIS log (tab '{log_tab}') for details. Go to <a href='file:///{path}'>{path}</a> to see the reports that were generated."
).format(details_msg=details_msg,
path=normalize_local_url(save_into_folder),
log_tab=self.LOG_TAB)
self.logger.warning_msg(__name__, msg)
def download_java_complete(self):
if self.java_dependency.fetcher_task and not self.java_dependency.fetcher_task.isCanceled(
):
if self.java_dependency.check_if_dependency_is_valid():
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Java was successfully configured!"), 5)
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You have just canceled the Java dependency download."), 5)
def download_report_complete(self):
if self.report_dependency.fetcher_task and not self.report_dependency.fetcher_task.isCanceled(
):
if self.report_dependency.check_if_dependency_is_valid():
self.logger.info_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"Report dependency was successfully configured!"), 5)
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"ReportGenerator",
"You have just canceled the report dependency download."),
5)
class BaseAllocateParcelsInitialPanelWidget(QgsPanelWidget, WIDGET_UI):
allocate_parcels_to_receiver_panel_requested = pyqtSignal(dict) # {parcel_fid: parcel_number}
configure_receivers_panel_requested = pyqtSignal()
split_data_for_receivers_panel_requested = pyqtSignal()
STATUS_COL = 1
def __init__(self, parent, controller):
QgsPanelWidget.__init__(self, parent)
self.setupUi(self)
self.parent = parent
self._controller = controller
self.logger = Logger()
self.app = AppInterface()
self.setDockMode(True)
self.setPanelTitle(QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Allocate parcels"))
self.parent.setWindowTitle(QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Allocate parcels"))
self.tbl_parcels.resizeColumnsToContents()
self.txt_search.valueChanged.connect(self.search_value_changed)
self.tbl_parcels.itemSelectionChanged.connect(self.selection_changed)
self.btn_allocate.clicked.connect(self.call_allocate_parcels_to_receiver_panel)
self.btn_configure_receivers.clicked.connect(self.configure_receivers_panel_requested)
self.btn_show_summary.clicked.connect(self.split_data_for_receivers_panel_requested)
self.chk_show_only_not_allocated.stateChanged.connect(self.chk_check_state_changed)
self.btn_reallocate.clicked.connect(self.reallocate_clicked)
self.connect_to_plot_selection(True)
self.__parcel_data = dict() # {parcel_fid: (parcel_number, surveyor_name)}
self.__selected_items = dict() # {parcel_fid: parcel_number}
def _parcel_data(self, refresh_parcel_data=False):
if not self.__parcel_data or refresh_parcel_data:
self.__parcel_data = self._controller.get_parcel_receiver_data()
return self.__parcel_data
def fill_data(self, refresh_parcel_data=False):
self.update_selected_items() # Save selection
self.tbl_parcels.blockSignals(True) # We don't want to get itemSelectionChanged here
self.tbl_parcels.clearContents()
self.tbl_parcels.blockSignals(False) # We don't want to get itemSelectionChanged here
# Build the parcel_data dict taking configuration (search string, chk filter) into account
parcel_data = self._parcel_data(refresh_parcel_data).copy()
if self.chk_show_only_not_allocated.isChecked():
parcel_data = {k:v for k,v in parcel_data.items() if not v[1]} # v: (parcel_number, surveyor)
parcel_data = self.filter_data_by_search_string(parcel_data)
self.tbl_parcels.setRowCount(len(parcel_data))
self.tbl_parcels.setSortingEnabled(False)
self.tbl_parcels.blockSignals(True) # We don't want to get itemSelectionChanged here
for row, data in enumerate(parcel_data.items()):
parcel_number, receiver = data[1]
self.fill_row(data[0], parcel_number, receiver, row)
self.tbl_parcels.blockSignals(False) # We don't want to get itemSelectionChanged here
self.tbl_parcels.setSortingEnabled(True)
self.tbl_parcels.resizeColumnsToContents()
def fill_row(self, parcel_fid, parcel_number, receiver, row):
item = QTableWidgetItem(parcel_number)
item.setData(Qt.UserRole, parcel_fid)
self.tbl_parcels.setItem(row, 0, item)
item2 = QTableWidgetItem(receiver or '')
if not receiver:
item2.setBackground(QBrush(NOT_ALLOCATED_PARCEL_COLOR))
self.tbl_parcels.setItem(row, self.STATUS_COL, item2)
if parcel_fid in self.__selected_items:
item.setSelected(True)
item2.setSelected(True)
def filter_data_by_search_string(self, parcel_data):
value = self.txt_search.value().strip()
if value and len(value) > 1:
parcel_data = {k:v for k,v in parcel_data.items() if value in v[0]}
return parcel_data
def search_value_changed(self, value):
self.fill_data()
def chk_check_state_changed(self, state):
self.fill_data()
def update_selected_items(self):
"""Update the internal selected_items dict"""
selected_gui_items = [item.data(Qt.UserRole) for item in self.tbl_parcels.selectedItems()]
for row in range(self.tbl_parcels.rowCount()):
item = self.tbl_parcels.item(row, 0)
fid = item.data(Qt.UserRole)
if fid in selected_gui_items:
self.__selected_items[fid] = item.text()
else:
if fid in self.__selected_items:
# It was selected before, but not anymore
del self.__selected_items[fid]
def selection_changed(self):
"""React upon manual selection in the table widget"""
self.update_selected_items()
self.connect_to_plot_selection(False) # This plot selection should not trigger a table view selection refresh
self._controller.update_plot_selection(list(self.__selected_items.keys()))
self.connect_to_plot_selection(True)
def update_parcel_selection(self, selected, deselected, clear_and_select):
"""React upon a plot selection"""
self.tbl_parcels.blockSignals(True) # We don't want to get itemSelectionChanged here
self.tbl_parcels.clearSelection() # Reset GUI selection
self.__selected_items = dict() # Reset internal selection dict
parcel_ids = self._controller.get_parcel_numbers_from_selected_plots()
for parcel_id in parcel_ids:
if parcel_id in self._parcel_data():
parcel_number = self._parcel_data()[parcel_id][0]
items = self.tbl_parcels.findItems(parcel_number, Qt.MatchExactly)
if items:
items[0].setSelected(True) # Select item in column 0
self.tbl_parcels.item(items[0].row(), self.STATUS_COL).setSelected(True) # Select item in column 1
else: # parcel is not currently shown, so select it in internal dict
if parcel_id in self._parcel_data():
self.__selected_items[parcel_id] = parcel_number
self.tbl_parcels.blockSignals(False)
self.update_selected_items() # Update the internal selection dict
def connect_to_plot_selection(self, connect):
if connect:
self._controller.plot_layer().selectionChanged.connect(self.update_parcel_selection)
else:
try:
self._controller.plot_layer().selectionChanged.disconnect(self.update_parcel_selection)
except (TypeError, RuntimeError): # Layer in C++ could be already deleted...
pass
def close_panel(self):
# Disconnect signals
self.connect_to_plot_selection(False)
def panel_accepted_clear_message_bar(self):
self.logger.clear_message_bar()
def panel_accepted_refresh_parcel_data(self):
"""Slot for refreshing parcel data when it has changed in other panels"""
self.panel_accepted_clear_message_bar()
self.fill_data(True)
def panel_accepted_refresh_and_clear_selection(self):
self.panel_accepted_refresh_parcel_data() # Refresh data in table widget, as it might be out of sync with newly added layers
self.tbl_parcels.clearSelection() # Selection might be remembered from the status before converting to offline
def call_allocate_parcels_to_receiver_panel(self):
# Make sure that all selected items are not yet allocated, otherwise, allow users to deallocate selected
already_allocated = list() # [parcel_fid1, ...]
for parcel_fid, parcel_number in self.__selected_items.items():
if parcel_fid in self._parcel_data():
if self._parcel_data()[parcel_fid][1]: # surveyor_name
already_allocated.append(parcel_fid)
if already_allocated:
msg = QMessageBox(self)
msg.setIcon(QMessageBox.Question)
msg.setText(QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget",
"Some selected parcels are already allocated!\n\nWhat would you like to do with selected parcels that are already allocated?"))
msg.setWindowTitle(QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Warning"))
msg.setStandardButtons(QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel)
msg.button(QMessageBox.Yes).setText(
QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Deselect them and continue"))
msg.button(QMessageBox.No).setText(
QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Reallocate and continue"))
reply = msg.exec_()
if reply == QMessageBox.Yes: # Ignore
# Remove selection of allocated parcels, reload table widget data and continue
for allocated_parcel_id in already_allocated:
if allocated_parcel_id in self._parcel_data():
items = self.tbl_parcels.findItems(self._parcel_data()[allocated_parcel_id][0], Qt.MatchExactly)
if items: # Item is currently shown, so deselect it in GUI
items[0].setSelected(False) # Deselect item in column 0
self.tbl_parcels.item(items[0].row(), self.STATUS_COL).setSelected(False) # Deselect item in column 1
else: # Item is not currently shown, deselected in internal selection dict
if allocated_parcel_id in self.__selected_items:
del self.__selected_items[allocated_parcel_id]
self.fill_data()
if not self.__selected_items:
self.logger.warning_msg(__name__, QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget",
"Ignoring selected parcels, there are none to be allocated! First select some!"), 10)
return
elif reply == QMessageBox.No: # Reallocate
# Preserve the selected_items dict, but remove allocation before continuing
if not self.discard_parcel_allocation(already_allocated):
return
else: # QMessageBox.Cancel
return
if self.__selected_items:
self.allocate_parcels_to_receiver_panel_requested.emit(self.__selected_items)
else:
self.logger.warning_msg(__name__, QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "First select some parcels to be allocated."), 5)
def discard_parcel_allocation(self, parcel_fids):
res = self._controller.discard_parcel_allocation(parcel_fids)
if res:
self.fill_data(True) # Refresh parcel data
self.logger.success_msg(__name__, QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget",
"Selected parcels are now not allocated!"))
else:
self.logger.warning_msg(__name__, QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget",
"There were troubles reallocating parcels!"))
return res
def reallocate_clicked(self):
if not self.__selected_items:
self.logger.warning_msg(__name__, QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget",
"First select some parcels."), 5)
return
# Get selected parcels that are already allocated
already_allocated = list() # [parcel_fid1, ...]
for parcel_fid, parcel_number in self.__selected_items.items():
if parcel_fid in self._parcel_data():
if self._parcel_data()[parcel_fid][1]: # surveyor_name
already_allocated.append(parcel_fid)
if already_allocated:
# Ask for confirmation
reply = QMessageBox.question(self,
QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Do you confirm?"),
QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget",
"Are you sure you want to remove the allocation of selected parcels?"),
QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.Yes:
self.discard_parcel_allocation(already_allocated)
else:
self.logger.info_msg(__name__, QCoreApplication.translate("AllocateParcelsFieldDataCapturePanelWidget", "Selected parcels are not yet allocated, so we cannot reallocate them."))
class QualityRuleController(QObject):
open_report_called = pyqtSignal(QualityRuleResultLog) # log result
quality_rule_layer_removed = pyqtSignal()
refresh_error_layer_symbology = pyqtSignal(QgsVectorLayer)
total_progress_changed = pyqtSignal(int) # Progress value
def __init__(self, db):
QObject.__init__(self)
self.app = AppInterface()
self.logger = Logger()
self.__db = db
self.__tr_dict = TranslatableConfigStrings(
).get_translatable_config_strings()
# Hierarquical dict of qrs and qr groups
self.__qrs_tree_data = dict() # {type: {qr_key1: qr_obj1, ...}, ...}
# Hierarquical dict of qrs and qr groups with general results
self.__general_results_tree_data = dict(
) # {type: {qr_obj1: qr_results1, ...}, ...}
# Hierarchical dict of qrs and their corresponding error instances
# feature1: {uuids, rel_uuids, error_type, nombre_ili_obj, details, values, fixed, exception, geom_fks}
self.__error_results_data = dict() # {qr_key1: {t_id1: feature1}}
self.__qr_results_dir_path = '' # Dir path where results will be stored
self.__selected_qrs = list() # QRs to be validated (at least 1)
self.__selected_qr = None # QR selected by the user to show its corresponding errors (exactly 1)
self.__qr_engine = None # Once set, we can reuse it
self.__qrs_results = None # QualityRulesExecutionResult object
# To cache layers from QR DB
self.__error_layer = None
self.__point_layer = None
self.__line_layer = None
self.__polygon_layer = None
# Cache by t_id (built on demand): {t_id1: 'Error', t_id2: 'Corregido', t_id3: 'Exception'}
self.__error_state_dict = dict()
def get_tr_string(self, key):
return self.__tr_dict.get(key, key)
def validate_qrs(self):
if self.__qr_engine is None:
self.__qr_engine = QualityRuleEngine(self.__db,
self.__selected_qrs,
self.app.settings.tolerance,
self.__qr_results_dir_path)
self.__qr_engine.progress_changed.connect(
self.total_progress_changed)
else:
self.__qr_engine.initialize(self.__db, self.__selected_qrs,
self.app.settings.tolerance,
self.__qr_results_dir_path)
#self.__qr_engine.qr_logger.show_message_emitted.connect(self.show_log_quality_message)
#self.__qr_engine.qr_logger.show_button_emitted.connect(self.show_log_quality_button)
#self.__qr_engine.qr_logger.set_initial_progress_emitted.connect(self.set_log_quality_initial_progress)
#self.__qr_engine.qr_logger.set_final_progress_emitted.connect(self.set_log_quality_final_progress)
use_roads = bool(QSettings().value(
'Asistente-LADM-COL/quality/use_roads', DEFAULT_USE_ROADS_VALUE,
bool))
options = {QR_IGACR3006: {'use_roads': use_roads}}
res, msg, qrs_res = self.__qr_engine.validate_quality_rules(options)
if not res:
return res, msg, None
self.__qrs_results = qrs_res
self.__connect_layer_willbedeleted_signals(
) # Note: Call it after validate_quality_rules!
res_u, msg_u, output_qr_dir = QualityErrorDBUtils.get_quality_validation_output_path(
self.__qr_results_dir_path, self.__qr_engine.get_timestamp())
if len(self.__selected_qrs) == 1:
pre_text = QCoreApplication.translate(
"QualityRules", "The quality rule was checked!")
else:
pre_text = QCoreApplication.translate(
"QualityRules",
"All the {} quality rules were checked!").format(
len(self.__selected_qrs))
post_text = QCoreApplication.translate(
"QualityRules",
"Both a PDF report and a GeoPackage database with errors can be found in <a href='file:///{}'>{}</a>."
).format(normalize_local_url(output_qr_dir), output_qr_dir)
self.logger.success_msg(__name__, "{} {}".format(pre_text, post_text))
self.__emit_refresh_error_layer_symbology()
return res, msg, self.__qrs_results
def __connect_layer_willbedeleted_signals(self):
"""
Iterate QR DB layers from the layer tree and connect their layerwillberemoved signals.
If a QR DB layer is removed, we'll react in the GUI.
"""
group = QualityErrorDBUtils.get_quality_error_group(
self.__qr_engine.get_timestamp())
if group:
for tree_layer in group.findLayers():
try:
tree_layer.layer().willBeDeleted.disconnect(
self.quality_rule_layer_removed)
except:
pass
tree_layer.layer().willBeDeleted.connect(
self.quality_rule_layer_removed)
def disconnect_layer_willberemoved_signals(self):
group = QualityErrorDBUtils.get_quality_error_group(
self.__qr_engine.get_timestamp(), False)
if group:
for tree_layer in group.findLayers():
try:
tree_layer.layer().willBeDeleted.disconnect(
self.quality_rule_layer_removed)
except:
pass
def get_qr_result(self, qr_key):
"""
Return the QRExecutionResult object for the given qr_key.
It first attempts to find it in the __qrs_results dict, but, chances are,
the whole set of QRs hasn't been validated when this method is called,
so, as a last resort, we go for the tree_data, which is updated each time
a QR gets its result.
"""
if self.__qrs_results is not None:
return self.__qrs_results.result(qr_key)
for type, qr_dict in self.__general_results_tree_data.items():
for k, v in qr_dict.items():
if k.id() == qr_key:
return self.__general_results_tree_data[type][k]
return None
def __reset_qrs_results(self):
# To be used when we are returning to select QRs (i.e., to the initial panel)
self.__qrs_results = None
def __get_qrs_per_role_and_models(self):
return QualityRuleRegistry().get_qrs_per_role_and_models(self.__db)
def load_tree_data(self, mode):
"""
Builds a hierarchical dict by qr type: {qr_type1: {qr_key1: qr_obj1, ...}, ...}
Tree data for panel 1.
:params mode: Value from EnumQualityRulePanelMode (either VALIDATE or READ).
For VALIDATE we load QRs from registry (filtered by role and current db models).
For READ we load QRs from the DB itself.
"""
if mode == EnumQualityRulePanelMode.VALIDATE:
qrs = self.__get_qrs_per_role_and_models(
) # Dict of qr key and qr objects.
else:
qrs = dict() # TODO: Read QRs from the QR DB
for qr_key, qr_obj in qrs.items():
type = qr_obj.type()
if type not in self.__qrs_tree_data:
self.__qrs_tree_data[type] = {qr_key: qr_obj}
else:
self.__qrs_tree_data[type][qr_key] = qr_obj
def get_qrs_tree_data(self):
return self.__qrs_tree_data
def set_qr_dir_path(self, path):
self.__qr_results_dir_path = path
def set_selected_qrs(self, selected_qrs):
# We sort them because the engine needs the QRs sorted for the PDF report
for type, qr_dict in self.__qrs_tree_data.items():
for qr_key, qr_obj in qr_dict.items():
if qr_key in selected_qrs:
self.__selected_qrs.append(qr_key)
def get_selected_qrs(self):
return self.__selected_qrs
def __reset_selected_qrs(self):
# To be used when we are returning to select QRs (i.e., to the initial panel)
self.__selected_qrs = list()
def reset_vars_for_general_results_panel(self):
# Initialize variables when we leave the general results panel
self.__reset_general_results_tree_data()
self.__reset_selected_qrs()
self.__reset_qrs_results()
self.__reset_layers()
# Call it before removing QR DB group to avoid triggering parent.layer_removed() slot again.
self.disconnect_layer_willberemoved_signals()
# When we leave the GRP, we remove the QR DB group from layer tree,
# because we won't be working anymore with that QR DB
QualityErrorDBUtils.remove_quality_error_group(
self.__qr_engine.get_timestamp())
def reset_vars_for_error_results_panel(self):
# Initialize variables when we leave the error results panel
self.__reset_error_results_data()
self.__reset_selected_qr()
self.__reset_error_state_dict()
self.__reset_layers()
def load_general_results_tree_data(self):
"""
Builds a hierarchical dict by qr type: {type: {qr_obj1: qr_results1, ...}, ...}
Tree data for panel 2.
"""
for type, qr_dict in self.__qrs_tree_data.items():
for qr_key, qr_obj in qr_dict.items():
if qr_key in self.__selected_qrs:
if type not in self.__general_results_tree_data:
self.__general_results_tree_data[type] = {qr_obj: None}
else:
self.__general_results_tree_data[type][qr_obj] = None
def get_general_results_tree_data(self):
return self.__general_results_tree_data
def __reset_general_results_tree_data(self):
# To be used when we are returning to select QRs (i.e., to the initial panel)
self.__general_results_tree_data = dict()
def set_qr_validation_result(self, qr, qr_result):
"""
When a QR has its validation result after validation,
we can store it in our custom dict by using this method.
"""
for type, qr_dict in self.__general_results_tree_data.items():
for k, v in qr_dict.items():
if k == qr:
self.__general_results_tree_data[type][k] = qr_result
def open_report(self):
if self.__qr_engine:
log_result = self.__qr_engine.qr_logger.get_log_result()
self.open_report_called.emit(log_result)
def set_selected_qr(self, qr_key):
self.__selected_qr = QualityRuleRegistry().get_quality_rule(qr_key)
return self.__selected_qr is not None # We should not be able to continue if we don't find the QR
def get_selected_qr(self):
return self.__selected_qr
def load_error_results_data(self):
"""
Go to table and bring data to the dict.
We should keep this dict updated with changes from the user.
From time to time we reflect this dict changes in the original data source.
"""
db = self.__qr_engine.get_db_quality()
names = db.names
layers = {names.ERR_QUALITY_ERROR_T: None, names.ERR_RULE_TYPE_T: None}
self.app.core.get_layers(db, layers, load=False)
if not layers:
self.logger.critical(
__name__,
"Quality error layers ('{}') not found!".format(",".join(
list(layers.keys()))))
return
# First go for the selected quality error's t_id
features = LADMData.get_features_from_t_ids(
layers[names.ERR_RULE_TYPE_T], names.ERR_RULE_TYPE_T_CODE_F,
[self.__selected_qr.id()])
t_id = features[0][names.T_ID_F] if features else None
if not t_id:
self.logger.critical(
__name__, "Quality error rule ('{}') not found!".format(
self.__selected_qr.id()))
return
# Now go for all features that match the selected quality rule
features = LADMData.get_features_from_t_ids(
layers[names.ERR_QUALITY_ERROR_T],
names.ERR_QUALITY_ERROR_T_RULE_TYPE_F, [t_id])
self.__error_results_data[self.__selected_qr.id()] = {
feature[names.T_ID_F]: feature
for feature in features
}
def get_error_results_data(self):
# Get the subdict {t_id1: feature1, ...} corresponding to selected qr
return self.__error_results_data.get(
self.__selected_qr.id() if self.__selected_qr else '', dict())
def __reset_error_results_data(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__error_results_data = dict()
def error_t_id(self, feature):
return feature[self.__qr_engine.get_db_quality().names.T_ID_F]
def is_fixed_error(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(
state_t_id) == LADMNames.ERR_ERROR_STATE_D_FIXED_V
def is_error(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(
state_t_id) == LADMNames.ERR_ERROR_STATE_D_ERROR_V
def is_exception(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(
state_t_id) == LADMNames.ERR_ERROR_STATE_D_EXCEPTION_V
def uuid_objs(self, feature):
return "\n".join(feature[self.__qr_engine.get_db_quality().names.
ERR_QUALITY_ERROR_T_OBJECT_IDS_F])
def ili_obj_name(self, feature):
ili_name = feature[self.__qr_engine.get_db_quality().names.
ERR_QUALITY_ERROR_T_ILI_NAME_F]
return ili_name.split(".")[-1] if ili_name else ''
def error_type_code_and_display(self, feature):
db = self.__qr_engine.get_db_quality()
names = db.names
layer = self.app.core.get_layer(db, names.ERR_ERROR_TYPE_T, load=False)
features = LADMData.get_features_from_t_ids(
layer, names.T_ID_F,
[feature[db.names.ERR_QUALITY_ERROR_T_ERROR_TYPE_F]]) # tid
return features[0][
names.
ERR_ERROR_TYPE_T_CODE_F] if features else QCoreApplication.translate(
"QualityRules", "No error type found!"
), features[0][
names.
ERR_ERROR_TYPE_T_DESCRIPTION_F] if features else QCoreApplication.translate(
"QualityRules", "No error description found!")
def error_details_and_values(self, feature):
res = ""
db = self.__qr_engine.get_db_quality()
details = feature[db.names.ERR_QUALITY_ERROR_T_DETAILS_F]
values = feature[db.names.ERR_QUALITY_ERROR_T_VALUES_F]
if details:
res = details
if values:
try:
res_values = json.loads(values)
if type(res_values) is dict:
items = ""
for k, v in res_values.items():
items = res + "{}: {}\n".format(k, v)
res_values = items.strip()
else:
res_values = str(res_values)
except json.decoder.JSONDecodeError as e:
res_values = values
res = res_values if not res else "{}\n\n{}".format(res, res_values)
return res
def error_state(self, feature):
db = self.__qr_engine.get_db_quality()
state_t_id = feature[db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F]
return self.__get_error_state_value(state_t_id)
def __get_error_state_value(self, state_t_id):
if state_t_id not in self.__error_state_dict:
db = self.__qr_engine.get_db_quality()
self.__error_state_dict[state_t_id] = LADMData(
).get_domain_value_from_code(db, db.names.ERR_ERROR_STATE_D,
state_t_id)
return self.__error_state_dict.get(state_t_id, "")
def __get_error_state_t_id(self, state_value):
# Use __error_state_dict to read cached values, but this time we have the value,
# not the key, so check in dict values and if not found, go for its t_id
if state_value not in self.__error_state_dict.values():
db = self.__qr_engine.get_db_quality()
t_id = LADMData().get_domain_code_from_value(
db, db.names.ERR_ERROR_STATE_D, state_value)
self.__error_state_dict[t_id] = state_value
# Get key by value in a dict:
return next((k for k in self.__error_state_dict
if self.__error_state_dict[k] == state_value), None)
def __get_error_layer(self):
if not self.__error_layer:
db = self.__qr_engine.get_db_quality()
self.__error_layer = self.app.core.get_layer(
db, db.names.ERR_QUALITY_ERROR_T)
return self.__error_layer
def __get_point_error_layer(self):
if not self.__point_layer:
db = self.__qr_engine.get_db_quality()
self.__point_layer = self.app.core.get_layer(
db, db.names.ERR_POINT_T)
return self.__point_layer
def __get_line_error_layer(self):
if not self.__line_layer:
db = self.__qr_engine.get_db_quality()
self.__line_layer = self.app.core.get_layer(
db, db.names.ERR_LINE_T)
return self.__line_layer
def __get_polygon_error_layer(self):
if not self.__polygon_layer:
db = self.__qr_engine.get_db_quality()
self.__polygon_layer = self.app.core.get_layer(
db, db.names.ERR_POLYGON_T)
return self.__polygon_layer
def __reset_layers(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__error_layer = None
self.__point_layer = None
self.__line_layer = None
self.__polygon_layer = None
def __reset_selected_qr(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__selected_qr = None
def __reset_error_state_dict(self):
# To be used when we are returning to select QR results (i.e., to the general results panel)
self.__error_state_dict = dict()
def __error_related_geometries(self, error_t_ids):
# Prefered geometry types are polygons, lines, points, in that order
db = self.__qr_engine.get_db_quality()
error_data = self.get_error_results_data()
dict_layer_fids = dict()
for error_t_id in error_t_ids:
feature = error_data.get(error_t_id, None)
if feature:
polygon = feature[db.names.ERR_QUALITY_ERROR_T_POLYGON_F]
line = feature[db.names.ERR_QUALITY_ERROR_T_LINE_F]
point = feature[db.names.ERR_QUALITY_ERROR_T_POINT_F]
if polygon:
if 'polygon' in dict_layer_fids:
dict_layer_fids['polygon']['fids'].append(polygon)
else:
dict_layer_fids['polygon'] = {
'layer': self.__get_polygon_error_layer(),
'fids': [polygon]
}
elif line:
if 'line' in dict_layer_fids:
dict_layer_fids['line']['fids'].append(line)
else:
dict_layer_fids['line'] = {
'layer': self.__get_line_error_layer(),
'fids': [line]
}
elif point:
if 'point' in dict_layer_fids:
dict_layer_fids['point']['fids'].append(point)
else:
dict_layer_fids['point'] = {
'layer': self.__get_point_error_layer(),
'fids': [point]
}
return dict_layer_fids
def highlight_geometries(self, t_ids):
res_geometries = self.__error_related_geometries(t_ids)
if res_geometries:
# First zoom to geometries
if len(res_geometries) == 1: # Only one geometry type related
for geom_type, dict_layer_fids in res_geometries.items(
): # We know this will be called just once
self.app.gui.zoom_to_feature_ids(dict_layer_fids['layer'],
dict_layer_fids['fids'])
else: # Multiple geometry types were found, so combine the extents and then zoom to it
combined_extent = QgsRectangle()
for geom_type, dict_layer_fids in res_geometries.items():
combined_extent.combineExtentWith(
self.app.core.get_extent_from_feature_ids(
dict_layer_fids['layer'], dict_layer_fids['fids']))
self.app.gui.zoom_to_extent(combined_extent)
# Now highlight geometries
for geom_type, dict_layer_fids in res_geometries.items():
self.app.gui.flash_features(dict_layer_fids['layer'],
dict_layer_fids['fids'],
flashes=5)
def get_uuids_display_name(self):
names = self.__qr_engine.get_db_quality().names
res = self.__selected_qr.field_mapping(names).get(
names.ERR_QUALITY_ERROR_T_OBJECT_IDS_F, '')
return res if res else QCoreApplication.translate(
"QualityRules", "UUIDs")
def set_fixed_error(self, error_t_id, fixed):
# Save to the intermediate dict of data and to the underlying data source whether an error is fixed or not
db = self.__qr_engine.get_db_quality()
idx_state = self.__get_error_layer().fields().indexOf(
db.names.ERR_QUALITY_ERROR_T_ERROR_STATE_F)
value = LADMNames.ERR_ERROR_STATE_D_FIXED_V if fixed else LADMNames.ERR_ERROR_STATE_D_ERROR_V
fixed_or_error_t_id = self.__get_error_state_t_id(value)
if fixed_or_error_t_id is None:
self.logger.critical(
__name__,
"The error state t_id couldn't be found for value '{}'!".
format(value))
return
# Save to dict
self.get_error_results_data()[error_t_id].setAttribute(
idx_state, fixed_or_error_t_id)
fids = LADMData.get_fids_from_key_values(self.__get_error_layer(),
db.names.T_ID_F, [error_t_id])
# Save to underlying data source
if fids:
res = self.__get_error_layer().dataProvider(
).changeAttributeValues(
{fids[0]: {
idx_state: fixed_or_error_t_id
}})
if not res:
self.logger.critical(__name__,
"Error modifying the error state value!")
else:
self.logger.critical(
__name__, "Error with t_id '' not found!".format(error_t_id))
def __emit_refresh_error_layer_symbology(self):
if self.__get_point_error_layer().featureCount():
self.refresh_error_layer_symbology.emit(
self.__get_point_error_layer())
if self.__get_line_error_layer().featureCount():
self.refresh_error_layer_symbology.emit(
self.__get_line_error_layer())
if self.__get_polygon_error_layer().featureCount():
self.refresh_error_layer_symbology.emit(
self.__get_polygon_error_layer())
class AppGUIInterface(QObject):
add_indicators_requested = pyqtSignal(
str, QgsLayerTreeNode.NodeType) # node name, node type
def __init__(self, iface):
QObject.__init__(self)
self.iface = iface
self.logger = Logger()
def trigger_add_feature(self):
self.iface.actionAddFeature().trigger()
def trigger_vertex_tool(self):
self.iface.actionVertexTool().trigger()
def create_progress_message_bar(self, text, progress):
progressMessageBar = self.iface.messageBar().createMessage(
PLUGIN_NAME, text)
progressMessageBar.layout().addWidget(progress)
self.iface.messageBar().pushWidget(progressMessageBar, Qgis.Info)
def refresh_layer_symbology(self, layer_id):
self.iface.layerTreeView().refreshLayerSymbology(layer_id)
def refresh_map(self):
self.iface.mapCanvas().refresh()
def redraw_all_layers(self):
self.iface.mapCanvas().redrawAllLayers()
def freeze_map(self, frozen):
self.iface.mapCanvas().freeze(frozen)
def activate_layer(self, layer):
self.iface.layerTreeView().setCurrentLayer(layer)
def set_node_visibility(self, node, visible=True):
# Modes may eventually be layer_id, group_name, layer, group
if node is not None:
node.setItemVisibilityChecked(visible)
def remove_error_group(self):
group = self.get_error_layers_group()
parent = group.parent()
parent.removeChildNode(group)
def clear_status_bar(self):
self.iface.statusBarIface().clearMessage()
def add_error_layer(self, db, error_layer):
group = self.get_error_layers_group()
# Check if layer is loaded and remove it
layers = group.findLayers()
for layer in layers:
if layer.name() == error_layer.name():
group.removeLayer(layer.layer())
break
added_layer = QgsProject.instance().addMapLayer(error_layer, False)
index = QgisModelBakerUtils().get_suggested_index_for_layer(
added_layer, group)
added_layer = group.insertLayer(index, added_layer).layer()
if added_layer.isSpatial():
# db connection is none because we are using a memory layer
SymbologyUtils().set_layer_style_from_qml(db,
added_layer,
is_error_layer=True)
if isinstance(added_layer.renderer(),
QgsCategorizedSymbolRenderer):
# Remove empty style categories as they just make difficult to understand validation errors
unique_values = added_layer.uniqueValues(
added_layer.fields().indexOf(
QCoreApplication.translate("QualityRule",
"codigo_error")))
renderer = added_layer.renderer()
for cat in reversed(renderer.categories()
): # To be safe while removing categories
if cat.value() not in unique_values:
renderer.deleteCategory(
renderer.categoryIndexForValue(cat.value()))
added_layer.setRenderer(added_layer.renderer().clone())
return added_layer
def get_error_layers_group(self):
"""
Get the topology errors group. If it exists but is placed in another
position rather than the top, it moves the group to the top.
"""
root = QgsProject.instance().layerTreeRoot()
translated_strings = TranslatableConfigStrings.get_translatable_config_strings(
)
group = root.findGroup(translated_strings[ERROR_LAYER_GROUP])
if group is None:
group = root.insertGroup(0, translated_strings[ERROR_LAYER_GROUP])
self.add_indicators_requested.emit(
translated_strings[ERROR_LAYER_GROUP],
QgsLayerTreeNode.NodeGroup)
elif not self.iface.layerTreeView().layerTreeModel().node2index(
group).row() == 0 or type(group.parent()) is QgsLayerTreeGroup:
group_clone = group.clone()
root.insertChildNode(0, group_clone)
parent = group.parent()
parent.removeChildNode(group)
group = group_clone
return group
def add_indicators(self, node_name, node_type, payload):
"""
Adds all indicators for a node in layer tree. It searches for the proper node and its config.
:param node_name: Key to get the config and possibly, the node (see payload)
:param node_type: QgsLayerTreeNode.NodeType
:param payload: If the node is a LADM layer, we need the layer object, as the name is not enough to disambiguate
between layers from different connections
"""
# First get the node
node = None
root = QgsProject.instance().layerTreeRoot()
if node_type == QgsLayerTreeNode.NodeGroup:
node = root.findGroup(node_name)
elif node_type == QgsLayerTreeNode.NodeLayer:
if payload:
node = root.findLayer(payload) # Search by QgsMapLayer
else: # Get the first layer matching the node name
layers = QgsProject.instance().mapLayersByName(node_name)
if layers:
node = root.findLayer(layers[0])
if not node:
self.logger.warning(
__name__,
"Node not found for adding indicators! ({}, {})".format(
node_name, node_type))
return # No node, no party
# Then, get the config
indicators_config = LayerTreeIndicatorConfig().get_indicators_config(
node_name, node_type)
if not indicators_config:
self.logger.warning(
__name__,
"Configuration for indicators not found for node '{}'!".format(
node_name))
# And finally...
for config in indicators_config:
self.add_indicator(node, config)
def add_indicator(self, node, config):
"""
Adds a single indicator for the node, based on a config dict
:param node: Layer tree node
:param config: Dictionary with required data to set the indicator
"""
indicator = QgsLayerTreeViewIndicator(self.iface.layerTreeView())
indicator.setToolTip(config[INDICATOR_TOOLTIP])
indicator.setIcon(config[INDICATOR_ICON])
indicator.clicked.connect(config[INDICATOR_SLOT])
self.iface.layerTreeView().addIndicator(node, indicator)
def export_error_group(self):
"""Exports the error group to GeoPackage"""
group = self.get_error_layers_group()
if group:
layers = group.findLayerIds()
if not layers:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"AppGUIInterface",
"There are no error layers to export!"))
return
filename, matched_filter = QFileDialog.getSaveFileName(
self.iface.mainWindow(),
QCoreApplication.translate(
"AppGUIInterface",
"Where do you want to save your GeoPackage?"), ".",
QCoreApplication.translate("AppGUIInterface",
"GeoPackage (*.gpkg)"))
if filename:
if not filename.endswith(".gpkg") and filename:
filename = filename + ".gpkg"
feedback = CustomFeedbackWithErrors()
try:
msg = QCoreApplication.translate(
"AppGUIInterface",
"Exporting quality errors to GeoPackage...")
with ProcessWithStatus(msg):
processing.run("native:package", {
'LAYERS': layers,
'OUTPUT': filename,
'OVERWRITE': False,
'SAVE_STYLES': True
},
feedback=feedback)
except QgsProcessingException as e:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"AppGUIInterface",
"The quality errors could not be exported. Details: {}"
.format(feedback.msg)))
return
self.logger.success_msg(
__name__,
QCoreApplication.translate(
"AppGUIInterface",
"The quality errors have been exported to GeoPackage!")
)
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"AppGUIInterface",
"Export to GeoPackage was cancelled. No output file was selected."
), 5)
else:
self.logger.warning_msg(
__name__,
QCoreApplication.translate(
"AppGUIInterface",
"There is no quality error group to export!"), 5)
def set_error_group_visibility(self, visible):
self.set_node_visibility(self.get_error_layers_group(), visible)
def set_layer_visibility(self, layer, visible):
node = QgsProject.instance().layerTreeRoot().findLayer(layer.id())
self.set_node_visibility(node, visible)
def error_group_exists(self):
root = QgsProject.instance().layerTreeRoot()
translated_strings = TranslatableConfigStrings.get_translatable_config_strings(
)
return root.findGroup(
translated_strings[ERROR_LAYER_GROUP]) is not None
@pyqtSlot()
def clear_message_bar(self):
self.iface.messageBar().clearWidgets()
def zoom_full(self):
self.iface.zoomFull()
def zoom_to_active_layer(self):
self.iface.zoomToActiveLayer()
def zoom_to_selected(self):
self.iface.actionZoomToSelected().trigger()
def show_message(self, msg, level, duration=5):
self.clear_message_bar(
) # Remove previous messages before showing a new one
self.iface.messageBar().pushMessage("Asistente LADM-COL", msg, level,
duration)
def show_status_bar_message(self, msg, duration):
self.iface.statusBarIface().showMessage(msg, duration)
def add_tabified_dock_widget(self, area, dock_widget):
"""
Adds the dock_widget to the given area, making sure it is tabified if other dock widgets exist.
:param area: Value of the Qt.DockWidgetArea enum
:param dock_widget: QDockWidget object
"""
if Qgis.QGIS_VERSION_INT >= 31300: # Use native addTabifiedDockWidget
self.iface.addTabifiedDockWidget(area, dock_widget, raiseTab=True)
else: # Use plugin's addTabifiedDockWidget, which does not raise the new tab
dock_widgets = list()
for dw in self.iface.mainWindow().findChildren(QDockWidget):
if dw.isVisible() and self.iface.mainWindow().dockWidgetArea(
dw) == area:
dock_widgets.append(dw)
self.iface.mainWindow().addDockWidget(
area,
dock_widget) # We add the dock widget, then attempt to tabify
if dock_widgets:
self.logger.debug(
__name__, "Tabifying dock widget {}...".format(
dock_widget.windowTitle()))
self.iface.mainWindow().tabifyDockWidget(
dock_widgets[0],
dock_widget) # No way to prefer one Dock Widget
