def save(self, url):
self._saving = True
self.savingChanged.emit()
self._savingProgress = 0
self.savingProgressChanged.emit()
p = QProcess(self)
p.setProcessChannelMode(QProcess.ForwardedErrorChannel)
stdout_buffer = b""
def ready_read_stdout():
nonlocal stdout_buffer
stdout_buffer += p.readAllStandardOutput().data()
*messages, stdout_buffer = stdout_buffer.split(b"\n")
for message in messages:
progress = json.loads(messages[-1].decode())
self._savingProgress = progress["fraction"]
self.savingProgressChanged.emit()
p.readyReadStandardOutput.connect(ready_read_stdout)
def finished(status):
self._saving = False
self.savingChanged.emit()
if status != 0:
self.savingError.emit()
p.finished.connect(finished)
args = ["-c", "from djpdf.scans2pdf import main; main()",
os.path.abspath(url.toLocalFile())]
if self._verbose:
args.append("--verbose")
p.start(sys.executable, args)
p.write(json.dumps([p._data for p in self._pages]).encode())
p.closeWriteChannel()
def save(self, url):
self._saving = True
self.savingChanged.emit()
self._savingProgress = 0
self.savingProgressChanged.emit()
p = QProcess(self)
p.setProcessChannelMode(QProcess.ForwardedErrorChannel)
stdout_buffer = b""
def ready_read_stdout():
nonlocal stdout_buffer
stdout_buffer += p.readAllStandardOutput().data()
*messages, stdout_buffer = stdout_buffer.split(b"\n")
for message in messages:
progress = json.loads(messages[-1].decode())
self._savingProgress = progress["fraction"]
self.savingProgressChanged.emit()
p.readyReadStandardOutput.connect(ready_read_stdout)
p.finished.connect(self._process_finished)
args = ["-c", "from djpdf.scans2pdf import main; main()",
os.path.abspath(url.toLocalFile())]
if self._verbose:
args.append("--verbose")
p.start(sys.executable, args)
self._process = p
p.write(json.dumps([p._data for p in self._pages]).encode())
p.closeWriteChannel()
class QProcessExecutionManager(ExecutionManager):
"""Class to manage tool instance execution using a PySide2 QProcess."""
def __init__(self, logger, program=None, args=None, silent=False, semisilent=False):
"""Class constructor.
Args:
logger (LoggerInterface): a logger instance
program (str): Path to program to run in the subprocess (e.g. julia.exe)
args (list): List of argument for the program (e.g. path to script file)
silent (bool): Whether or not to emit logger msg signals
"""
super().__init__(logger)
self._program = program
self._args = args
self._silent = silent # Do not show Event Log nor Process Log messages
self._semisilent = semisilent # Do not show Event Log messages but show Process Log messages
self.process_failed = False
self.process_failed_to_start = False
self._user_stopped = False
self._process = QProcess(self)
self.process_output = None # stdout when running silent
self.error_output = None # stderr when running silent
self.data_to_inject = None
def program(self):
"""Program getter method."""
return self._program
def args(self):
"""Program argument getter method."""
return self._args
# noinspection PyUnresolvedReferences
def start_execution(self, workdir=None):
"""Starts the execution of a command in a QProcess.
Args:
workdir (str): Work directory
"""
if workdir is not None:
self._process.setWorkingDirectory(workdir)
self._process.started.connect(self.process_started)
self._process.finished.connect(self.on_process_finished)
if not self._silent and not self._semisilent: # Loud
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.errorOccurred.connect(self.on_process_error)
self._process.stateChanged.connect(self.on_state_changed)
elif self._semisilent: # semi-silent
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.start(self._program, self._args)
if not self._process.waitForStarted(msecs=10000): # This blocks until process starts or timeout happens
self.process_failed = True
self.process_failed_to_start = True
self._process.deleteLater()
self._process = None
self.data_to_inject = None
self.execution_finished.emit(-9998)
if self.data_to_inject is not None:
self.inject_data_to_write_channel()
def inject_data_to_write_channel(self):
"""Writes data to process write channel and closes it afterwards."""
self._process.write(json.dumps(self.data_to_inject).encode("utf-8"))
self._process.write(b'\n')
self._process.closeWriteChannel()
def wait_for_process_finished(self, msecs=30000):
"""Wait for subprocess to finish.
Return:
True if process finished successfully, False otherwise
"""
if not self._process:
return False
if self.process_failed or self.process_failed_to_start:
return False
if not self._process.waitForFinished(msecs):
self.process_failed = True
self._process.close()
self._process = None
return False
return True
@Slot()
def process_started(self):
"""Run when subprocess has started."""
@Slot("QProcess::ProcessState")
def on_state_changed(self, new_state):
"""Runs when QProcess state changes.
Args:
new_state (QProcess::ProcessState): Process state number
"""
if new_state == QProcess.Starting:
self._logger.msg.emit("\tStarting program <b>{0}</b>".format(self._program))
arg_str = " ".join(self._args)
self._logger.msg.emit("\tArguments: <b>{0}</b>".format(arg_str))
elif new_state == QProcess.Running:
self._logger.msg_warning.emit("\tExecution is in progress. See Process Log for messages " "(stdout&stderr)")
elif new_state == QProcess.NotRunning:
# logging.debug("Process is not running")
pass
else:
self._logger.msg_error.emit("Process is in an unspecified state")
logging.error("QProcess unspecified state: %s", new_state)
@Slot("QProcess::ProcessError")
def on_process_error(self, process_error):
"""Runs if there is an error in the running QProcess.
Args:
process_error (QProcess::ProcessError): Process error number
"""
if process_error == QProcess.FailedToStart:
self.process_failed = True
self.process_failed_to_start = True
self._logger.msg_error.emit("Process failed to start")
elif process_error == QProcess.Timedout:
self.process_failed = True
self._logger.msg_error.emit("Timed out")
elif process_error == QProcess.Crashed:
self.process_failed = True
if not self._user_stopped:
self._logger.msg_error.emit("Process crashed")
elif process_error == QProcess.WriteError:
self._logger.msg_error.emit("Process WriteError")
elif process_error == QProcess.ReadError:
self._logger.msg_error.emit("Process ReadError")
elif process_error == QProcess.UnknownError:
self._logger.msg_error.emit("Unknown error in process")
else:
self._logger.msg_error.emit("Unspecified error in process: {0}".format(process_error))
self.teardown_process()
def teardown_process(self):
"""Tears down the QProcess in case a QProcess.ProcessError occurred.
Emits execution_finished signal."""
# self._logger.msg.emit("Tearing down process")
if not self._process:
pass
else:
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
if out is not None:
self._logger.msg_proc.emit(out.strip())
if errout is not None:
self._logger.msg_proc.emit(errout.strip())
self._process.deleteLater()
self._process = None
self.data_to_inject = None
self.execution_finished.emit(-9998)
def stop_execution(self):
"""See base class."""
self._logger.msg_error.emit("Terminating process")
self._user_stopped = True
self.process_failed = True
if not self._process:
return
try:
self._process.terminate()
except Exception as ex: # pylint: disable=broad-except
self._logger.msg_error.emit("[{0}] exception when terminating process".format(ex))
logging.exception("Exception in closing QProcess: %s", ex)
finally:
self._process.deleteLater()
self._process = None
self.data_to_inject = None
@Slot(int, "QProcess::ExitStatus")
def on_process_finished(self, exit_code, exit_status):
"""Runs when subprocess has finished.
Args:
exit_code (int): Return code from external program (only valid for normal exits)
exit_status (QProcess.ExitStatus): Crash or normal exit
"""
# logging.debug("Error that occurred last: {0}".format(self._process.error()))
if not self._process:
return
if exit_status == QProcess.CrashExit:
if not self._silent:
self._logger.msg_error.emit("\tProcess crashed")
exit_code = -1
elif exit_status == QProcess.NormalExit:
pass
else:
if not self._silent:
self._logger.msg_error.emit("Unknown QProcess exit status [{0}]".format(exit_status))
exit_code = -1
if not exit_code == 0:
self.process_failed = True
if not self._user_stopped:
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
if out is not None:
if not self._silent:
self._logger.msg_proc.emit(out.strip())
else:
self.process_output = out.strip()
self.error_output = errout.strip()
else:
self._logger.msg.emit("*** Terminating process ***")
# Delete QProcess
self._process.deleteLater()
self._process = None
self.data_to_inject = None
self.execution_finished.emit(exit_code)
@Slot()
def on_ready_stdout(self):
"""Emit data from stdout."""
if not self._process:
return
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
self._logger.msg_proc.emit(out.strip())
@Slot()
def on_ready_stderr(self):
"""Emit data from stderr."""
if not self._process:
return
err = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
self._logger.msg_proc_error.emit(err.strip())
class Importer(ProjectItem):
def __init__(self, name, description, mappings, x, y, toolbox, project, logger, cancel_on_error=True):
"""Importer class.
Args:
name (str): Project item name
description (str): Project item description
mappings (list): List where each element contains two dicts (path dict and mapping dict)
x (float): Initial icon scene X coordinate
y (float): Initial icon scene Y coordinate
toolbox (ToolboxUI): QMainWindow instance
project (SpineToolboxProject): the project this item belongs to
logger (LoggerInterface): a logger instance
cancel_on_error (bool): if True the item's execution will stop on import error
"""
super().__init__(name, description, x, y, project, logger)
# Make logs subdirectory for this item
self._toolbox = toolbox
self.logs_dir = os.path.join(self.data_dir, "logs")
try:
create_dir(self.logs_dir)
except OSError:
self._logger.msg_error.emit(f"[OSError] Creating directory {self.logs_dir} failed. Check permissions.")
# Variables for saving selections when item is (de)activated
if not mappings:
mappings = list()
# convert table_types and table_row_types keys to int since json always has strings as keys.
for _, mapping in mappings:
table_types = mapping.get("table_types", {})
mapping["table_types"] = {
table_name: {int(col): t for col, t in col_types.items()}
for table_name, col_types in table_types.items()
}
table_row_types = mapping.get("table_row_types", {})
mapping["table_row_types"] = {
table_name: {int(row): t for row, t in row_types.items()}
for table_name, row_types in table_row_types.items()
}
# Convert serialized paths to absolute in mappings
self.settings = self.deserialize_mappings(mappings, self._project.project_dir)
# self.settings is now a dictionary, where elements have the absolute path as the key and the mapping as value
self.cancel_on_error = cancel_on_error
self.resources_from_downstream = list()
self.file_model = QStandardItemModel()
self.importer_process = None
self.all_files = [] # All source files
self.unchecked_files = [] # Unchecked source files
# connector class
self._preview_widget = {} # Key is the filepath, value is the ImportPreviewWindow instance
@staticmethod
def item_type():
"""See base class."""
return "Importer"
@staticmethod
def category():
"""See base class."""
return "Importers"
@Slot(QStandardItem, name="_handle_file_model_item_changed")
def _handle_file_model_item_changed(self, item):
if item.checkState() == Qt.Checked:
self.unchecked_files.remove(item.text())
self._logger.msg.emit(f"<b>{self.name}:</b> Source file '{item.text()}' will be processed at execution.")
elif item.checkState() != Qt.Checked:
self.unchecked_files.append(item.text())
self._logger.msg.emit(
f"<b>{self.name}:</b> Source file '{item.text()}' will *NOT* be processed at execution."
)
def make_signal_handler_dict(self):
"""Returns a dictionary of all shared signals and their handlers.
This is to enable simpler connecting and disconnecting."""
s = super().make_signal_handler_dict()
s[self._properties_ui.toolButton_open_dir.clicked] = lambda checked=False: self.open_directory()
s[self._properties_ui.pushButton_import_editor.clicked] = self._handle_import_editor_clicked
s[self._properties_ui.treeView_files.doubleClicked] = self._handle_files_double_clicked
return s
def activate(self):
"""Restores selections, cancel on error checkbox and connects signals."""
self._properties_ui.cancel_on_error_checkBox.setCheckState(Qt.Checked if self.cancel_on_error else Qt.Unchecked)
self.restore_selections()
super().connect_signals()
def deactivate(self):
"""Saves selections and disconnects signals."""
self.save_selections()
if not super().disconnect_signals():
logging.error("Item %s deactivation failed.", self.name)
return False
return True
def restore_selections(self):
"""Restores selections into shared widgets when this project item is selected."""
self._properties_ui.label_name.setText(self.name)
self._properties_ui.treeView_files.setModel(self.file_model)
self.file_model.itemChanged.connect(self._handle_file_model_item_changed)
def save_selections(self):
"""Saves selections in shared widgets for this project item into instance variables."""
self._properties_ui.treeView_files.setModel(None)
self.file_model.itemChanged.disconnect(self._handle_file_model_item_changed)
def update_name_label(self):
"""Update Importer properties tab name label. Used only when renaming project items."""
self._properties_ui.label_name.setText(self.name)
@Slot(bool, name="_handle_import_editor_clicked")
def _handle_import_editor_clicked(self, checked=False):
"""Opens Import editor for the file selected in list view."""
index = self._properties_ui.treeView_files.currentIndex()
self.open_import_editor(index)
@Slot("QModelIndex", name="_handle_files_double_clicked")
def _handle_files_double_clicked(self, index):
"""Opens Import editor for the double clicked index."""
self.open_import_editor(index)
def open_import_editor(self, index):
"""Opens Import editor for the given index."""
importee = index.data()
if importee is None:
self._logger.msg_error.emit("Please select a source file from the list first.")
return
if not os.path.exists(importee):
self._logger.msg_error.emit(f"Invalid path: {importee}")
return
# Raise current form for the selected file if any
preview_widget = self._preview_widget.get(importee, None)
if preview_widget:
if preview_widget.windowState() & Qt.WindowMinimized:
# Remove minimized status and restore window with the previous state (maximized/normal state)
preview_widget.setWindowState(preview_widget.windowState() & ~Qt.WindowMinimized | Qt.WindowActive)
preview_widget.activateWindow()
else:
preview_widget.raise_()
return
# Create a new form for the selected file
settings = self.get_settings(importee)
# Try and get connector from settings
source_type = settings.get("source_type", None)
if source_type is not None:
connector = _CONNECTOR_NAME_TO_CLASS[source_type]
else:
# Ask user
connector = self.get_connector(importee)
if not connector:
# Aborted by the user
return
self._logger.msg.emit(f"Opening Import editor for file: {importee}")
preview_widget = self._preview_widget[importee] = ImportPreviewWindow(
self, importee, connector, settings, self._toolbox
)
preview_widget.settings_updated.connect(lambda s, importee=importee: self.save_settings(s, importee))
preview_widget.connection_failed.connect(lambda m, importee=importee: self._connection_failed(m, importee))
preview_widget.destroyed.connect(lambda o=None, importee=importee: self._preview_destroyed(importee))
preview_widget.start_ui()
def get_connector(self, importee):
"""Shows a QDialog to select a connector for the given source file.
Mimics similar routine in `spine_io.widgets.import_widget.ImportDialog`
Args:
importee (str): Path to file acting as an importee
Returns:
Asynchronous data reader class for the given importee
"""
connector_list = [CSVConnector, ExcelConnector, GdxConnector] # add others as needed
connector_names = [c.DISPLAY_NAME for c in connector_list]
dialog = QDialog(self._toolbox)
dialog.setLayout(QVBoxLayout())
connector_list_wg = QListWidget()
connector_list_wg.addItems(connector_names)
# Set current item in `connector_list_wg` based on file extension
_filename, file_extension = os.path.splitext(importee)
if file_extension.lower().startswith(".xls"):
row = connector_list.index(ExcelConnector)
elif file_extension.lower() == ".csv":
row = connector_list.index(CSVConnector)
elif file_extension.lower() == ".gdx":
row = connector_list.index(GdxConnector)
else:
row = None
if row:
connector_list_wg.setCurrentRow(row)
button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
button_box.button(QDialogButtonBox.Ok).clicked.connect(dialog.accept)
button_box.button(QDialogButtonBox.Cancel).clicked.connect(dialog.reject)
connector_list_wg.doubleClicked.connect(dialog.accept)
dialog.layout().addWidget(connector_list_wg)
dialog.layout().addWidget(button_box)
_dirname, filename = os.path.split(importee)
dialog.setWindowTitle("Select connector for '{}'".format(filename))
answer = dialog.exec_()
if answer:
row = connector_list_wg.currentIndex().row()
return connector_list[row]
def select_connector_type(self, index):
"""Opens dialog to select connector type for the given index."""
importee = index.data()
connector = self.get_connector(importee)
if not connector:
# Aborted by the user
return
settings = self.get_settings(importee)
settings["source_type"] = connector.__name__
def _connection_failed(self, msg, importee):
self._logger.msg.emit(msg)
preview_widget = self._preview_widget.pop(importee, None)
if preview_widget:
preview_widget.close()
def get_settings(self, importee):
"""Returns the mapping dictionary for the file in given path.
Args:
importee (str): Absolute path to a file, whose mapping is queried
Returns:
dict: Mapping dictionary for the requested importee or an empty dict if not found
"""
importee_settings = None
for p in self.settings:
if p == importee:
importee_settings = self.settings[p]
if not importee_settings:
return {}
return importee_settings
def save_settings(self, settings, importee):
"""Updates an existing mapping or adds a new mapping
(settings) after closing the import preview window.
Args:
settings (dict): Updated mapping (settings) dictionary
importee (str): Absolute path to a file, whose mapping has been updated
"""
if importee in self.settings.keys():
self.settings[importee].update(settings)
else:
self.settings[importee] = settings
def _preview_destroyed(self, importee):
"""Destroys preview widget instance for the given importee.
Args:
importee (str): Absolute path to a file, whose preview widget is destroyed
"""
self._preview_widget.pop(importee, None)
def update_file_model(self, items):
"""Adds given list of items to the file model. If None or
an empty list is given, the model is cleared.
Args:
items (set): Set of absolute file paths
"""
self.all_files = items
self.file_model.clear()
self.file_model.setHorizontalHeaderItem(0, QStandardItem("Source files")) # Add header
if items is not None:
for item in items:
qitem = QStandardItem(item)
qitem.setEditable(False)
qitem.setCheckable(True)
if item in self.unchecked_files:
qitem.setCheckState(Qt.Unchecked)
else:
qitem.setCheckState(Qt.Checked)
qitem.setData(QFileIconProvider().icon(QFileInfo(item)), Qt.DecorationRole)
self.file_model.appendRow(qitem)
def _run_importer_program(self, args):
"""Starts and runs the importer program in a separate process.
Args:
args (list): List of arguments for the importer program
"""
self.importer_process = QProcess()
self.importer_process.readyReadStandardOutput.connect(self._log_importer_process_stdout)
self.importer_process.readyReadStandardError.connect(self._log_importer_process_stderr)
self.importer_process.finished.connect(self.importer_process.deleteLater)
program_path = os.path.abspath(importer_program.__file__)
self.importer_process.start(sys.executable, [program_path])
self.importer_process.waitForStarted()
self.importer_process.write(json.dumps(args).encode("utf-8"))
self.importer_process.write(b'\n')
self.importer_process.closeWriteChannel()
if self.importer_process.state() == QProcess.Running:
loop = QEventLoop()
self.importer_process.finished.connect(loop.quit)
loop.exec_()
return self.importer_process.exitCode()
@Slot()
def _log_importer_process_stdout(self):
output = str(self.importer_process.readAllStandardOutput().data(), "utf-8").strip()
self._logger.msg.emit(f"<b>{self.name}</b>: {output}")
@Slot()
def _log_importer_process_stderr(self):
output = str(self.importer_process.readAllStandardError().data(), "utf-8").strip()
self._logger.msg_error.emit(f"<b>{self.name}</b>: {output}")
def execute_backward(self, resources):
"""See base class."""
self.resources_from_downstream = resources.copy()
return True
def execute_forward(self, resources):
"""See base class."""
args = [
[f for f in self.all_files if f not in self.unchecked_files],
self.settings,
[r.url for r in self.resources_from_downstream if r.type_ == "database"],
self.logs_dir,
self._properties_ui.cancel_on_error_checkBox.isChecked(),
]
exit_code = self._run_importer_program(args)
return exit_code == 0
def stop_execution(self):
"""Stops executing this Importer."""
super().stop_execution()
if not self.importer_process:
return
self.importer_process.kill()
def _do_handle_dag_changed(self, resources):
"""See base class."""
file_list = [r.path for r in resources if r.type_ == "file" and not r.metadata.get("future")]
self._notify_if_duplicate_file_paths(file_list)
self.update_file_model(set(file_list))
if not file_list:
self.add_notification(
"This Importer does not have any input data. "
"Connect Data Connections to this Importer to use their data as input."
)
def item_dict(self):
"""Returns a dictionary corresponding to this item."""
d = super().item_dict()
# Serialize mappings before saving
d["mappings"] = self.serialize_mappings(self.settings, self._project.project_dir)
d["cancel_on_error"] = self._properties_ui.cancel_on_error_checkBox.isChecked()
return d
def notify_destination(self, source_item):
"""See base class."""
if source_item.item_type() == "Data Connection":
self._logger.msg.emit(
"Link established. You can define mappings on data from "
f"<b>{source_item.name}</b> using item <b>{self.name}</b>."
)
elif source_item.item_type() == "Data Store":
# Does this type of link do anything?
self._logger.msg.emit("Link established.")
else:
super().notify_destination(source_item)
@staticmethod
def default_name_prefix():
"""see base class"""
return "Importer"
def tear_down(self):
"""Closes all preview widgets."""
for widget in self._preview_widget.values():
widget.close()
def _notify_if_duplicate_file_paths(self, file_list):
"""Adds a notification if file_list contains duplicate entries."""
file_counter = Counter(file_list)
duplicates = list()
for file_name, count in file_counter.items():
if count > 1:
duplicates.append(file_name)
if duplicates:
self.add_notification("Duplicate input files from upstream items:<br>{}".format("<br>".join(duplicates)))
@staticmethod
def upgrade_from_no_version_to_version_1(item_name, old_item_dict, old_project_dir):
"""Converts mappings to a list, where each element contains two dictionaries,
the serialized path dictionary and the mapping dictionary for the file in that
path."""
new_importer = dict(old_item_dict)
mappings = new_importer.get("mappings", {})
list_of_mappings = list()
paths = list(mappings.keys())
for path in paths:
mapping = mappings[path]
if "source_type" in mapping and mapping["source_type"] == "CSVConnector":
_fix_csv_connector_settings(mapping)
new_path = serialize_path(path, old_project_dir)
if new_path["relative"]:
new_path["path"] = os.path.join(".spinetoolbox", "items", new_path["path"])
list_of_mappings.append([new_path, mapping])
new_importer["mappings"] = list_of_mappings
return new_importer
@staticmethod
def deserialize_mappings(mappings, project_path):
"""Returns mapping settings as dict with absolute paths as keys.
Args:
mappings (list): List where each element contains two dictionaries (path dict and mapping dict)
project_path (str): Path to project directory
Returns:
dict: Dictionary with absolute paths as keys and mapping settings as values
"""
abs_path_mappings = {}
for source, mapping in mappings:
abs_path_mappings[deserialize_path(source, project_path)] = mapping
return abs_path_mappings
@staticmethod
def serialize_mappings(mappings, project_path):
"""Returns a list of mappings, where each element contains two dictionaries,
the 'serialized' path in a dictionary and the mapping dictionary.
Args:
mappings (dict): Dictionary with mapping specifications
project_path (str): Path to project directory
Returns:
list: List where each element contains two dictionaries.
"""
serialized_mappings = list()
for source, mapping in mappings.items(): # mappings is a dict with absolute paths as keys and mapping as values
serialized_mappings.append([serialize_path(source, project_path), mapping])
return serialized_mappings