class EmcExe_Qt(EmcExe):
""" PySide2 implementation of the EmcExec """
def __init__(self, *args, **kargs):
super().__init__(*args, **kargs)
self._proc = QProcess()
self._proc.errorOccurred.connect(self._error_cb)
if self._done_cb:
self._proc.finished.connect(self._finished_cb)
if self._grab_output:
self._proc.readyReadStandardOutput.connect(self._stdout_cb)
if self._params:
self._proc.start(self._cmd, self._params)
else:
self._proc.start(self._cmd)
def delete(self) -> None:
super().delete()
if self._proc and self._proc.state() == QProcess.Running:
self._proc.kill()
self._proc = None
def _finished_cb(self, exit_code):
self._call_user_callback(exit_code)
def _error_cb(self, error):
if self._proc and not self.deleted:
self._call_user_callback(-1)
def _stdout_cb(self):
if self._proc and not self.deleted:
self._out_buffer.append(self._proc.readAllStandardOutput().data())
class QProcessExecutionManager(ExecutionManager):
"""Class to manage tool instance execution using a PySide2 QProcess."""
def __init__(self, logger, program="", args=None, silent=False, semisilent=False):
"""Class constructor.
Args:
logger (LoggerInterface): a logger instance
program (str): Path to program to run in the subprocess (e.g. julia.exe)
args (list, optional): List of argument for the program (e.g. path to script file)
silent (bool): Whether or not to emit logger msg signals
semisilent (bool): If True, show Process Log messages
"""
super().__init__(logger)
self._program = program
self._args = args if args is not None else []
self._silent = silent # Do not show Event Log nor Process Log messages
self._semisilent = semisilent # Do not show Event Log messages but show Process Log messages
self.process_failed = False
self.process_failed_to_start = False
self.user_stopped = False
self._process = QProcess(self)
self.process_output = None # stdout when running silent
self.process_error = None # stderr when running silent
self._out_chunks = []
self._err_chunks = []
def program(self):
"""Program getter method."""
return self._program
def args(self):
"""Program argument getter method."""
return self._args
# noinspection PyUnresolvedReferences
def start_execution(self, workdir=None):
"""Starts the execution of a command in a QProcess.
Args:
workdir (str, optional): Work directory
"""
if workdir is not None:
self._process.setWorkingDirectory(workdir)
self._process.started.connect(self.process_started)
self._process.finished.connect(self.on_process_finished)
if not self._silent and not self._semisilent: # Loud
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.errorOccurred.connect(self.on_process_error)
self._process.stateChanged.connect(self.on_state_changed)
elif self._semisilent: # semi-silent
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.start(self._program, self._args)
if self._process is not None and not self._process.waitForStarted(msecs=10000):
self.process_failed = True
self.process_failed_to_start = True
self._process.deleteLater()
self._process = None
self.execution_finished.emit(-9998)
def wait_for_process_finished(self, msecs=30000):
"""Wait for subprocess to finish.
Args:
msecs (int): Timeout in milliseconds
Return:
True if process finished successfully, False otherwise
"""
if self._process is None:
return False
if self.process_failed or self.process_failed_to_start:
return False
if not self._process.waitForFinished(msecs):
self.process_failed = True
self._process.close()
self._process = None
return False
return True
@Slot()
def process_started(self):
"""Run when subprocess has started."""
@Slot(int)
def on_state_changed(self, new_state):
"""Runs when QProcess state changes.
Args:
new_state (int): Process state number (``QProcess::ProcessState``)
"""
if new_state == QProcess.Starting:
self._logger.msg.emit("\tStarting program <b>{0}</b>".format(self._program))
arg_str = " ".join(self._args)
self._logger.msg.emit("\tArguments: <b>{0}</b>".format(arg_str))
elif new_state == QProcess.Running:
self._logger.msg_warning.emit("\tExecution in progress...")
elif new_state == QProcess.NotRunning:
# logging.debug("Process is not running")
pass
else:
self._logger.msg_error.emit("Process is in an unspecified state")
logging.error("QProcess unspecified state: %s", new_state)
@Slot(int)
def on_process_error(self, process_error):
"""Runs if there is an error in the running QProcess.
Args:
process_error (int): Process error number (``QProcess::ProcessError``)
"""
if process_error == QProcess.FailedToStart:
self.process_failed = True
self.process_failed_to_start = True
self._logger.msg_error.emit("Process failed to start")
elif process_error == QProcess.Timedout:
self.process_failed = True
self._logger.msg_error.emit("Timed out")
elif process_error == QProcess.Crashed:
self.process_failed = True
if not self.user_stopped:
self._logger.msg_error.emit("Process crashed")
elif process_error == QProcess.WriteError:
self._logger.msg_error.emit("Process WriteError")
elif process_error == QProcess.ReadError:
self._logger.msg_error.emit("Process ReadError")
elif process_error == QProcess.UnknownError:
self._logger.msg_error.emit("Unknown error in process")
else:
self._logger.msg_error.emit("Unspecified error in process: {0}".format(process_error))
self.teardown_process()
def teardown_process(self):
"""Tears down the QProcess in case a QProcess.ProcessError occurred.
Emits execution_finished signal."""
if not self._process:
pass
else:
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
if out is not None:
self._logger.msg_proc.emit(out.strip())
if errout is not None:
self._logger.msg_proc.emit(errout.strip())
self._process.deleteLater()
self._process = None
self.execution_finished.emit(-9998)
def stop_execution(self):
"""See base class."""
self.user_stopped = True
self.process_failed = True
if not self._process:
return
try:
self._process.kill()
if not self._process.waitForFinished(5000):
self._process.finished.emit(-1, -1)
self._process.deleteLater()
except Exception as ex: # pylint: disable=broad-except
self._logger.msg_error.emit("[{0}] exception when terminating process".format(ex))
logging.exception("Exception in closing QProcess: %s", ex)
finally:
self._process = None
@Slot(int, int)
def on_process_finished(self, exit_code, exit_status):
"""Runs when subprocess has finished.
Args:
exit_code (int): Return code from external program (only valid for normal exits)
exit_status (int): Crash or normal exit (``QProcess::ExitStatus``)
"""
if not self._process:
return
if exit_status == QProcess.CrashExit:
if not self._silent:
self._logger.msg_error.emit("\tProcess crashed")
exit_code = -1
elif exit_status == QProcess.NormalExit:
pass
else:
if not self._silent:
self._logger.msg_error.emit("Unknown QProcess exit status [{0}]".format(exit_status))
exit_code = -1
if not exit_code == 0:
self.process_failed = True
if not self.user_stopped:
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
if out is not None:
if not self._silent:
self._logger.msg_proc.emit(out.strip())
else:
self.process_output = out.strip()
self.process_error = errout.strip()
else:
self._logger.msg.emit("*** Terminating process ***")
# Delete QProcess
self._process.deleteLater()
self._process = None
self.execution_finished.emit(exit_code)
@Slot()
def on_ready_stdout(self):
"""Emit data from stdout."""
if not self._process:
return
self._process.setReadChannel(QProcess.StandardOutput)
chunk = self._process.readLine().data()
self._out_chunks.append(chunk)
if not chunk.endswith(b"\n"):
return
line = b"".join(self._out_chunks)
line = str(line, "unicode_escape", errors="replace").strip()
self._logger.msg_proc.emit(line)
self._out_chunks.clear()
@Slot()
def on_ready_stderr(self):
"""Emit data from stderr."""
if not self._process:
return
self._process.setReadChannel(QProcess.StandardError)
chunk = self._process.readLine().data()
self._err_chunks.append(chunk)
if not chunk.endswith(b"\n"):
return
line = b"".join(self._err_chunks)
line = str(line, "utf-8", errors="replace").strip()
self._logger.msg_proc_error.emit(line)
self._err_chunks.clear()
class CliWidget(QDialog):
def __init__(self, title: str, program: str, args: List[str],
commands: List[str]):
super().__init__()
self.setWindowTitle(title)
self.program = program
self.args = args
self.layout = QGridLayout()
self.output = QTextEdit()
self.output.acceptRichText = True
self.input = QLineEdit()
self.completer = QCompleter(commands, self)
self.completer.setCaseSensitivity(Qt.CaseInsensitive)
self.input.setCompleter(self.completer)
self.input.setFocus()
self.process = QProcess()
self.process.setProgram(self.program)
self.process.setCurrentReadChannel(0)
# noinspection PyUnresolvedReferences
self.process.readyReadStandardError.connect(self.handle_error)
# noinspection PyUnresolvedReferences
self.process.readyReadStandardOutput.connect(self.handle_output)
self.layout.addWidget(self.output)
self.layout.addWidget(self.input)
self.setLayout(self.layout)
self.connect(self.input, SIGNAL("returnPressed(void)"),
self.execute_user_command)
self.connect(self.completer, SIGNAL("activated(const QString&)"),
self.input.clear, Qt.QueuedConnection)
def execute_user_command(self):
cmd = str(self.input.text())
self.run_command(cmd)
def run_command(self, cmd: str):
log.info('run_command', program=self.program, args=self.args, cmd=cmd)
self.output.append(f'> {cmd}\n')
self.input.clear()
self.process.kill()
args = list(self.args)
args.append(cmd)
self.process.setArguments(args)
self.process.start()
def handle_error(self):
output: QByteArray = self.process.readAllStandardError()
message = output.data().decode('utf-8').strip()
self.output.append(message)
def handle_output(self):
output: QByteArray = self.process.readAllStandardOutput()
message = output.data().decode('utf-8').strip()
if message.startswith('{') or message.startswith('['):
formatter = HtmlFormatter()
formatter.noclasses = True
formatter.linenos = False
formatter.nobackground = True
message = highlight(message, JsonLexer(), formatter)
self.output.insertHtml(message)
else:
self.output.append(message)
# This is just for generating the command lists in constants
# commands = None
# if '== Blockchain ==' in message:
# commands = self.parse_bitcoin_cli_commands(message)
# elif 'lncli [global options] command [command options]' in message:
# commands = self.parse_lncli_commands(message)
# if commands is not None:
# log.debug('commands', commands=commands)
max_scroll = self.output.verticalScrollBar().maximum()
self.output.verticalScrollBar().setValue(max_scroll)
def parse_bitcoin_cli_commands(self, message: str):
log.debug('parse_bitcoin_cli_commands')
commands = []
for line in message.split(sep='\n'):
line = line.strip()
if not line or line.startswith('=='):
continue
command = line.split()[0]
command = command.strip()
commands.append(command)
return commands
def parse_lncli_commands(self, message: str):
log.debug('parse_lncli_commands')
at_commands = False
commands = []
for line in message.split(sep='\n'):
line = line.strip()
if not at_commands:
if 'COMMANDS:' in line:
at_commands = True
log.debug('commands line', line=line)
continue
elif 'GLOBAL OPTIONS' in line:
return commands
elif line.endswith(':') or not line:
continue
command = line.split()[0]
command = command.strip().replace(',', '')
commands.append(command)
return commands
def show(self):
self.showMaximized()
self.input.setFocus()
self.run_command('help')
class WebMediaView(MediaView):
def __init__(self, media, parent):
super(WebMediaView, self).__init__(media, parent)
self.widget = QWidget(parent)
self.process = QProcess(self.widget)
self.process.setObjectName('%s-process' % self.objectName())
self.std_out = []
self.errors = []
self.stopping = False
self.mute = False
self.widget.setGeometry(media['geometry'])
self.connect(self.process, SIGNAL('error()'), self.process_error)
self.connect(self.process, SIGNAL('finished()'), self.process_finished)
self.connect(self.process, SIGNAL('started()'), self.process_started)
self.set_default_widget_prop()
self.stop_timer = QTimer(self)
self.stop_timer.setSingleShot(True)
self.stop_timer.setInterval(1000)
self.stop_timer.timeout.connect(self.process_timeout)
self.rect = self.widget.geometry()
@Slot()
def process_timeout(self):
os.kill(self.process.pid(), signal.SIGTERM)
self.stopping = False
if not self.is_started():
self.started_signal.emit()
super(WebMediaView, self).stop()
@Slot(object)
def process_error(self, err):
print('---- process error ----')
self.errors.append(err)
self.stop()
@Slot()
def process_finished(self):
self.stop()
@Slot()
def process_started(self):
self.stop_timer.stop()
if float(self.duration) > 0:
self.play_timer.setInterval(int(float(self.duration) * 1000))
self.play_timer.start()
self.started_signal.emit()
pass
@Slot()
def play(self):
self.finished = 0
self.widget.show()
self.widget.raise_()
#---- kong ----
url = self.options['uri']
args = [
str(self.rect.left()),
str(self.rect.top()),
str(self.rect.width()),
str(self.rect.height()),
QUrl.fromPercentEncoding(QByteArray(url.encode('utf-8')))
]
#self.process.start('dist/web.exe', args) # for windows
#self.process.start('./dist/web', args) # for RPi
self.stop_timer.start()
#----
@Slot()
def stop(self, delete_widget=False):
#---- kong ----
if not self.widget:
return False
if self.stopping or self.is_finished():
return False
self.stop_timer.start()
self.stopping = True
if self.process.state() == QProcess.ProcessState.Running:
#---- kill process ----
self.process.terminate() # for windows
self.process.kill() # for linux
#os.system('pkill web') # for RPi
#----
self.process.waitForFinished()
self.process.close()
super(WebMediaView, self).stop(delete_widget)
self.stopping = False
self.stop_timer.stop()
return True
class Importer(ProjectItem):
def __init__(self, name, description, mappings, x, y, toolbox, project, logger, cancel_on_error=True):
"""Importer class.
Args:
name (str): Project item name
description (str): Project item description
mappings (list): List where each element contains two dicts (path dict and mapping dict)
x (float): Initial icon scene X coordinate
y (float): Initial icon scene Y coordinate
toolbox (ToolboxUI): QMainWindow instance
project (SpineToolboxProject): the project this item belongs to
logger (LoggerInterface): a logger instance
cancel_on_error (bool): if True the item's execution will stop on import error
"""
super().__init__(name, description, x, y, project, logger)
# Make logs subdirectory for this item
self._toolbox = toolbox
self.logs_dir = os.path.join(self.data_dir, "logs")
try:
create_dir(self.logs_dir)
except OSError:
self._logger.msg_error.emit(f"[OSError] Creating directory {self.logs_dir} failed. Check permissions.")
# Variables for saving selections when item is (de)activated
if not mappings:
mappings = list()
# convert table_types and table_row_types keys to int since json always has strings as keys.
for _, mapping in mappings:
table_types = mapping.get("table_types", {})
mapping["table_types"] = {
table_name: {int(col): t for col, t in col_types.items()}
for table_name, col_types in table_types.items()
}
table_row_types = mapping.get("table_row_types", {})
mapping["table_row_types"] = {
table_name: {int(row): t for row, t in row_types.items()}
for table_name, row_types in table_row_types.items()
}
# Convert serialized paths to absolute in mappings
self.settings = self.deserialize_mappings(mappings, self._project.project_dir)
# self.settings is now a dictionary, where elements have the absolute path as the key and the mapping as value
self.cancel_on_error = cancel_on_error
self.resources_from_downstream = list()
self.file_model = QStandardItemModel()
self.importer_process = None
self.all_files = [] # All source files
self.unchecked_files = [] # Unchecked source files
# connector class
self._preview_widget = {} # Key is the filepath, value is the ImportPreviewWindow instance
@staticmethod
def item_type():
"""See base class."""
return "Importer"
@staticmethod
def category():
"""See base class."""
return "Importers"
@Slot(QStandardItem, name="_handle_file_model_item_changed")
def _handle_file_model_item_changed(self, item):
if item.checkState() == Qt.Checked:
self.unchecked_files.remove(item.text())
self._logger.msg.emit(f"<b>{self.name}:</b> Source file '{item.text()}' will be processed at execution.")
elif item.checkState() != Qt.Checked:
self.unchecked_files.append(item.text())
self._logger.msg.emit(
f"<b>{self.name}:</b> Source file '{item.text()}' will *NOT* be processed at execution."
)
def make_signal_handler_dict(self):
"""Returns a dictionary of all shared signals and their handlers.
This is to enable simpler connecting and disconnecting."""
s = super().make_signal_handler_dict()
s[self._properties_ui.toolButton_open_dir.clicked] = lambda checked=False: self.open_directory()
s[self._properties_ui.pushButton_import_editor.clicked] = self._handle_import_editor_clicked
s[self._properties_ui.treeView_files.doubleClicked] = self._handle_files_double_clicked
return s
def activate(self):
"""Restores selections, cancel on error checkbox and connects signals."""
self._properties_ui.cancel_on_error_checkBox.setCheckState(Qt.Checked if self.cancel_on_error else Qt.Unchecked)
self.restore_selections()
super().connect_signals()
def deactivate(self):
"""Saves selections and disconnects signals."""
self.save_selections()
if not super().disconnect_signals():
logging.error("Item %s deactivation failed.", self.name)
return False
return True
def restore_selections(self):
"""Restores selections into shared widgets when this project item is selected."""
self._properties_ui.label_name.setText(self.name)
self._properties_ui.treeView_files.setModel(self.file_model)
self.file_model.itemChanged.connect(self._handle_file_model_item_changed)
def save_selections(self):
"""Saves selections in shared widgets for this project item into instance variables."""
self._properties_ui.treeView_files.setModel(None)
self.file_model.itemChanged.disconnect(self._handle_file_model_item_changed)
def update_name_label(self):
"""Update Importer properties tab name label. Used only when renaming project items."""
self._properties_ui.label_name.setText(self.name)
@Slot(bool, name="_handle_import_editor_clicked")
def _handle_import_editor_clicked(self, checked=False):
"""Opens Import editor for the file selected in list view."""
index = self._properties_ui.treeView_files.currentIndex()
self.open_import_editor(index)
@Slot("QModelIndex", name="_handle_files_double_clicked")
def _handle_files_double_clicked(self, index):
"""Opens Import editor for the double clicked index."""
self.open_import_editor(index)
def open_import_editor(self, index):
"""Opens Import editor for the given index."""
importee = index.data()
if importee is None:
self._logger.msg_error.emit("Please select a source file from the list first.")
return
if not os.path.exists(importee):
self._logger.msg_error.emit(f"Invalid path: {importee}")
return
# Raise current form for the selected file if any
preview_widget = self._preview_widget.get(importee, None)
if preview_widget:
if preview_widget.windowState() & Qt.WindowMinimized:
# Remove minimized status and restore window with the previous state (maximized/normal state)
preview_widget.setWindowState(preview_widget.windowState() & ~Qt.WindowMinimized | Qt.WindowActive)
preview_widget.activateWindow()
else:
preview_widget.raise_()
return
# Create a new form for the selected file
settings = self.get_settings(importee)
# Try and get connector from settings
source_type = settings.get("source_type", None)
if source_type is not None:
connector = _CONNECTOR_NAME_TO_CLASS[source_type]
else:
# Ask user
connector = self.get_connector(importee)
if not connector:
# Aborted by the user
return
self._logger.msg.emit(f"Opening Import editor for file: {importee}")
preview_widget = self._preview_widget[importee] = ImportPreviewWindow(
self, importee, connector, settings, self._toolbox
)
preview_widget.settings_updated.connect(lambda s, importee=importee: self.save_settings(s, importee))
preview_widget.connection_failed.connect(lambda m, importee=importee: self._connection_failed(m, importee))
preview_widget.destroyed.connect(lambda o=None, importee=importee: self._preview_destroyed(importee))
preview_widget.start_ui()
def get_connector(self, importee):
"""Shows a QDialog to select a connector for the given source file.
Mimics similar routine in `spine_io.widgets.import_widget.ImportDialog`
Args:
importee (str): Path to file acting as an importee
Returns:
Asynchronous data reader class for the given importee
"""
connector_list = [CSVConnector, ExcelConnector, GdxConnector] # add others as needed
connector_names = [c.DISPLAY_NAME for c in connector_list]
dialog = QDialog(self._toolbox)
dialog.setLayout(QVBoxLayout())
connector_list_wg = QListWidget()
connector_list_wg.addItems(connector_names)
# Set current item in `connector_list_wg` based on file extension
_filename, file_extension = os.path.splitext(importee)
if file_extension.lower().startswith(".xls"):
row = connector_list.index(ExcelConnector)
elif file_extension.lower() == ".csv":
row = connector_list.index(CSVConnector)
elif file_extension.lower() == ".gdx":
row = connector_list.index(GdxConnector)
else:
row = None
if row:
connector_list_wg.setCurrentRow(row)
button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
button_box.button(QDialogButtonBox.Ok).clicked.connect(dialog.accept)
button_box.button(QDialogButtonBox.Cancel).clicked.connect(dialog.reject)
connector_list_wg.doubleClicked.connect(dialog.accept)
dialog.layout().addWidget(connector_list_wg)
dialog.layout().addWidget(button_box)
_dirname, filename = os.path.split(importee)
dialog.setWindowTitle("Select connector for '{}'".format(filename))
answer = dialog.exec_()
if answer:
row = connector_list_wg.currentIndex().row()
return connector_list[row]
def select_connector_type(self, index):
"""Opens dialog to select connector type for the given index."""
importee = index.data()
connector = self.get_connector(importee)
if not connector:
# Aborted by the user
return
settings = self.get_settings(importee)
settings["source_type"] = connector.__name__
def _connection_failed(self, msg, importee):
self._logger.msg.emit(msg)
preview_widget = self._preview_widget.pop(importee, None)
if preview_widget:
preview_widget.close()
def get_settings(self, importee):
"""Returns the mapping dictionary for the file in given path.
Args:
importee (str): Absolute path to a file, whose mapping is queried
Returns:
dict: Mapping dictionary for the requested importee or an empty dict if not found
"""
importee_settings = None
for p in self.settings:
if p == importee:
importee_settings = self.settings[p]
if not importee_settings:
return {}
return importee_settings
def save_settings(self, settings, importee):
"""Updates an existing mapping or adds a new mapping
(settings) after closing the import preview window.
Args:
settings (dict): Updated mapping (settings) dictionary
importee (str): Absolute path to a file, whose mapping has been updated
"""
if importee in self.settings.keys():
self.settings[importee].update(settings)
else:
self.settings[importee] = settings
def _preview_destroyed(self, importee):
"""Destroys preview widget instance for the given importee.
Args:
importee (str): Absolute path to a file, whose preview widget is destroyed
"""
self._preview_widget.pop(importee, None)
def update_file_model(self, items):
"""Adds given list of items to the file model. If None or
an empty list is given, the model is cleared.
Args:
items (set): Set of absolute file paths
"""
self.all_files = items
self.file_model.clear()
self.file_model.setHorizontalHeaderItem(0, QStandardItem("Source files")) # Add header
if items is not None:
for item in items:
qitem = QStandardItem(item)
qitem.setEditable(False)
qitem.setCheckable(True)
if item in self.unchecked_files:
qitem.setCheckState(Qt.Unchecked)
else:
qitem.setCheckState(Qt.Checked)
qitem.setData(QFileIconProvider().icon(QFileInfo(item)), Qt.DecorationRole)
self.file_model.appendRow(qitem)
def _run_importer_program(self, args):
"""Starts and runs the importer program in a separate process.
Args:
args (list): List of arguments for the importer program
"""
self.importer_process = QProcess()
self.importer_process.readyReadStandardOutput.connect(self._log_importer_process_stdout)
self.importer_process.readyReadStandardError.connect(self._log_importer_process_stderr)
self.importer_process.finished.connect(self.importer_process.deleteLater)
program_path = os.path.abspath(importer_program.__file__)
self.importer_process.start(sys.executable, [program_path])
self.importer_process.waitForStarted()
self.importer_process.write(json.dumps(args).encode("utf-8"))
self.importer_process.write(b'\n')
self.importer_process.closeWriteChannel()
if self.importer_process.state() == QProcess.Running:
loop = QEventLoop()
self.importer_process.finished.connect(loop.quit)
loop.exec_()
return self.importer_process.exitCode()
@Slot()
def _log_importer_process_stdout(self):
output = str(self.importer_process.readAllStandardOutput().data(), "utf-8").strip()
self._logger.msg.emit(f"<b>{self.name}</b>: {output}")
@Slot()
def _log_importer_process_stderr(self):
output = str(self.importer_process.readAllStandardError().data(), "utf-8").strip()
self._logger.msg_error.emit(f"<b>{self.name}</b>: {output}")
def execute_backward(self, resources):
"""See base class."""
self.resources_from_downstream = resources.copy()
return True
def execute_forward(self, resources):
"""See base class."""
args = [
[f for f in self.all_files if f not in self.unchecked_files],
self.settings,
[r.url for r in self.resources_from_downstream if r.type_ == "database"],
self.logs_dir,
self._properties_ui.cancel_on_error_checkBox.isChecked(),
]
exit_code = self._run_importer_program(args)
return exit_code == 0
def stop_execution(self):
"""Stops executing this Importer."""
super().stop_execution()
if not self.importer_process:
return
self.importer_process.kill()
def _do_handle_dag_changed(self, resources):
"""See base class."""
file_list = [r.path for r in resources if r.type_ == "file" and not r.metadata.get("future")]
self._notify_if_duplicate_file_paths(file_list)
self.update_file_model(set(file_list))
if not file_list:
self.add_notification(
"This Importer does not have any input data. "
"Connect Data Connections to this Importer to use their data as input."
)
def item_dict(self):
"""Returns a dictionary corresponding to this item."""
d = super().item_dict()
# Serialize mappings before saving
d["mappings"] = self.serialize_mappings(self.settings, self._project.project_dir)
d["cancel_on_error"] = self._properties_ui.cancel_on_error_checkBox.isChecked()
return d
def notify_destination(self, source_item):
"""See base class."""
if source_item.item_type() == "Data Connection":
self._logger.msg.emit(
"Link established. You can define mappings on data from "
f"<b>{source_item.name}</b> using item <b>{self.name}</b>."
)
elif source_item.item_type() == "Data Store":
# Does this type of link do anything?
self._logger.msg.emit("Link established.")
else:
super().notify_destination(source_item)
@staticmethod
def default_name_prefix():
"""see base class"""
return "Importer"
def tear_down(self):
"""Closes all preview widgets."""
for widget in self._preview_widget.values():
widget.close()
def _notify_if_duplicate_file_paths(self, file_list):
"""Adds a notification if file_list contains duplicate entries."""
file_counter = Counter(file_list)
duplicates = list()
for file_name, count in file_counter.items():
if count > 1:
duplicates.append(file_name)
if duplicates:
self.add_notification("Duplicate input files from upstream items:<br>{}".format("<br>".join(duplicates)))
@staticmethod
def upgrade_from_no_version_to_version_1(item_name, old_item_dict, old_project_dir):
"""Converts mappings to a list, where each element contains two dictionaries,
the serialized path dictionary and the mapping dictionary for the file in that
path."""
new_importer = dict(old_item_dict)
mappings = new_importer.get("mappings", {})
list_of_mappings = list()
paths = list(mappings.keys())
for path in paths:
mapping = mappings[path]
if "source_type" in mapping and mapping["source_type"] == "CSVConnector":
_fix_csv_connector_settings(mapping)
new_path = serialize_path(path, old_project_dir)
if new_path["relative"]:
new_path["path"] = os.path.join(".spinetoolbox", "items", new_path["path"])
list_of_mappings.append([new_path, mapping])
new_importer["mappings"] = list_of_mappings
return new_importer
@staticmethod
def deserialize_mappings(mappings, project_path):
"""Returns mapping settings as dict with absolute paths as keys.
Args:
mappings (list): List where each element contains two dictionaries (path dict and mapping dict)
project_path (str): Path to project directory
Returns:
dict: Dictionary with absolute paths as keys and mapping settings as values
"""
abs_path_mappings = {}
for source, mapping in mappings:
abs_path_mappings[deserialize_path(source, project_path)] = mapping
return abs_path_mappings
@staticmethod
def serialize_mappings(mappings, project_path):
"""Returns a list of mappings, where each element contains two dictionaries,
the 'serialized' path in a dictionary and the mapping dictionary.
Args:
mappings (dict): Dictionary with mapping specifications
project_path (str): Path to project directory
Returns:
list: List where each element contains two dictionaries.
"""
serialized_mappings = list()
for source, mapping in mappings.items(): # mappings is a dict with absolute paths as keys and mapping as values
serialized_mappings.append([serialize_path(source, project_path), mapping])
return serialized_mappings
