def run_handler(self): self.py_ = G.config.installed_apps[self.pack_name].get('py_') if not self.py_: QMessageBox.warning(self.parent, "提示", "未选择Python解释器") self.act_setting_slot() return if not os.path.exists(self.py_) or not os.path.isfile(self.py_): QMessageBox.warning(self.parent, "提示", f"{self.py_} 不存在") return try: self.entry_, requirement_ = self.get_build() except Exception as e: QMessageBox.warning(self.parent, "提示", str(e)) return ##检测依赖 p = QProcess() p.start(' '.join(([self.py_, "-m", 'pip', "freeze"]))) p.waitForFinished() out = p.readAllStandardOutput().data().decode() output = out.splitlines() with open(requirement_, 'r') as f: requirement = f.read().splitlines() dissatisfy, version_less = diff_pip(output, requirement) if dissatisfy: msgbox = QMessageBox(self.parent) msgbox.setWindowTitle("缺少依赖") msgbox.setText("\n".join(dissatisfy[:15]) + "\n...") yes = msgbox.addButton('立即安装', QMessageBox.AcceptRole) no = msgbox.addButton('稍后', QMessageBox.RejectRole) msgbox.setDefaultButton(yes) reply = msgbox.exec_() if reply == QMessageBox.AcceptRole: self.requirement_ = dissatisfy self.install_handler() return # run TipDialog("正在启动...") cmd = ' '.join([self.py_, self.entry_]) QProcess().startDetached(cmd)
def handle_output(self, process: QProcess): output: QByteArray = process.readAllStandardOutput() message = output.data().decode('utf-8').strip() if message.startswith('{') or message.startswith('['): formatter = HtmlFormatter() formatter.noclasses = True formatter.linenos = False formatter.nobackground = True message = highlight(message, JsonLexer(), formatter) self.output.insertHtml(message) else: self.output.append(message) # This is just for generating the command lists in constants # commands = None # if '== Blockchain ==' in message: # commands = self.parse_litecoin_cli_commands(message) # elif 'lncli [global options] command [command options]' in message: # commands = self.parse_lncli_commands(message) # if commands is not None: # log.debug('commands', commands=commands) max_scroll = self.output.verticalScrollBar().maximum() self.output.verticalScrollBar().setValue(max_scroll)
class QProcessExecutionManager(ExecutionManager): """Class to manage tool instance execution using a PySide2 QProcess.""" def __init__(self, logger, program="", args=None, silent=False, semisilent=False): """Class constructor. Args: logger (LoggerInterface): a logger instance program (str): Path to program to run in the subprocess (e.g. julia.exe) args (list, optional): List of argument for the program (e.g. path to script file) silent (bool): Whether or not to emit logger msg signals semisilent (bool): If True, show Process Log messages """ super().__init__(logger) self._program = program self._args = args if args is not None else [] self._silent = silent # Do not show Event Log nor Process Log messages self._semisilent = semisilent # Do not show Event Log messages but show Process Log messages self.process_failed = False self.process_failed_to_start = False self.user_stopped = False self._process = QProcess(self) self.process_output = None # stdout when running silent self.process_error = None # stderr when running silent self._out_chunks = [] self._err_chunks = [] def program(self): """Program getter method.""" return self._program def args(self): """Program argument getter method.""" return self._args # noinspection PyUnresolvedReferences def start_execution(self, workdir=None): """Starts the execution of a command in a QProcess. Args: workdir (str, optional): Work directory """ if workdir is not None: self._process.setWorkingDirectory(workdir) self._process.started.connect(self.process_started) self._process.finished.connect(self.on_process_finished) if not self._silent and not self._semisilent: # Loud self._process.readyReadStandardOutput.connect(self.on_ready_stdout) self._process.readyReadStandardError.connect(self.on_ready_stderr) self._process.errorOccurred.connect(self.on_process_error) self._process.stateChanged.connect(self.on_state_changed) elif self._semisilent: # semi-silent self._process.readyReadStandardOutput.connect(self.on_ready_stdout) self._process.readyReadStandardError.connect(self.on_ready_stderr) self._process.start(self._program, self._args) if self._process is not None and not self._process.waitForStarted(msecs=10000): self.process_failed = True self.process_failed_to_start = True self._process.deleteLater() self._process = None self.execution_finished.emit(-9998) def wait_for_process_finished(self, msecs=30000): """Wait for subprocess to finish. Args: msecs (int): Timeout in milliseconds Return: True if process finished successfully, False otherwise """ if self._process is None: return False if self.process_failed or self.process_failed_to_start: return False if not self._process.waitForFinished(msecs): self.process_failed = True self._process.close() self._process = None return False return True @Slot() def process_started(self): """Run when subprocess has started.""" @Slot(int) def on_state_changed(self, new_state): """Runs when QProcess state changes. Args: new_state (int): Process state number (``QProcess::ProcessState``) """ if new_state == QProcess.Starting: self._logger.msg.emit("\tStarting program <b>{0}</b>".format(self._program)) arg_str = " ".join(self._args) self._logger.msg.emit("\tArguments: <b>{0}</b>".format(arg_str)) elif new_state == QProcess.Running: self._logger.msg_warning.emit("\tExecution in progress...") elif new_state == QProcess.NotRunning: # logging.debug("Process is not running") pass else: self._logger.msg_error.emit("Process is in an unspecified state") logging.error("QProcess unspecified state: %s", new_state) @Slot(int) def on_process_error(self, process_error): """Runs if there is an error in the running QProcess. Args: process_error (int): Process error number (``QProcess::ProcessError``) """ if process_error == QProcess.FailedToStart: self.process_failed = True self.process_failed_to_start = True self._logger.msg_error.emit("Process failed to start") elif process_error == QProcess.Timedout: self.process_failed = True self._logger.msg_error.emit("Timed out") elif process_error == QProcess.Crashed: self.process_failed = True if not self.user_stopped: self._logger.msg_error.emit("Process crashed") elif process_error == QProcess.WriteError: self._logger.msg_error.emit("Process WriteError") elif process_error == QProcess.ReadError: self._logger.msg_error.emit("Process ReadError") elif process_error == QProcess.UnknownError: self._logger.msg_error.emit("Unknown error in process") else: self._logger.msg_error.emit("Unspecified error in process: {0}".format(process_error)) self.teardown_process() def teardown_process(self): """Tears down the QProcess in case a QProcess.ProcessError occurred. Emits execution_finished signal.""" if not self._process: pass else: out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace") errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace") if out is not None: self._logger.msg_proc.emit(out.strip()) if errout is not None: self._logger.msg_proc.emit(errout.strip()) self._process.deleteLater() self._process = None self.execution_finished.emit(-9998) def stop_execution(self): """See base class.""" self.user_stopped = True self.process_failed = True if not self._process: return try: self._process.kill() if not self._process.waitForFinished(5000): self._process.finished.emit(-1, -1) self._process.deleteLater() except Exception as ex: # pylint: disable=broad-except self._logger.msg_error.emit("[{0}] exception when terminating process".format(ex)) logging.exception("Exception in closing QProcess: %s", ex) finally: self._process = None @Slot(int, int) def on_process_finished(self, exit_code, exit_status): """Runs when subprocess has finished. Args: exit_code (int): Return code from external program (only valid for normal exits) exit_status (int): Crash or normal exit (``QProcess::ExitStatus``) """ if not self._process: return if exit_status == QProcess.CrashExit: if not self._silent: self._logger.msg_error.emit("\tProcess crashed") exit_code = -1 elif exit_status == QProcess.NormalExit: pass else: if not self._silent: self._logger.msg_error.emit("Unknown QProcess exit status [{0}]".format(exit_status)) exit_code = -1 if not exit_code == 0: self.process_failed = True if not self.user_stopped: out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace") errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace") if out is not None: if not self._silent: self._logger.msg_proc.emit(out.strip()) else: self.process_output = out.strip() self.process_error = errout.strip() else: self._logger.msg.emit("*** Terminating process ***") # Delete QProcess self._process.deleteLater() self._process = None self.execution_finished.emit(exit_code) @Slot() def on_ready_stdout(self): """Emit data from stdout.""" if not self._process: return self._process.setReadChannel(QProcess.StandardOutput) chunk = self._process.readLine().data() self._out_chunks.append(chunk) if not chunk.endswith(b"\n"): return line = b"".join(self._out_chunks) line = str(line, "unicode_escape", errors="replace").strip() self._logger.msg_proc.emit(line) self._out_chunks.clear() @Slot() def on_ready_stderr(self): """Emit data from stderr.""" if not self._process: return self._process.setReadChannel(QProcess.StandardError) chunk = self._process.readLine().data() self._err_chunks.append(chunk) if not chunk.endswith(b"\n"): return line = b"".join(self._err_chunks) line = str(line, "utf-8", errors="replace").strip() self._logger.msg_proc_error.emit(line) self._err_chunks.clear()
class QProcessExecutionManager(ExecutionManager): """Class to manage tool instance execution using a PySide2 QProcess.""" def __init__(self, logger, program=None, args=None, silent=False, semisilent=False): """Class constructor. Args: logger (LoggerInterface): a logger instance program (str): Path to program to run in the subprocess (e.g. julia.exe) args (list): List of argument for the program (e.g. path to script file) silent (bool): Whether or not to emit logger msg signals """ super().__init__(logger) self._program = program self._args = args self._silent = silent # Do not show Event Log nor Process Log messages self._semisilent = semisilent # Do not show Event Log messages but show Process Log messages self.process_failed = False self.process_failed_to_start = False self._user_stopped = False self._process = QProcess(self) self.process_output = None # stdout when running silent self.error_output = None # stderr when running silent def program(self): """Program getter method.""" return self._program def args(self): """Program argument getter method.""" return self._args # noinspection PyUnresolvedReferences def start_execution(self, workdir=None): """Starts the execution of a command in a QProcess. Args: workdir (str): Work directory """ if workdir is not None: self._process.setWorkingDirectory(workdir) self._process.started.connect(self.process_started) self._process.finished.connect(self.on_process_finished) if not self._silent and not self._semisilent: # Loud self._process.readyReadStandardOutput.connect(self.on_ready_stdout) self._process.readyReadStandardError.connect(self.on_ready_stderr) self._process.error.connect(self.on_process_error) # errorOccurred available in Qt 5.6 self._process.stateChanged.connect(self.on_state_changed) elif self._semisilent: # semi-silent self._process.readyReadStandardOutput.connect(self.on_ready_stdout) self._process.readyReadStandardError.connect(self.on_ready_stderr) self._process.start(self._program, self._args) if not self._process.waitForStarted(msecs=10000): # This blocks until process starts or timeout happens self.process_failed = True self.process_failed_to_start = True self._process.deleteLater() self._process = None self.execution_finished.emit(-9998) def wait_for_process_finished(self, msecs=30000): """Wait for subprocess to finish. Return: True if process finished successfully, False otherwise """ if not self._process: return False if self.process_failed or self.process_failed_to_start: return False if not self._process.waitForFinished(msecs): self.process_failed = True self._process.close() self._process = None return False return True @Slot(name="process_started") def process_started(self): """Run when subprocess has started.""" @Slot("QProcess::ProcessState", name="on_state_changed") def on_state_changed(self, new_state): """Runs when QProcess state changes. Args: new_state (QProcess::ProcessState): Process state number """ if new_state == QProcess.Starting: self._logger.msg.emit("\tStarting program <b>{0}</b>".format(self._program)) arg_str = " ".join(self._args) self._logger.msg.emit("\tArguments: <b>{0}</b>".format(arg_str)) elif new_state == QProcess.Running: self._logger.msg_warning.emit( "\tExecution is in progress. See Process Log for messages " "(stdout&stderr)" ) elif new_state == QProcess.NotRunning: # logging.debug("QProcess is not running") pass else: self._logger.msg_error.emit("Process is in an unspecified state") logging.error("QProcess unspecified state: %s", new_state) @Slot("QProcess::ProcessError", name="'on_process_error") def on_process_error(self, process_error): """Run if there is an error in the running QProcess. Args: process_error (QProcess::ProcessError): Process error number """ if process_error == QProcess.FailedToStart: self.process_failed = True self.process_failed_to_start = True elif process_error == QProcess.Timedout: self.process_failed = True self._logger.msg_error.emit("Timed out") elif process_error == QProcess.Crashed: self.process_failed = True if not self._user_stopped: self._logger.msg_error.emit("Process crashed") elif process_error == QProcess.WriteError: self._logger.msg_error.emit("Process WriteError") elif process_error == QProcess.ReadError: self._logger.msg_error.emit("Process ReadError") elif process_error == QProcess.UnknownError: self._logger.msg_error.emit("Unknown error in process") else: self._logger.msg_error.emit("Unspecified error in process: {0}".format(process_error)) def stop_execution(self): """See base class.""" self._logger.msg_error.emit("Terminating process") self._user_stopped = True self.process_failed = True if not self._process: return try: self._process.terminate() except Exception as ex: # pylint: disable=broad-except self._logger.msg_error.emit("[{0}] exception when terminating process".format(ex)) logging.exception("Exception in closing QProcess: %s", ex) finally: self._process.deleteLater() self._process = None @Slot(int) def on_process_finished(self, exit_code): """Runs when subprocess has finished. Args: exit_code (int): Return code from external program (only valid for normal exits) """ # logging.debug("Error that occurred last: {0}".format(self._process.error())) if not self._process: return exit_status = self._process.exitStatus() # Normal or crash exit if exit_status == QProcess.CrashExit: if not self._silent: self._logger.msg_error.emit("\tProcess crashed") exit_code = -1 elif exit_status == QProcess.NormalExit: pass else: if not self._silent: self._logger.msg_error.emit("Unknown QProcess exit status [{0}]".format(exit_status)) exit_code = -1 if not exit_code == 0: self.process_failed = True if not self._user_stopped: out = str(self._process.readAllStandardOutput().data(), "utf-8") errout = str(self._process.readAllStandardError().data(), "utf-8") if out is not None: if not self._silent: self._logger.msg_proc.emit(out.strip()) else: self.process_output = out.strip() self.error_output = errout.strip() else: self._logger.msg.emit("*** Terminating process ***") # Delete QProcess self._process.deleteLater() self._process = None self.execution_finished.emit(exit_code) @Slot(name="on_ready_stdout") def on_ready_stdout(self): """Emit data from stdout.""" if not self._process: return out = str(self._process.readAllStandardOutput().data(), "utf-8") self._logger.msg_proc.emit(out.strip()) @Slot(name="on_ready_stderr") def on_ready_stderr(self): """Emit data from stderr.""" if not self._process: return err = str(self._process.readAllStandardError().data(), "utf-8") self._logger.msg_proc_error.emit(err.strip())
def runProcess(command, arguments): process = QProcess() process.start(command, arguments) process.waitForFinished() std_output = process.readAllStandardOutput().data().decode('utf-8') return std_output.split('\n')
class TclInterfaceHandler(QObject): # The TCP server IP address HOST = "127.0.0.1" # TCP server port PORT = 9955 commandExecutionFinished = Signal(str) faultInjectionFinished = Signal(str) #--------------------------------------------------------------------------------- # @brief Class constructor. #--------------------------------------------------------------------------------- def __init__(self): super(TclInterfaceHandler, self).__init__() self.operation = Operation.NONE self.frame_address = "00000000" self.frames = "1" self.tcpClientSocket = QtNetwork.QTcpSocket() self.tcpClientSocketStatus = SocketStatus.DISCONNECTED self.startVivado() return #--------------------------------------------------------------------------------- # @brief This function should be called when the application terminates. #--------------------------------------------------------------------------------- def exit(self): self.vivadoProcess.kill() vivado_command_to_kill = "cmd.exe /C Taskkill /IM vivado.exe /F" process = QProcess() process.start(vivado_command_to_kill) process.waitForFinished(5000) tcp_command_to_kill = "cmd.exe /C netstat -ano | find '0.0.0.0:9955'" return #********************************************************************************** # TCP Client #********************************************************************************** #--------------------------------------------------------------------------------- # @brief This function starts the TCP client. #--------------------------------------------------------------------------------- def clientStart(self): if self.tcpClientSocketStatus == SocketStatus.CONNECTED: self.tcpClientSocket.abort() self.tcpClientSocket.connectToHost(TclInterfaceHandler.HOST, TclInterfaceHandler.PORT) self.tcpClientSocket.waitForConnected(3000) self.tcpClientSocket.readyRead.connect(self.clientReadReady) self.tcpClientSocket.error.connect(self.clientError) self.tcpClientSocketStatus = SocketStatus.CONNECTED return #--------------------------------------------------------------------------------- # @brief This function is called by the TCP client when it has data ready to be read. #--------------------------------------------------------------------------------- def clientReadReady(self): message = QTextStream(self.tcpClientSocket).readAll() if Operation.READBACK_CAPTURE.name in message: XtclLog.writeLine( "================ FPGA readback capture finished ================ ", XtclCommon.blue) elif Operation.READBACK_VERIFY.name in message: XtclLog.writeLine( "================ FPGA readback verify finished ================ ", XtclCommon.blue) #verify = filecmp.cmp(XtclSettings.readbackFilePathGolden, self.readbackFile, False) elif Operation.READBACK.name in message: XtclLog.writeLine( "================ FPGA readback finished ================ ", XtclCommon.blue) elif Operation.CONFIGURATION.name in message: XtclLog.writeLine( "================ FPGA configuration finished ================ ", XtclCommon.blue) elif Operation.FRAMES_READ.name in message or Operation.FAULT_INJECTION_READ.name in message: XtclLog.writeLine( "================ FPGA frame readback finished ================ ", XtclCommon.blue) elif Operation.FRAMES_WRITE.name in message or Operation.FAULT_INJECTION_WRITE.name in message: XtclLog.writeLine( "================ FPGA frame write finished ================ ", XtclCommon.blue) elif Operation.READ_FIFO.name in message: XtclLog.writeLine( "================ Reading internal FIFO finished ================ ", XtclCommon.blue) elif Operation.RESET_FIFO.name in message: XtclLog.writeLine( "================ Reseting internal FIFO finished ================ ", XtclCommon.blue) elif Operation.READ_HEARTBEAT.name in message: XtclLog.writeLine( "================ Reading heartbeat finished ================ ", XtclCommon.blue) elif Operation.LOGIC_STATUS.name in message: XtclLog.writeLine( "================ Reading active logic status finished ================ ", XtclCommon.blue) elif Operation.REGISTER_READ.name in message: XtclLog.writeLine( "================ Reading configuration register finished ================ ", XtclCommon.blue) elif Operation.REGISTER_WRITE.name in message: XtclLog.writeLine( "================ Writing configuration register finished ================ ", XtclCommon.blue) else: XtclLog.writeLine(message, XtclCommon.red) self.operation = Operation.NONE if "FAULT_INJECTION" in message: self.faultInjectionFinished.emit(message) else: self.commandExecutionFinished.emit(message) return #--------------------------------------------------------------------------------- # @brief Callback function for the client error. #--------------------------------------------------------------------------------- def clientError(self, socketError): if socketError == QtNetwork.QAbstractSocket.RemoteHostClosedError: pass elif socketError == QtNetwork.QAbstractSocket.HostNotFoundError: XtclLog.writeLine( "The host was not found. Please check the host name and port settings", XtclCommon.red) elif socketError == QtNetwork.QAbstractSocket.ConnectionRefusedError: XtclLog.writeLine( "The connection was refused by the peer. Make sure the " "server is running, and check that the host name " "and port settings are correct.", TclInterfaceHandler.red) else: XtclLog.writeLine( "The following error occurred: %s." % self.tcpSocket.errorString(), XtclCommon.red) return #--------------------------------------------------------------------------------- # @brief Send data to the TCP server #--------------------------------------------------------------------------------- def clientSend(self, data): bytesArray = bytes(data, 'utf-8') message = QByteArray.fromRawData(bytesArray) self.tcpClientSocket.write(message) self.tcpClientSocket.flush() return #********************************************************************************** # Vivado interface #********************************************************************************** #--------------------------------------------------------------------------------- # @brief This function starts the Vivado instanse. Please replace the command # with the appropriate command for the running platform #--------------------------------------------------------------------------------- def startVivado(self): self.operation = Operation.NONE TclInterfaceHandler.isOperationFinished = False XtclLog.writeLine( "================ Starting Vivado process ================ ", XtclCommon.blue) XtclLog.write("PLEASE WAIT UNTIL YOU SEE \"# vwait forever\" MESSAGE!", XtclCommon.red) command = XtclSettings.vivadoDirectory + "/vivado.bat -nojournal -nolog -mode batch -source jtag_configuration_engine.tcl" # Create runner self.vivadoProcess = QProcess() self.vivadoProcess.readyReadStandardError.connect(self.errorInfo) self.vivadoProcess.readyReadStandardOutput.connect( self.readAllStandardOutput) self.vivadoProcess.finished.connect(self.finished) self.vivadoProcess.start(command) return #--------------------------------------------------------------------------------- # @brief Callback function for the error of the Vivado process #--------------------------------------------------------------------------------- def errorInfo(self): info = self.vivadoProcess.readAllStandardError() info_text = QTextStream(info).readAll() XtclLog.write(info_text) return #--------------------------------------------------------------------------------- # @brief Callback function to rediarect the output of the Vivado process #--------------------------------------------------------------------------------- def readAllStandardOutput(self): info = self.vivadoProcess.readAllStandardOutput() info_text = QTextStream(info).readAll() XtclLog.write(info_text) return #--------------------------------------------------------------------------------- # @brief Callback function for the termination event of the Vivado process #--------------------------------------------------------------------------------- def finished(self, exitCode, exitStatus): return #********************************************************************************** # Interface commands #********************************************************************************** #--------------------------------------------------------------------------------- # @brief This function configures the FPGA # @param bitstream_filepath: The full path of the bitstream file # @param mask_filepath: The full path of the mask file #--------------------------------------------------------------------------------- def configure(self, bitstream_filepath, mask_filepath): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Starting FPGA configuration ================ ", XtclCommon.blue) self.clientSend(Operation.CONFIGURATION.name + "%" + bitstream_filepath + "%" + mask_filepath) self.operation = Operation.CONFIGURATION return #--------------------------------------------------------------------------------- # @brief This function reads-back the FPGA # @param filename: If provided should be the full path of the file where the readback # data will be saved. Othervise the readback data is saved in a # timestamp-based file name inside the "readback-files" of the # working folder. #--------------------------------------------------------------------------------- def readback(self, filename=None): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() if filename == None: readbackFileName = "readback-" + str( datetime.datetime.now().timestamp()) + ".rbd" else: readbackFileName = filename self.readbackFile = XtclSettings.workingDirectory + '/readback-files/' + readbackFileName XtclLog.writeLine( "================ Starting FPGA readback ================ ", XtclCommon.blue) XtclLog.writeLine(readbackFileName, XtclCommon.blue) self.clientSend(Operation.READBACK.name + "%" + self.readbackFile) self.operation = Operation.READBACK return #--------------------------------------------------------------------------------- # @brief This function reads-back the FPGA using the capture mode # @note The readback data is saved in a timestamp-based file name inside the "readback-capture-files" # of the working folder. #--------------------------------------------------------------------------------- def readbackCapture(self): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() readbackFileName = "readbackCapture-" + str( datetime.datetime.now().timestamp()) + ".rbd" self.readbackFile = XtclSettings.workingDirectory + '/readback-capture-files/' + readbackFileName XtclLog.writeLine( "================ Starting FPGA readback capture ================ ", XtclCommon.blue) XtclLog.writeLine(readbackFileName, XtclCommon.blue) self.clientSend(Operation.READBACK_CAPTURE.name + "%" + self.readbackFile) self.operation = Operation.READBACK_CAPTURE return #--------------------------------------------------------------------------------- # @brief This function verifies the FPGA. # @note The FPGA device should be configured before issuing this command. #--------------------------------------------------------------------------------- def readbackVerify(self): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Starting FPGA readback verify ================ ", XtclCommon.blue) self.clientSend(Operation.READBACK_VERIFY.name) self.operation = Operation.READBACK_VERIFY return #--------------------------------------------------------------------------------- # @brief This function reads the internal FIFO of the interface logic. #--------------------------------------------------------------------------------- def readInternalFifo(self): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Reading internal FIFO ================ ", XtclCommon.blue) self.clientSend(Operation.READ_FIFO.name) self.operation = Operation.READ_FIFO return #--------------------------------------------------------------------------------- # @brief This function resets the internal FIFO of the interface logic. #--------------------------------------------------------------------------------- def resetInternalFifo(self): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Reseting internal FIFO ================ ", XtclCommon.blue) self.clientSend(Operation.RESET_FIFO.name) self.operation = Operation.RESET_FIFO return #--------------------------------------------------------------------------------- # @brief This function reads the heartbeat signal the interface logic. #--------------------------------------------------------------------------------- def readHeartbeat(self): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Reading heartbeat ================ ", XtclCommon.blue) self.clientSend(Operation.READ_HEARTBEAT.name) self.operation = Operation.READ_HEARTBEAT return #--------------------------------------------------------------------------------- # @brief This function writes frames in configuration memory of the FPGA # @param frame_address: The frame address in HEX format (i.e. 00001002). # @param frame_file: The full path of the file which holds the frame data to be written. # @param append_dummy_frame: True to append a dummy frame after writing the frame data. # Set it to false if the file contains also the dummy frame. # @param is_frame_data_in_hex_format: Trus if the content of the @ref frame_file is in HEX foramt. # @param frames: The number of frames to be written. This should be the same as the frames # inside the @ref frame_file (do not include the dummy frame). # @param reset_fifo: True to reset the FIFO at the end of writing. # @param is_fault_injection: Trus if this write is for fault injection experiment. #--------------------------------------------------------------------------------- def writeFrames(self, frame_address, frame_file, append_dummy_frame=True, is_frame_data_in_hex_format=True, frames=1, reset_fifo=False, is_fault_injection=False): self.frame_address = str(frame_address) self.frames = str(frames) XtclLog.writeLine( "================ Starting FPGA frame write ================ ", XtclCommon.blue) append_dummy_frame_ = str(int(append_dummy_frame)) is_frame_data_in_hex_format_ = str(int(is_frame_data_in_hex_format)) reset_fifo_ = str(int(reset_fifo)) if is_fault_injection == True: command = Operation.FAULT_INJECTION_WRITE.name + "%" + frame_file + "%" + self.frame_address + "%" + self.frames + "%" + append_dummy_frame_ + "%" + is_frame_data_in_hex_format_ + "%" + reset_fifo_ else: command = Operation.FRAMES_WRITE.name + "%" + frame_file + "%" + self.frame_address + "%" + self.frames + "%" + append_dummy_frame_ + "%" + is_frame_data_in_hex_format_ + "%" + reset_fifo_ self.clientSend(command) self.operation = Operation.FRAMES_WRITE return #--------------------------------------------------------------------------------- # @brief This function reads frames from the configuration memory of the FPGA # @param frame_address: The frame address in HEX format (i.e. 00001002). # @param frames: The number of frames to be written. This should be the same as the frames # inside the @ref frame_file (do not include the dummy frame). # @param readback_file: If provided should be the full path of the file where the readback # data will be saved. Othervise the readback data is saved in a # timestamp-based file name inside the "readback-frame-files" of the # working folder. # @param is_fault_injection: Trus if this write is for fault injection experiment. #--------------------------------------------------------------------------------- def readFrames(self, frame_address, frames=1, readback_file=None, is_fault_injection=False): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() self.frame_address = str(frame_address) self.frames = str(frames) XtclLog.writeLine( "================ Starting FPGA frame read ================ ", XtclCommon.blue) if readback_file == None: readbackFileName = "readbackBlock-0x" + str( self.frame_address) + "-" + str( datetime.datetime.now().timestamp()) + ".rbd" self.readbackFile = XtclSettings.workingDirectory + '/readback-frame-files/' + readbackFileName else: self.readbackFile = XtclSettings.workingDirectory + '/readback-frame-files/' + readback_file if is_fault_injection == True: command = Operation.FAULT_INJECTION_READ.name + "%" + self.readbackFile + "%" + self.frame_address + "%" + self.frames else: command = Operation.FRAMES_READ.name + "%" + self.readbackFile + "%" + self.frame_address + "%" + self.frames self.clientSend(command) self.operation = Operation.FRAMES_READ return #--------------------------------------------------------------------------------- # @brief This function reads the status signal of the user logic #--------------------------------------------------------------------------------- def readLogicStatus(self): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Reading active logic status ================ ", XtclCommon.blue) self.clientSend(Operation.LOGIC_STATUS.name) self.operation = Operation.LOGIC_STATUS return #--------------------------------------------------------------------------------- # @brief This function reads a configuration register # @param register_address: The frame of the register in 5-bit format (i.e. 01010) of in HEX format (0A). #--------------------------------------------------------------------------------- def readRegister(self, register_address): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Reading configuration register ================ ", XtclCommon.blue) self.register_address = str(register_address) command = Operation.REGISTER_READ.name + "%" + self.register_address self.clientSend(command) self.operation = Operation.REGISTER_READ return #--------------------------------------------------------------------------------- # @brief This function writes a configuration register # @param register_address: The frame of the register in 5-bit format (i.e. 01010) of in HEX format (0A). # @param register_value: The register value to be written in 32-bit HEX format (i.e. A000029B) #--------------------------------------------------------------------------------- def writeRegister(self, register_address, register_value): if self.tcpClientSocketStatus != SocketStatus.CONNECTED: self.clientStart() XtclLog.writeLine( "================ Writing configuration register ================ ", XtclCommon.blue) self.register_address = str(register_address) self.register_value = str(register_value) command = Operation.REGISTER_WRITE.name + "%" + self.register_address + "%" + self.register_value self.clientSend(command) self.operation = Operation.REGISTER_WRITE return
class QSubProcess(QObject): """Class to handle starting, running, and finishing PySide2 QProcesses.""" subprocess_finished_signal = Signal(int, name="subprocess_finished_signal") def __init__(self, toolbox, program=None, args=None, silent=False): """Class constructor. Args: toolbox (ToolboxUI): Instance of Main UI class. program (str): Path to program to run in the subprocess (e.g. julia.exe) args (list): List of argument for the program (e.g. path to script file) silent (bool): Whether or not to emit toolbox msg signals """ super().__init__() self._toolbox = toolbox self._program = program self._args = args self._silent = silent self.process_failed = False self.process_failed_to_start = False self._user_stopped = False self._process = QProcess(self) self.output = None def program(self): """Program getter method.""" return self._program def args(self): """Program argument getter method.""" return self._args # noinspection PyUnresolvedReferences def start_process(self, workdir=None): """Start the execution of a command in a QProcess. Args: workdir (str): Directory for the script (at least with Julia this is a must) """ if workdir is not None: self._process.setWorkingDirectory(workdir) self._process.started.connect(self.process_started) self._process.finished.connect(self.process_finished) if not self._silent: self._process.readyReadStandardOutput.connect(self.on_ready_stdout) self._process.readyReadStandardError.connect(self.on_ready_stderr) self._process.error.connect( self.on_process_error) # errorOccurred available in Qt 5.6 self._process.stateChanged.connect(self.on_state_changed) # self._toolbox.msg.emit("\tStarting program: <b>{0}</b>".format(self._program)) self._process.start(self._program, self._args) if not self._process.waitForStarted( msecs=10000 ): # This blocks until process starts or timeout happens self.process_failed = True self.process_failed_to_start = True self._process.deleteLater() self._process = None self.subprocess_finished_signal.emit(-9998) def wait_for_finished(self, msecs=30000): """Wait for subprocess to finish. Return: True if process finished successfully, False otherwise """ if not self._process: return False if self.process_failed or self.process_failed_to_start: return False if not self._process.waitForFinished(msecs): self.process_failed = True self._process.close() self._process = None return False return True @Slot(name="process_started") def process_started(self): """Run when subprocess has started.""" pass # self._toolbox.msg.emit("\tSubprocess started...") @Slot("QProcess::ProcessState", name="on_state_changed") def on_state_changed(self, new_state): """Runs when QProcess state changes. Args: new_state (QProcess::ProcessState): Process state number """ if new_state == QProcess.Starting: self._toolbox.msg.emit("\tStarting program <b>{0}</b>".format( self._program)) arg_str = " ".join(self._args) self._toolbox.msg.emit("\tArguments: <b>{0}</b>".format(arg_str)) elif new_state == QProcess.Running: self._toolbox.msg_warning.emit( "\tExecution is in progress. See Process Log for messages " "(stdout&stderr)") elif new_state == QProcess.NotRunning: # logging.debug("QProcess is not running") pass else: self._toolbox.msg_error.emit("Process is in an unspecified state") logging.error("QProcess unspecified state: {0}".format(new_state)) @Slot("QProcess::ProcessError", name="'on_process_error") def on_process_error(self, process_error): """Run if there is an error in the running QProcess. Args: process_error (QProcess::ProcessError): Process error number """ if process_error == QProcess.FailedToStart: # self._toolbox.msg_error.emit("Failed to start") self.process_failed = True self.process_failed_to_start = True elif process_error == QProcess.Timedout: self.process_failed = True self._toolbox.msg_error.emit("Timed out") elif process_error == QProcess.Crashed: self.process_failed = True self._toolbox.msg_error.emit("Process crashed") elif process_error == QProcess.WriteError: self._toolbox.msg_error.emit("Process WriteError") elif process_error == QProcess.ReadError: self._toolbox.msg_error.emit("Process ReadError") elif process_error == QProcess.UnknownError: self._toolbox.msg_error.emit("Unknown error in process") else: self._toolbox.msg_error.emit( "Unspecified error in process: {0}".format(process_error)) def terminate_process(self): """Shutdown simulation in a QProcess.""" self._toolbox.msg_error.emit("<br/>Terminating process") # logging.debug("Terminating QProcess nr.{0}. ProcessState:{1} and ProcessError:{2}" # .format(self._process.processId(), self._process.state(), self._process.error())) self._user_stopped = True self.process_failed = True try: self._process.close() except Exception as ex: self._toolbox.msg_error.emit( "[{0}] exception when terminating process".format(ex)) logging.exception("Exception in closing QProcess: {}".format(ex)) @Slot(int, name="process_finished") def process_finished(self, exit_code): """Run when subprocess has finished. Args: exit_code (int): Return code from external program (only valid for normal exits) """ # logging.debug("Error that occurred last: {0}".format(self._process.error())) exit_status = self._process.exitStatus() # Normal or crash exit if exit_status == QProcess.CrashExit: if not self._silent: self._toolbox.msg_error.emit("\tProcess crashed") exit_code = -1 elif exit_status == QProcess.NormalExit: if not self._silent: self._toolbox.msg.emit("\tProcess finished") else: if not self._silent: self._toolbox.msg_error.emit( "Unknown QProcess exit status [{0}]".format(exit_status)) exit_code = -1 if not exit_code == 0: self.process_failed = True if not self._user_stopped: out = str(self._process.readAllStandardOutput().data(), "utf-8") if out is not None: if not self._silent: self._toolbox.msg_proc.emit(out.strip()) else: self.output = out.strip() else: self._toolbox.msg.emit("*** Terminating process ***") # Delete QProcess self._process.deleteLater() self._process = None self.subprocess_finished_signal.emit(exit_code) @Slot(name="on_ready_stdout") def on_ready_stdout(self): """Emit data from stdout.""" out = str(self._process.readAllStandardOutput().data(), "utf-8") self._toolbox.msg_proc.emit(out.strip()) @Slot(name="on_ready_stderr") def on_ready_stderr(self): """Emit data from stderr.""" err = str(self._process.readAllStandardError().data(), "utf-8") self._toolbox.msg_proc_error.emit(err.strip())
class SerialControl(QObject): sig_keyseq_pressed = Signal(str) sig_CPU_comout = Signal(int) sig_CPU_comin = Signal(str) sig_port_change = Signal(str) sig_button_pressed = Signal(int) sig_terminal_open = Signal(bool) sig_firmware_update = Signal(str) def __init__(self, cpu, monitor_frame, terminal_frame, usb_frame, statusbar, config, sig_update, config_file_path): QObject.__init__(self) self.cpu = cpu self.monitor = monitor_frame self.terminal = terminal_frame self.statusbar = statusbar self.config = config self.ser_port = None self.sig_update = sig_update self.fd_thread = None self.fwth = None self.monitor_frame = monitor_frame self.usb_frame = usb_frame self.config_file_path = config_file_path # Connect signal self.sig_keyseq_pressed.connect(self.on_key_pressed) self.sig_CPU_comout.connect(self.on_comout) self.sig_CPU_comin.connect(self.on_comin) self.sig_port_change.connect(self.on_port_change) self.sig_button_pressed.connect(self.on_button_dispatch) self.sig_terminal_open.connect(self.on_terminal_open) self.sig_firmware_update.connect(self.on_firmware_update) self.monitor_frame.sig_keyseq_pressed = self.sig_keyseq_pressed self.monitor_frame.sig_button_pressed = self.sig_button_pressed self.cpu.sig_CPU_comout = self.sig_CPU_comout self.cpu.sig_CPU_comin = self.sig_CPU_comin self.usb_frame.usb_combo.sig_port_change = self.sig_port_change self.usb_frame.sig_button_pressed = self.sig_button_pressed self.usb_frame.sig_firmware_update = self.sig_firmware_update self.terminal.sig_terminal_open = self.sig_terminal_open # Disable fw flash button if udr2 binary is nor present self.udr2 = f"cli/udr2-{sys.platform}" if sys.platform != "win32" else "cli\\udr2-win32.exe" if not os.path.isfile(self.udr2): self.usb_frame.firmware_btn.setEnabled(False) self.init_serial() def init_serial(self, do_refresh=True): is_thread = InitSerialThread(self) is_thread.update_combo = self.usb_frame.usb_combo.set_ports is_thread.do_refresh = do_refresh is_thread.start() def init_OK(self): self.statusbar.sig_temp_message.emit("Serial port Initialized") # # Signal Handling # @Slot(str) def on_key_pressed(self, key): if self.cpu.rx is None and len(key) == 1: self.cpu.rx = key self.monitor_frame.serial_in.setText(key) @Slot(str) def on_comin(self, char): """Char is handled by CPU, we free the slot""" self.monitor_frame.serial_in.setText(" ") @Slot(int) def on_comout(self, byte): """Append the char to the console""" self.monitor_frame.append_serial_out(byte) @Slot(str) def on_port_change(self, port): self.config.set("serial", "port", port) if self.ser_port: self.ser_port.port = port with open(self.config_file_path, 'w') as configfile: self.config.write(configfile) self.init_serial(False) @Slot(int) def on_button_dispatch(self, btn_nbr): if btn_nbr == 0: self.to_digirule() elif btn_nbr == 1: self.from_digirule() elif btn_nbr == 2: self.init_serial() elif btn_nbr == 3: self.on_clear_button() @Slot(bool) def on_terminal_open(self, is_open): if is_open: # Terminal window is open : create the terminal serial thread self.statusbar.sig_temp_message.emit("open terminal thread") self.terminal.serth = SerialThread(self) # Start serial thread self.terminal.serth.start() else: # Terminal window is closed : quit the terminal serial thread if self.terminal.serth: self.terminal.serth.running = False sleep(0.5) self.terminal.serth = None def on_clear_button(self): self.monitor_frame.clear() # Clear the serial in/out content self.cpu.rx = None self.cpu.tx = None def to_digirule(self): if self.ser_port is None: self.statusbar.sig_temp_message.emit( "Error : No serial port configured") else: dump = ram2hex(self.cpu.ram) self.statusbar.sig_temp_message.emit("Dumpimg memory on port " + self.ser_port.port) try: self.ser_port.open() except serial.serialutil.SerialException as ex: self.statusbar.sig_temp_message.emit(ex.strerror) else: for line in dump.splitlines(): self.ser_port.write(line.encode("utf-8")) sleep(0.1) sleep(2) self.ser_port.close() self.statusbar.sig_temp_message.emit("Memory sent") def from_digirule(self): """Launch receive sequence in background""" if self.ser_port: self.statusbar.sig_temp_message.emit( "Waiting to receive Digirule on " + self.ser_port.port) self.fd_thread = FromDigiruleThread(self) self.fd_thread.start() else: self.init_serial() @Slot(str) def on_firmware_update(self, filepath): if self.ser_port: self.proc = QProcess(self) self.proc.readyReadStandardOutput.connect(self.stdoutReady) self.proc.readyReadStandardError.connect(self.stderrReady) if sys.platform == "win32": command = f'{self.udr2} --program {self.ser_port.port} < {filepath}' self.usb_frame.out.write( "Firmware update started, please wait ") # displays running dots on windows to pretend it is not stalled self.bullshitTimer = QTimer() self.bullshitTimer.timeout.connect(self.stdoutBullshit) self.bullshitTimer.start(1000) self.proc.setProcessChannelMode(QProcess.MergedChannels) self.proc.start('cmd.exe', ['/c', command]) else: command = f'{self.udr2} --program {self.ser_port.port} < "{filepath}"' self.bullshitTimer = None self.proc.start('bash', ['-c', command]) # print(command) def stdoutBullshit(self): self.usb_frame.out.write(".") def stdoutReady(self): if self.bullshitTimer: self.usb_frame.out.write("\n") self.bullshitTimer.stop() text = str(self.proc.readAllStandardOutput()) self.usb_frame.out.write(eval(text).decode('iso8859-1')) def stderrReady(self): text = str(self.proc.readAllStandardError()) self.usb_frame.out.write(eval(text).decode('iso8859-1'))
class MainWindow(QWidget): def __init__(self): QWidget.__init__(self) self.label = QLabel() self.label.setText('Drag and drop nk file') self.button = QPushButton('Disable') self.progressBar = QProgressBar() self.progressBar.setRange(0, 100) self.text_area = QPlainTextEdit() self.text_area.setReadOnly(True) master_lay = QVBoxLayout() master_lay.addWidget(self.label) master_lay.addWidget(self.progressBar) master_lay.addWidget(self.text_area) master_lay.addWidget(self.button) self.process = None self.setAcceptDrops(True) self.setLayout(master_lay) self.setMinimumSize(500, 250) self.setWindowTitle('Nuke Disable') self.button.clicked.connect(self.execute) def message(self, s): self.text_area.appendPlainText(s) def dragEnterEvent(self, e): if e.mimeData().hasUrls(): e.acceptProposedAction() def dropEvent(self, e): for url in e.mimeData().urls(): file_name = url.toLocalFile() self.label.setText(file_name) print("Dropped file: " + file_name) def execute(self): wrk_file_path = self.label.text() if wrk_file_path.endswith('nk'): if self.process is None: self.process = QProcess() self.process.readyReadStandardOutput.connect( self.handle_stdout) self.process.readyReadStandardError.connect(self.handle_stderr) self.process.stateChanged.connect(self.handle_state) self.process.finished.connect(self.process_finished) self.process.start('"{}" -it {} {}'.format( nuke_path, python_file, wrk_file_path)) else: msgBox = QMessageBox() msgBox.setText("Please Drag a nuke file") msgBox.exec_() def handle_stderr(self): data = self.process.readAllStandardError() stderr = bytes(data).decode() # Extract progress if it is in the data. progress = simple_percent_parser(stderr) if progress: self.progressBar.setStyleSheet( "QProgressBar::chunk" "{" "background-color: QLinearGradient( x1: 0, y1: 0, x2: 1, y2: 0," "stop: 0 #FF4E50," "stop: 1 #F9D423);" "}") self.progressBar.setValue(progress) self.message(stderr) def handle_stdout(self): data = self.process.readAllStandardOutput() stdout = bytes(data).decode() self.message(stdout) def handle_state(self, state): states = { QProcess.NotRunning: 'Not running', QProcess.Starting: 'Starting', QProcess.Running: 'Running', } state_name = states[state] self.message("State changed: {}".format(state_name)) def process_finished(self): self.message("Process finished.") self.process = None
class QKonsol(QPlainTextEdit): userTextEntry = "" commandList = ["cd " + QDir.homePath()] length = 0 history = -1 def __init__(self, parent=None): super().__init__() self.setParent(parent) self.setWindowTitle(self.tr("Terminal")) self.setCursorWidth(7) self.setContextMenuPolicy(Qt.NoContextMenu) font = self.font() font.setFamily("Consolas") font.setPointSize(10) self.setFont(font) self.setUndoRedoEnabled(False) palette = QPalette() palette.setColor(QPalette.Base, Qt.black) palette.setColor(QPalette.Text, Qt.white) palette.setColor(QPalette.Highlight, Qt.white) palette.setColor(QPalette.HighlightedText, Qt.black) self.setFrameShape(QFrame.NoFrame) self.setPalette(palette) self.resize(720, 480) self.process = QProcess(self) self.process.setProcessChannelMode(QProcess.MergedChannels) self.process.setReadChannel(QProcess.StandardOutput) self.process.readyReadStandardOutput.connect(self.readStandartOutput) self.process.readyReadStandardError.connect( lambda: print(self.readAllStandartError())) self.cursorPositionChanged.connect(self.cursorPosition) self.textChanged.connect(self.whatText) if sys.platform == "win32": self.process.start("cmd.exe", [], mode=QProcess.ReadWrite) else: self.process.start( "bash", ["-i"], mode=QProcess.ReadWrite) # bash -i interactive mode def readStandartOutput(self): if sys.platform == "win32": st = self.process.readAllStandardOutput().data().decode( str(ctypes.cdll.kernel32.GetConsoleOutputCP())) else: st = self.process.readAllStandardOutput().data().decode("utf-8") # print(repr(st), self.commandList) if not st.startswith(self.commandList[-1]): self.appendPlainText(st) def __line_end(self): if sys.platform == "win32": return "\r\n" elif sys.platform == "linux": return "\n" elif sys.platform == "darwin": return "\r" def keyPressEvent(self, event: QKeyEvent): if event.key() in (Qt.Key_Enter, Qt.Key_Return): if self.commandList[ -1] != self.userTextEntry and self.userTextEntry != "": self.commandList.append(self.userTextEntry) self.length = len(self.userTextEntry + self.__line_end()) self.process.writeData(self.userTextEntry + self.__line_end(), self.length) self.userTextEntry = "" elif event.key() == Qt.Key_Backspace: if self.userTextEntry == "": return else: self.userTextEntry = self.userTextEntry[:-1] super().keyPressEvent(event) elif event.key() == Qt.Key_Up: if -len(self.commandList) < self.history: self.history -= 1 print(self.commandList[self.history]) return elif event.key() == Qt.Key_Down: if self.history < -1: self.history += 1 print(self.commandList[self.history]) return elif event.key() == Qt.Key_Delete: return elif event.modifiers() == Qt.ControlModifier: super().keyPressEvent(event) else: super().keyPressEvent(event) self.userTextEntry += event.text() def cursorPosition(self): pass # print(self.textCursor().position()) def whatText(self): pass # print(self.blockCount()) def insertFromMimeData(self, source): super().insertFromMimeData(source) self.userTextEntry += source.text() def mouseReleaseEvent(self, event): super().mousePressEvent(event) cur = self.textCursor() if event.button() == Qt.LeftButton: cur.movePosition(QTextCursor.End, QTextCursor.MoveAnchor, 1) self.setTextCursor(cur) def closeEvent(self, event): self.process.close()
class CliWidget(QDialog): def __init__(self, title: str, program: str, args: List[str], commands: List[str]): super().__init__() self.setWindowTitle(title) self.program = program self.args = args self.layout = QGridLayout() self.output = QTextEdit() self.output.acceptRichText = True self.input = QLineEdit() self.completer = QCompleter(commands, self) self.completer.setCaseSensitivity(Qt.CaseInsensitive) self.input.setCompleter(self.completer) self.input.setFocus() self.process = QProcess() self.process.setProgram(self.program) self.process.setCurrentReadChannel(0) # noinspection PyUnresolvedReferences self.process.readyReadStandardError.connect(self.handle_error) # noinspection PyUnresolvedReferences self.process.readyReadStandardOutput.connect(self.handle_output) self.layout.addWidget(self.output) self.layout.addWidget(self.input) self.setLayout(self.layout) self.connect(self.input, SIGNAL("returnPressed(void)"), self.execute_user_command) self.connect(self.completer, SIGNAL("activated(const QString&)"), self.input.clear, Qt.QueuedConnection) def execute_user_command(self): cmd = str(self.input.text()) self.run_command(cmd) def run_command(self, cmd: str): log.info('run_command', program=self.program, args=self.args, cmd=cmd) self.output.append(f'> {cmd}\n') self.input.clear() self.process.kill() args = list(self.args) args.append(cmd) self.process.setArguments(args) self.process.start() def handle_error(self): output: QByteArray = self.process.readAllStandardError() message = output.data().decode('utf-8').strip() self.output.append(message) def handle_output(self): output: QByteArray = self.process.readAllStandardOutput() message = output.data().decode('utf-8').strip() if message.startswith('{') or message.startswith('['): formatter = HtmlFormatter() formatter.noclasses = True formatter.linenos = False formatter.nobackground = True message = highlight(message, JsonLexer(), formatter) self.output.insertHtml(message) else: self.output.append(message) # This is just for generating the command lists in constants # commands = None # if '== Blockchain ==' in message: # commands = self.parse_bitcoin_cli_commands(message) # elif 'lncli [global options] command [command options]' in message: # commands = self.parse_lncli_commands(message) # if commands is not None: # log.debug('commands', commands=commands) max_scroll = self.output.verticalScrollBar().maximum() self.output.verticalScrollBar().setValue(max_scroll) def parse_bitcoin_cli_commands(self, message: str): log.debug('parse_bitcoin_cli_commands') commands = [] for line in message.split(sep='\n'): line = line.strip() if not line or line.startswith('=='): continue command = line.split()[0] command = command.strip() commands.append(command) return commands def parse_lncli_commands(self, message: str): log.debug('parse_lncli_commands') at_commands = False commands = [] for line in message.split(sep='\n'): line = line.strip() if not at_commands: if 'COMMANDS:' in line: at_commands = True log.debug('commands line', line=line) continue elif 'GLOBAL OPTIONS' in line: return commands elif line.endswith(':') or not line: continue command = line.split()[0] command = command.strip().replace(',', '') commands.append(command) return commands def show(self): self.showMaximized() self.input.setFocus() self.run_command('help')
class PyManageWidget(QWidget, Ui_Form): """python解释器以及pip包""" def __init__(self, home, app_name=None): super(self.__class__, self).__init__() self.setupUi(self) self.setStyleSheet(qss) self.home = home self.interpreter = None self.p = QProcess() self.thread_pool = QThreadPool() # btn self.py_setting_btn.clicked.connect(self.py_setting_slot) self.set_app_py.clicked.connect(self.set_app_py_slot) # self.py_box.currentTextChanged.connect(self.py_change_slot) self.pip_list.setContextMenuPolicy(Qt.CustomContextMenu) self.pip_list.customContextMenuRequested.connect(self.generate_menu) ####右键菜单 # self.load_py() if not app_name: """用于设置窗口""" self.app_name.hide() self.set_app_py.hide() self.cur_py.hide() self.label_3.hide() else: """用于应用窗口""" self.app_name.setText(app_name) self.app_name.setStyleSheet("""color:#BE9117""") self.cur_py.setStyleSheet("""color:#168AD5""") path = G.config.installed_apps[app_name].get('py_', '未选择') for name, p in G.config.python_path.items(): if path == p: self.cur_py.setText(name) else: self.cur_py.setText(path) def load_py(self): """加载PY列表""" self.py_box.clear() for k, v in G.config.python_path.items(): self.py_box.addItem(k) def load_pip(self, py_): """加载pip包列表""" self.pip_list.clear() # self.p.readyReadStandardOutput.connect(self.readout) self.p.finished.connect(self.pip_finish_slot) self.p.start(' '.join([py_, '-m', 'pip', 'freeze'])) def pip_finish_slot(self): try: output = self.p.readAllStandardOutput().data().decode() for i in output.splitlines(): self.pip_list.addItem(i) except RuntimeError: pass def generate_menu(self, pos): row_num = -1 for i in self.pip_list.selectionModel().selection().indexes(): row_num = i.row() if row_num < 0: return menu = QMenu() item1 = menu.addAction("uninstall") action = menu.exec_(self.pip_list.mapToGlobal(pos)) row_data = self.pip_list.item(row_num) if action == item1 and row_data: pack = row_data.text() py_ = self.path.text() self.thread_pool.start(Worker(self.uninstall_pip, py_, pack)) self.pip_list.takeItem(row_num) def uninstall_pip(self, py_, pack): cmd = [py_, '-m', 'pip', 'uninstall', pack, '-y'] subprocess.check_output(cmd) def set_app_py_slot(self): """设置为当前app解释器""" py_ = self.path.text() G.config.installed_apps[self.app_name.text()].update({"py_": py_}) G.config.to_file() self.cur_py.setText(self.py_box.currentText()) TipDialog("设置成功") def py_setting_slot(self): self.interpreter = InterpreterWidget(self) self.interpreter.exec_() def py_change_slot(self, name): if name.strip(): path = G.config.python_path[name] self.path.setText(path) self.load_pip(path) def window_cleanup(self): """关闭的事后处理""" if self.interpreter: self.interpreter.window_cleanup() self.close()
class Standard(object): """Action download ---|--> install --|--> == run v v \\ cancel cancel uninstall """ # common name = "" # 应用名称 desc = "" # 应用描述 icon = "" # 应用图标 app_url = "" # 应用地址 versions = {} # 应用版本及下载地址 # installed install_version = "" # 选择安装的版本 app_folder = "" # 应用安装路径 py_ = "" # 解释器 (G 中实时获取) entry_ = "" # 启动文件 (build.json实时获取) requirement_ = "" # 缺失依赖 def __init__(self, parent: QWidget, **kwargs): self.cls_name = self.__class__.__name__ self.parent = parent self.action = kwargs.get("action", Actions.DOWNLOAD) self.thread_pool = QThreadPool() self.check(**kwargs) self.div: AppDiv self.div_init() self.count = 0 self.start_time = 0 self.cancel = False self.process = QProcess(self.parent) self.process.readyReadStandardOutput.connect(self.on_readoutput) self.process.readyReadStandardError.connect(self.on_readerror) def check(self, **kwargs): """检查已下载应用参数""" self.app_folder = kwargs.get("app_folder") # 应用安装路径 self.install_version = kwargs.get('install_version') # 应用安装路径 self.app_info(**kwargs) def app_info(self, **kwargs): raise NotImplementedError @property def pack_name(self): """安装后id""" return self.cls_name + "_" + self.install_version.replace('.', '_') @property def ui_name(self): """ui中id""" return self.cls_name + "_app" def _transfer(self, widget, value=None): if widget == 'bar': self.div.job.progressbar_signal.emit(value) elif widget == 'msg': self.div.job.msg_signal.emit(value) elif widget == 'switch': self.div.job.switch_signal.emit() elif widget == 'action': self.div.job.action_signal.emit(value) def before_handle(self): self.action = Actions.CANCEL self.cancel = False self.div.action.setText(Actions.to_zn(self.action)) self._transfer('switch') def _tip(self, msg): self.parent.mainwindow.job.msg_box_signal.emit({"msg": str(msg)}) def div_init(self): self.div = AppDiv(self.parent) self.div.icon.setStyleSheet(f"image: url({self.icon});") self.div.name.setText(self.name) self.div.action.setText(Actions.to_zn(self.action)) self.div.action.clicked.connect(self.action_handler) if self.action == Actions.DOWNLOAD: for i in self.versions.keys(): act = QAction(i, self.parent) setattr(self.div, f"act_{'_'.join([j for j in i if i.isalnum()])}", act) self.div.menu.addAction(act) self.div.menu.triggered[QAction].connect( self.version_action_triggered) self.div.desc.setText(self.desc) self.div.desc.url = self.app_url # 可点击 setattr(self.parent, self.ui_name, self) self.parent.apps_layout.addLayout(self.div.layout) elif self.action == Actions.RUN: act_uninstall = QAction(Actions.to_zn(Actions.UNINSTALL), self.parent) act_setting = QAction("解释器", self.parent) act_upgrade = QAction(Actions.to_zn(Actions.UPGRADE), self.parent) setattr(self.div, f"act_uninstall", act_uninstall) setattr(self.div, f"act_setting", act_setting) setattr(self.div, f"act_upgrade", act_upgrade) self.div.menu.addAction(act_setting) self.div.menu.addAction(act_upgrade) self.div.menu.addAction(act_uninstall) self.div.menu.triggered[QAction].connect( self.menu_action_triggered) self.div.desc.setText(self.install_version) setattr(self.parent, self.pack_name, self) self.parent.installed_layout.addLayout(self.div.layout) def version_action_triggered(self, q): """点击版本号直接下载""" if self.action == Actions.CANCEL: return self.install_version = q.text() self.action_handler() def menu_action_triggered(self, q): """卸载/更新处理""" if self.action == Actions.CANCEL: return act = q.text() if act == "解释器": self.act_setting_slot() elif Actions.to_en(act) == Actions.UNINSTALL: self.uninstall_handler() elif Actions.to_en(act) == Actions.UPGRADE: self.upgrade_handler() def act_setting_slot(self): self.py_manage = PyManageWidget(self.parent, self.pack_name) self.py_manage.show() @before_download def download_handler(self): """ 版本号 下载目录 """ url = self.versions[self.install_version] postfix = os.path.splitext(url)[-1] # .zip self.app_folder = os.path.join(G.config.install_path, self.pack_name) file_temp = self.app_folder + postfix # 压缩文件路径 ## 文件续传 if os.path.exists(file_temp): local_file = os.path.getsize(file_temp) headers = {'Range': 'bytes=%d-' % local_file} mode = 'ab' else: local_file = 0 headers = {} mode = 'wb' # download self._transfer("msg", "获取中...") try: response = requests.get(url, stream=True, headers=headers) response.raise_for_status() except Exception as e: return False content_size = float(response.headers.get('Content-Length', 0)) self._transfer("bar", dict(range=[0, content_size])) # save with open(file_temp, mode) as file: chunk_size = 1024 for data in response.iter_content(chunk_size=chunk_size): if self.cancel or G.pool_done: return False file.write(data) ## self.count += 1 ##show current = chunk_size * self.count + local_file if content_size: self._transfer("bar", dict(value=current)) speed = format_size(current / (time.time() - self.start_time)) self._transfer( "msg", f"{round(current / 1024, 2)}KB/{round(content_size / 1024, 2) or '-'}KB | {speed}/s" ) extract(file_temp) # 解压 return True def on_download_callback(self, res): self._transfer('switch') if res is True: data = { "cls_name": self.cls_name, "install_version": self.install_version, "action": Actions.RUN, "app_folder": self.app_folder, "py_": "" } G.config.installed_apps.update({self.pack_name: data}) G.config.to_file() self.div.add_installed_layout(data) elif res is False: pass self.action = Actions.DOWNLOAD self._transfer("action", Actions.to_zn(self.action)) def get_build(self): """ :return: path 路径 """ path = find_file(self.app_folder, 'build.json') if path: try: with open(path[0], 'r') as f: build = json.load(f) entry = find_file(self.app_folder, build['entry'])[0] requirement = find_file(self.app_folder, build['requirement'])[0] except KeyError: raise Exception('请确保build.json中含有entry和requirement') except IndexError: raise Exception("未找到entry文件或requirement文件") except json.JSONDecodeError: raise Exception("build.json 有错误") return entry, requirement else: raise Exception("未找到文件build.json") @before_install def install_handler(self): """解析 build.json""" for line in self.requirement_: line = line.strip().replace('==', '>=') cmd_ = [self.py_, "-m", "pip", "install", line ] + G.config.get_pypi_source() if self.cancel or G.pool_done: return False self.process.start(" ".join(cmd_)) self.process.waitForFinished() return True def on_readoutput(self): output = self.process.readAllStandardOutput().data().decode() self._transfer("msg", output.replace('\n', '')) def on_readerror(self): error = self.process.readAllStandardError().data().decode() self._tip(error) def on_install_callback(self, res): self._transfer('switch') self.action = Actions.RUN self._transfer("action", Actions.to_zn(self.action)) def run_handler(self): self.py_ = G.config.installed_apps[self.pack_name].get('py_') if not self.py_: QMessageBox.warning(self.parent, "提示", "未选择Python解释器") self.act_setting_slot() return if not os.path.exists(self.py_) or not os.path.isfile(self.py_): QMessageBox.warning(self.parent, "提示", f"{self.py_} 不存在") return try: self.entry_, requirement_ = self.get_build() except Exception as e: QMessageBox.warning(self.parent, "提示", str(e)) return ##检测依赖 p = QProcess() p.start(' '.join(([self.py_, "-m", 'pip', "freeze"]))) p.waitForFinished() out = p.readAllStandardOutput().data().decode() output = out.splitlines() with open(requirement_, 'r') as f: requirement = f.read().splitlines() dissatisfy, version_less = diff_pip(output, requirement) if dissatisfy: msgbox = QMessageBox(self.parent) msgbox.setWindowTitle("缺少依赖") msgbox.setText("\n".join(dissatisfy[:15]) + "\n...") yes = msgbox.addButton('立即安装', QMessageBox.AcceptRole) no = msgbox.addButton('稍后', QMessageBox.RejectRole) msgbox.setDefaultButton(yes) reply = msgbox.exec_() if reply == QMessageBox.AcceptRole: self.requirement_ = dissatisfy self.install_handler() return # run TipDialog("正在启动...") cmd = ' '.join([self.py_, self.entry_]) QProcess().startDetached(cmd) def upgrade_handler(self): pass @before_uninstall def uninstall_handler(self): try: if os.path.exists(self.app_folder) and os.path.isdir( self.app_folder): shutil.rmtree(self.app_folder) except Exception as e: self._tip({"msg": str(e)}) finally: for name, attr in self.div.__dict__.items(): if name not in self.div.not_widget: attr.deleteLater() G.config.installed_apps.pop(self.pack_name, None) G.config.to_file() def cancel_handler(self): self.cancel = True self._transfer("msg", "Releasing...") def action_handler(self): if self.action == Actions.DOWNLOAD: self.download_handler() # self.git_download_handler() elif self.action == Actions.CANCEL: self.cancel_handler() elif self.action == Actions.RUN: self.run_handler()
class Importer(ProjectItem): def __init__(self, name, description, mappings, x, y, toolbox, project, logger, cancel_on_error=True): """Importer class. Args: name (str): Project item name description (str): Project item description mappings (list): List where each element contains two dicts (path dict and mapping dict) x (float): Initial icon scene X coordinate y (float): Initial icon scene Y coordinate toolbox (ToolboxUI): QMainWindow instance project (SpineToolboxProject): the project this item belongs to logger (LoggerInterface): a logger instance cancel_on_error (bool): if True the item's execution will stop on import error """ super().__init__(name, description, x, y, project, logger) # Make logs subdirectory for this item self._toolbox = toolbox self.logs_dir = os.path.join(self.data_dir, "logs") try: create_dir(self.logs_dir) except OSError: self._logger.msg_error.emit(f"[OSError] Creating directory {self.logs_dir} failed. Check permissions.") # Variables for saving selections when item is (de)activated if not mappings: mappings = list() # convert table_types and table_row_types keys to int since json always has strings as keys. for _, mapping in mappings: table_types = mapping.get("table_types", {}) mapping["table_types"] = { table_name: {int(col): t for col, t in col_types.items()} for table_name, col_types in table_types.items() } table_row_types = mapping.get("table_row_types", {}) mapping["table_row_types"] = { table_name: {int(row): t for row, t in row_types.items()} for table_name, row_types in table_row_types.items() } # Convert serialized paths to absolute in mappings self.settings = self.deserialize_mappings(mappings, self._project.project_dir) # self.settings is now a dictionary, where elements have the absolute path as the key and the mapping as value self.cancel_on_error = cancel_on_error self.resources_from_downstream = list() self.file_model = QStandardItemModel() self.importer_process = None self.all_files = [] # All source files self.unchecked_files = [] # Unchecked source files # connector class self._preview_widget = {} # Key is the filepath, value is the ImportPreviewWindow instance @staticmethod def item_type(): """See base class.""" return "Importer" @staticmethod def category(): """See base class.""" return "Importers" @Slot(QStandardItem, name="_handle_file_model_item_changed") def _handle_file_model_item_changed(self, item): if item.checkState() == Qt.Checked: self.unchecked_files.remove(item.text()) self._logger.msg.emit(f"<b>{self.name}:</b> Source file '{item.text()}' will be processed at execution.") elif item.checkState() != Qt.Checked: self.unchecked_files.append(item.text()) self._logger.msg.emit( f"<b>{self.name}:</b> Source file '{item.text()}' will *NOT* be processed at execution." ) def make_signal_handler_dict(self): """Returns a dictionary of all shared signals and their handlers. This is to enable simpler connecting and disconnecting.""" s = super().make_signal_handler_dict() s[self._properties_ui.toolButton_open_dir.clicked] = lambda checked=False: self.open_directory() s[self._properties_ui.pushButton_import_editor.clicked] = self._handle_import_editor_clicked s[self._properties_ui.treeView_files.doubleClicked] = self._handle_files_double_clicked return s def activate(self): """Restores selections, cancel on error checkbox and connects signals.""" self._properties_ui.cancel_on_error_checkBox.setCheckState(Qt.Checked if self.cancel_on_error else Qt.Unchecked) self.restore_selections() super().connect_signals() def deactivate(self): """Saves selections and disconnects signals.""" self.save_selections() if not super().disconnect_signals(): logging.error("Item %s deactivation failed.", self.name) return False return True def restore_selections(self): """Restores selections into shared widgets when this project item is selected.""" self._properties_ui.label_name.setText(self.name) self._properties_ui.treeView_files.setModel(self.file_model) self.file_model.itemChanged.connect(self._handle_file_model_item_changed) def save_selections(self): """Saves selections in shared widgets for this project item into instance variables.""" self._properties_ui.treeView_files.setModel(None) self.file_model.itemChanged.disconnect(self._handle_file_model_item_changed) def update_name_label(self): """Update Importer properties tab name label. Used only when renaming project items.""" self._properties_ui.label_name.setText(self.name) @Slot(bool, name="_handle_import_editor_clicked") def _handle_import_editor_clicked(self, checked=False): """Opens Import editor for the file selected in list view.""" index = self._properties_ui.treeView_files.currentIndex() self.open_import_editor(index) @Slot("QModelIndex", name="_handle_files_double_clicked") def _handle_files_double_clicked(self, index): """Opens Import editor for the double clicked index.""" self.open_import_editor(index) def open_import_editor(self, index): """Opens Import editor for the given index.""" importee = index.data() if importee is None: self._logger.msg_error.emit("Please select a source file from the list first.") return if not os.path.exists(importee): self._logger.msg_error.emit(f"Invalid path: {importee}") return # Raise current form for the selected file if any preview_widget = self._preview_widget.get(importee, None) if preview_widget: if preview_widget.windowState() & Qt.WindowMinimized: # Remove minimized status and restore window with the previous state (maximized/normal state) preview_widget.setWindowState(preview_widget.windowState() & ~Qt.WindowMinimized | Qt.WindowActive) preview_widget.activateWindow() else: preview_widget.raise_() return # Create a new form for the selected file settings = self.get_settings(importee) # Try and get connector from settings source_type = settings.get("source_type", None) if source_type is not None: connector = _CONNECTOR_NAME_TO_CLASS[source_type] else: # Ask user connector = self.get_connector(importee) if not connector: # Aborted by the user return self._logger.msg.emit(f"Opening Import editor for file: {importee}") preview_widget = self._preview_widget[importee] = ImportPreviewWindow( self, importee, connector, settings, self._toolbox ) preview_widget.settings_updated.connect(lambda s, importee=importee: self.save_settings(s, importee)) preview_widget.connection_failed.connect(lambda m, importee=importee: self._connection_failed(m, importee)) preview_widget.destroyed.connect(lambda o=None, importee=importee: self._preview_destroyed(importee)) preview_widget.start_ui() def get_connector(self, importee): """Shows a QDialog to select a connector for the given source file. Mimics similar routine in `spine_io.widgets.import_widget.ImportDialog` Args: importee (str): Path to file acting as an importee Returns: Asynchronous data reader class for the given importee """ connector_list = [CSVConnector, ExcelConnector, GdxConnector] # add others as needed connector_names = [c.DISPLAY_NAME for c in connector_list] dialog = QDialog(self._toolbox) dialog.setLayout(QVBoxLayout()) connector_list_wg = QListWidget() connector_list_wg.addItems(connector_names) # Set current item in `connector_list_wg` based on file extension _filename, file_extension = os.path.splitext(importee) if file_extension.lower().startswith(".xls"): row = connector_list.index(ExcelConnector) elif file_extension.lower() == ".csv": row = connector_list.index(CSVConnector) elif file_extension.lower() == ".gdx": row = connector_list.index(GdxConnector) else: row = None if row: connector_list_wg.setCurrentRow(row) button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) button_box.button(QDialogButtonBox.Ok).clicked.connect(dialog.accept) button_box.button(QDialogButtonBox.Cancel).clicked.connect(dialog.reject) connector_list_wg.doubleClicked.connect(dialog.accept) dialog.layout().addWidget(connector_list_wg) dialog.layout().addWidget(button_box) _dirname, filename = os.path.split(importee) dialog.setWindowTitle("Select connector for '{}'".format(filename)) answer = dialog.exec_() if answer: row = connector_list_wg.currentIndex().row() return connector_list[row] def select_connector_type(self, index): """Opens dialog to select connector type for the given index.""" importee = index.data() connector = self.get_connector(importee) if not connector: # Aborted by the user return settings = self.get_settings(importee) settings["source_type"] = connector.__name__ def _connection_failed(self, msg, importee): self._logger.msg.emit(msg) preview_widget = self._preview_widget.pop(importee, None) if preview_widget: preview_widget.close() def get_settings(self, importee): """Returns the mapping dictionary for the file in given path. Args: importee (str): Absolute path to a file, whose mapping is queried Returns: dict: Mapping dictionary for the requested importee or an empty dict if not found """ importee_settings = None for p in self.settings: if p == importee: importee_settings = self.settings[p] if not importee_settings: return {} return importee_settings def save_settings(self, settings, importee): """Updates an existing mapping or adds a new mapping (settings) after closing the import preview window. Args: settings (dict): Updated mapping (settings) dictionary importee (str): Absolute path to a file, whose mapping has been updated """ if importee in self.settings.keys(): self.settings[importee].update(settings) else: self.settings[importee] = settings def _preview_destroyed(self, importee): """Destroys preview widget instance for the given importee. Args: importee (str): Absolute path to a file, whose preview widget is destroyed """ self._preview_widget.pop(importee, None) def update_file_model(self, items): """Adds given list of items to the file model. If None or an empty list is given, the model is cleared. Args: items (set): Set of absolute file paths """ self.all_files = items self.file_model.clear() self.file_model.setHorizontalHeaderItem(0, QStandardItem("Source files")) # Add header if items is not None: for item in items: qitem = QStandardItem(item) qitem.setEditable(False) qitem.setCheckable(True) if item in self.unchecked_files: qitem.setCheckState(Qt.Unchecked) else: qitem.setCheckState(Qt.Checked) qitem.setData(QFileIconProvider().icon(QFileInfo(item)), Qt.DecorationRole) self.file_model.appendRow(qitem) def _run_importer_program(self, args): """Starts and runs the importer program in a separate process. Args: args (list): List of arguments for the importer program """ self.importer_process = QProcess() self.importer_process.readyReadStandardOutput.connect(self._log_importer_process_stdout) self.importer_process.readyReadStandardError.connect(self._log_importer_process_stderr) self.importer_process.finished.connect(self.importer_process.deleteLater) program_path = os.path.abspath(importer_program.__file__) self.importer_process.start(sys.executable, [program_path]) self.importer_process.waitForStarted() self.importer_process.write(json.dumps(args).encode("utf-8")) self.importer_process.write(b'\n') self.importer_process.closeWriteChannel() if self.importer_process.state() == QProcess.Running: loop = QEventLoop() self.importer_process.finished.connect(loop.quit) loop.exec_() return self.importer_process.exitCode() @Slot() def _log_importer_process_stdout(self): output = str(self.importer_process.readAllStandardOutput().data(), "utf-8").strip() self._logger.msg.emit(f"<b>{self.name}</b>: {output}") @Slot() def _log_importer_process_stderr(self): output = str(self.importer_process.readAllStandardError().data(), "utf-8").strip() self._logger.msg_error.emit(f"<b>{self.name}</b>: {output}") def execute_backward(self, resources): """See base class.""" self.resources_from_downstream = resources.copy() return True def execute_forward(self, resources): """See base class.""" args = [ [f for f in self.all_files if f not in self.unchecked_files], self.settings, [r.url for r in self.resources_from_downstream if r.type_ == "database"], self.logs_dir, self._properties_ui.cancel_on_error_checkBox.isChecked(), ] exit_code = self._run_importer_program(args) return exit_code == 0 def stop_execution(self): """Stops executing this Importer.""" super().stop_execution() if not self.importer_process: return self.importer_process.kill() def _do_handle_dag_changed(self, resources): """See base class.""" file_list = [r.path for r in resources if r.type_ == "file" and not r.metadata.get("future")] self._notify_if_duplicate_file_paths(file_list) self.update_file_model(set(file_list)) if not file_list: self.add_notification( "This Importer does not have any input data. " "Connect Data Connections to this Importer to use their data as input." ) def item_dict(self): """Returns a dictionary corresponding to this item.""" d = super().item_dict() # Serialize mappings before saving d["mappings"] = self.serialize_mappings(self.settings, self._project.project_dir) d["cancel_on_error"] = self._properties_ui.cancel_on_error_checkBox.isChecked() return d def notify_destination(self, source_item): """See base class.""" if source_item.item_type() == "Data Connection": self._logger.msg.emit( "Link established. You can define mappings on data from " f"<b>{source_item.name}</b> using item <b>{self.name}</b>." ) elif source_item.item_type() == "Data Store": # Does this type of link do anything? self._logger.msg.emit("Link established.") else: super().notify_destination(source_item) @staticmethod def default_name_prefix(): """see base class""" return "Importer" def tear_down(self): """Closes all preview widgets.""" for widget in self._preview_widget.values(): widget.close() def _notify_if_duplicate_file_paths(self, file_list): """Adds a notification if file_list contains duplicate entries.""" file_counter = Counter(file_list) duplicates = list() for file_name, count in file_counter.items(): if count > 1: duplicates.append(file_name) if duplicates: self.add_notification("Duplicate input files from upstream items:<br>{}".format("<br>".join(duplicates))) @staticmethod def upgrade_from_no_version_to_version_1(item_name, old_item_dict, old_project_dir): """Converts mappings to a list, where each element contains two dictionaries, the serialized path dictionary and the mapping dictionary for the file in that path.""" new_importer = dict(old_item_dict) mappings = new_importer.get("mappings", {}) list_of_mappings = list() paths = list(mappings.keys()) for path in paths: mapping = mappings[path] if "source_type" in mapping and mapping["source_type"] == "CSVConnector": _fix_csv_connector_settings(mapping) new_path = serialize_path(path, old_project_dir) if new_path["relative"]: new_path["path"] = os.path.join(".spinetoolbox", "items", new_path["path"]) list_of_mappings.append([new_path, mapping]) new_importer["mappings"] = list_of_mappings return new_importer @staticmethod def deserialize_mappings(mappings, project_path): """Returns mapping settings as dict with absolute paths as keys. Args: mappings (list): List where each element contains two dictionaries (path dict and mapping dict) project_path (str): Path to project directory Returns: dict: Dictionary with absolute paths as keys and mapping settings as values """ abs_path_mappings = {} for source, mapping in mappings: abs_path_mappings[deserialize_path(source, project_path)] = mapping return abs_path_mappings @staticmethod def serialize_mappings(mappings, project_path): """Returns a list of mappings, where each element contains two dictionaries, the 'serialized' path in a dictionary and the mapping dictionary. Args: mappings (dict): Dictionary with mapping specifications project_path (str): Path to project directory Returns: list: List where each element contains two dictionaries. """ serialized_mappings = list() for source, mapping in mappings.items(): # mappings is a dict with absolute paths as keys and mapping as values serialized_mappings.append([serialize_path(source, project_path), mapping]) return serialized_mappings