class MainWindow(QMainWindow):
def __init__(self):
super().__init__()
# Hold process reference.
self.p = None
layout = QVBoxLayout()
self.text = QPlainTextEdit()
layout.addWidget(self.text)
self.progress = QProgressBar()
layout.addWidget(self.progress)
btn_run = QPushButton("Execute")
btn_run.clicked.connect(self.start)
layout.addWidget(btn_run)
w = QWidget()
w.setLayout(layout)
self.setCentralWidget(w)
self.show()
def start(self):
if self.p is not None:
return
self.p = QProcess()
self.p.readyReadStandardOutput.connect(self.handle_stdout)
self.p.readyReadStandardError.connect(self.handle_stderr)
self.p.stateChanged.connect(self.handle_state)
self.p.finished.connect(self.cleanup)
self.p.start("python", ["dummy_script.py"])
def handle_stderr(self):
result = bytes(self.p.readAllStandardError()).decode("utf8")
progress = simple_percent_parser(result)
self.progress.setValue(progress)
def handle_stdout(self):
result = bytes(self.p.readAllStandardOutput()).decode("utf8")
data = extract_vars(result)
self.text.appendPlainText(str(data))
def handle_state(self, state):
self.statusBar().showMessage(STATES[state])
def cleanup(self):
self.p = None
class Form(QPlainTextEdit):
processFinished = Signal()
def __init__(self, fileName, parent=None):
super(Form, self).__init__(parent)
self._fileName = fileName
self.setStyleSheet("font-family: Source Code Pro; font-size: 16px; ")
self._process = QProcess()
self._process.readyReadStandardOutput.connect(self._processStdOut)
self._process.readyReadStandardError.connect(self._processStdErr)
self._process.finished.connect(self._processFinished)
def run(self, args):
self._append("> " + shlex.join(args.args) + "\n")
self._lastPos = 0
self._process.setWorkingDirectory(args.dir)
self._process.setProgram(args.args[0])
self._process.setArguments(args.args[1:])
self._process.start()
@Slot()
def putTextToFile(self):
open(self._fileName, "w").write(self.toPlainText())
def keyPressEvent(self, event):
k = event.key()
if not self.textCursor().position() < self._lastPos:
super().keyPressEvent(event)
if k == Qt.Key_Return or k == Qt.Key_Enter:
self._process.write(
bytes(self.toPlainText()[self._lastPos:], "utf-8"))
self._lastPos = self.textCursor().position()
def _processStdOut(self):
self._append(str(self._process.readAllStandardOutput(), "utf-8"))
def _processStdErr(self):
self._append(str(self._process.readAllStandardError(), "utf-8"))
def _append(self, output):
self.moveCursor(QTextCursor.End)
self.insertPlainText(output)
self.moveCursor(QTextCursor.End)
self._lastPos = self.textCursor().position()
def _processFinished(self, exitCode):
self._append("\nProcess Finished with exit code: " + str(exitCode) +
"\n")
self.processFinished.emit()
class QProcessExecutionManager(ExecutionManager):
"""Class to manage tool instance execution using a PySide2 QProcess."""
def __init__(self, logger, program="", args=None, silent=False, semisilent=False):
"""Class constructor.
Args:
logger (LoggerInterface): a logger instance
program (str): Path to program to run in the subprocess (e.g. julia.exe)
args (list, optional): List of argument for the program (e.g. path to script file)
silent (bool): Whether or not to emit logger msg signals
semisilent (bool): If True, show Process Log messages
"""
super().__init__(logger)
self._program = program
self._args = args if args is not None else []
self._silent = silent # Do not show Event Log nor Process Log messages
self._semisilent = semisilent # Do not show Event Log messages but show Process Log messages
self.process_failed = False
self.process_failed_to_start = False
self.user_stopped = False
self._process = QProcess(self)
self.process_output = None # stdout when running silent
self.process_error = None # stderr when running silent
self._out_chunks = []
self._err_chunks = []
def program(self):
"""Program getter method."""
return self._program
def args(self):
"""Program argument getter method."""
return self._args
# noinspection PyUnresolvedReferences
def start_execution(self, workdir=None):
"""Starts the execution of a command in a QProcess.
Args:
workdir (str, optional): Work directory
"""
if workdir is not None:
self._process.setWorkingDirectory(workdir)
self._process.started.connect(self.process_started)
self._process.finished.connect(self.on_process_finished)
if not self._silent and not self._semisilent: # Loud
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.errorOccurred.connect(self.on_process_error)
self._process.stateChanged.connect(self.on_state_changed)
elif self._semisilent: # semi-silent
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.start(self._program, self._args)
if self._process is not None and not self._process.waitForStarted(msecs=10000):
self.process_failed = True
self.process_failed_to_start = True
self._process.deleteLater()
self._process = None
self.execution_finished.emit(-9998)
def wait_for_process_finished(self, msecs=30000):
"""Wait for subprocess to finish.
Args:
msecs (int): Timeout in milliseconds
Return:
True if process finished successfully, False otherwise
"""
if self._process is None:
return False
if self.process_failed or self.process_failed_to_start:
return False
if not self._process.waitForFinished(msecs):
self.process_failed = True
self._process.close()
self._process = None
return False
return True
@Slot()
def process_started(self):
"""Run when subprocess has started."""
@Slot(int)
def on_state_changed(self, new_state):
"""Runs when QProcess state changes.
Args:
new_state (int): Process state number (``QProcess::ProcessState``)
"""
if new_state == QProcess.Starting:
self._logger.msg.emit("\tStarting program <b>{0}</b>".format(self._program))
arg_str = " ".join(self._args)
self._logger.msg.emit("\tArguments: <b>{0}</b>".format(arg_str))
elif new_state == QProcess.Running:
self._logger.msg_warning.emit("\tExecution in progress...")
elif new_state == QProcess.NotRunning:
# logging.debug("Process is not running")
pass
else:
self._logger.msg_error.emit("Process is in an unspecified state")
logging.error("QProcess unspecified state: %s", new_state)
@Slot(int)
def on_process_error(self, process_error):
"""Runs if there is an error in the running QProcess.
Args:
process_error (int): Process error number (``QProcess::ProcessError``)
"""
if process_error == QProcess.FailedToStart:
self.process_failed = True
self.process_failed_to_start = True
self._logger.msg_error.emit("Process failed to start")
elif process_error == QProcess.Timedout:
self.process_failed = True
self._logger.msg_error.emit("Timed out")
elif process_error == QProcess.Crashed:
self.process_failed = True
if not self.user_stopped:
self._logger.msg_error.emit("Process crashed")
elif process_error == QProcess.WriteError:
self._logger.msg_error.emit("Process WriteError")
elif process_error == QProcess.ReadError:
self._logger.msg_error.emit("Process ReadError")
elif process_error == QProcess.UnknownError:
self._logger.msg_error.emit("Unknown error in process")
else:
self._logger.msg_error.emit("Unspecified error in process: {0}".format(process_error))
self.teardown_process()
def teardown_process(self):
"""Tears down the QProcess in case a QProcess.ProcessError occurred.
Emits execution_finished signal."""
if not self._process:
pass
else:
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
if out is not None:
self._logger.msg_proc.emit(out.strip())
if errout is not None:
self._logger.msg_proc.emit(errout.strip())
self._process.deleteLater()
self._process = None
self.execution_finished.emit(-9998)
def stop_execution(self):
"""See base class."""
self.user_stopped = True
self.process_failed = True
if not self._process:
return
try:
self._process.kill()
if not self._process.waitForFinished(5000):
self._process.finished.emit(-1, -1)
self._process.deleteLater()
except Exception as ex: # pylint: disable=broad-except
self._logger.msg_error.emit("[{0}] exception when terminating process".format(ex))
logging.exception("Exception in closing QProcess: %s", ex)
finally:
self._process = None
@Slot(int, int)
def on_process_finished(self, exit_code, exit_status):
"""Runs when subprocess has finished.
Args:
exit_code (int): Return code from external program (only valid for normal exits)
exit_status (int): Crash or normal exit (``QProcess::ExitStatus``)
"""
if not self._process:
return
if exit_status == QProcess.CrashExit:
if not self._silent:
self._logger.msg_error.emit("\tProcess crashed")
exit_code = -1
elif exit_status == QProcess.NormalExit:
pass
else:
if not self._silent:
self._logger.msg_error.emit("Unknown QProcess exit status [{0}]".format(exit_status))
exit_code = -1
if not exit_code == 0:
self.process_failed = True
if not self.user_stopped:
out = str(self._process.readAllStandardOutput().data(), "utf-8", errors="replace")
errout = str(self._process.readAllStandardError().data(), "utf-8", errors="replace")
if out is not None:
if not self._silent:
self._logger.msg_proc.emit(out.strip())
else:
self.process_output = out.strip()
self.process_error = errout.strip()
else:
self._logger.msg.emit("*** Terminating process ***")
# Delete QProcess
self._process.deleteLater()
self._process = None
self.execution_finished.emit(exit_code)
@Slot()
def on_ready_stdout(self):
"""Emit data from stdout."""
if not self._process:
return
self._process.setReadChannel(QProcess.StandardOutput)
chunk = self._process.readLine().data()
self._out_chunks.append(chunk)
if not chunk.endswith(b"\n"):
return
line = b"".join(self._out_chunks)
line = str(line, "unicode_escape", errors="replace").strip()
self._logger.msg_proc.emit(line)
self._out_chunks.clear()
@Slot()
def on_ready_stderr(self):
"""Emit data from stderr."""
if not self._process:
return
self._process.setReadChannel(QProcess.StandardError)
chunk = self._process.readLine().data()
self._err_chunks.append(chunk)
if not chunk.endswith(b"\n"):
return
line = b"".join(self._err_chunks)
line = str(line, "utf-8", errors="replace").strip()
self._logger.msg_proc_error.emit(line)
self._err_chunks.clear()
class QProcessExecutionManager(ExecutionManager):
"""Class to manage tool instance execution using a PySide2 QProcess."""
def __init__(self, logger, program=None, args=None, silent=False, semisilent=False):
"""Class constructor.
Args:
logger (LoggerInterface): a logger instance
program (str): Path to program to run in the subprocess (e.g. julia.exe)
args (list): List of argument for the program (e.g. path to script file)
silent (bool): Whether or not to emit logger msg signals
"""
super().__init__(logger)
self._program = program
self._args = args
self._silent = silent # Do not show Event Log nor Process Log messages
self._semisilent = semisilent # Do not show Event Log messages but show Process Log messages
self.process_failed = False
self.process_failed_to_start = False
self._user_stopped = False
self._process = QProcess(self)
self.process_output = None # stdout when running silent
self.error_output = None # stderr when running silent
def program(self):
"""Program getter method."""
return self._program
def args(self):
"""Program argument getter method."""
return self._args
# noinspection PyUnresolvedReferences
def start_execution(self, workdir=None):
"""Starts the execution of a command in a QProcess.
Args:
workdir (str): Work directory
"""
if workdir is not None:
self._process.setWorkingDirectory(workdir)
self._process.started.connect(self.process_started)
self._process.finished.connect(self.on_process_finished)
if not self._silent and not self._semisilent: # Loud
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.error.connect(self.on_process_error) # errorOccurred available in Qt 5.6
self._process.stateChanged.connect(self.on_state_changed)
elif self._semisilent: # semi-silent
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.start(self._program, self._args)
if not self._process.waitForStarted(msecs=10000): # This blocks until process starts or timeout happens
self.process_failed = True
self.process_failed_to_start = True
self._process.deleteLater()
self._process = None
self.execution_finished.emit(-9998)
def wait_for_process_finished(self, msecs=30000):
"""Wait for subprocess to finish.
Return:
True if process finished successfully, False otherwise
"""
if not self._process:
return False
if self.process_failed or self.process_failed_to_start:
return False
if not self._process.waitForFinished(msecs):
self.process_failed = True
self._process.close()
self._process = None
return False
return True
@Slot(name="process_started")
def process_started(self):
"""Run when subprocess has started."""
@Slot("QProcess::ProcessState", name="on_state_changed")
def on_state_changed(self, new_state):
"""Runs when QProcess state changes.
Args:
new_state (QProcess::ProcessState): Process state number
"""
if new_state == QProcess.Starting:
self._logger.msg.emit("\tStarting program <b>{0}</b>".format(self._program))
arg_str = " ".join(self._args)
self._logger.msg.emit("\tArguments: <b>{0}</b>".format(arg_str))
elif new_state == QProcess.Running:
self._logger.msg_warning.emit(
"\tExecution is in progress. See Process Log for messages " "(stdout&stderr)"
)
elif new_state == QProcess.NotRunning:
# logging.debug("QProcess is not running")
pass
else:
self._logger.msg_error.emit("Process is in an unspecified state")
logging.error("QProcess unspecified state: %s", new_state)
@Slot("QProcess::ProcessError", name="'on_process_error")
def on_process_error(self, process_error):
"""Run if there is an error in the running QProcess.
Args:
process_error (QProcess::ProcessError): Process error number
"""
if process_error == QProcess.FailedToStart:
self.process_failed = True
self.process_failed_to_start = True
elif process_error == QProcess.Timedout:
self.process_failed = True
self._logger.msg_error.emit("Timed out")
elif process_error == QProcess.Crashed:
self.process_failed = True
if not self._user_stopped:
self._logger.msg_error.emit("Process crashed")
elif process_error == QProcess.WriteError:
self._logger.msg_error.emit("Process WriteError")
elif process_error == QProcess.ReadError:
self._logger.msg_error.emit("Process ReadError")
elif process_error == QProcess.UnknownError:
self._logger.msg_error.emit("Unknown error in process")
else:
self._logger.msg_error.emit("Unspecified error in process: {0}".format(process_error))
def stop_execution(self):
"""See base class."""
self._logger.msg_error.emit("Terminating process")
self._user_stopped = True
self.process_failed = True
if not self._process:
return
try:
self._process.terminate()
except Exception as ex: # pylint: disable=broad-except
self._logger.msg_error.emit("[{0}] exception when terminating process".format(ex))
logging.exception("Exception in closing QProcess: %s", ex)
finally:
self._process.deleteLater()
self._process = None
@Slot(int)
def on_process_finished(self, exit_code):
"""Runs when subprocess has finished.
Args:
exit_code (int): Return code from external program (only valid for normal exits)
"""
# logging.debug("Error that occurred last: {0}".format(self._process.error()))
if not self._process:
return
exit_status = self._process.exitStatus() # Normal or crash exit
if exit_status == QProcess.CrashExit:
if not self._silent:
self._logger.msg_error.emit("\tProcess crashed")
exit_code = -1
elif exit_status == QProcess.NormalExit:
pass
else:
if not self._silent:
self._logger.msg_error.emit("Unknown QProcess exit status [{0}]".format(exit_status))
exit_code = -1
if not exit_code == 0:
self.process_failed = True
if not self._user_stopped:
out = str(self._process.readAllStandardOutput().data(), "utf-8")
errout = str(self._process.readAllStandardError().data(), "utf-8")
if out is not None:
if not self._silent:
self._logger.msg_proc.emit(out.strip())
else:
self.process_output = out.strip()
self.error_output = errout.strip()
else:
self._logger.msg.emit("*** Terminating process ***")
# Delete QProcess
self._process.deleteLater()
self._process = None
self.execution_finished.emit(exit_code)
@Slot(name="on_ready_stdout")
def on_ready_stdout(self):
"""Emit data from stdout."""
if not self._process:
return
out = str(self._process.readAllStandardOutput().data(), "utf-8")
self._logger.msg_proc.emit(out.strip())
@Slot(name="on_ready_stderr")
def on_ready_stderr(self):
"""Emit data from stderr."""
if not self._process:
return
err = str(self._process.readAllStandardError().data(), "utf-8")
self._logger.msg_proc_error.emit(err.strip())
class TclInterfaceHandler(QObject):
# The TCP server IP address
HOST = "127.0.0.1"
# TCP server port
PORT = 9955
commandExecutionFinished = Signal(str)
faultInjectionFinished = Signal(str)
#---------------------------------------------------------------------------------
# @brief Class constructor.
#---------------------------------------------------------------------------------
def __init__(self):
super(TclInterfaceHandler, self).__init__()
self.operation = Operation.NONE
self.frame_address = "00000000"
self.frames = "1"
self.tcpClientSocket = QtNetwork.QTcpSocket()
self.tcpClientSocketStatus = SocketStatus.DISCONNECTED
self.startVivado()
return
#---------------------------------------------------------------------------------
# @brief This function should be called when the application terminates.
#---------------------------------------------------------------------------------
def exit(self):
self.vivadoProcess.kill()
vivado_command_to_kill = "cmd.exe /C Taskkill /IM vivado.exe /F"
process = QProcess()
process.start(vivado_command_to_kill)
process.waitForFinished(5000)
tcp_command_to_kill = "cmd.exe /C netstat -ano | find '0.0.0.0:9955'"
return
#**********************************************************************************
# TCP Client
#**********************************************************************************
#---------------------------------------------------------------------------------
# @brief This function starts the TCP client.
#---------------------------------------------------------------------------------
def clientStart(self):
if self.tcpClientSocketStatus == SocketStatus.CONNECTED:
self.tcpClientSocket.abort()
self.tcpClientSocket.connectToHost(TclInterfaceHandler.HOST,
TclInterfaceHandler.PORT)
self.tcpClientSocket.waitForConnected(3000)
self.tcpClientSocket.readyRead.connect(self.clientReadReady)
self.tcpClientSocket.error.connect(self.clientError)
self.tcpClientSocketStatus = SocketStatus.CONNECTED
return
#---------------------------------------------------------------------------------
# @brief This function is called by the TCP client when it has data ready to be read.
#---------------------------------------------------------------------------------
def clientReadReady(self):
message = QTextStream(self.tcpClientSocket).readAll()
if Operation.READBACK_CAPTURE.name in message:
XtclLog.writeLine(
"================ FPGA readback capture finished ================ ",
XtclCommon.blue)
elif Operation.READBACK_VERIFY.name in message:
XtclLog.writeLine(
"================ FPGA readback verify finished ================ ",
XtclCommon.blue)
#verify = filecmp.cmp(XtclSettings.readbackFilePathGolden, self.readbackFile, False)
elif Operation.READBACK.name in message:
XtclLog.writeLine(
"================ FPGA readback finished ================ ",
XtclCommon.blue)
elif Operation.CONFIGURATION.name in message:
XtclLog.writeLine(
"================ FPGA configuration finished ================ ",
XtclCommon.blue)
elif Operation.FRAMES_READ.name in message or Operation.FAULT_INJECTION_READ.name in message:
XtclLog.writeLine(
"================ FPGA frame readback finished ================ ",
XtclCommon.blue)
elif Operation.FRAMES_WRITE.name in message or Operation.FAULT_INJECTION_WRITE.name in message:
XtclLog.writeLine(
"================ FPGA frame write finished ================ ",
XtclCommon.blue)
elif Operation.READ_FIFO.name in message:
XtclLog.writeLine(
"================ Reading internal FIFO finished ================ ",
XtclCommon.blue)
elif Operation.RESET_FIFO.name in message:
XtclLog.writeLine(
"================ Reseting internal FIFO finished ================ ",
XtclCommon.blue)
elif Operation.READ_HEARTBEAT.name in message:
XtclLog.writeLine(
"================ Reading heartbeat finished ================ ",
XtclCommon.blue)
elif Operation.LOGIC_STATUS.name in message:
XtclLog.writeLine(
"================ Reading active logic status finished ================ ",
XtclCommon.blue)
elif Operation.REGISTER_READ.name in message:
XtclLog.writeLine(
"================ Reading configuration register finished ================ ",
XtclCommon.blue)
elif Operation.REGISTER_WRITE.name in message:
XtclLog.writeLine(
"================ Writing configuration register finished ================ ",
XtclCommon.blue)
else:
XtclLog.writeLine(message, XtclCommon.red)
self.operation = Operation.NONE
if "FAULT_INJECTION" in message:
self.faultInjectionFinished.emit(message)
else:
self.commandExecutionFinished.emit(message)
return
#---------------------------------------------------------------------------------
# @brief Callback function for the client error.
#---------------------------------------------------------------------------------
def clientError(self, socketError):
if socketError == QtNetwork.QAbstractSocket.RemoteHostClosedError:
pass
elif socketError == QtNetwork.QAbstractSocket.HostNotFoundError:
XtclLog.writeLine(
"The host was not found. Please check the host name and port settings",
XtclCommon.red)
elif socketError == QtNetwork.QAbstractSocket.ConnectionRefusedError:
XtclLog.writeLine(
"The connection was refused by the peer. Make sure the "
"server is running, and check that the host name "
"and port settings are correct.", TclInterfaceHandler.red)
else:
XtclLog.writeLine(
"The following error occurred: %s." %
self.tcpSocket.errorString(), XtclCommon.red)
return
#---------------------------------------------------------------------------------
# @brief Send data to the TCP server
#---------------------------------------------------------------------------------
def clientSend(self, data):
bytesArray = bytes(data, 'utf-8')
message = QByteArray.fromRawData(bytesArray)
self.tcpClientSocket.write(message)
self.tcpClientSocket.flush()
return
#**********************************************************************************
# Vivado interface
#**********************************************************************************
#---------------------------------------------------------------------------------
# @brief This function starts the Vivado instanse. Please replace the command
# with the appropriate command for the running platform
#---------------------------------------------------------------------------------
def startVivado(self):
self.operation = Operation.NONE
TclInterfaceHandler.isOperationFinished = False
XtclLog.writeLine(
"================ Starting Vivado process ================ ",
XtclCommon.blue)
XtclLog.write("PLEASE WAIT UNTIL YOU SEE \"# vwait forever\" MESSAGE!",
XtclCommon.red)
command = XtclSettings.vivadoDirectory + "/vivado.bat -nojournal -nolog -mode batch -source jtag_configuration_engine.tcl"
# Create runner
self.vivadoProcess = QProcess()
self.vivadoProcess.readyReadStandardError.connect(self.errorInfo)
self.vivadoProcess.readyReadStandardOutput.connect(
self.readAllStandardOutput)
self.vivadoProcess.finished.connect(self.finished)
self.vivadoProcess.start(command)
return
#---------------------------------------------------------------------------------
# @brief Callback function for the error of the Vivado process
#---------------------------------------------------------------------------------
def errorInfo(self):
info = self.vivadoProcess.readAllStandardError()
info_text = QTextStream(info).readAll()
XtclLog.write(info_text)
return
#---------------------------------------------------------------------------------
# @brief Callback function to rediarect the output of the Vivado process
#---------------------------------------------------------------------------------
def readAllStandardOutput(self):
info = self.vivadoProcess.readAllStandardOutput()
info_text = QTextStream(info).readAll()
XtclLog.write(info_text)
return
#---------------------------------------------------------------------------------
# @brief Callback function for the termination event of the Vivado process
#---------------------------------------------------------------------------------
def finished(self, exitCode, exitStatus):
return
#**********************************************************************************
# Interface commands
#**********************************************************************************
#---------------------------------------------------------------------------------
# @brief This function configures the FPGA
# @param bitstream_filepath: The full path of the bitstream file
# @param mask_filepath: The full path of the mask file
#---------------------------------------------------------------------------------
def configure(self, bitstream_filepath, mask_filepath):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Starting FPGA configuration ================ ",
XtclCommon.blue)
self.clientSend(Operation.CONFIGURATION.name + "%" +
bitstream_filepath + "%" + mask_filepath)
self.operation = Operation.CONFIGURATION
return
#---------------------------------------------------------------------------------
# @brief This function reads-back the FPGA
# @param filename: If provided should be the full path of the file where the readback
# data will be saved. Othervise the readback data is saved in a
# timestamp-based file name inside the "readback-files" of the
# working folder.
#---------------------------------------------------------------------------------
def readback(self, filename=None):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
if filename == None:
readbackFileName = "readback-" + str(
datetime.datetime.now().timestamp()) + ".rbd"
else:
readbackFileName = filename
self.readbackFile = XtclSettings.workingDirectory + '/readback-files/' + readbackFileName
XtclLog.writeLine(
"================ Starting FPGA readback ================ ",
XtclCommon.blue)
XtclLog.writeLine(readbackFileName, XtclCommon.blue)
self.clientSend(Operation.READBACK.name + "%" + self.readbackFile)
self.operation = Operation.READBACK
return
#---------------------------------------------------------------------------------
# @brief This function reads-back the FPGA using the capture mode
# @note The readback data is saved in a timestamp-based file name inside the "readback-capture-files"
# of the working folder.
#---------------------------------------------------------------------------------
def readbackCapture(self):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
readbackFileName = "readbackCapture-" + str(
datetime.datetime.now().timestamp()) + ".rbd"
self.readbackFile = XtclSettings.workingDirectory + '/readback-capture-files/' + readbackFileName
XtclLog.writeLine(
"================ Starting FPGA readback capture ================ ",
XtclCommon.blue)
XtclLog.writeLine(readbackFileName, XtclCommon.blue)
self.clientSend(Operation.READBACK_CAPTURE.name + "%" +
self.readbackFile)
self.operation = Operation.READBACK_CAPTURE
return
#---------------------------------------------------------------------------------
# @brief This function verifies the FPGA.
# @note The FPGA device should be configured before issuing this command.
#---------------------------------------------------------------------------------
def readbackVerify(self):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Starting FPGA readback verify ================ ",
XtclCommon.blue)
self.clientSend(Operation.READBACK_VERIFY.name)
self.operation = Operation.READBACK_VERIFY
return
#---------------------------------------------------------------------------------
# @brief This function reads the internal FIFO of the interface logic.
#---------------------------------------------------------------------------------
def readInternalFifo(self):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Reading internal FIFO ================ ",
XtclCommon.blue)
self.clientSend(Operation.READ_FIFO.name)
self.operation = Operation.READ_FIFO
return
#---------------------------------------------------------------------------------
# @brief This function resets the internal FIFO of the interface logic.
#---------------------------------------------------------------------------------
def resetInternalFifo(self):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Reseting internal FIFO ================ ",
XtclCommon.blue)
self.clientSend(Operation.RESET_FIFO.name)
self.operation = Operation.RESET_FIFO
return
#---------------------------------------------------------------------------------
# @brief This function reads the heartbeat signal the interface logic.
#---------------------------------------------------------------------------------
def readHeartbeat(self):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Reading heartbeat ================ ",
XtclCommon.blue)
self.clientSend(Operation.READ_HEARTBEAT.name)
self.operation = Operation.READ_HEARTBEAT
return
#---------------------------------------------------------------------------------
# @brief This function writes frames in configuration memory of the FPGA
# @param frame_address: The frame address in HEX format (i.e. 00001002).
# @param frame_file: The full path of the file which holds the frame data to be written.
# @param append_dummy_frame: True to append a dummy frame after writing the frame data.
# Set it to false if the file contains also the dummy frame.
# @param is_frame_data_in_hex_format: Trus if the content of the @ref frame_file is in HEX foramt.
# @param frames: The number of frames to be written. This should be the same as the frames
# inside the @ref frame_file (do not include the dummy frame).
# @param reset_fifo: True to reset the FIFO at the end of writing.
# @param is_fault_injection: Trus if this write is for fault injection experiment.
#---------------------------------------------------------------------------------
def writeFrames(self,
frame_address,
frame_file,
append_dummy_frame=True,
is_frame_data_in_hex_format=True,
frames=1,
reset_fifo=False,
is_fault_injection=False):
self.frame_address = str(frame_address)
self.frames = str(frames)
XtclLog.writeLine(
"================ Starting FPGA frame write ================ ",
XtclCommon.blue)
append_dummy_frame_ = str(int(append_dummy_frame))
is_frame_data_in_hex_format_ = str(int(is_frame_data_in_hex_format))
reset_fifo_ = str(int(reset_fifo))
if is_fault_injection == True:
command = Operation.FAULT_INJECTION_WRITE.name + "%" + frame_file + "%" + self.frame_address + "%" + self.frames + "%" + append_dummy_frame_ + "%" + is_frame_data_in_hex_format_ + "%" + reset_fifo_
else:
command = Operation.FRAMES_WRITE.name + "%" + frame_file + "%" + self.frame_address + "%" + self.frames + "%" + append_dummy_frame_ + "%" + is_frame_data_in_hex_format_ + "%" + reset_fifo_
self.clientSend(command)
self.operation = Operation.FRAMES_WRITE
return
#---------------------------------------------------------------------------------
# @brief This function reads frames from the configuration memory of the FPGA
# @param frame_address: The frame address in HEX format (i.e. 00001002).
# @param frames: The number of frames to be written. This should be the same as the frames
# inside the @ref frame_file (do not include the dummy frame).
# @param readback_file: If provided should be the full path of the file where the readback
# data will be saved. Othervise the readback data is saved in a
# timestamp-based file name inside the "readback-frame-files" of the
# working folder.
# @param is_fault_injection: Trus if this write is for fault injection experiment.
#---------------------------------------------------------------------------------
def readFrames(self,
frame_address,
frames=1,
readback_file=None,
is_fault_injection=False):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
self.frame_address = str(frame_address)
self.frames = str(frames)
XtclLog.writeLine(
"================ Starting FPGA frame read ================ ",
XtclCommon.blue)
if readback_file == None:
readbackFileName = "readbackBlock-0x" + str(
self.frame_address) + "-" + str(
datetime.datetime.now().timestamp()) + ".rbd"
self.readbackFile = XtclSettings.workingDirectory + '/readback-frame-files/' + readbackFileName
else:
self.readbackFile = XtclSettings.workingDirectory + '/readback-frame-files/' + readback_file
if is_fault_injection == True:
command = Operation.FAULT_INJECTION_READ.name + "%" + self.readbackFile + "%" + self.frame_address + "%" + self.frames
else:
command = Operation.FRAMES_READ.name + "%" + self.readbackFile + "%" + self.frame_address + "%" + self.frames
self.clientSend(command)
self.operation = Operation.FRAMES_READ
return
#---------------------------------------------------------------------------------
# @brief This function reads the status signal of the user logic
#---------------------------------------------------------------------------------
def readLogicStatus(self):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Reading active logic status ================ ",
XtclCommon.blue)
self.clientSend(Operation.LOGIC_STATUS.name)
self.operation = Operation.LOGIC_STATUS
return
#---------------------------------------------------------------------------------
# @brief This function reads a configuration register
# @param register_address: The frame of the register in 5-bit format (i.e. 01010) of in HEX format (0A).
#---------------------------------------------------------------------------------
def readRegister(self, register_address):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Reading configuration register ================ ",
XtclCommon.blue)
self.register_address = str(register_address)
command = Operation.REGISTER_READ.name + "%" + self.register_address
self.clientSend(command)
self.operation = Operation.REGISTER_READ
return
#---------------------------------------------------------------------------------
# @brief This function writes a configuration register
# @param register_address: The frame of the register in 5-bit format (i.e. 01010) of in HEX format (0A).
# @param register_value: The register value to be written in 32-bit HEX format (i.e. A000029B)
#---------------------------------------------------------------------------------
def writeRegister(self, register_address, register_value):
if self.tcpClientSocketStatus != SocketStatus.CONNECTED:
self.clientStart()
XtclLog.writeLine(
"================ Writing configuration register ================ ",
XtclCommon.blue)
self.register_address = str(register_address)
self.register_value = str(register_value)
command = Operation.REGISTER_WRITE.name + "%" + self.register_address + "%" + self.register_value
self.clientSend(command)
self.operation = Operation.REGISTER_WRITE
return
class QSubProcess(QObject):
"""Class to handle starting, running, and finishing PySide2 QProcesses."""
subprocess_finished_signal = Signal(int, name="subprocess_finished_signal")
def __init__(self, toolbox, program=None, args=None, silent=False):
"""Class constructor.
Args:
toolbox (ToolboxUI): Instance of Main UI class.
program (str): Path to program to run in the subprocess (e.g. julia.exe)
args (list): List of argument for the program (e.g. path to script file)
silent (bool): Whether or not to emit toolbox msg signals
"""
super().__init__()
self._toolbox = toolbox
self._program = program
self._args = args
self._silent = silent
self.process_failed = False
self.process_failed_to_start = False
self._user_stopped = False
self._process = QProcess(self)
self.output = None
def program(self):
"""Program getter method."""
return self._program
def args(self):
"""Program argument getter method."""
return self._args
# noinspection PyUnresolvedReferences
def start_process(self, workdir=None):
"""Start the execution of a command in a QProcess.
Args:
workdir (str): Directory for the script (at least with Julia this is a must)
"""
if workdir is not None:
self._process.setWorkingDirectory(workdir)
self._process.started.connect(self.process_started)
self._process.finished.connect(self.process_finished)
if not self._silent:
self._process.readyReadStandardOutput.connect(self.on_ready_stdout)
self._process.readyReadStandardError.connect(self.on_ready_stderr)
self._process.error.connect(
self.on_process_error) # errorOccurred available in Qt 5.6
self._process.stateChanged.connect(self.on_state_changed)
# self._toolbox.msg.emit("\tStarting program: <b>{0}</b>".format(self._program))
self._process.start(self._program, self._args)
if not self._process.waitForStarted(
msecs=10000
): # This blocks until process starts or timeout happens
self.process_failed = True
self.process_failed_to_start = True
self._process.deleteLater()
self._process = None
self.subprocess_finished_signal.emit(-9998)
def wait_for_finished(self, msecs=30000):
"""Wait for subprocess to finish.
Return:
True if process finished successfully, False otherwise
"""
if not self._process:
return False
if self.process_failed or self.process_failed_to_start:
return False
if not self._process.waitForFinished(msecs):
self.process_failed = True
self._process.close()
self._process = None
return False
return True
@Slot(name="process_started")
def process_started(self):
"""Run when subprocess has started."""
pass
# self._toolbox.msg.emit("\tSubprocess started...")
@Slot("QProcess::ProcessState", name="on_state_changed")
def on_state_changed(self, new_state):
"""Runs when QProcess state changes.
Args:
new_state (QProcess::ProcessState): Process state number
"""
if new_state == QProcess.Starting:
self._toolbox.msg.emit("\tStarting program <b>{0}</b>".format(
self._program))
arg_str = " ".join(self._args)
self._toolbox.msg.emit("\tArguments: <b>{0}</b>".format(arg_str))
elif new_state == QProcess.Running:
self._toolbox.msg_warning.emit(
"\tExecution is in progress. See Process Log for messages "
"(stdout&stderr)")
elif new_state == QProcess.NotRunning:
# logging.debug("QProcess is not running")
pass
else:
self._toolbox.msg_error.emit("Process is in an unspecified state")
logging.error("QProcess unspecified state: {0}".format(new_state))
@Slot("QProcess::ProcessError", name="'on_process_error")
def on_process_error(self, process_error):
"""Run if there is an error in the running QProcess.
Args:
process_error (QProcess::ProcessError): Process error number
"""
if process_error == QProcess.FailedToStart:
# self._toolbox.msg_error.emit("Failed to start")
self.process_failed = True
self.process_failed_to_start = True
elif process_error == QProcess.Timedout:
self.process_failed = True
self._toolbox.msg_error.emit("Timed out")
elif process_error == QProcess.Crashed:
self.process_failed = True
self._toolbox.msg_error.emit("Process crashed")
elif process_error == QProcess.WriteError:
self._toolbox.msg_error.emit("Process WriteError")
elif process_error == QProcess.ReadError:
self._toolbox.msg_error.emit("Process ReadError")
elif process_error == QProcess.UnknownError:
self._toolbox.msg_error.emit("Unknown error in process")
else:
self._toolbox.msg_error.emit(
"Unspecified error in process: {0}".format(process_error))
def terminate_process(self):
"""Shutdown simulation in a QProcess."""
self._toolbox.msg_error.emit("<br/>Terminating process")
# logging.debug("Terminating QProcess nr.{0}. ProcessState:{1} and ProcessError:{2}"
# .format(self._process.processId(), self._process.state(), self._process.error()))
self._user_stopped = True
self.process_failed = True
try:
self._process.close()
except Exception as ex:
self._toolbox.msg_error.emit(
"[{0}] exception when terminating process".format(ex))
logging.exception("Exception in closing QProcess: {}".format(ex))
@Slot(int, name="process_finished")
def process_finished(self, exit_code):
"""Run when subprocess has finished.
Args:
exit_code (int): Return code from external program (only valid for normal exits)
"""
# logging.debug("Error that occurred last: {0}".format(self._process.error()))
exit_status = self._process.exitStatus() # Normal or crash exit
if exit_status == QProcess.CrashExit:
if not self._silent:
self._toolbox.msg_error.emit("\tProcess crashed")
exit_code = -1
elif exit_status == QProcess.NormalExit:
if not self._silent:
self._toolbox.msg.emit("\tProcess finished")
else:
if not self._silent:
self._toolbox.msg_error.emit(
"Unknown QProcess exit status [{0}]".format(exit_status))
exit_code = -1
if not exit_code == 0:
self.process_failed = True
if not self._user_stopped:
out = str(self._process.readAllStandardOutput().data(), "utf-8")
if out is not None:
if not self._silent:
self._toolbox.msg_proc.emit(out.strip())
else:
self.output = out.strip()
else:
self._toolbox.msg.emit("*** Terminating process ***")
# Delete QProcess
self._process.deleteLater()
self._process = None
self.subprocess_finished_signal.emit(exit_code)
@Slot(name="on_ready_stdout")
def on_ready_stdout(self):
"""Emit data from stdout."""
out = str(self._process.readAllStandardOutput().data(), "utf-8")
self._toolbox.msg_proc.emit(out.strip())
@Slot(name="on_ready_stderr")
def on_ready_stderr(self):
"""Emit data from stderr."""
err = str(self._process.readAllStandardError().data(), "utf-8")
self._toolbox.msg_proc_error.emit(err.strip())
def handle_error(self, process: QProcess):
output: QByteArray = process.readAllStandardError()
message = output.data().decode('utf-8').strip()
self.output.append(message)
class SerialControl(QObject):
sig_keyseq_pressed = Signal(str)
sig_CPU_comout = Signal(int)
sig_CPU_comin = Signal(str)
sig_port_change = Signal(str)
sig_button_pressed = Signal(int)
sig_terminal_open = Signal(bool)
sig_firmware_update = Signal(str)
def __init__(self, cpu, monitor_frame, terminal_frame, usb_frame,
statusbar, config, sig_update, config_file_path):
QObject.__init__(self)
self.cpu = cpu
self.monitor = monitor_frame
self.terminal = terminal_frame
self.statusbar = statusbar
self.config = config
self.ser_port = None
self.sig_update = sig_update
self.fd_thread = None
self.fwth = None
self.monitor_frame = monitor_frame
self.usb_frame = usb_frame
self.config_file_path = config_file_path
# Connect signal
self.sig_keyseq_pressed.connect(self.on_key_pressed)
self.sig_CPU_comout.connect(self.on_comout)
self.sig_CPU_comin.connect(self.on_comin)
self.sig_port_change.connect(self.on_port_change)
self.sig_button_pressed.connect(self.on_button_dispatch)
self.sig_terminal_open.connect(self.on_terminal_open)
self.sig_firmware_update.connect(self.on_firmware_update)
self.monitor_frame.sig_keyseq_pressed = self.sig_keyseq_pressed
self.monitor_frame.sig_button_pressed = self.sig_button_pressed
self.cpu.sig_CPU_comout = self.sig_CPU_comout
self.cpu.sig_CPU_comin = self.sig_CPU_comin
self.usb_frame.usb_combo.sig_port_change = self.sig_port_change
self.usb_frame.sig_button_pressed = self.sig_button_pressed
self.usb_frame.sig_firmware_update = self.sig_firmware_update
self.terminal.sig_terminal_open = self.sig_terminal_open
# Disable fw flash button if udr2 binary is nor present
self.udr2 = f"cli/udr2-{sys.platform}" if sys.platform != "win32" else "cli\\udr2-win32.exe"
if not os.path.isfile(self.udr2):
self.usb_frame.firmware_btn.setEnabled(False)
self.init_serial()
def init_serial(self, do_refresh=True):
is_thread = InitSerialThread(self)
is_thread.update_combo = self.usb_frame.usb_combo.set_ports
is_thread.do_refresh = do_refresh
is_thread.start()
def init_OK(self):
self.statusbar.sig_temp_message.emit("Serial port Initialized")
#
# Signal Handling
#
@Slot(str)
def on_key_pressed(self, key):
if self.cpu.rx is None and len(key) == 1:
self.cpu.rx = key
self.monitor_frame.serial_in.setText(key)
@Slot(str)
def on_comin(self, char):
"""Char is handled by CPU, we free the slot"""
self.monitor_frame.serial_in.setText(" ")
@Slot(int)
def on_comout(self, byte):
"""Append the char to the console"""
self.monitor_frame.append_serial_out(byte)
@Slot(str)
def on_port_change(self, port):
self.config.set("serial", "port", port)
if self.ser_port:
self.ser_port.port = port
with open(self.config_file_path, 'w') as configfile:
self.config.write(configfile)
self.init_serial(False)
@Slot(int)
def on_button_dispatch(self, btn_nbr):
if btn_nbr == 0:
self.to_digirule()
elif btn_nbr == 1:
self.from_digirule()
elif btn_nbr == 2:
self.init_serial()
elif btn_nbr == 3:
self.on_clear_button()
@Slot(bool)
def on_terminal_open(self, is_open):
if is_open:
# Terminal window is open : create the terminal serial thread
self.statusbar.sig_temp_message.emit("open terminal thread")
self.terminal.serth = SerialThread(self) # Start serial thread
self.terminal.serth.start()
else:
# Terminal window is closed : quit the terminal serial thread
if self.terminal.serth:
self.terminal.serth.running = False
sleep(0.5)
self.terminal.serth = None
def on_clear_button(self):
self.monitor_frame.clear() # Clear the serial in/out content
self.cpu.rx = None
self.cpu.tx = None
def to_digirule(self):
if self.ser_port is None:
self.statusbar.sig_temp_message.emit(
"Error : No serial port configured")
else:
dump = ram2hex(self.cpu.ram)
self.statusbar.sig_temp_message.emit("Dumpimg memory on port " +
self.ser_port.port)
try:
self.ser_port.open()
except serial.serialutil.SerialException as ex:
self.statusbar.sig_temp_message.emit(ex.strerror)
else:
for line in dump.splitlines():
self.ser_port.write(line.encode("utf-8"))
sleep(0.1)
sleep(2)
self.ser_port.close()
self.statusbar.sig_temp_message.emit("Memory sent")
def from_digirule(self):
"""Launch receive sequence in background"""
if self.ser_port:
self.statusbar.sig_temp_message.emit(
"Waiting to receive Digirule on " + self.ser_port.port)
self.fd_thread = FromDigiruleThread(self)
self.fd_thread.start()
else:
self.init_serial()
@Slot(str)
def on_firmware_update(self, filepath):
if self.ser_port:
self.proc = QProcess(self)
self.proc.readyReadStandardOutput.connect(self.stdoutReady)
self.proc.readyReadStandardError.connect(self.stderrReady)
if sys.platform == "win32":
command = f'{self.udr2} --program {self.ser_port.port} < {filepath}'
self.usb_frame.out.write(
"Firmware update started, please wait ")
# displays running dots on windows to pretend it is not stalled
self.bullshitTimer = QTimer()
self.bullshitTimer.timeout.connect(self.stdoutBullshit)
self.bullshitTimer.start(1000)
self.proc.setProcessChannelMode(QProcess.MergedChannels)
self.proc.start('cmd.exe', ['/c', command])
else:
command = f'{self.udr2} --program {self.ser_port.port} < "{filepath}"'
self.bullshitTimer = None
self.proc.start('bash', ['-c', command])
# print(command)
def stdoutBullshit(self):
self.usb_frame.out.write(".")
def stdoutReady(self):
if self.bullshitTimer:
self.usb_frame.out.write("\n")
self.bullshitTimer.stop()
text = str(self.proc.readAllStandardOutput())
self.usb_frame.out.write(eval(text).decode('iso8859-1'))
def stderrReady(self):
text = str(self.proc.readAllStandardError())
self.usb_frame.out.write(eval(text).decode('iso8859-1'))
class MainWindow(QWidget):
def __init__(self):
QWidget.__init__(self)
self.label = QLabel()
self.label.setText('Drag and drop nk file')
self.button = QPushButton('Disable')
self.progressBar = QProgressBar()
self.progressBar.setRange(0, 100)
self.text_area = QPlainTextEdit()
self.text_area.setReadOnly(True)
master_lay = QVBoxLayout()
master_lay.addWidget(self.label)
master_lay.addWidget(self.progressBar)
master_lay.addWidget(self.text_area)
master_lay.addWidget(self.button)
self.process = None
self.setAcceptDrops(True)
self.setLayout(master_lay)
self.setMinimumSize(500, 250)
self.setWindowTitle('Nuke Disable')
self.button.clicked.connect(self.execute)
def message(self, s):
self.text_area.appendPlainText(s)
def dragEnterEvent(self, e):
if e.mimeData().hasUrls():
e.acceptProposedAction()
def dropEvent(self, e):
for url in e.mimeData().urls():
file_name = url.toLocalFile()
self.label.setText(file_name)
print("Dropped file: " + file_name)
def execute(self):
wrk_file_path = self.label.text()
if wrk_file_path.endswith('nk'):
if self.process is None:
self.process = QProcess()
self.process.readyReadStandardOutput.connect(
self.handle_stdout)
self.process.readyReadStandardError.connect(self.handle_stderr)
self.process.stateChanged.connect(self.handle_state)
self.process.finished.connect(self.process_finished)
self.process.start('"{}" -it {} {}'.format(
nuke_path, python_file, wrk_file_path))
else:
msgBox = QMessageBox()
msgBox.setText("Please Drag a nuke file")
msgBox.exec_()
def handle_stderr(self):
data = self.process.readAllStandardError()
stderr = bytes(data).decode()
# Extract progress if it is in the data.
progress = simple_percent_parser(stderr)
if progress:
self.progressBar.setStyleSheet(
"QProgressBar::chunk"
"{"
"background-color: QLinearGradient( x1: 0, y1: 0, x2: 1, y2: 0,"
"stop: 0 #FF4E50,"
"stop: 1 #F9D423);"
"}")
self.progressBar.setValue(progress)
self.message(stderr)
def handle_stdout(self):
data = self.process.readAllStandardOutput()
stdout = bytes(data).decode()
self.message(stdout)
def handle_state(self, state):
states = {
QProcess.NotRunning: 'Not running',
QProcess.Starting: 'Starting',
QProcess.Running: 'Running',
}
state_name = states[state]
self.message("State changed: {}".format(state_name))
def process_finished(self):
self.message("Process finished.")
self.process = None
class CliWidget(QDialog):
def __init__(self, title: str, program: str, args: List[str],
commands: List[str]):
super().__init__()
self.setWindowTitle(title)
self.program = program
self.args = args
self.layout = QGridLayout()
self.output = QTextEdit()
self.output.acceptRichText = True
self.input = QLineEdit()
self.completer = QCompleter(commands, self)
self.completer.setCaseSensitivity(Qt.CaseInsensitive)
self.input.setCompleter(self.completer)
self.input.setFocus()
self.process = QProcess()
self.process.setProgram(self.program)
self.process.setCurrentReadChannel(0)
# noinspection PyUnresolvedReferences
self.process.readyReadStandardError.connect(self.handle_error)
# noinspection PyUnresolvedReferences
self.process.readyReadStandardOutput.connect(self.handle_output)
self.layout.addWidget(self.output)
self.layout.addWidget(self.input)
self.setLayout(self.layout)
self.connect(self.input, SIGNAL("returnPressed(void)"),
self.execute_user_command)
self.connect(self.completer, SIGNAL("activated(const QString&)"),
self.input.clear, Qt.QueuedConnection)
def execute_user_command(self):
cmd = str(self.input.text())
self.run_command(cmd)
def run_command(self, cmd: str):
log.info('run_command', program=self.program, args=self.args, cmd=cmd)
self.output.append(f'> {cmd}\n')
self.input.clear()
self.process.kill()
args = list(self.args)
args.append(cmd)
self.process.setArguments(args)
self.process.start()
def handle_error(self):
output: QByteArray = self.process.readAllStandardError()
message = output.data().decode('utf-8').strip()
self.output.append(message)
def handle_output(self):
output: QByteArray = self.process.readAllStandardOutput()
message = output.data().decode('utf-8').strip()
if message.startswith('{') or message.startswith('['):
formatter = HtmlFormatter()
formatter.noclasses = True
formatter.linenos = False
formatter.nobackground = True
message = highlight(message, JsonLexer(), formatter)
self.output.insertHtml(message)
else:
self.output.append(message)
# This is just for generating the command lists in constants
# commands = None
# if '== Blockchain ==' in message:
# commands = self.parse_bitcoin_cli_commands(message)
# elif 'lncli [global options] command [command options]' in message:
# commands = self.parse_lncli_commands(message)
# if commands is not None:
# log.debug('commands', commands=commands)
max_scroll = self.output.verticalScrollBar().maximum()
self.output.verticalScrollBar().setValue(max_scroll)
def parse_bitcoin_cli_commands(self, message: str):
log.debug('parse_bitcoin_cli_commands')
commands = []
for line in message.split(sep='\n'):
line = line.strip()
if not line or line.startswith('=='):
continue
command = line.split()[0]
command = command.strip()
commands.append(command)
return commands
def parse_lncli_commands(self, message: str):
log.debug('parse_lncli_commands')
at_commands = False
commands = []
for line in message.split(sep='\n'):
line = line.strip()
if not at_commands:
if 'COMMANDS:' in line:
at_commands = True
log.debug('commands line', line=line)
continue
elif 'GLOBAL OPTIONS' in line:
return commands
elif line.endswith(':') or not line:
continue
command = line.split()[0]
command = command.strip().replace(',', '')
commands.append(command)
return commands
def show(self):
self.showMaximized()
self.input.setFocus()
self.run_command('help')
class VidConvertWindow(QWidget, Ui_Form):
"""this is the main class for video converter"""
def __init__(self):
super().__init__()
self.setupUi(self)
# class variable declarations
self.available_formats = []
self.selected_files = []
self.process_argument = ""
self.current_file_duration = None
self.duration_re = re.compile(r'Duration: ([0-9:.]+)')
self.time_re = re.compile(r'time=\s*([0-9:.]+) ')
self.process = QProcess()
self.kill_process = QProcess()
self.file_picker = QFileDialog(self)
self.output_dir = ""
self.current_file_idx = 0
self.conversion_started = False
self.output_folder_picker = QFileDialog(self)
self.output_folder_picker.setFileMode(QFileDialog.DirectoryOnly)
# listview & models
self.file_list_model = QStandardItemModel(self.listViewFiles)
self.listViewFiles.setModel(self.file_list_model)
self.type_list_model = QStandardItemModel(self.listViewTypes)
self.listViewTypes.setModel(self.type_list_model)
# signals & slots
self.process.readyReadStandardError.connect(self.read_output)
self.file_list_model.itemChanged.connect(self.update_selected_files)
self.btnStop.clicked.connect(self.stop_convertion)
self.btnAdd.clicked.connect(self.add_files)
self.btnConvert.clicked.connect(self.start_convertion)
# call post_init
self.post_init()
def post_init(self):
"""runs after init"""
self.btnConvert.setIcon(QPixmap('./icons/start.ico'))
self.btnAdd.setIcon(QPixmap('./icons/file.ico'))
self.btnStop.setIcon(QPixmap('./icons/stop.ico'))
self.progressBarCurrent.setValue(0)
self.progressBarTotal.setValue(0)
self.setWindowTitle("Simple Video Converter")
self.setGeometry(100, 100, 640, 480)
for filetype in self.available_formats:
self.add_item2model(filetype, self.type_list_model)
def add_item2model(self, filename: str, model: QStandardItemModel):
"""sample listview code"""
list_item = QStandardItem(filename)
list_item.setCheckable(True)
list_item.setEditable(False)
list_item.setSelectable(False)
model.appendRow(list_item)
def update_selected_files(self, item):
"""selects the checked items"""
if item.checkState() == Qt.CheckState.Checked and item.text(
) not in self.selected_files:
self.selected_files.append(item.text())
else:
self.selected_files.remove(item.text())
if DEBUG:
print(self.selected_files)
def add_files(self):
"""opens file picker for choosing files"""
self.file_picker.setFileMode(QFileDialog.ExistingFiles)
self.file_picker.setNameFilter("Videos (*.mp4 *.mkv *.mov)")
self.file_picker.setViewMode(QFileDialog.Detail)
if self.file_picker.exec_():
files_selected = self.file_picker.selectedFiles()
for file in files_selected:
self.add_item2model(file, self.file_list_model)
if DEBUG:
print(file)
def get_file_name(self, idx: int, suffix: str):
"""returns the filename from path (static)"""
return Path(self.selected_files[idx]).with_suffix(suffix).name
def start_convertion(self):
"""implement conversion task"""
# check if output dir is already defined else get outdir
if not self.conversion_started:
self.get_output_dir()
self.progressBarTotal.setMaximum(len(self.selected_files))
self.btnConvert.setEnabled(False)
self.conversion_started = True
# setting up arguments
current_file_name = self.selected_files[self.current_file_idx]
current_outfile_name = Path(self.output_dir).joinpath(
self.get_file_name(self.current_file_idx, ".avi"))
self.process_argument = " -i {} {}".format(current_file_name,
current_outfile_name)
# create a process everytime it's called
# NOTE: this is a local instance of the process so it changes after every call
process = QProcess()
process.readyReadStandardError.connect(
lambda: self.parse_output(process))
process.finished.connect(self.recursion_handler)
process.started.connect(lambda: self.ref_process(process))
process.start("ffmpeg", self.process_argument.split())
def ref_process(self, process):
self.process = process
def get_output_dir(self):
""" get the output directory """
if self.output_folder_picker.exec_():
self.output_dir = self.output_folder_picker.selectedFiles()[0]
def recursion_handler(self):
"""controls the multiple process iterations"""
# prepare next file for xonversion
self.current_file_idx += 1
self.current_file_duration = None
self.progressBarCurrent.setValue(0)
self.progressBarTotal.setValue(self.current_file_idx)
# check if the number of files converted exceed total number
if self.current_file_idx == len(self.selected_files):
print("conversion complete!")
self.btnConvert.setEnabled(True)
self.conversion_started = False
return
# if everything okay, start conversion again
self.start_convertion()
@staticmethod
def parse_time(time):
"""parsing time format to second"""
_t = list(map(float, time.split(":")))
return _t[0] * 3600 + _t[1] * 60 + _t[2]
def parse_output(self, process):
""" parses current progress """
# update progress
data = process.readAllStandardError().data().decode("utf-8")
if self.current_file_duration is None:
match = self.duration_re.search(data)
if match:
self.current_file_duration = self.parse_time(match.group(1))
else:
match = self.time_re.search(data)
if match:
current_progress = self.parse_time(match.group(1))
self.progressBarCurrent.setValue(
min((current_progress / self.current_file_duration) * 100,
100))
# deprecated
def read_output(self):
"""process the output of ffmpeg"""
data = self.process.readAllStandardError().data().decode("utf-8")
if self.current_file_duration is None:
match = self.duration_re.search(data)
if match:
self.current_file_duration = self.parse_time(match.group(1))
else:
match = self.time_re.search(data)
if match:
current_progress = self.parse_time(match.group(1))
self.progressBarCurrent.setValue(
min((current_progress / self.current_file_duration) * 100,
100))
def stop_convertion(self):
"""stop running coversion task"""
# print("Not implemented")
print("testing stop")
# pid = self.process.processId()
if sys.platform == 'linux':
self.kill_process.start("pkill", "ffmpeg".split())
self.process.terminate()
class Standard(object):
"""Action
download ---|--> install --|--> == run
v v \\
cancel cancel uninstall
"""
# common
name = "" # 应用名称
desc = "" # 应用描述
icon = "" # 应用图标
app_url = "" # 应用地址
versions = {} # 应用版本及下载地址
# installed
install_version = "" # 选择安装的版本
app_folder = "" # 应用安装路径
py_ = "" # 解释器 (G 中实时获取)
entry_ = "" # 启动文件 (build.json实时获取)
requirement_ = "" # 缺失依赖
def __init__(self, parent: QWidget, **kwargs):
self.cls_name = self.__class__.__name__
self.parent = parent
self.action = kwargs.get("action", Actions.DOWNLOAD)
self.thread_pool = QThreadPool()
self.check(**kwargs)
self.div: AppDiv
self.div_init()
self.count = 0
self.start_time = 0
self.cancel = False
self.process = QProcess(self.parent)
self.process.readyReadStandardOutput.connect(self.on_readoutput)
self.process.readyReadStandardError.connect(self.on_readerror)
def check(self, **kwargs):
"""检查已下载应用参数"""
self.app_folder = kwargs.get("app_folder") # 应用安装路径
self.install_version = kwargs.get('install_version') # 应用安装路径
self.app_info(**kwargs)
def app_info(self, **kwargs):
raise NotImplementedError
@property
def pack_name(self):
"""安装后id"""
return self.cls_name + "_" + self.install_version.replace('.', '_')
@property
def ui_name(self):
"""ui中id"""
return self.cls_name + "_app"
def _transfer(self, widget, value=None):
if widget == 'bar':
self.div.job.progressbar_signal.emit(value)
elif widget == 'msg':
self.div.job.msg_signal.emit(value)
elif widget == 'switch':
self.div.job.switch_signal.emit()
elif widget == 'action':
self.div.job.action_signal.emit(value)
def before_handle(self):
self.action = Actions.CANCEL
self.cancel = False
self.div.action.setText(Actions.to_zn(self.action))
self._transfer('switch')
def _tip(self, msg):
self.parent.mainwindow.job.msg_box_signal.emit({"msg": str(msg)})
def div_init(self):
self.div = AppDiv(self.parent)
self.div.icon.setStyleSheet(f"image: url({self.icon});")
self.div.name.setText(self.name)
self.div.action.setText(Actions.to_zn(self.action))
self.div.action.clicked.connect(self.action_handler)
if self.action == Actions.DOWNLOAD:
for i in self.versions.keys():
act = QAction(i, self.parent)
setattr(self.div,
f"act_{'_'.join([j for j in i if i.isalnum()])}", act)
self.div.menu.addAction(act)
self.div.menu.triggered[QAction].connect(
self.version_action_triggered)
self.div.desc.setText(self.desc)
self.div.desc.url = self.app_url # 可点击
setattr(self.parent, self.ui_name, self)
self.parent.apps_layout.addLayout(self.div.layout)
elif self.action == Actions.RUN:
act_uninstall = QAction(Actions.to_zn(Actions.UNINSTALL),
self.parent)
act_setting = QAction("解释器", self.parent)
act_upgrade = QAction(Actions.to_zn(Actions.UPGRADE), self.parent)
setattr(self.div, f"act_uninstall", act_uninstall)
setattr(self.div, f"act_setting", act_setting)
setattr(self.div, f"act_upgrade", act_upgrade)
self.div.menu.addAction(act_setting)
self.div.menu.addAction(act_upgrade)
self.div.menu.addAction(act_uninstall)
self.div.menu.triggered[QAction].connect(
self.menu_action_triggered)
self.div.desc.setText(self.install_version)
setattr(self.parent, self.pack_name, self)
self.parent.installed_layout.addLayout(self.div.layout)
def version_action_triggered(self, q):
"""点击版本号直接下载"""
if self.action == Actions.CANCEL:
return
self.install_version = q.text()
self.action_handler()
def menu_action_triggered(self, q):
"""卸载/更新处理"""
if self.action == Actions.CANCEL:
return
act = q.text()
if act == "解释器":
self.act_setting_slot()
elif Actions.to_en(act) == Actions.UNINSTALL:
self.uninstall_handler()
elif Actions.to_en(act) == Actions.UPGRADE:
self.upgrade_handler()
def act_setting_slot(self):
self.py_manage = PyManageWidget(self.parent, self.pack_name)
self.py_manage.show()
@before_download
def download_handler(self):
"""
版本号
下载目录
"""
url = self.versions[self.install_version]
postfix = os.path.splitext(url)[-1] # .zip
self.app_folder = os.path.join(G.config.install_path, self.pack_name)
file_temp = self.app_folder + postfix # 压缩文件路径
## 文件续传
if os.path.exists(file_temp):
local_file = os.path.getsize(file_temp)
headers = {'Range': 'bytes=%d-' % local_file}
mode = 'ab'
else:
local_file = 0
headers = {}
mode = 'wb'
# download
self._transfer("msg", "获取中...")
try:
response = requests.get(url, stream=True, headers=headers)
response.raise_for_status()
except Exception as e:
return False
content_size = float(response.headers.get('Content-Length', 0))
self._transfer("bar", dict(range=[0, content_size]))
# save
with open(file_temp, mode) as file:
chunk_size = 1024
for data in response.iter_content(chunk_size=chunk_size):
if self.cancel or G.pool_done:
return False
file.write(data) ##
self.count += 1
##show
current = chunk_size * self.count + local_file
if content_size:
self._transfer("bar", dict(value=current))
speed = format_size(current / (time.time() - self.start_time))
self._transfer(
"msg",
f"{round(current / 1024, 2)}KB/{round(content_size / 1024, 2) or '-'}KB | {speed}/s"
)
extract(file_temp) # 解压
return True
def on_download_callback(self, res):
self._transfer('switch')
if res is True:
data = {
"cls_name": self.cls_name,
"install_version": self.install_version,
"action": Actions.RUN,
"app_folder": self.app_folder,
"py_": ""
}
G.config.installed_apps.update({self.pack_name: data})
G.config.to_file()
self.div.add_installed_layout(data)
elif res is False:
pass
self.action = Actions.DOWNLOAD
self._transfer("action", Actions.to_zn(self.action))
def get_build(self):
"""
:return: path 路径
"""
path = find_file(self.app_folder, 'build.json')
if path:
try:
with open(path[0], 'r') as f:
build = json.load(f)
entry = find_file(self.app_folder, build['entry'])[0]
requirement = find_file(self.app_folder,
build['requirement'])[0]
except KeyError:
raise Exception('请确保build.json中含有entry和requirement')
except IndexError:
raise Exception("未找到entry文件或requirement文件")
except json.JSONDecodeError:
raise Exception("build.json 有错误")
return entry, requirement
else:
raise Exception("未找到文件build.json")
@before_install
def install_handler(self):
"""解析 build.json"""
for line in self.requirement_:
line = line.strip().replace('==', '>=')
cmd_ = [self.py_, "-m", "pip", "install", line
] + G.config.get_pypi_source()
if self.cancel or G.pool_done:
return False
self.process.start(" ".join(cmd_))
self.process.waitForFinished()
return True
def on_readoutput(self):
output = self.process.readAllStandardOutput().data().decode()
self._transfer("msg", output.replace('\n', ''))
def on_readerror(self):
error = self.process.readAllStandardError().data().decode()
self._tip(error)
def on_install_callback(self, res):
self._transfer('switch')
self.action = Actions.RUN
self._transfer("action", Actions.to_zn(self.action))
def run_handler(self):
self.py_ = G.config.installed_apps[self.pack_name].get('py_')
if not self.py_:
QMessageBox.warning(self.parent, "提示", "未选择Python解释器")
self.act_setting_slot()
return
if not os.path.exists(self.py_) or not os.path.isfile(self.py_):
QMessageBox.warning(self.parent, "提示", f"{self.py_} 不存在")
return
try:
self.entry_, requirement_ = self.get_build()
except Exception as e:
QMessageBox.warning(self.parent, "提示", str(e))
return
##检测依赖
p = QProcess()
p.start(' '.join(([self.py_, "-m", 'pip', "freeze"])))
p.waitForFinished()
out = p.readAllStandardOutput().data().decode()
output = out.splitlines()
with open(requirement_, 'r') as f:
requirement = f.read().splitlines()
dissatisfy, version_less = diff_pip(output, requirement)
if dissatisfy:
msgbox = QMessageBox(self.parent)
msgbox.setWindowTitle("缺少依赖")
msgbox.setText("\n".join(dissatisfy[:15]) + "\n...")
yes = msgbox.addButton('立即安装', QMessageBox.AcceptRole)
no = msgbox.addButton('稍后', QMessageBox.RejectRole)
msgbox.setDefaultButton(yes)
reply = msgbox.exec_()
if reply == QMessageBox.AcceptRole:
self.requirement_ = dissatisfy
self.install_handler()
return
# run
TipDialog("正在启动...")
cmd = ' '.join([self.py_, self.entry_])
QProcess().startDetached(cmd)
def upgrade_handler(self):
pass
@before_uninstall
def uninstall_handler(self):
try:
if os.path.exists(self.app_folder) and os.path.isdir(
self.app_folder):
shutil.rmtree(self.app_folder)
except Exception as e:
self._tip({"msg": str(e)})
finally:
for name, attr in self.div.__dict__.items():
if name not in self.div.not_widget:
attr.deleteLater()
G.config.installed_apps.pop(self.pack_name, None)
G.config.to_file()
def cancel_handler(self):
self.cancel = True
self._transfer("msg", "Releasing...")
def action_handler(self):
if self.action == Actions.DOWNLOAD:
self.download_handler()
# self.git_download_handler()
elif self.action == Actions.CANCEL:
self.cancel_handler()
elif self.action == Actions.RUN:
self.run_handler()
class Importer(ProjectItem):
def __init__(self, name, description, mappings, x, y, toolbox, project, logger, cancel_on_error=True):
"""Importer class.
Args:
name (str): Project item name
description (str): Project item description
mappings (list): List where each element contains two dicts (path dict and mapping dict)
x (float): Initial icon scene X coordinate
y (float): Initial icon scene Y coordinate
toolbox (ToolboxUI): QMainWindow instance
project (SpineToolboxProject): the project this item belongs to
logger (LoggerInterface): a logger instance
cancel_on_error (bool): if True the item's execution will stop on import error
"""
super().__init__(name, description, x, y, project, logger)
# Make logs subdirectory for this item
self._toolbox = toolbox
self.logs_dir = os.path.join(self.data_dir, "logs")
try:
create_dir(self.logs_dir)
except OSError:
self._logger.msg_error.emit(f"[OSError] Creating directory {self.logs_dir} failed. Check permissions.")
# Variables for saving selections when item is (de)activated
if not mappings:
mappings = list()
# convert table_types and table_row_types keys to int since json always has strings as keys.
for _, mapping in mappings:
table_types = mapping.get("table_types", {})
mapping["table_types"] = {
table_name: {int(col): t for col, t in col_types.items()}
for table_name, col_types in table_types.items()
}
table_row_types = mapping.get("table_row_types", {})
mapping["table_row_types"] = {
table_name: {int(row): t for row, t in row_types.items()}
for table_name, row_types in table_row_types.items()
}
# Convert serialized paths to absolute in mappings
self.settings = self.deserialize_mappings(mappings, self._project.project_dir)
# self.settings is now a dictionary, where elements have the absolute path as the key and the mapping as value
self.cancel_on_error = cancel_on_error
self.resources_from_downstream = list()
self.file_model = QStandardItemModel()
self.importer_process = None
self.all_files = [] # All source files
self.unchecked_files = [] # Unchecked source files
# connector class
self._preview_widget = {} # Key is the filepath, value is the ImportPreviewWindow instance
@staticmethod
def item_type():
"""See base class."""
return "Importer"
@staticmethod
def category():
"""See base class."""
return "Importers"
@Slot(QStandardItem, name="_handle_file_model_item_changed")
def _handle_file_model_item_changed(self, item):
if item.checkState() == Qt.Checked:
self.unchecked_files.remove(item.text())
self._logger.msg.emit(f"<b>{self.name}:</b> Source file '{item.text()}' will be processed at execution.")
elif item.checkState() != Qt.Checked:
self.unchecked_files.append(item.text())
self._logger.msg.emit(
f"<b>{self.name}:</b> Source file '{item.text()}' will *NOT* be processed at execution."
)
def make_signal_handler_dict(self):
"""Returns a dictionary of all shared signals and their handlers.
This is to enable simpler connecting and disconnecting."""
s = super().make_signal_handler_dict()
s[self._properties_ui.toolButton_open_dir.clicked] = lambda checked=False: self.open_directory()
s[self._properties_ui.pushButton_import_editor.clicked] = self._handle_import_editor_clicked
s[self._properties_ui.treeView_files.doubleClicked] = self._handle_files_double_clicked
return s
def activate(self):
"""Restores selections, cancel on error checkbox and connects signals."""
self._properties_ui.cancel_on_error_checkBox.setCheckState(Qt.Checked if self.cancel_on_error else Qt.Unchecked)
self.restore_selections()
super().connect_signals()
def deactivate(self):
"""Saves selections and disconnects signals."""
self.save_selections()
if not super().disconnect_signals():
logging.error("Item %s deactivation failed.", self.name)
return False
return True
def restore_selections(self):
"""Restores selections into shared widgets when this project item is selected."""
self._properties_ui.label_name.setText(self.name)
self._properties_ui.treeView_files.setModel(self.file_model)
self.file_model.itemChanged.connect(self._handle_file_model_item_changed)
def save_selections(self):
"""Saves selections in shared widgets for this project item into instance variables."""
self._properties_ui.treeView_files.setModel(None)
self.file_model.itemChanged.disconnect(self._handle_file_model_item_changed)
def update_name_label(self):
"""Update Importer properties tab name label. Used only when renaming project items."""
self._properties_ui.label_name.setText(self.name)
@Slot(bool, name="_handle_import_editor_clicked")
def _handle_import_editor_clicked(self, checked=False):
"""Opens Import editor for the file selected in list view."""
index = self._properties_ui.treeView_files.currentIndex()
self.open_import_editor(index)
@Slot("QModelIndex", name="_handle_files_double_clicked")
def _handle_files_double_clicked(self, index):
"""Opens Import editor for the double clicked index."""
self.open_import_editor(index)
def open_import_editor(self, index):
"""Opens Import editor for the given index."""
importee = index.data()
if importee is None:
self._logger.msg_error.emit("Please select a source file from the list first.")
return
if not os.path.exists(importee):
self._logger.msg_error.emit(f"Invalid path: {importee}")
return
# Raise current form for the selected file if any
preview_widget = self._preview_widget.get(importee, None)
if preview_widget:
if preview_widget.windowState() & Qt.WindowMinimized:
# Remove minimized status and restore window with the previous state (maximized/normal state)
preview_widget.setWindowState(preview_widget.windowState() & ~Qt.WindowMinimized | Qt.WindowActive)
preview_widget.activateWindow()
else:
preview_widget.raise_()
return
# Create a new form for the selected file
settings = self.get_settings(importee)
# Try and get connector from settings
source_type = settings.get("source_type", None)
if source_type is not None:
connector = _CONNECTOR_NAME_TO_CLASS[source_type]
else:
# Ask user
connector = self.get_connector(importee)
if not connector:
# Aborted by the user
return
self._logger.msg.emit(f"Opening Import editor for file: {importee}")
preview_widget = self._preview_widget[importee] = ImportPreviewWindow(
self, importee, connector, settings, self._toolbox
)
preview_widget.settings_updated.connect(lambda s, importee=importee: self.save_settings(s, importee))
preview_widget.connection_failed.connect(lambda m, importee=importee: self._connection_failed(m, importee))
preview_widget.destroyed.connect(lambda o=None, importee=importee: self._preview_destroyed(importee))
preview_widget.start_ui()
def get_connector(self, importee):
"""Shows a QDialog to select a connector for the given source file.
Mimics similar routine in `spine_io.widgets.import_widget.ImportDialog`
Args:
importee (str): Path to file acting as an importee
Returns:
Asynchronous data reader class for the given importee
"""
connector_list = [CSVConnector, ExcelConnector, GdxConnector] # add others as needed
connector_names = [c.DISPLAY_NAME for c in connector_list]
dialog = QDialog(self._toolbox)
dialog.setLayout(QVBoxLayout())
connector_list_wg = QListWidget()
connector_list_wg.addItems(connector_names)
# Set current item in `connector_list_wg` based on file extension
_filename, file_extension = os.path.splitext(importee)
if file_extension.lower().startswith(".xls"):
row = connector_list.index(ExcelConnector)
elif file_extension.lower() == ".csv":
row = connector_list.index(CSVConnector)
elif file_extension.lower() == ".gdx":
row = connector_list.index(GdxConnector)
else:
row = None
if row:
connector_list_wg.setCurrentRow(row)
button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
button_box.button(QDialogButtonBox.Ok).clicked.connect(dialog.accept)
button_box.button(QDialogButtonBox.Cancel).clicked.connect(dialog.reject)
connector_list_wg.doubleClicked.connect(dialog.accept)
dialog.layout().addWidget(connector_list_wg)
dialog.layout().addWidget(button_box)
_dirname, filename = os.path.split(importee)
dialog.setWindowTitle("Select connector for '{}'".format(filename))
answer = dialog.exec_()
if answer:
row = connector_list_wg.currentIndex().row()
return connector_list[row]
def select_connector_type(self, index):
"""Opens dialog to select connector type for the given index."""
importee = index.data()
connector = self.get_connector(importee)
if not connector:
# Aborted by the user
return
settings = self.get_settings(importee)
settings["source_type"] = connector.__name__
def _connection_failed(self, msg, importee):
self._logger.msg.emit(msg)
preview_widget = self._preview_widget.pop(importee, None)
if preview_widget:
preview_widget.close()
def get_settings(self, importee):
"""Returns the mapping dictionary for the file in given path.
Args:
importee (str): Absolute path to a file, whose mapping is queried
Returns:
dict: Mapping dictionary for the requested importee or an empty dict if not found
"""
importee_settings = None
for p in self.settings:
if p == importee:
importee_settings = self.settings[p]
if not importee_settings:
return {}
return importee_settings
def save_settings(self, settings, importee):
"""Updates an existing mapping or adds a new mapping
(settings) after closing the import preview window.
Args:
settings (dict): Updated mapping (settings) dictionary
importee (str): Absolute path to a file, whose mapping has been updated
"""
if importee in self.settings.keys():
self.settings[importee].update(settings)
else:
self.settings[importee] = settings
def _preview_destroyed(self, importee):
"""Destroys preview widget instance for the given importee.
Args:
importee (str): Absolute path to a file, whose preview widget is destroyed
"""
self._preview_widget.pop(importee, None)
def update_file_model(self, items):
"""Adds given list of items to the file model. If None or
an empty list is given, the model is cleared.
Args:
items (set): Set of absolute file paths
"""
self.all_files = items
self.file_model.clear()
self.file_model.setHorizontalHeaderItem(0, QStandardItem("Source files")) # Add header
if items is not None:
for item in items:
qitem = QStandardItem(item)
qitem.setEditable(False)
qitem.setCheckable(True)
if item in self.unchecked_files:
qitem.setCheckState(Qt.Unchecked)
else:
qitem.setCheckState(Qt.Checked)
qitem.setData(QFileIconProvider().icon(QFileInfo(item)), Qt.DecorationRole)
self.file_model.appendRow(qitem)
def _run_importer_program(self, args):
"""Starts and runs the importer program in a separate process.
Args:
args (list): List of arguments for the importer program
"""
self.importer_process = QProcess()
self.importer_process.readyReadStandardOutput.connect(self._log_importer_process_stdout)
self.importer_process.readyReadStandardError.connect(self._log_importer_process_stderr)
self.importer_process.finished.connect(self.importer_process.deleteLater)
program_path = os.path.abspath(importer_program.__file__)
self.importer_process.start(sys.executable, [program_path])
self.importer_process.waitForStarted()
self.importer_process.write(json.dumps(args).encode("utf-8"))
self.importer_process.write(b'\n')
self.importer_process.closeWriteChannel()
if self.importer_process.state() == QProcess.Running:
loop = QEventLoop()
self.importer_process.finished.connect(loop.quit)
loop.exec_()
return self.importer_process.exitCode()
@Slot()
def _log_importer_process_stdout(self):
output = str(self.importer_process.readAllStandardOutput().data(), "utf-8").strip()
self._logger.msg.emit(f"<b>{self.name}</b>: {output}")
@Slot()
def _log_importer_process_stderr(self):
output = str(self.importer_process.readAllStandardError().data(), "utf-8").strip()
self._logger.msg_error.emit(f"<b>{self.name}</b>: {output}")
def execute_backward(self, resources):
"""See base class."""
self.resources_from_downstream = resources.copy()
return True
def execute_forward(self, resources):
"""See base class."""
args = [
[f for f in self.all_files if f not in self.unchecked_files],
self.settings,
[r.url for r in self.resources_from_downstream if r.type_ == "database"],
self.logs_dir,
self._properties_ui.cancel_on_error_checkBox.isChecked(),
]
exit_code = self._run_importer_program(args)
return exit_code == 0
def stop_execution(self):
"""Stops executing this Importer."""
super().stop_execution()
if not self.importer_process:
return
self.importer_process.kill()
def _do_handle_dag_changed(self, resources):
"""See base class."""
file_list = [r.path for r in resources if r.type_ == "file" and not r.metadata.get("future")]
self._notify_if_duplicate_file_paths(file_list)
self.update_file_model(set(file_list))
if not file_list:
self.add_notification(
"This Importer does not have any input data. "
"Connect Data Connections to this Importer to use their data as input."
)
def item_dict(self):
"""Returns a dictionary corresponding to this item."""
d = super().item_dict()
# Serialize mappings before saving
d["mappings"] = self.serialize_mappings(self.settings, self._project.project_dir)
d["cancel_on_error"] = self._properties_ui.cancel_on_error_checkBox.isChecked()
return d
def notify_destination(self, source_item):
"""See base class."""
if source_item.item_type() == "Data Connection":
self._logger.msg.emit(
"Link established. You can define mappings on data from "
f"<b>{source_item.name}</b> using item <b>{self.name}</b>."
)
elif source_item.item_type() == "Data Store":
# Does this type of link do anything?
self._logger.msg.emit("Link established.")
else:
super().notify_destination(source_item)
@staticmethod
def default_name_prefix():
"""see base class"""
return "Importer"
def tear_down(self):
"""Closes all preview widgets."""
for widget in self._preview_widget.values():
widget.close()
def _notify_if_duplicate_file_paths(self, file_list):
"""Adds a notification if file_list contains duplicate entries."""
file_counter = Counter(file_list)
duplicates = list()
for file_name, count in file_counter.items():
if count > 1:
duplicates.append(file_name)
if duplicates:
self.add_notification("Duplicate input files from upstream items:<br>{}".format("<br>".join(duplicates)))
@staticmethod
def upgrade_from_no_version_to_version_1(item_name, old_item_dict, old_project_dir):
"""Converts mappings to a list, where each element contains two dictionaries,
the serialized path dictionary and the mapping dictionary for the file in that
path."""
new_importer = dict(old_item_dict)
mappings = new_importer.get("mappings", {})
list_of_mappings = list()
paths = list(mappings.keys())
for path in paths:
mapping = mappings[path]
if "source_type" in mapping and mapping["source_type"] == "CSVConnector":
_fix_csv_connector_settings(mapping)
new_path = serialize_path(path, old_project_dir)
if new_path["relative"]:
new_path["path"] = os.path.join(".spinetoolbox", "items", new_path["path"])
list_of_mappings.append([new_path, mapping])
new_importer["mappings"] = list_of_mappings
return new_importer
@staticmethod
def deserialize_mappings(mappings, project_path):
"""Returns mapping settings as dict with absolute paths as keys.
Args:
mappings (list): List where each element contains two dictionaries (path dict and mapping dict)
project_path (str): Path to project directory
Returns:
dict: Dictionary with absolute paths as keys and mapping settings as values
"""
abs_path_mappings = {}
for source, mapping in mappings:
abs_path_mappings[deserialize_path(source, project_path)] = mapping
return abs_path_mappings
@staticmethod
def serialize_mappings(mappings, project_path):
"""Returns a list of mappings, where each element contains two dictionaries,
the 'serialized' path in a dictionary and the mapping dictionary.
Args:
mappings (dict): Dictionary with mapping specifications
project_path (str): Path to project directory
Returns:
list: List where each element contains two dictionaries.
"""
serialized_mappings = list()
for source, mapping in mappings.items(): # mappings is a dict with absolute paths as keys and mapping as values
serialized_mappings.append([serialize_path(source, project_path), mapping])
return serialized_mappings
