def at_exit(workers_pool: QThreadPool, logging_worker: LoggingWorker,
name: str):
"""
The instance deconstruction handler is meant to be used with weakref.finalize() conforming with the requirement
to have no reference to the target object (so it doesn't contain any instance reference and also is decorated as
'staticmethod')
"""
# Wait forever for all the jobs to complete. Currently, we cannot abort them gracefully
workers_pool.waitForDone(msecs=-1)
logging_worker.stopped.set(
) # post the event in the logging worker to inform it...
logging_worker.thread.wait() # ...and wait for it to exit, too
module_logger.debug(f"destroyed {name}")
class QThreadData(QThread):
def __init__(self, target: list):
QThread.__init__(self)
self.target = target
self.pool = QThreadPool()
def __del__(self):
self.wait()
def run(self):
for item in self.target:
logger.debug('start ' + item.__name__)
worker = Worker(item)
self.pool.start(worker)
self.pool.waitForDone()
logger.debug('thread finished')
class View(QWidget):
previous = Signal()
next = Signal()
def __init__(self, window):
super(View, self).__init__(window)
self.setFocusPolicy(Qt.StrongFocus)
self.shiftKey = False
self.ctrlKey = False
self.lastMousePos = QPoint()
self.lastTabletPos = QPoint()
self.mode = 'add'
self.maskOnly = False
self.refresh = QTimer(self)
self.refresh.setSingleShot(True)
self.refresh.timeout.connect(self.repaint)
self.addCursor = makeCursor('images/cursor-add.png',
QColor.fromRgbF(0.5, 0.5, 1.0))
self.delCursor = makeCursor('images/cursor-del.png',
QColor.fromRgbF(1.0, 0.5, 0.5))
self.setCursor(self.addCursor)
self.imagefile = None
self.maskfile = None
self.image = QImage()
self.mask = QImage(self.image.size(), QImage.Format_RGB32)
self.mask.fill(Qt.black)
self.changed = False
self.update()
self.path = list()
self.load_threads = QThreadPool()
self.load_threads.setMaxThreadCount(4)
def load(self, filename):
self.load_threads.start(LoadTask(self, filename))
def save(self):
if self.maskfile and self.changed:
self.load_threads.waitForDone()
if self.maskfile and self.changed:
bitmap = self.mask.createMaskFromColor(
QColor.fromRgbF(1.0, 0.0, 1.0).rgb())
bitmap.save(str(self.maskfile), "PNG")
self.changed = False
def update(self):
widgetRatio = self.width() / self.height()
aspectRatio = self.image.width() / max(1, self.image.height())
if aspectRatio >= widgetRatio:
width = self.width()
height = self.width() / aspectRatio
else:
width = self.height() * aspectRatio
height = self.height()
self.rc = QRectF((self.width() - width) / 2.0,
(self.height() - height) / 2.0, width, height)
self.repaint()
def resizeEvent(self, event):
self.update()
def paintEvent(self, event):
p = QPainter(self.mask)
for (mode, p1, p2, weight) in self.path:
if mode == 'add':
p.setPen(
QPen(QColor.fromRgbF(1.0, 0.0, 1.0), (weight * 10.0)**2,
Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin))
else:
p.setPen(
QPen(QColor.fromRgbF(0.0, 0.0, 0.0), (weight * 10.0)**2,
Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin))
p.drawLine(realCoords(p1, self.mask.rect()),
realCoords(p2, self.mask.rect()))
self.changed = True
self.path = list()
p.end()
p = QPainter(self)
p.setCompositionMode(QPainter.CompositionMode_SourceOver)
if not self.maskOnly:
p.drawImage(self.rc, self.image)
p.setCompositionMode(QPainter.CompositionMode_Plus)
p.drawImage(self.rc, self.mask)
p.end()
def closeEvent(self, event):
self.refresh.stop()
event.accept()
def enterEvent(self, event):
self.setFocus(Qt.OtherFocusReason)
def keyPressEvent(self, event):
k = event.key()
if k == Qt.Key_Shift:
self.shiftKey = True
if k == Qt.Key_Control:
self.ctrlKey = True
if k == Qt.Key_Space:
self.maskOnly = not self.maskOnly
self.repaint()
def keyReleaseEvent(self, event):
k = event.key()
mod = event.modifiers()
if k == Qt.Key_Shift:
self.shiftKey = False
if k == Qt.Key_Control:
self.ctrlKey = False
def mousePressEvent(self, event):
x = event.x()
y = event.y()
self.lastMousePos = event.pos()
if event.button() == Qt.ExtraButton1:
if self.mode == 'add':
self.mode = 'del'
self.setCursor(self.delCursor)
else:
self.mode = 'add'
self.setCursor(self.addCursor)
elif event.button() == Qt.ExtraButton2:
self.maskOnly = not self.maskOnly
self.repaint()
elif event.button() == Qt.ExtraButton3:
self.previous.emit()
elif event.button() == Qt.ExtraButton4:
self.next.emit()
def mouseMoveEvent(self, event):
x = event.x()
y = event.y()
dx = x - self.lastMousePos.x()
dy = y - self.lastMousePos.y()
self.lastMousePos = event.pos()
# if event.buttons() & Qt.LeftButton:
# elif event.buttons() & Qt.MiddleButton:
# elif event.buttons() & Qt.RightButton:
def wheelEvent(self, event):
dx = event.angleDelta().x() / 8
dy = event.angleDelta().y() / 8
# self.cameraZoom.emit(dy / 15)
def tabletEvent(self, event):
if event.device() == QTabletEvent.Stylus and event.pointerType(
) == QTabletEvent.Pen:
if event.type() == QEvent.TabletPress:
self.tabletPressEvent(event)
elif event.type() == QEvent.TabletRelease:
self.tabletReleaseEvent(event)
elif event.type() == QEvent.TabletMove:
if event.pressure() > 0.0:
self.tabletMoveEvent(event)
else:
print('tabletEvent', event.device(), event.type(),
event.pointerType())
else:
print('tabletEvent', event.device(), event.type(),
event.pointerType())
def tabletPressEvent(self, event):
if event.buttons() & Qt.LeftButton:
self.lastTabletPos = normalizeCoords(event.posF(), self.rc)
if event.buttons() & Qt.MiddleButton:
if self.mode == 'add':
self.mode = 'del'
self.setCursor(self.delCursor)
else:
self.mode = 'add'
self.setCursor(self.addCursor)
if event.buttons() & Qt.RightButton:
self.next.emit()
def tabletReleaseEvent(self, event):
self.lastTabletPos = normalizeCoords(event.posF(), self.rc)
def tabletMoveEvent(self, event):
self.path.append((self.mode, self.lastTabletPos,
normalizeCoords(event.posF(),
self.rc), event.pressure()))
self.lastTabletPos = normalizeCoords(event.posF(), self.rc)
if not self.refresh.isActive():
self.refresh.start(50)
class ViolinGUI(QMainWindow):
"""Main Window Widget for ViolinGUI."""
style = {
'title': 'QLabel {font-size: 18pt; font-weight: 600}',
'header': 'QLabel {font-size: 12pt; font-weight: 520}',
'label': 'QLabel {font-size: 10pt}',
'button': 'QPushButton {font-size: 10pt}',
'run button': 'QPushButton {font-size: 18pt; font-weight: 600}',
'line edit': 'QLineEdit {font-size: 10pt}',
'checkbox': 'QCheckBox {font-size: 10pt}',
'drop down': 'QComboBox {font-size: 10pt}'
}
def __init__(self) -> None:
"""ViolinGUI Constructor. Defines all aspects of the GUI."""
# ## Setup section
# Inherits from QMainWindow
super().__init__()
self.rootdir = get_project_root()
# QMainWindow basic properties
self.setWindowTitle("SCOUTS - Violins")
self.setWindowIcon(
QIcon(
os.path.abspath(os.path.join(self.rootdir, 'src',
'scouts.ico'))))
# Creates QWidget as QMainWindow's central widget
self.page = QWidget(self)
self.setCentralWidget(self.page)
# Miscellaneous initialization values
self.threadpool = QThreadPool() # Threadpool for workers
self.population_df = None # DataFrame of whole population (raw data)
self.summary_df = None # DataFrame indicating which SCOUTS output corresponds to which rule
self.summary_path = None # path to all DataFrames generated by SCOUTS
self.main_layout = QVBoxLayout(self.page)
# Title section
# Title
self.title = QLabel(self.page)
self.title.setText('SCOUTS - Violins')
self.title.setStyleSheet(self.style['title'])
self.title.adjustSize()
self.main_layout.addWidget(self.title)
# ## Input section
# Input header
self.input_header = QLabel(self.page)
self.input_header.setText('Load data')
self.input_header.setStyleSheet(self.style['header'])
self.input_header.adjustSize()
self.main_layout.addWidget(self.input_header)
# Input/Output frame
self.input_frame = QFrame(self.page)
self.input_frame.setFrameShape(QFrame.StyledPanel)
self.input_frame.setLayout(QFormLayout())
self.main_layout.addWidget(self.input_frame)
# Raw data button
self.input_button = QPushButton(self.page)
self.input_button.setStyleSheet(self.style['button'])
self.set_icon(self.input_button, 'x-office-spreadsheet')
self.input_button.setObjectName('file')
self.input_button.setText(' Load raw data file')
self.input_button.setToolTip(
'Load raw data file (the file given to SCOUTS as the input file)')
self.input_button.clicked.connect(self.get_path)
# SCOUTS results button
self.output_button = QPushButton(self.page)
self.output_button.setStyleSheet(self.style['button'])
self.set_icon(self.output_button, 'folder')
self.output_button.setObjectName('folder')
self.output_button.setText(' Load SCOUTS results')
self.output_button.setToolTip(
'Load data from SCOUTS analysis '
'(the folder given to SCOUTS as the output folder)')
self.output_button.clicked.connect(self.get_path)
# Add widgets above to input frame Layout
self.input_frame.layout().addRow(self.input_button)
self.input_frame.layout().addRow(self.output_button)
# ## Samples section
# Samples header
self.samples_header = QLabel(self.page)
self.samples_header.setText('Select sample names')
self.samples_header.setStyleSheet(self.style['header'])
self.samples_header.adjustSize()
self.main_layout.addWidget(self.samples_header)
# Samples frame
self.samples_frame = QFrame(self.page)
self.samples_frame.setFrameShape(QFrame.StyledPanel)
self.samples_frame.setLayout(QFormLayout())
self.main_layout.addWidget(self.samples_frame)
# Samples label
self.samples_label = QLabel(self.page)
self.samples_label.setText(
'Write sample names delimited by semicolons below.\nEx: Control;Treat_01;Pac-03'
)
self.samples_label.setStyleSheet(self.style['label'])
# Sample names line edit
self.sample_names = QLineEdit(self.page)
self.sample_names.setStyleSheet(self.style['line edit'])
# Add widgets above to samples frame Layout
self.samples_frame.layout().addRow(self.samples_label)
self.samples_frame.layout().addRow(self.sample_names)
# ## Analysis section
# Analysis header
self.analysis_header = QLabel(self.page)
self.analysis_header.setText('Plot parameters')
self.analysis_header.setStyleSheet(self.style['header'])
self.analysis_header.adjustSize()
self.main_layout.addWidget(self.analysis_header)
# Analysis frame
self.analysis_frame = QFrame(self.page)
self.analysis_frame.setFrameShape(QFrame.StyledPanel)
self.analysis_frame.setLayout(QFormLayout())
self.main_layout.addWidget(self.analysis_frame)
# Analysis labels
self.analysis_label_01 = QLabel(self.page)
self.analysis_label_01.setText('Compare')
self.analysis_label_01.setStyleSheet(self.style['label'])
self.analysis_label_02 = QLabel(self.page)
self.analysis_label_02.setText('with')
self.analysis_label_02.setStyleSheet(self.style['label'])
self.analysis_label_03 = QLabel(self.page)
self.analysis_label_03.setText('for marker')
self.analysis_label_03.setStyleSheet(self.style['label'])
self.analysis_label_04 = QLabel(self.page)
self.analysis_label_04.setText('Outlier type')
self.analysis_label_04.setStyleSheet(self.style['label'])
# Analysis drop-down boxes
self.drop_down_01 = QComboBox(self.page)
self.drop_down_01.addItems([
'whole population', 'non-outliers', 'top outliers',
'bottom outliers', 'none'
])
self.drop_down_01.setStyleSheet(self.style['drop down'])
self.drop_down_01.setCurrentIndex(2)
self.drop_down_02 = QComboBox(self.page)
self.drop_down_02.addItems([
'whole population', 'non-outliers', 'top outliers',
'bottom outliers', 'none'
])
self.drop_down_02.setStyleSheet(self.style['drop down'])
self.drop_down_02.setCurrentIndex(0)
self.drop_down_03 = QComboBox(self.page)
self.drop_down_03.setStyleSheet(self.style['drop down'])
self.drop_down_04 = QComboBox(self.page)
self.drop_down_04.addItems(['OutS', 'OutR'])
self.drop_down_04.setStyleSheet(self.style['drop down'])
# Add widgets above to samples frame Layout
self.analysis_frame.layout().addRow(self.analysis_label_01,
self.drop_down_01)
self.analysis_frame.layout().addRow(self.analysis_label_02,
self.drop_down_02)
self.analysis_frame.layout().addRow(self.analysis_label_03,
self.drop_down_03)
self.analysis_frame.layout().addRow(self.analysis_label_04,
self.drop_down_04)
self.legend_checkbox = QCheckBox(self.page)
self.legend_checkbox.setText('Add legend to the plot')
self.legend_checkbox.setStyleSheet(self.style['checkbox'])
self.main_layout.addWidget(self.legend_checkbox)
# Plot button (stand-alone)
self.plot_button = QPushButton(self.page)
self.set_icon(self.plot_button, 'system-run')
self.plot_button.setText(' Plot')
self.plot_button.setToolTip(
'Plot data after loading the input data and selecting parameters')
self.plot_button.setStyleSheet(self.style['run button'])
self.plot_button.setEnabled(False)
self.plot_button.clicked.connect(self.run_plot)
self.main_layout.addWidget(self.plot_button)
# ## Secondary Window
# This is used to plot the violins only
self.secondary_window = QMainWindow(self)
self.secondary_window.resize(720, 720)
self.dynamic_canvas = DynamicCanvas(self.secondary_window,
width=6,
height=6,
dpi=120)
self.secondary_window.setCentralWidget(self.dynamic_canvas)
self.secondary_window.addToolBar(
NavBar(self.dynamic_canvas, self.secondary_window))
def set_icon(self, widget: QWidget, icon: str) -> None:
"""Associates an icon to a widget."""
i = QIcon()
i.addPixmap(
QPixmap(
os.path.abspath(
os.path.join(self.rootdir, 'src', 'default_icons',
f'{icon}.svg'))))
widget.setIcon(QIcon.fromTheme(icon, i))
def get_path(self) -> None:
"""Opens a dialog box and loads the corresponding data into memory, depending on the caller widget."""
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
query = None
func = None
if self.sender().objectName() == 'file':
query, _ = QFileDialog.getOpenFileName(self,
"Select file",
"",
"All Files (*)",
options=options)
func = self.load_scouts_input_data
elif self.sender().objectName() == 'folder':
query = QFileDialog.getExistingDirectory(self,
"Select Directory",
options=options)
func = self.load_scouts_results
if query:
self.load_data(query, func)
def load_data(self, query: str, func: Callable) -> None:
"""Loads input data into memory, while displaying a loading message as a separate worker."""
worker = Worker(func=func, query=query)
message = self.loading_message()
worker.signals.started.connect(message.show)
worker.signals.started.connect(self.page.setDisabled)
worker.signals.error.connect(self.generic_error_message)
worker.signals.error.connect(message.destroy)
worker.signals.failed.connect(self.plot_button.setDisabled)
worker.signals.success.connect(message.destroy)
worker.signals.success.connect(self.enable_plot)
worker.signals.finished.connect(self.page.setEnabled)
self.threadpool.start(worker)
def loading_message(self) -> QDialog:
"""Returns the message box to be displayed while the user waits for the input data to load."""
message = QDialog(self)
message.setWindowTitle('Loading')
message.resize(300, 50)
label = QLabel('loading DataFrame into memory...', message)
label.setStyleSheet(self.style['label'])
label.adjustSize()
label.setAlignment(Qt.AlignCenter)
label.move(int((message.width() - label.width()) / 2),
int((message.height() - label.height()) / 2))
return message
def load_scouts_input_data(self, query: str) -> None:
"""Loads data for whole population prior to SCOUTS into memory (used for plotting the whole population)."""
try:
self.population_df = pd.read_excel(query, index_col=0)
except XLRDError:
self.population_df = pd.read_csv(query, index_col=0)
self.drop_down_03.clear()
self.drop_down_03.addItems(list(self.population_df.columns))
self.drop_down_03.setCurrentIndex(0)
def load_scouts_results(self, query: str) -> None:
"""Loads the SCOUTS summary file into memory, in order to dynamically locate SCOUTS output files later when
the user chooses which data to plot."""
self.summary_df = pd.read_excel(os.path.join(query, 'summary.xlsx'),
index_col=None)
self.summary_path = query
def enable_plot(self) -> None:
"""Enables plot button if all necessary files are placed in memory."""
if isinstance(self.summary_df, pd.DataFrame) and isinstance(
self.population_df, pd.DataFrame):
self.plot_button.setEnabled(True)
def run_plot(self) -> None:
"""Sets and starts the plot worker."""
worker = Worker(func=self.plot)
worker.signals.error.connect(self.generic_error_message)
worker.signals.success.connect(self.secondary_window.show)
self.threadpool.start(worker)
def plot(self) -> None:
"""Logic for plotting data based on user selection of populations, markers, etc."""
# Clear figure currently on plot
self.dynamic_canvas.axes.cla()
# Initialize values and get parameters from GUI
columns = ['sample', 'marker', 'population', 'expression']
samples = self.parse_sample_names()
pop_01 = self.drop_down_01.currentText()
pop_02 = self.drop_down_02.currentText()
pops_to_analyse = [pop_01, pop_02]
marker = self.drop_down_03.currentText()
cutoff_from_reference = True if self.drop_down_04.currentText(
) == 'OutR' else False
violin_df = pd.DataFrame(columns=columns)
# Start fetching data from files
# Whole population
for pop in pops_to_analyse:
if pop == 'whole population':
for partial_df in self.yield_violin_values(
df=self.population_df,
population='whole population',
samples=samples,
marker=marker,
columns=columns):
violin_df = violin_df.append(partial_df)
# Other comparisons
elif pop != 'none':
for file_number in self.yield_selected_file_numbers(
summary_df=self.summary_df,
population=pop,
cutoff_from_reference=cutoff_from_reference,
marker=marker):
df_path = os.path.join(self.summary_path, 'data',
f'{"%04d" % file_number}.')
try:
sample_df = pd.read_excel(df_path + 'xlsx',
index_col=0)
except FileNotFoundError:
sample_df = pd.read_csv(df_path + 'csv', index_col=0)
if not sample_df.empty:
for partial_df in self.yield_violin_values(
df=sample_df,
population=pop,
samples=samples,
marker=marker,
columns=columns):
violin_df = violin_df.append(partial_df)
# Plot data
pops_to_analyse = [p for p in pops_to_analyse if p != 'none']
violin_df = violin_df[violin_df['marker'] == marker]
for pop in pops_to_analyse:
pop_subset = violin_df.loc[violin_df['population'] == pop]
for sample in samples:
sample_subset = pop_subset.loc[pop_subset['sample'] == sample]
sat = 1.0 - samples.index(sample) / (len(samples) + 1)
self.dynamic_canvas.update_figure(
subset_by_sample=sample_subset,
pop=pop,
sat=sat,
samples=samples)
# Draw plotted data on canvas
if self.legend_checkbox.isChecked():
self.dynamic_canvas.add_legend()
self.dynamic_canvas.axes.set_title(
f'{marker} expression - {self.drop_down_04.currentText()}')
self.dynamic_canvas.fig.canvas.draw()
def parse_sample_names(self) -> List[str]:
"""Parse sample names from the QLineEdit Widget."""
return self.sample_names.text().split(';')
def generic_error_message(self, error: Tuple[Exception, str]) -> None:
"""Error message box used to display any error message (including traceback) for any uncaught errors."""
name, trace = error
QMessageBox.critical(
self, 'An error occurred!',
f"Error: {str(name)}\n\nfull traceback:\n{trace}")
def closeEvent(self, event: QEvent) -> None:
"""Defines the message box for when the user wants to quit ViolinGUI."""
title = 'Quit Application'
mes = "Are you sure you want to quit?"
reply = QMessageBox.question(self, title, mes,
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No)
if reply == QMessageBox.Yes:
self.setEnabled(False)
self.threadpool.waitForDone()
event.accept()
else:
event.ignore()
@staticmethod
def yield_violin_values(df: pd.DataFrame, population: str,
samples: List[str], marker: str,
columns: List[str]) -> pd.DataFrame:
"""Returns a DataFrame from expression values, along with information of sample, marker and population. This
DataFrame is appended to the violin plot DataFrame in order to simplify plotting the violins afterwards."""
for sample in samples:
series = df.loc[df.index.str.contains(sample)].loc[:, marker]
yield pd.DataFrame(
{
'sample': sample,
'marker': marker,
'population': population,
'expression': series
},
columns=columns)
@staticmethod
def yield_selected_file_numbers(
summary_df: pd.DataFrame, population: str,
cutoff_from_reference: bool,
marker: str) -> Generator[pd.DataFrame, None, None]:
"""Yields file numbers from DataFrames resulting from SCOUTS analysis. DataFrames are yielded based on
global values, i.e. the comparisons the user wants to perform."""
cutoff = 'sample'
if cutoff_from_reference is True:
cutoff = 'reference'
for index, (file_number, cutoff_from, reference, outliers_for,
category) in summary_df.iterrows():
if cutoff_from == cutoff and outliers_for == marker and category == population:
yield file_number
class TabDisplays(QTabWidget):
def __init__(self, parent=None):
super(TabDisplays, self).__init__(parent)
# Initialize logging
logging_conf_file = os.path.join(os.path.dirname(__file__),
'cfg/aecgviewer_aecg_logging.conf')
logging.config.fileConfig(logging_conf_file)
self.logger = logging.getLogger(__name__)
self.studyindex_info = aecg.tools.indexer.StudyInfo()
self.validator = QWidget()
self.studyinfo = QWidget()
self.statistics = QWidget()
self.waveforms = QWidget()
self.waveforms.setAccessibleName("Waveforms")
self.scatterplot = QWidget()
self.histogram = QWidget()
self.trends = QWidget()
self.xmlviewer = QWidget()
self.xml_display = QTextEdit(self.xmlviewer)
self.options = QWidget()
self.aecg_display_area = QScrollArea()
self.cbECGLayout = QComboBox()
self.aecg_display = EcgDisplayWidget(self.aecg_display_area)
self.aecg_display_area.setWidget(self.aecg_display)
self.addTab(self.validator, "Study information")
self.addTab(self.waveforms, "Waveforms")
self.addTab(self.xmlviewer, "XML")
self.addTab(self.options, "Options")
self.setup_validator()
self.setup_waveforms()
self.setup_xmlviewer()
self.setup_options()
size = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
size.setHeightForWidth(False)
self.setSizePolicy(size)
# Initialized a threpool with 2 threads 1 for the GUI, 1 for long
# tasks, so GUI remains responsive
self.threadpool = QThreadPool()
self.threadpool.setMaxThreadCount(2)
self.validator_worker = None
self.indexing_timer = QElapsedTimer()
def setup_validator(self):
self.directory_indexer = None # aecg.indexing.DirectoryIndexer()
self.validator_layout_container = QWidget()
self.validator_layout = QFormLayout()
self.validator_form_layout = QFormLayout(
self.validator_layout_container)
self.validator_grid_layout = QGridLayout()
self.study_info_file = QLineEdit()
self.study_info_file.setToolTip("Study index file")
self.study_info_description = QLineEdit()
self.study_info_description.setToolTip("Description")
self.app_type = QLineEdit()
self.app_type.setToolTip("Application type (e.g., NDA, IND, BLA, IDE)")
self.app_num = QLineEdit()
self.app_num.setToolTip("Six-digit application number")
self.app_num.setValidator(QIntValidator(self.app_num))
self.study_id = QLineEdit()
self.study_id.setToolTip("Study identifier")
self.study_sponsor = QLineEdit()
self.study_sponsor.setToolTip("Sponsor of the study")
self.study_annotation_aecg_cb = QComboBox()
self.study_annotation_aecg_cb.addItems(
["Rhythm", "Derived beat", "Holter-rhythm", "Holter-derived"])
self.study_annotation_aecg_cb.setToolTip(
"Waveforms used to perform the ECG measurements (i.e., "
"annotations)\n"
"\tRhythm: annotations in a rhythm strip or discrete ECG "
"extraction (e.g., 10-s strips)\n"
"\tDerived beat: annotations in a representative beat derived "
"from a rhythm strip\n"
"\tHolter-rhythm: annotations in a the analysis window of a "
"continuous recording\n"
"\tHolter-derived: annotations in a representative beat derived "
"from analysis window of a continuous recording\n")
self.study_annotation_lead_cb = QComboBox()
self.ui_leads = ["GLOBAL"] + aecg.STD_LEADS[0:12] +\
[aecg.KNOWN_NON_STD_LEADS[1]] + aecg.STD_LEADS[12:15] + ["Other"]
self.study_annotation_lead_cb.addItems(self.ui_leads)
self.study_annotation_lead_cb.setToolTip(
"Primary analysis lead annotated per protocol. There could be "
"annotations in other leads also, but only the primary lead should"
" be selected here.\n"
"Select global if all leads were used at the "
"same time (e.g., superimposed on screen).\n"
"Select other if the primary lead used is not in the list.")
self.study_numsubjects = QLineEdit()
self.study_numsubjects.setToolTip(
"Number of subjects with ECGs in the study")
self.study_numsubjects.setValidator(
QIntValidator(self.study_numsubjects))
self.study_aecgpersubject = QLineEdit()
self.study_aecgpersubject.setToolTip(
"Number of scheduled ECGs (or analysis windows) per subject as "
"specified in the study protocol.\n"
"Enter average number of ECGs "
"per subject if the protocol does not specify a fixed number of "
"ECGs per subject.")
self.study_aecgpersubject.setValidator(
QIntValidator(self.study_aecgpersubject))
self.study_numaecg = QLineEdit()
self.study_numaecg.setToolTip(
"Total number of aECG XML files in the study")
self.study_numaecg.setValidator(QIntValidator(self.study_numaecg))
self.study_annotation_numbeats = QLineEdit()
self.study_annotation_numbeats.setToolTip(
"Minimum number of beats annotated in each ECG or analysis window"
".\nEnter 1 if annotations were done in the derived beat.")
self.study_annotation_numbeats.setValidator(
QIntValidator(self.study_annotation_numbeats))
self.aecg_numsubjects = QLineEdit()
self.aecg_numsubjects.setToolTip(
"Number of subjects found across the provided aECG XML files")
self.aecg_numsubjects.setReadOnly(True)
self.aecg_aecgpersubject = QLineEdit()
self.aecg_aecgpersubject.setToolTip(
"Average number of ECGs per subject found across the provided "
"aECG XML files")
self.aecg_aecgpersubject.setReadOnly(True)
self.aecg_numaecg = QLineEdit()
self.aecg_numaecg.setToolTip(
"Number of aECG XML files found in the study aECG directory")
self.aecg_numaecg.setReadOnly(True)
self.subjects_less_aecgs = QLineEdit()
self.subjects_less_aecgs.setToolTip(
"Percentage of subjects with less aECGs than specified per "
"protocol")
self.subjects_less_aecgs.setReadOnly(True)
self.subjects_more_aecgs = QLineEdit()
self.subjects_more_aecgs.setToolTip(
"Percentage of subjects with more aECGs than specified per "
"protocol")
self.subjects_more_aecgs.setReadOnly(True)
self.aecgs_no_annotations = QLineEdit()
self.aecgs_no_annotations.setToolTip(
"Percentage of aECGs with no annotations")
self.aecgs_no_annotations.setReadOnly(True)
self.aecgs_less_qt_in_primary_lead = QLineEdit()
self.aecgs_less_qt_in_primary_lead.setToolTip(
"Percentage of aECGs with less QT intervals in the primary lead "
"than specified per protocol")
self.aecgs_less_qt_in_primary_lead.setReadOnly(True)
self.aecgs_less_qts = QLineEdit()
self.aecgs_less_qts.setToolTip(
"Percentage of aECGs with less QT intervals than specified per "
"protocol")
self.aecgs_less_qts.setReadOnly(True)
self.aecgs_annotations_multiple_leads = QLineEdit()
self.aecgs_annotations_multiple_leads.setToolTip(
"Percentage of aECGs with QT annotations in multiple leads")
self.aecgs_annotations_multiple_leads.setReadOnly(True)
self.aecgs_annotations_no_primary_lead = QLineEdit()
self.aecgs_annotations_no_primary_lead.setToolTip(
"Percentage of aECGs with QT annotations not in the primary lead")
self.aecgs_annotations_no_primary_lead.setReadOnly(True)
self.aecgs_with_errors = QLineEdit()
self.aecgs_with_errors.setToolTip("Number of aECG files with errors")
self.aecgs_with_errors.setReadOnly(True)
self.aecgs_potentially_digitized = QLineEdit()
self.aecgs_potentially_digitized.setToolTip(
"Number of aECG files potentially digitized (i.e., with more than "
"5% of samples missing)")
self.aecgs_potentially_digitized.setReadOnly(True)
self.study_dir = QLineEdit()
self.study_dir.setToolTip("Directory containing the aECG files")
self.study_dir_button = QPushButton("...")
self.study_dir_button.clicked.connect(self.select_study_dir)
self.study_dir_button.setToolTip("Open select directory dialog")
self.validator_form_layout.addRow("Application Type", self.app_type)
self.validator_form_layout.addRow("Application Number", self.app_num)
self.validator_form_layout.addRow("Study name/ID", self.study_id)
self.validator_form_layout.addRow("Sponsor", self.study_sponsor)
self.validator_form_layout.addRow("Study description",
self.study_info_description)
self.validator_form_layout.addRow("Annotations in",
self.study_annotation_aecg_cb)
self.validator_form_layout.addRow("Annotations primary lead",
self.study_annotation_lead_cb)
self.validator_grid_layout.addWidget(QLabel(""), 0, 0)
self.validator_grid_layout.addWidget(
QLabel("Per study protocol or report"), 0, 1)
self.validator_grid_layout.addWidget(QLabel("Found in aECG files"), 0,
2)
self.validator_grid_layout.addWidget(QLabel("Number of subjects"), 1,
0)
self.validator_grid_layout.addWidget(self.study_numsubjects, 1, 1)
self.validator_grid_layout.addWidget(self.aecg_numsubjects, 1, 2)
self.validator_grid_layout.addWidget(
QLabel("Number of aECG per subject"), 2, 0)
self.validator_grid_layout.addWidget(self.study_aecgpersubject, 2, 1)
self.validator_grid_layout.addWidget(self.aecg_aecgpersubject, 2, 2)
self.validator_grid_layout.addWidget(QLabel("Total number of aECG"), 3,
0)
self.validator_grid_layout.addWidget(self.study_numaecg, 3, 1)
self.validator_grid_layout.addWidget(self.aecg_numaecg, 3, 2)
self.validator_grid_layout.addWidget(
QLabel("Number of beats per aECG"), 4, 0)
self.validator_grid_layout.addWidget(self.study_annotation_numbeats, 4,
1)
self.validator_grid_layout.addWidget(
QLabel("Subjects with fewer ECGs"), 5, 1)
self.validator_grid_layout.addWidget(self.subjects_less_aecgs, 5, 2)
self.validator_grid_layout.addWidget(QLabel("Subjects with more ECGs"),
6, 1)
self.validator_grid_layout.addWidget(self.subjects_more_aecgs, 6, 2)
self.validator_grid_layout.addWidget(
QLabel("aECGs without annotations"), 7, 1)
self.validator_grid_layout.addWidget(self.aecgs_no_annotations, 7, 2)
self.validator_grid_layout.addWidget(
QLabel("aECGs without expected number of QTs in primary lead"), 8,
1)
self.validator_grid_layout.addWidget(
self.aecgs_less_qt_in_primary_lead, 8, 2)
self.validator_grid_layout.addWidget(
QLabel("aECGs without expected number of QTs"), 9, 1)
self.validator_grid_layout.addWidget(self.aecgs_less_qts, 9, 2)
self.validator_grid_layout.addWidget(
QLabel("aECGs annotated in multiple leads"), 10, 1)
self.validator_grid_layout.addWidget(
self.aecgs_annotations_multiple_leads, 10, 2)
self.validator_grid_layout.addWidget(
QLabel("aECGs with annotations not in primary lead"), 11, 1)
self.validator_grid_layout.addWidget(
self.aecgs_annotations_no_primary_lead, 11, 2)
self.validator_grid_layout.addWidget(QLabel("aECGs with errors"), 12,
1)
self.validator_grid_layout.addWidget(self.aecgs_with_errors, 12, 2)
self.validator_grid_layout.addWidget(
QLabel("Potentially digitized aECGs"), 13, 1)
self.validator_grid_layout.addWidget(self.aecgs_potentially_digitized,
13, 2)
self.validator_form_layout.addRow(self.validator_grid_layout)
tmp = QHBoxLayout()
tmp.addWidget(self.study_dir)
tmp.addWidget(self.study_dir_button)
self.validator_form_layout.addRow("Study aECGs directory", tmp)
self.validator_form_layout.addRow("Study index file",
self.study_info_file)
self.validator_layout.addWidget(self.validator_layout_container)
self.validator_effective_dirs = QLabel("")
self.validator_effective_dirs.setWordWrap(True)
self.validator_layout.addWidget(self.validator_effective_dirs)
self.val_button = QPushButton("Generate/update study index")
self.val_button.clicked.connect(self.importstudy_dialog)
self.validator_layout.addWidget(self.val_button)
self.cancel_val_button = QPushButton("Cancel study index generation")
self.cancel_val_button.clicked.connect(self.cancel_validator)
self.cancel_val_button.setEnabled(False)
self.validator_layout.addWidget(self.cancel_val_button)
self.validator_pl = QLabel("")
self.validator_layout.addWidget(self.validator_pl)
self.validator_pb = QProgressBar()
self.validator_layout.addWidget(self.validator_pb)
self.validator.setLayout(self.validator_layout)
self.stop_indexing = False
self.lastindexing_starttime = None
self.update_validator_effective_dirs()
def effective_aecgs_dir(self, navwidget, silent=False):
aecgs_effective_dir = self.study_dir.text()
if navwidget.project_loaded != '':
# Path specified in the GUI
potential_aecgs_dirs = [self.study_dir.text()]
# StudyDir path from current working directory
potential_aecgs_dirs += [self.studyindex_info.StudyDir]
# StudyDir path from directory where the index is located
potential_aecgs_dirs += [
os.path.join(os.path.dirname(navwidget.project_loaded),
self.studyindex_info.StudyDir)
]
# StudyDir replaced with the directory where the index is located
potential_aecgs_dirs += [os.path.dirname(navwidget.project_loaded)]
dir_found = False
# Get xml and zip filenames from first element in the index
aecg_xml_file = navwidget.data_index["AECGXML"][0]
zipfile = ""
if aecg_xml_file != "":
zipfile = navwidget.data_index["ZIPFILE"][0]
for p in potential_aecgs_dirs:
testfn = os.path.join(p, aecg_xml_file)
if zipfile != "":
testfn = os.path.join(p, zipfile)
if os.path.isfile(testfn):
dir_found = True
aecgs_effective_dir = p
break
if not silent:
if not dir_found:
QMessageBox.warning(
self, f"Study aECGs directory not found",
f"The following paths were checked:"
f"{[','.join(p) for p in potential_aecgs_dirs]} and "
f"none of them is valid")
elif p != self.study_dir.text():
QMessageBox.warning(
self, f"Study aECGs directory not found",
f"The path specified in the study aECGs directory is "
f"not valid and {p} is being used instead. Check and "
f"update the path in Study aECGs directory textbox if "
f"the suggested path is not the adequate path")
return aecgs_effective_dir
def update_validator_effective_dirs(self):
msg = f"Working directory: {os.getcwd()}"
if self.parent() is not None:
if isinstance(self.parent(), QSplitter):
navwidget = self.parent().parent()
else: # Tabs widget has not been allocated the QSplitter yet
navwidget = self.parent()
project_loaded = navwidget.project_loaded
if project_loaded != '':
msg = f"{msg}\nLoaded project index: "\
f"{navwidget.project_loaded}"
effective_aecgs_path = self.effective_aecgs_dir(navwidget)
msg = f"{msg}\nEffective study aECGs directory: "\
f"{effective_aecgs_path}"
else:
msg = f"{msg}\nLoaded project index: None"
else:
msg = f"{msg}\nLoaded project index: None"
self.validator_effective_dirs.setText(msg)
def load_study_info(self, fileName):
self.study_info_file.setText(fileName)
try:
study_info = pd.read_excel(fileName, sheet_name="Info")
self.studyindex_info = aecg.tools.indexer.StudyInfo()
self.studyindex_info.__dict__.update(
study_info.set_index("Property").transpose().reset_index(
drop=True).to_dict('index')[0])
sponsor = ""
description = ""
if self.studyindex_info.Sponsor is not None and\
isinstance(self.studyindex_info.Sponsor, str):
sponsor = self.studyindex_info.Sponsor
if self.studyindex_info.Description is not None and\
isinstance(self.studyindex_info.Description, str):
description = self.studyindex_info.Description
self.study_sponsor.setText(sponsor)
self.study_info_description.setText(description)
self.app_type.setText(self.studyindex_info.AppType)
self.app_num.setText(f"{int(self.studyindex_info.AppNum):06d}")
self.study_id.setText(self.studyindex_info.StudyID)
self.study_numsubjects.setText(str(self.studyindex_info.NumSubj))
self.study_aecgpersubject.setText(
str(self.studyindex_info.NECGSubj))
self.study_numaecg.setText(str(self.studyindex_info.TotalECGs))
anns_in = self.studyindex_info.AnMethod.upper()
idx = 0
if anns_in == "RHYTHM":
idx = 0
elif anns_in == "DERIVED":
idx = 1
elif anns_in == "HOLTER_RHYTHM":
idx = 2
elif anns_in == "HOLTER_MEDIAN_BEAT":
idx = 3
else:
idx = int(anns_in) - 1
self.study_annotation_aecg_cb.setCurrentIndex(idx)
the_lead = self.studyindex_info.AnLead
idx = self.study_annotation_lead_cb.findText(str(the_lead))
if idx == -1:
idx = self.study_annotation_lead_cb.findText("MDC_ECG_LEAD_" +
str(the_lead))
if idx == -1:
idx = int(the_lead)
self.study_annotation_lead_cb.setCurrentIndex(idx)
self.study_annotation_numbeats.setText(
str(self.studyindex_info.AnNbeats))
if self.studyindex_info.StudyDir == "":
self.studyindex_info.StudyDir = os.path.dirname(fileName)
self.study_dir.setText(self.studyindex_info.StudyDir)
self.update_validator_effective_dirs()
self.setCurrentWidget(self.validator)
except Exception as ex:
QMessageBox.critical(
self, "Import study error",
"Error reading the study information file: '" + fileName + "'")
def setup_waveforms(self):
wflayout = QVBoxLayout()
# ECG plot layout selection box
self.cbECGLayout.addItems(
['12-lead stacked', '3x4 + lead II rhythm', 'Superimposed'])
self.cbECGLayout.currentIndexChanged.connect(
self.ecgplotlayout_changed)
# Zoom buttons
blayout = QHBoxLayout()
pb_zoomin = QPushButton()
pb_zoomin.setText("Zoom +")
pb_zoomin.clicked.connect(self.zoom_in)
pb_zoomreset = QPushButton()
pb_zoomreset.setText("Zoom 1:1")
pb_zoomreset.clicked.connect(self.zoom_reset)
pb_zoomout = QPushButton()
pb_zoomout.setText("Zoom -")
pb_zoomout.clicked.connect(self.zoom_out)
blayout.addWidget(self.cbECGLayout)
blayout.addWidget(pb_zoomout)
blayout.addWidget(pb_zoomreset)
blayout.addWidget(pb_zoomin)
wflayout.addLayout(blayout)
# Add QScrollArea to main layout of waveforms tab
self.aecg_display_area.setWidgetResizable(False)
wflayout.addWidget(self.aecg_display_area)
self.waveforms.setLayout(wflayout)
size = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred)
size.setHeightForWidth(False)
self.aecg_display_area.setSizePolicy(size)
self.waveforms.setSizePolicy(size)
def setup_xmlviewer(self):
wf_layout = QHBoxLayout()
wf_layout.addWidget(self.xml_display)
self.xmlviewer.setLayout(wf_layout)
def setup_options(self):
self.options_layout = QFormLayout()
self.aecg_schema_filename = QLineEdit(aecg.get_aecg_schema_location())
self.options_layout.addRow("aECG XML schema path",
self.aecg_schema_filename)
self.save_index_every_n_aecgs = QSpinBox()
self.save_index_every_n_aecgs.setMinimum(0)
self.save_index_every_n_aecgs.setMaximum(50000)
self.save_index_every_n_aecgs.setValue(0)
self.save_index_every_n_aecgs.setSingleStep(100)
self.save_index_every_n_aecgs.setSuffix(" aECGs")
self.save_index_every_n_aecgs.setToolTip(
"Set o 0 to save the study index file only after its generation "
"is completed.\nOtherwise, the file is saved everytime the "
" specified number of ECGs have been appended to the index.")
self.options_layout.addRow("Save index every ",
self.save_index_every_n_aecgs)
self.save_all_intervals_cb = QCheckBox("")
self.save_all_intervals_cb.setChecked(False)
self.options_layout.addRow("Save individual beat intervals",
self.save_all_intervals_cb)
self.parallel_processing_cb = QCheckBox("")
self.parallel_processing_cb.setChecked(True)
self.options_layout.addRow("Parallel processing of files",
self.parallel_processing_cb)
self.options.setLayout(self.options_layout)
def zoom_in(self):
self.aecg_display.apply_zoom(self.aecg_display.zoom_factor + 0.1)
def zoom_out(self):
self.aecg_display.apply_zoom(self.aecg_display.zoom_factor - 0.1)
def zoom_reset(self):
self.aecg_display.apply_zoom(1.0)
def ecgplotlayout_changed(self, i):
self.aecg_display.update_aecg_plot(
ecg_layout=aecg.utils.ECG_plot_layout(i + 1))
def update_search_progress(self, i, n):
self.validator_pl.setText(
f"Searching aECGs in directory ({n} XML files found)")
def update_progress(self, i, n):
j = i
m = n
if i <= 1:
j = 1
if self.validator_pb.value() > 0:
j = self.validator_pb.value() + 1
m = self.validator_pb.maximum()
running_time = self.indexing_timer.elapsed() * 1e-3 # in seconds
time_per_item = running_time / j
# reamining = seconds per item so far * total pending items to process
remaining_time = time_per_item * (m - j)
eta = datetime.datetime.now() +\
datetime.timedelta(seconds=round(remaining_time, 0))
self.validator_pl.setText(
f"Validating aECG {j}/{m} | "
f"Execution time: "
f"{str(datetime.timedelta(0,seconds=round(running_time)))} | "
f"{round(1/time_per_item,2)} aECGs per second | "
f"ETA: {eta.isoformat(timespec='seconds')}")
self.validator_pb.setValue(j)
if self.save_index_every_n_aecgs.value() > 0 and\
len(self.directory_indexer.studyindex) % \
self.save_index_every_n_aecgs.value() == 0:
self.save_validator_results(
pd.concat(self.directory_indexer.studyindex,
ignore_index=True))
def save_validator_results(self, res):
if res.shape[0] > 0:
self.studyindex_info = aecg.tools.indexer.StudyInfo()
self.studyindex_info.StudyDir = self.study_dir.text()
self.studyindex_info.IndexFile = self.study_info_file.text()
self.studyindex_info.Sponsor = self.study_sponsor.text()
self.studyindex_info.Description =\
self.study_info_description.text()
self.studyindex_info.Date = self.lastindexing_starttime.isoformat()
self.studyindex_info.End_date = datetime.datetime.now().isoformat()
self.studyindex_info.Version = aecg.__version__
self.studyindex_info.AppType = self.app_type.text()
self.studyindex_info.AppNum = f"{int(self.app_num.text()):06d}"
self.studyindex_info.StudyID = self.study_id.text()
self.studyindex_info.NumSubj = int(self.study_numsubjects.text())
self.studyindex_info.NECGSubj = int(
self.study_aecgpersubject.text())
self.studyindex_info.TotalECGs = int(self.study_numaecg.text())
anmethod = aecg.tools.indexer.AnnotationMethod(
self.study_annotation_aecg_cb.currentIndex())
self.studyindex_info.AnMethod = anmethod.name
self.studyindex_info.AnLead =\
self.study_annotation_lead_cb.currentText()
self.studyindex_info.AnNbeats = int(
self.study_annotation_numbeats.text())
# Calculate stats
study_stats = aecg.tools.indexer.StudyStats(
self.studyindex_info, res)
# Save to file
aecg.tools.indexer.save_study_index(self.studyindex_info, res,
study_stats)
validator_data_ready = Signal()
def save_validator_results_and_load_index(self, res):
self.save_validator_results(res)
self.validator_data_ready.emit()
def indexer_validator_results(self, res):
self.studyindex_df = pd.concat([self.studyindex_df, res],
ignore_index=True)
def subindex_thread_complete(self):
return
def index_directory_thread_complete(self):
tmp = self.validator_pl.text().replace("ETA:", "Completed: ").replace(
"Validating", "Validated")
self.validator_pl.setText(tmp)
self.val_button.setEnabled(True)
self.cancel_val_button.setEnabled(False)
self.validator_layout_container.setEnabled(True)
def index_directory(self, progress_callback):
self.lastindexing_starttime = datetime.datetime.now()
self.indexing_timer.start()
studyindex_df = []
n_cores = os.cpu_count()
aecg_schema = None
if self.aecg_schema_filename.text() != "":
aecg_schema = self.aecg_schema_filename.text()
if self.parallel_processing_cb.isChecked():
studyindex_df = self.directory_indexer.index_directory(
self.save_all_intervals_cb.isChecked(), aecg_schema, n_cores,
progress_callback)
else:
studyindex_df = self.directory_indexer.index_directory(
self.save_all_intervals_cb.isChecked(), aecg_schema, 1,
progress_callback)
return studyindex_df
def importstudy_dialog(self):
dirName = os.path.normpath(self.study_dir.text())
if dirName != "":
if os.path.exists(dirName):
self.directory_indexer = aecg.indexing.DirectoryIndexer()
self.directory_indexer.set_aecg_dir(
dirName, self.update_search_progress)
self.validator_pb.setMaximum(self.directory_indexer.num_files)
self.validator_pb.reset()
self.stop_indexing = False
self.validator_layout_container.setEnabled(False)
self.val_button.setEnabled(False)
self.cancel_val_button.setEnabled(True)
self.validator_worker = Worker(self.index_directory)
self.validator_worker.signals.result.connect(
self.save_validator_results_and_load_index)
self.validator_worker.signals.finished.connect(
self.index_directory_thread_complete)
self.validator_worker.signals.progress.connect(
self.update_progress)
# Execute
self.threadpool.start(self.validator_worker)
else:
QMessageBox.critical(
self, "Directory not found",
f"Specified study directory not found:\n{dirName}")
else:
QMessageBox.critical(self, "Import study error",
"Study directory cannot be empty")
def cancel_validator(self):
self.cancel_val_button.setEnabled(False)
self.stop_indexing = True
self.directory_indexer.cancel_indexing = True
self.threadpool.waitForDone(3000)
self.val_button.setEnabled(True)
def select_study_dir(self):
cd = self.study_dir.text()
if cd == "":
cd = "."
dir = QFileDialog.getExistingDirectory(
self, "Open Directory", cd,
QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks)
if dir != "":
self.study_dir.setText(dir)
class MainWindow(QObject):
def __init__(self, ui_file, parent=None):
super(MainWindow, self).__init__(parent)
ui_file = QFile(ui_file)
ui_file.open(QFile.ReadOnly)
loader = QUiLoader()
self.window = loader.load(ui_file)
ui_file.close()
dir_btn: QPushButton = self.window.findChild(QPushButton,
"dir_choose_btn")
dir_btn.clicked.connect(self.dir_btn_handler)
self.dirPath: QLineEdit = self.window.findChild(QLineEdit, "dirPath")
self.repeatFiles_dir: QLineEdit = self.window.findChild(
QLineEdit, "repeatFiles_dir")
self.singleFiles_dir: QLineEdit = self.window.findChild(
QLineEdit, "singleFiles_dir")
self.moveSingleFiles: QCheckBox = self.window.findChild(
QCheckBox, "moveSingleFiles")
self.moveRepeatFiles: QCheckBox = self.window.findChild(
QCheckBox, "moveRepeatFiles")
self.sortBar: QProgressBar = self.window.findChild(
QProgressBar, "sortBar")
self.sortBar.setValue(0)
self.moveBar: QProgressBar = self.window.findChild(
QProgressBar, "moveBar")
self.moveBar.setValue(0)
self.startButton: QPushButton = self.window.findChild(
QPushButton, "startButton")
self.startButton.clicked.connect(self.start_handler)
self.stopButton: QPushButton = self.window.findChild(
QPushButton, "stopButton")
self.stopButton.clicked.connect(self.stop_handler)
self.worker: Worker = None
self.running = False
self.thread_pool = QThreadPool()
print("Multi threading with maximum %d threads" %
self.thread_pool.maxThreadCount())
self.msg_box = QMessageBox(parent=self.window)
self.msg_box.setIcon(QMessageBox.Critical)
self.msg_box.setWindowTitle("Некорректные настройки")
self.window.destroyed.connect(self.stop_handler)
self.window.show()
def start_handler(self):
if self.thread_pool.activeThreadCount() > 0:
return
if self.check_inputs():
self.worker = Worker(self)
self.worker.signals.updated.connect(self.worker_upd_handler)
self.worker.running = True
self.thread_pool.start(self.worker)
return
def stop_handler(self):
self.worker.running = False
self.thread_pool.waitForDone(1000)
return
def worker_upd_handler(self, data):
if "sortBar" in data.keys():
self.sortBar.setValue(data["sortBar"])
if "moveBar" in data.keys():
self.moveBar.setValue(data["moveBar"])
def dir_btn_handler(self):
dialog = QFileDialog()
dirname = dialog.getExistingDirectory()
self.dirPath.setText(dirname)
def check_inputs(self):
errors = []
if not os.path.exists(self.dirPath.text()):
errors.append("Выбранная папка не существует")
if self.moveRepeatFiles.isChecked() and len(
self.repeatFiles_dir.text()) < 1:
errors.append(
"Введите название папки для переноса повторяющихся файлов")
if self.moveSingleFiles.isChecked() and len(
self.singleFiles_dir.text()) < 1:
errors.append(
"Введите название папки для переноса одиночных файлов")
if errors:
self.msg_box.setText("\n".join(errors))
self.msg_box.exec()
return False
return True
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, settings):
# mandatory
QMainWindow.__init__(self)
Ui_MainWindow.__init__(self)
self.trading_session_state = "TBD"
self.est = timezone('US/Eastern')
self.settingsWindow = SettingsWindow()
self.setupUi(self)
self.settings = settings
self.ibkrworker = IBKRWorker(self.settings)
self.threadpool = QThreadPool()
self.setWindowTitle("Algo Traider v 2.0")
sys.stderr = open('LOG/errorLog.txt', 'w')
self.create_open_positions_grid()
# setting a timer for Worker
self.uiTimer = QTimer()
self.uiTimer.timeout.connect(self.update_ui)
self.workerTimer = QTimer()
self.workerTimer.timeout.connect(self.run_worker)
self.server_timer = QTimer()
self.server_timer.timeout.connect(self.report_to_server)
self.server_timer.start(int(self.settings.INTERVALSERVER) * 1000)
# connecting a buttons
self.chbxProcess.stateChanged.connect(self.process_checked)
self.btnSettings.pressed.connect(self.show_settings)
self.statusbar.showMessage("Ready")
stock_names = [o.ticker for o in self.settings.CANDIDATES]
self.ibkrworker.stocks_data_from_server = get_market_data_from_server(
self.settings, stock_names)
self.update_console("Market data for " + str(len(stock_names)) +
" Candidates received from Server")
self.start_updating_candidates_and_connect()
StyleSheet = '''
#lcdPNLgreen {
border: 3px solid green;
}
#lcdPNLred {
border: 3px solid red;
}
'''
self.setStyleSheet(StyleSheet)
def update_candidates_info(self, status_callback, notification_callback):
today_dt = date.today()
for c in self.ibkrworker.stocks_data_from_server:
updated_dt = c['tiprank_updated'].date()
if today_dt != updated_dt:
# yahoo
notification_callback.emit('Update for ' + c['ticker'] +
' needed:')
notification_callback.emit('Checking for Yahoo statisticks...')
drop, change = get_yahoo_stats_for_candidate(
c['ticker'], notification_callback)
c['yahoo_avdropP'] = drop
c['yahoo_avspreadP'] = change
notification_callback.emit('Got ' + str(drop) +
' average daily drop and ' +
str(change) +
' average daily change.')
# tipranks
notification_callback.emit('Checking for Tiprank...')
rank = get_tiprank_rating_for_ticker(
c['ticker'], self.settings.PATHTOWEBDRIVER)
notification_callback.emit('Got rank of :' + str(rank))
c['tipranks'] = rank
c['tiprank_updated'] = today_dt
else:
notification_callback.emit(
'Data for ' + c['ticker'] +
' is up to the date,no update needed')
report_market_data_to_server(self.settings,
self.ibkrworker.stocks_data_from_server)
return 'done'
def start_updating_candidates_and_connect(self):
cand = Worker(self.update_candidates_info)
cand.signals.result.connect(self.connect_to_ibkr)
# connector.signals.status.connect(self.update_status)
cand.signals.notification.connect(self.update_console)
# Execute
self.threadpool.start(cand)
def connect_to_ibkr(self):
"""
Starts the connection to the IBKR terminal in separate thread
"""
self.update_console("Reporting connection to the server...")
print("Reporting connection to the server...")
result = report_login_to_server(self.settings)
self.update_console(result)
connector = Worker(self.ibkrworker.prepare_and_connect)
connector.signals.result.connect(self.connection_done)
connector.signals.status.connect(self.update_status)
connector.signals.notification.connect(self.update_console)
# Execute
self.threadpool.start(connector)
def process_checked(self):
"""
Starts the Timer with interval from Config file
"""
if self.chbxProcess.isChecked():
self.run_worker()
self.workerTimer.start(int(self.settings.INTERVALWORKER) * 1000)
else:
self.workerTimer.stop()
# noinspection PyUnresolvedReferences
def run_worker(self):
"""
Executed the Worker in separate thread
"""
# exec(open('restarter.py').read())
# sys.exit()
self.update_session_state()
currentTime = QTime().currentTime()
fromTime = QTime(int(self.settings.TECHFROMHOUR),
int(self.settings.TECHFROMMIN))
toTime = QTime(int(self.settings.TECHTOHOUR),
int(self.settings.TECHTOMIN))
sessionState = self.lblMarket.text()
if fromTime < currentTime < toTime:
print("Worker skept-Technical break : ",
fromTime.toString("hh:mm"), " to ", toTime.toString("hh:mm"))
self.update_console("Technical break untill " +
toTime.toString("hh:mm"))
else:
self.update_console("Starting Worker- UI Paused")
self.uiTimer.stop(
) # to not cause an errors when lists will be resetted
worker = Worker(
self.ibkrworker.process_positions_candidates
) # Any other args, kwargs are passed to the run function
worker.signals.result.connect(self.update_ui)
worker.signals.status.connect(self.update_status)
worker.signals.notification.connect(self.update_console)
# Execute
self.threadpool.start(worker)
def connection_done(self):
# add processing
self.update_ui()
if self.settings.AUTOSTART:
self.chbxProcess.setChecked(True)
# # report market data to server
# if self.settings.USESERVER:
# print("Reporting market data to the server...")
# result = report_market_data_to_server(self.settings, self.ibkrworker.app.candidatesLive)
# self.update_console(result)
def report_to_server(self):
"""
reports to the server
"""
if self.settings.USESERVER:
net_liquidation = self.ibkrworker.app.netLiquidation
if hasattr(self.ibkrworker.app, 'smaWithSafety'):
remaining_sma_with_safety = self.ibkrworker.app.smaWithSafety
else:
remaining_sma_with_safety = self.ibkrworker.app.sMa
excess_liquidity = self.ibkrworker.app.excessLiquidity
remaining_trades = self.ibkrworker.app.tradesRemaining
all_positions_value = 0
open_positions = self.ibkrworker.app.openPositions
open_orders = self.ibkrworker.app.openOrders
dailyPnl = self.ibkrworker.app.dailyPnl
tradinng_session_state = self.trading_session_state
data_for_report = [
self.settings, net_liquidation, remaining_sma_with_safety,
remaining_trades, all_positions_value, open_positions,
open_orders, dailyPnl,
self.ibkrworker.last_worker_execution_time,
datetime.now(self.est), self.trading_session_state,
excess_liquidity
]
worker = Worker(report_snapshot_to_server, self.settings,
data_for_report)
worker.signals.result.connect(self.process_server_response)
# Execute
self.threadpool.start(worker)
def process_server_response(self, r):
# trying to restart
if '$restart$' in r:
restart()
self.update_console(r)
def update_ui(self):
"""
Updates UI after connection/worker execution
"""
# main data
self.lAcc.setText(self.settings.ACCOUNT)
# self.lExcessLiquidity.setText(str(self.ibkrworker.app.excessLiquidity))
# self.lSma.setText(str(self.ibkrworker.app.sMa))
if hasattr(self.ibkrworker.app, 'smaWithSafety'):
self.lSma.setText(str(round(self.ibkrworker.app.smaWithSafety, 1)))
else:
self.lSma.setText(str(round(self.ibkrworker.app.sMa, 1)))
self.lMarketValue.setText(str(self.ibkrworker.app.netLiquidation))
self.lblAvailTrades.setText(str(self.ibkrworker.app.tradesRemaining))
self.lcdPNL.display(self.ibkrworker.app.dailyPnl)
if self.ibkrworker.app.dailyPnl > 0:
palette = self.lcdPNL.palette()
palette.setColor(palette.WindowText, QtGui.QColor(51, 153, 51))
self.lcdPNL.setPalette(palette)
elif self.ibkrworker.app.dailyPnl < 0:
palette = self.lcdPNL.palette()
palette.setColor(palette.WindowText, QtGui.QColor(255, 0, 0))
self.lcdPNL.setPalette(palette)
total_positions_value = 0
for p in self.ibkrworker.app.openPositions.values():
if hasattr(p, 'Value'):
total_positions_value += p["Value"]
self.lPositionsTotalValue.setText(str(round(total_positions_value, 1)))
self.update_open_positions()
self.update_live_candidates()
self.update_open_orders()
# everything disabled for safety - is now enabled
self.chbxProcess.setEnabled(True)
self.btnSettings.setEnabled(True)
self.update_session_state()
if not self.uiTimer.isActive():
self.update_console("UI resumed.")
self.uiTimer.start(int(self.settings.INTERVALUI) *
1000) # reset the ui timer
def update_session_state(self):
fmt = '%Y-%m-%d %H:%M:%S'
self.est_dateTime = datetime.now(self.est)
self.est_current_time = QTime(self.est_dateTime.hour,
self.est_dateTime.minute,
self.est_dateTime.second)
self.lblTime.setText(self.est_current_time.toString())
dStart = QTime(4, 00)
dEnd = QTime(20, 00)
tStart = QTime(9, 30)
tEnd = QTime(16, 0)
self.ibkrworker.check_if_holiday()
if not self.ibkrworker.trading_session_holiday:
if self.est_current_time > dStart and self.est_current_time <= tStart:
self.ibkrworker.trading_session_state = "Pre Market"
self.lblMarket.setText("Pre Market")
elif self.est_current_time > tStart and self.est_current_time <= tEnd:
self.ibkrworker.trading_session_state = "Open"
self.lblMarket.setText("Open")
elif self.est_current_time > tEnd and self.est_current_time <= dEnd:
self.ibkrworker.trading_session_state = "After Market"
self.lblMarket.setText("After Market")
else:
self.ibkrworker.trading_session_state = "Closed"
self.lblMarket.setText("Closed")
else:
self.ibkrworker.trading_session_state = "Holiday"
self.lblMarket.setText("Holiday")
self.trading_session_state = self.ibkrworker.trading_session_state
def progress_fn(self, n):
msgBox = QMessageBox()
msgBox.setText(str(n))
retval = msgBox.exec_()
def update_status(self, s):
"""
Updates StatusBar on event of Status
:param s:
"""
self.statusbar.showMessage(s)
def update_console(self, n):
"""
Adds Message to console- upon event
:param n:
"""
self.consoleOut.append(n)
self.log_message(n)
def log_message(self, message):
"""
Adds message to the standard log
:param message:
"""
with open(LOGFILE, "a") as f:
currentDt = datetime.now().strftime("%d-%b-%Y (%H:%M:%S.%f)")
message = "\n" + currentDt + '---' + message
f.write(message)
def update_live_candidates(self):
"""
Updates Candidates table
"""
liveCandidates = self.ibkrworker.app.candidatesLive
try:
line = 0
self.tCandidates.setRowCount(len(liveCandidates))
for k, v in liveCandidates.items():
self.tCandidates.setItem(line, 0, QTableWidgetItem(v['Stock']))
self.tCandidates.setItem(line, 1,
QTableWidgetItem(str(v['Open'])))
self.tCandidates.setItem(line, 2,
QTableWidgetItem(str(v['Close'])))
self.tCandidates.setItem(line, 3,
QTableWidgetItem(str(v['Bid'])))
self.tCandidates.setItem(line, 4,
QTableWidgetItem(str(v['Ask'])))
if v['Ask'] < v['target_price'] and v['Ask'] != -1:
self.tCandidates.item(line, 4).setBackground(
QtGui.QColor(0, 255, 0))
if v['target_price'] is float:
self.tCandidates.setItem(
line, 5,
QTableWidgetItem(str(round(v['target_price'], 2))))
else:
self.tCandidates.setItem(
line, 5, QTableWidgetItem(str(v['target_price'])))
self.tCandidates.setItem(
line, 6,
QTableWidgetItem(str(round(v['averagePriceDropP'], 2))))
if v['tipranksRank'] == '':
v['tipranksRank'] = 0
self.tCandidates.setItem(
line, 7, QTableWidgetItem(str(v['tipranksRank'])))
if int(v['tipranksRank']) > 7:
self.tCandidates.item(line, 7).setBackground(
QtGui.QColor(0, 255, 0))
self.tCandidates.setItem(
line, 8, QTableWidgetItem(str(v['LastUpdate'])))
line += 1
except Exception as e:
if hasattr(e, 'message'):
self.update_console("Error in updating Candidates: " +
str(e.message))
else:
self.update_console("Error in updating Candidates: " + str(e))
def update_open_positions(self):
"""
Updates Positions grid
"""
open_positions = self.ibkrworker.app.openPositions
allKeys = [*open_positions]
lastUpdatedWidget = 0
try:
for i in range(len(open_positions)): # Update positions Panels
widget = self.gp.itemAt(i).widget()
key = allKeys[i]
values = open_positions[key]
if 'stocks' in values.keys():
if values['stocks'] != 0:
candidate = next((x for x in self.settings.CANDIDATES
if x.ticker == key), None)
reason_of_candidate = "Bought manually"
if candidate is not None:
reason_of_candidate = candidate.reason
widget.update_view(key, values, reason_of_candidate)
widget.show()
lastUpdatedWidget = i
else:
widgetToRemove = self.gp.itemAt(i).widget()
widgetToRemove.hide()
else:
print("value not yet received")
for i in range(self.gp.count()): # Hide the rest of the panels
if i > lastUpdatedWidget:
widgetToRemove = self.gp.itemAt(i).widget()
widgetToRemove.hide()
except Exception as e:
if hasattr(e, 'message'):
self.update_console("Error in refreshing Positions: " +
str(e.message))
else:
self.update_console("Error in refreshing Positions: " + str(e))
def create_open_positions_grid(self):
"""
Creates Open positions grid with 99 Positions widgets
"""
counter = 0
col = 0
row = 0
for i in range(0, 99):
if counter % 3 == 0:
col = 0
row += 1
self.gp.addWidget(PositionPanel(), row, col)
counter += 1
col += 1
def update_open_orders(self):
"""
Updates Positions table
"""
openOrders = self.ibkrworker.app.openOrders
try:
line = 0
self.tOrders.setRowCount(len(openOrders))
for k, v in openOrders.items():
self.tOrders.setItem(line, 0, QTableWidgetItem(k))
self.tOrders.setItem(line, 1, QTableWidgetItem(v['Action']))
self.tOrders.setItem(line, 2, QTableWidgetItem(v['Type']))
line += 1
except Exception as e:
if hasattr(e, 'message'):
self.update_console("Error in Updating open Orders : " +
str(e.message))
else:
self.update_console("Error in Updating open Orders : " +
str(e))
def thread_complete(self):
"""
After threaded task finished
"""
print("TREAD COMPLETE (good or bad)!")
def show_settings(self):
# self.settingsWindow = SettingsWindowO(self.settings)
# self.settingsWindow.show() # Показываем окно
# # maybe not needed
# self.settingsWindow.changedSettings = False
self.settingsWindow.existingSettings = copy.deepcopy(self.settings)
self.settingsWindow.changedSettings = False
self.settingsWindow.ibkrClient = self.ibkrworker
if self.settingsWindow.exec_():
self.settings = self.settingsWindow.existingSettings
self.settings.write_config()
self.restart_all()
else:
print("Settings window Canceled")
self.settingsWindow = SettingsWindow()
def restart_all(self):
"""
Restarts everything after Save
"""
self.threadpool.waitForDone()
self.update_console("UI paused- for restart")
self.uiTimer.stop()
self.workerTimer.stop()
self.update_console("Configuration changed - restarting everything")
self.chbxProcess.setEnabled(False)
self.chbxProcess.setChecked(False)
self.btnSettings.setEnabled(False)
self.ibkrworker.app.disconnect()
while self.ibkrworker.app.isConnected():
print("waiting for disconnect")
time.sleep(1)
self.ibkrworker = None
self.ibkrworker = IBKRWorker(self.settings)
self.connect_to_ibkr()
i = 4
class ProjectListItem(QObject):
"""
The core functionality class - GUI representation of the Stm32pio project
"""
nameChanged = Signal() # properties notifiers
stateChanged = Signal()
stageChanged = Signal()
logAdded = Signal(str, int,
arguments=['message', 'level'
]) # send the log message to the front-end
actionDone = Signal(str, bool,
arguments=['action', 'success'
]) # emit when the action has executed
def __init__(self,
project_args: list = None,
project_kwargs: dict = None,
parent: QObject = None):
super().__init__(parent=parent)
if project_args is None:
project_args = []
if project_kwargs is None:
project_kwargs = {}
self.logger = logging.getLogger(f"{stm32pio.lib.__name__}.{id(self)}")
self.logger.setLevel(
logging.DEBUG if settings.get('verbose') else logging.INFO)
self.logging_worker = LoggingWorker(self.logger)
self.logging_worker.sendLog.connect(self.logAdded)
# QThreadPool can automatically queue new incoming tasks if a number of them are larger than maxThreadCount
self.workers_pool = QThreadPool()
self.workers_pool.setMaxThreadCount(1)
self.workers_pool.setExpiryTimeout(
-1) # tasks forever wait for the available spot
# These values are valid till the Stm32pio project does not initialize itself (or failed to)
self.project = None
self._name = 'Loading...'
self._state = {'LOADING': True}
self._current_stage = 'Loading...'
self.qml_ready = threading.Event(
) # the front and the back both should know when each other is initialized
self._finalizer = weakref.finalize(
self, self.at_exit) # register some kind of deconstruction handler
if project_args is not None:
if 'instance_options' not in project_kwargs:
project_kwargs['instance_options'] = {'logger': self.logger}
elif 'logger' not in project_kwargs['instance_options']:
project_kwargs['instance_options']['logger'] = self.logger
# Start the Stm32pio part initialization right after. It can take some time so we schedule it in a dedicated
# thread
self.init_thread = threading.Thread(target=self.init_project,
args=project_args,
kwargs=project_kwargs)
self.init_thread.start()
def init_project(self, *args, **kwargs) -> None:
"""
Initialize the underlying Stm32pio project.
Args:
*args: positional arguments of the Stm32pio constructor
**kwargs: keyword arguments of the Stm32pio constructor
"""
try:
self.project = stm32pio.lib.Stm32pio(*args, **kwargs)
except Exception as e:
# Error during the initialization
self.logger.exception(e,
exc_info=self.logger.isEnabledFor(
logging.DEBUG))
if len(args):
self._name = args[
0] # use project path string (probably) as a name
else:
self._name = 'No name'
self._state = {'INIT_ERROR': True}
self._current_stage = 'Initializing error'
else:
self._name = 'Project' # successful initialization. These values should not be used anymore
self._state = {}
self._current_stage = 'Initialized'
finally:
self.qml_ready.wait() # wait for the GUI to initialized
self.nameChanged.emit(
) # in any case we should notify the GUI part about the initialization ending
self.stageChanged.emit()
self.stateChanged.emit()
def at_exit(self):
module_logger.info(f"destroy {self.project}")
self.workers_pool.waitForDone(
msecs=-1
) # wait for all jobs to complete. Currently, we cannot abort them gracefully
self.logging_worker.stopped.set(
) # post the event in the logging worker to inform it...
self.logging_worker.thread.wait() # ...and wait for it to exit
@Property(str, notify=nameChanged)
def name(self):
if self.project is not None:
return self.project.path.name
else:
return self._name
@Property('QVariant', notify=stateChanged)
def state(self):
if self.project is not None:
# Convert to normal dict (JavaScript object) and exclude UNDEFINED key
return {
stage.name: value
for stage, value in self.project.state.items()
if stage != stm32pio.lib.ProjectStage.UNDEFINED
}
else:
return self._state
@Property(str, notify=stageChanged)
def current_stage(self):
if self.project is not None:
return str(self.project.state.current_stage)
else:
return self._current_stage
@Slot()
def qmlLoaded(self):
"""
Event signaling the complete loading of needed frontend components.
"""
self.qml_ready.set()
self.logging_worker.can_flush_log.set()
@Slot(str, 'QVariantList')
def run(self, action: str, args: list):
"""
Asynchronously perform Stm32pio actions (generate, build, etc.) (dispatch all business logic).
Args:
action: method name of the corresponding Stm32pio action
args: list of positional arguments for the action
"""
worker = ProjectActionWorker(getattr(self.project, action), args,
self.logger)
worker.actionDone.connect(self.stateChanged)
worker.actionDone.connect(self.stageChanged)
worker.actionDone.connect(self.actionDone)
self.workers_pool.start(
worker) # will automatically place to the queue