class DiscreteVariableEditor(VariableEditor):
def __init__(self, parent: QWidget, items: Tuple[str], callback: Callable):
super().__init__(parent, callback)
self._combo = QComboBox(
parent,
maximumWidth=180,
sizePolicy=QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
)
self._combo.addItems(items + ("?",))
self._combo.currentIndexChanged.connect(self.__on_index_changed)
self.layout().addWidget(self._combo)
@property
def value(self) -> Union[int, float]:
return self._map_to_var_values()
@value.setter
def value(self, value: float):
if np.isnan(value):
value = self._combo.model().rowCount() - 1
assert value == int(value)
self._combo.setCurrentIndex(int(value))
def __on_index_changed(self):
self.valueChanged.emit(self._map_to_var_values())
def _map_to_var_values(self) -> Union[int, float]:
n_values = self._combo.model().rowCount() - 1
current = self._combo.currentIndex()
return current if current < n_values else np.nan
def _(combo: QComboBox, value: str):
model: QAbstractItemModel = combo.model()
values = [
model.data(model.index(i, 0), role=Qt.EditRole)
for i in range(model.rowCount())
]
combo.setCurrentIndex(values.index(value))
class RecentPathsWComboMixin(RecentPathsWidgetMixin):
"""
Adds file combo handling to :obj:`RecentPathsWidgetMixin`.
The mixin constructs a combo box `self.file_combo` and provides a method
`set_file_list` for updating its content. The mixin also overloads the
inherited `add_path` and `select_file` to call `set_file_list`.
"""
def __init__(self):
super().__init__()
self.file_combo = \
QComboBox(self, sizeAdjustPolicy=QComboBox.AdjustToContents)
def add_path(self, filename):
"""Add (or move) a file name to the top of recent paths"""
super().add_path(filename)
self.set_file_list()
def select_file(self, n):
"""Move the n-th file to the top of the list"""
super().select_file(n)
self.set_file_list()
def set_file_list(self):
"""
Sets the items in the file list combo
"""
self._check_init()
self.file_combo.clear()
if not self.recent_paths:
self.file_combo.addItem("(none)")
self.file_combo.model().item(0).setEnabled(False)
else:
for i, recent in enumerate(self.recent_paths):
self.file_combo.addItem(recent.basename)
self.file_combo.model().item(i).setToolTip(recent.abspath)
if not os.path.exists(recent.abspath):
self.file_combo.setItemData(i, QBrush(Qt.red),
Qt.TextColorRole)
def workflowEnvChanged(self, key, value, oldvalue):
super().workflowEnvChanged(key, value, oldvalue)
if key == "basedir":
self.set_file_list()
def setenabled(cb: QComboBox, clu: bool, clu_op: bool):
model = cb.model()
assert isinstance(model, QStandardItemModel)
idx = cb.findData(Clustering.OrderedClustering, ClusteringRole)
assert idx != -1
model.item(idx).setEnabled(clu_op)
idx = cb.findData(Clustering.Clustering, ClusteringRole)
assert idx != -1
model.item(idx).setEnabled(clu)
class OWImportImages(widget.OWWidget):
name = "Import Images"
description = "Import images from a directory(s)"
icon = "icons/ImportImages.svg"
priority = 110
outputs = [("Data", Orange.data.Table)]
#: list of recent paths
recent_paths = settings.Setting([]) # type: List[RecentPath]
currentPath = settings.Setting(None)
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
# Modality = Qt.WindowModal
MaxRecentItems = 20
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.data = None
self._n_image_categories = 0
self._n_image_data = 0
self._n_skipped = 0
self.__invalidated = False
self.__pendingTask = None
vbox = gui.vBox(self.controlArea)
hbox = gui.hBox(vbox)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
icons = standard_icons(self)
browseaction = QAction(
"Open/Load Images",
self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=icons.dir_open_icon,
toolTip="Select a directory from which to load the images")
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction("Reload",
self,
icon=icons.reload_icon,
toolTip="Reload current image set")
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=True,
)
hbox.layout().addWidget(self.recent_cb)
hbox.layout().addWidget(browsebutton)
hbox.layout().addWidget(reloadbutton)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled()))
box = gui.vBox(vbox, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(text="No image set selected", wordWrap=True)
self.progress_widget = QProgressBar(minimum=0, maximum=0)
self.cancel_button = QPushButton(
"Cancel",
icon=icons.cancel_icon,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def __initRecentItemsModel(self):
if self.currentPath is not None and \
not os.path.isdir(self.currentPath):
self.currentPath = None
recent_paths = []
for item in self.recent_paths:
if os.path.isdir(item.abspath):
recent_paths.append(item)
recent_paths = recent_paths[:OWImportImages.MaxRecentItems]
recent_model = self.recent_cb.model()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.currentPath is not None and \
os.path.isdir(self.currentPath) and self.recent_paths and \
os.path.samefile(self.currentPath, self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
else:
self.currentPath = None
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(self.currentPath is not None)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
if OWImportImages.Modality == Qt.WindowModal:
dlg = QFileDialog(
self,
"Select Top Level Directory",
startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, "Select Top Level Directory", startdir)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No image set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self._n_image_data
ncategories = self._n_image_categories
n_skipped = self._n_skipped
if ncategories < 2:
text = "{} image{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} images / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root image path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.currentPath is not None and path is not None and \
os.path.isdir(self.currentPath) and os.path.isdir(path) and \
os.path.samefile(self.currentPath, path):
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a directory".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
if newindex >= 0:
self.currentPath = path
else:
self.currentPath = None
else:
self.currentPath = None
self.__actions.reload.setEnabled(self.currentPath is not None)
if self.__state == State.Processing:
self.cancel()
return success
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath(path, None, None)
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [
State.Done, State.NoState, State.Error, State.Cancelled
]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the image scan task
"""
if self.__state == State.Processing:
self.cancel()
self.data = None
self.start()
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if self.currentPath is None:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')".format(
self.__pendingTask.startdir))
self.cancel()
startdir = self.currentPath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(self, "__onReportProgress", (object, ))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettyfypath(arg.lastpath))
def commit(self):
"""
Commit a Table from the collected image meta data.
"""
self.send("Data", self.data)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
class OWLoadData(widget.OWWidget):
name = "Load Data"
icon = "icons/upload.svg"
priority = 10
class Outputs:
data = widget.Output("Data", Orange.data.Table)
class Information(widget.OWWidget.Information):
modified = widget.Msg(
"Uncommited changes\nPress 'Load data' to submit changes")
class Warning(widget.OWWidget.Warning):
sampling_in_effect = widget.Msg("Sampling is in effect.")
class Error(widget.OWWidget.Error):
row_annotation_mismatch = widget.Msg("Row annotation length mismatch\n"
"Expected {} rows got {}")
col_annotation_mismatch = widget.Msg(
"Column annotation length mismatch\n"
"Expected {} rows got {}")
_recent = settings.Setting([]) # type: List[str]
_recent_row_annotations = settings.Setting([]) # type: List[str]
_recent_col_annotations = settings.Setting([]) # type: List[str]
_cells_in_rows = settings.Setting(False)
_col_annotations_enabled = settings.Setting(False)
_row_annotations_enabled = settings.Setting(False)
_last_path = settings.Setting("") # type: str
_header_rows_count = settings.Setting(1) # type: int
_header_cols_count = settings.Setting(1) # type: int
_sample_rows_enabled = settings.Setting(False) # type: bool
_sample_cols_enabled = settings.Setting(False) # type: bool
_sample_cols_p = settings.Setting(10.0) # type: bool
_sample_rows_p = settings.Setting(10.0) # type: bool
settingsHandler = RunaroundSettingsHandler()
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
self._current_path = ""
icon_open_dir = self.style().standardIcon(QStyle.SP_DirOpenIcon)
# Top grid with file selection combo box
grid = QGridLayout()
lb = QLabel("File:")
lb.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
self.recent_combo = cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=20,
toolTip="Select a recent file")
self.recent_model = cb.model() # type: QStandardItemModel
self.recent_combo.activated[int].connect(self._select_recent)
browse = QPushButton("...",
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse)
# reload = QPushButton("Reload", autoDefault=False, icon=icon_reload)
grid.addWidget(lb, 0, 0, Qt.AlignVCenter)
grid.addWidget(cb, 0, 1)
grid.addWidget(browse, 0, 2)
# grid.addWidget(reload, 0, 3)
self.summary_label = label = QLabel("", self)
label.ensurePolished()
f = label.font()
if f.pointSizeF() != -1:
f.setPointSizeF(f.pointSizeF() * 5 / 6)
else:
f.setPixelSize(f.pixelSize() * 5 / 6)
label.setFont(f)
grid.addWidget(label, 1, 1, 1, 3)
self.controlArea.layout().addLayout(grid)
box = gui.widgetBox(self.controlArea,
"Headers and Row Labels",
spacing=-1)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_rows_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_rows_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_rows_count)
hl.addWidget(QLabel("Data starts with", box))
hl.addWidget(self.header_rows_spin)
hl.addWidget(QLabel("header row(s)", box))
hl.addStretch(10)
box.layout().addLayout(hl)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_cols_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_cols_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_cols_count)
hl.addWidget(QLabel("First", box))
hl.addWidget(self.header_cols_spin)
hl.addWidget(QLabel("column(s) are row labels", box))
hl.addStretch(10)
box.layout().addLayout(hl)
self.data_struct_box = box = gui.widgetBox(self.controlArea,
"Input Data Structure")
gui.radioButtons(box,
self,
"_cells_in_rows", [
"Genes in rows, samples in columns",
"Samples in rows, genes in columns"
],
callback=self._invalidate)
box = gui.widgetBox(self.controlArea, "Sample Data", spacing=-1)
grid = QGridLayout()
grid.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(grid)
self.sample_rows_cb = cb = QCheckBox(checked=self._sample_rows_enabled)
spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_rows_p,
enabled=self._sample_rows_enabled)
spin.valueChanged.connect(self.set_sample_rows_p)
suffix = QLabel("% of Samples", enabled=self._sample_rows_enabled)
cb.toggled.connect(self.set_sample_rows_enabled)
cb.toggled.connect(spin.setEnabled)
cb.toggled.connect(suffix.setEnabled)
grid.addWidget(cb, 0, 0)
grid.addWidget(spin, 0, 1)
grid.addWidget(suffix, 0, 2)
self.sample_cols_cb = cb = QCheckBox(checked=self._sample_cols_enabled)
spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_cols_p,
enabled=self._sample_cols_enabled)
spin.valueChanged.connect(self.set_sample_cols_p)
suffix = QLabel("% of genes", enabled=self._sample_cols_enabled)
cb.toggled.connect(self.set_sample_cols_enabled)
cb.toggled.connect(spin.setEnabled)
cb.toggled.connect(suffix.setEnabled)
grid.addWidget(cb, 1, 0)
grid.addWidget(spin, 1, 1)
grid.addWidget(suffix, 1, 2)
grid.setColumnStretch(3, 10)
self.annotation_files_box = box = gui.widgetBox(
self.controlArea, "Cell && Gene Annotation Files")
form = QFormLayout(
formAlignment=Qt.AlignLeft,
rowWrapPolicy=QFormLayout.WrapAllRows,
)
box.layout().addLayout(form)
self.row_annotations_cb = cb = QCheckBox(
"Cell annotations", checked=self._row_annotations_enabled)
self._row_annotations_w = w = QWidget(
enabled=self._row_annotations_enabled)
cb.toggled.connect(self.set_row_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.row_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.row_annotations_combo.activated.connect(self._invalidate)
hl.addWidget(self.row_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_row_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.col_annotations_cb = cb = QCheckBox(
"Gene annotations", checked=self._col_annotations_enabled)
self._col_annotations_w = w = QWidget(
enabled=self._col_annotations_enabled)
cb.toggled.connect(self.set_col_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.col_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.col_annotations_combo.activated.connect(self._invalidate)
hl.addWidget(self.col_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_col_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.controlArea.layout().addStretch(10)
self.load_data_button = button = VariableTextPushButton(
"Load data",
autoDefault=True,
textChoiceList=["Load data", "Reload"])
self.load_data_button.setAutoDefault(True)
button.clicked.connect(self.commit, Qt.QueuedConnection)
self.controlArea.layout().addWidget(button, alignment=Qt.AlignRight)
init_recent_paths_model(
self.recent_model,
[RecentPath.create(p, []) for p in self._recent],
)
init_recent_paths_model(
self.row_annotations_combo.model(),
[RecentPath.create(p, []) for p in self._recent_row_annotations])
init_recent_paths_model(
self.col_annotations_combo.model(),
[RecentPath.create(p, []) for p in self._recent_col_annotations])
self._update_summary()
self._update_warning()
if self._last_path != "" and os.path.exists(self._last_path):
self.set_current_path(self._last_path)
else:
self.recent_combo.setCurrentIndex(-1)
def _update_warning(self):
if (self._sample_rows_enabled and self._sample_rows_p < 100) or \
(self._sample_cols_enabled and self._sample_cols_p < 100):
self.Warning.sampling_in_effect()
else:
self.Warning.sampling_in_effect.clear()
def set_sample_rows_enabled(self, enabled):
if self._sample_rows_enabled != enabled:
self._sample_rows_enabled = enabled
self._update_warning()
self._invalidate()
def set_sample_cols_enabled(self, enabled):
if self._sample_cols_enabled != enabled:
self._sample_cols_enabled = enabled
self._update_warning()
self._invalidate()
def set_sample_rows_p(self, p):
if self._sample_rows_p != p:
self._sample_rows_p = p
self._update_warning()
self._invalidate()
def set_sample_cols_p(self, p):
if self._sample_cols_p != p:
self._sample_cols_p = p
self._update_warning()
self._invalidate()
def set_header_rows_count(self, n):
if self._header_rows_count != n:
self._header_rows_count = n
self.header_rows_spin.setValue(n)
self._invalidate()
def set_header_cols_count(self, n):
if self._header_cols_count != n:
self._header_cols_count = n
self.header_cols_spin.setValue(n)
self._invalidate()
def set_row_annotations_enabled(self, enabled):
if self._row_annotations_enabled != enabled:
self._row_annotations_enabled = enabled
self.row_annotations_cb.setChecked(enabled)
self._invalidate()
def set_col_annotations_enabled(self, enabled):
if self._col_annotations_enabled != enabled:
self._col_annotations_enabled = enabled
self.col_annotations_cb.setChecked(enabled)
self._invalidate()
def set_current_path(self, path):
if samepath(self._current_path, path):
return
model = self.recent_model
index = -1
pathitem = None
for i in range(model.rowCount()):
item = model.item(i)
data = item.data(Qt.UserRole) if item is not None else None
if isinstance(data, RecentPath) and samepath(path, data.abspath):
index, pathitem = i, data
break
rpaths = []
if pathitem is None:
assert index == -1
pathitem = RecentPath.create(path, rpaths)
if index != -1:
item = model.takeRow(index)
else:
item = RecentPath_asqstandarditem(pathitem)
opts = infer_options(path)
if path.endswith(".count"):
self.set_header_rows_count(1)
self.set_header_cols_count(1)
fixed_format = False
elif path.endswith(".mtx"):
self.set_header_rows_count(0)
self.set_header_cols_count(0)
fixed_format = False
self._cells_in_rows = False
else:
fixed_format = True
if opts.transposed is not None:
self._cells_in_rows = not opts.transposed
self.data_struct_box.setEnabled(fixed_format)
self.header_rows_spin.setEnabled(fixed_format)
self.header_cols_spin.setEnabled(fixed_format)
model.insertRow(0, item)
self._current_path = path
self.recent_combo.setCurrentIndex(0)
self._update_summary()
if opts.row_annotation_file is not None:
index = insert_recent_path(
self.row_annotations_combo.model(),
RecentPath.create(opts.row_annotation_file, []))
self.row_annotations_combo.setCurrentIndex(index)
self.set_row_annotations_enabled(True)
else:
self.row_annotations_combo.setCurrentIndex(-1)
if opts.column_annotation_file is not None:
index = insert_recent_path(
self.col_annotations_combo.model(),
RecentPath.create(opts.column_annotation_file, []))
self.col_annotations_combo.setCurrentIndex(index)
self.set_col_annotations_enabled(True)
else:
self.col_annotations_combo.setCurrentIndex(-1)
if path.endswith(".mtx") \
and opts.row_annotation_file is not None \
and opts.column_annotation_file is not None \
and os.path.basename(opts.row_annotation_file) == "barcodes.tsv" \
and os.path.basename(opts.column_annotation_file) == "genes.tsv":
# 10x gene-barcode matrix
# TODO: The genes/barcodes files should be unconditionally loaded
# alongside the mtx. The row/col annotations might be used to
# specify additional sources. For the time being they are put in
# the corresponding comboboxes and made uneditable.
self.annotation_files_box.setEnabled(False)
else:
self.annotation_files_box.setEnabled(True)
self._invalidate()
def _update_summary(self):
path = self._current_path
size = None
ncols = None
nrows = None
try:
st = os.stat(path)
except OSError:
pass
else:
size = st.st_size
if os.path.splitext(path)[1] == ".mtx":
try:
with open(path, "rb") as f:
nrows, ncols = scipy.io.mminfo(f)[:2]
except OSError:
pass
except ValueError:
pass
else:
try:
with open(path, "rt", encoding="latin-1") as f:
sep = separator_from_filename(path)
ncols = len(next(csv.reader(f, delimiter=sep)))
nrows = sum(1 for _ in f)
except OSError:
pass
except StopIteration:
pass
text = []
if size is not None:
text += [sizeformat(size)]
if nrows is not None:
text += ["{:n} rows".format(nrows)]
if nrows is not None:
text += ["{:n} columns".format(ncols)]
self.summary_label.setText(", ".join(text))
def current_path(self):
return self._current_path
def _select_recent(self, index):
# type: (int) -> None
# select a file from the recent list (entered via combo box `activate`)
assert 0 <= index < self.recent_model.rowCount()
item = self.recent_model.item(index)
pathitem = item.data(Qt.UserRole)
assert isinstance(pathitem, RecentPath)
self.set_current_path(pathitem.abspath)
@Slot()
def browse(self):
dlg = QFileDialog(self)
dlg.setAcceptMode(QFileDialog.AcceptOpen)
dlg.setFileMode(QFileDialog.ExistingFile)
filters = Formats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
f = dlg.selectedNameFilter()
filename = dlg.selectedFiles()[0]
self.set_current_path(filename)
@Slot()
def browse_row_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
f = dlg.selectedNameFilter()
filename = dlg.selectedFiles()[0]
m = self.row_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.row_annotations_combo.setCurrentIndex(index)
self._invalidate()
@Slot()
def browse_col_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
f = dlg.selectedNameFilter()
filename = dlg.selectedFiles()[0]
m = self.col_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.col_annotations_combo.setCurrentIndex(index)
self._invalidate()
def _invalidate(self):
self.set_modified(True)
def set_modified(self, modified):
if modified:
text = "Load data"
else:
text = "Reload"
self.load_data_button.setText(text)
self.load_data_button.setAutoDefault(modified)
# Setting autoDefault once also sets default which persists even after
# settings autoDefault back to False??
self.load_data_button.setDefault(modified)
self.Information.modified(shown=modified)
def commit(self):
path = self._current_path
if not path:
return
transpose = not self._cells_in_rows
row_annot = self.row_annotations_combo.currentData(Qt.UserRole)
col_annot = self.col_annotations_combo.currentData(Qt.UserRole)
if self._row_annotations_enabled and \
isinstance(row_annot, RecentPath) and \
os.path.exists(row_annot.abspath):
row_annot = row_annot.abspath # type: str
else:
row_annot = None
if self._col_annotations_enabled and \
isinstance(col_annot, RecentPath) and \
os.path.exists(col_annot.abspath):
col_annot = col_annot.abspath # type: str
else:
col_annot = None
meta_parts = [] # type: List[pd.DataFrame]
attrs = [] # type: List[ContinuousVariable]
metas = [] # type: List[StringVariable]
rstate = np.random.RandomState(0x667)
skip_row = skip_col = None
if self._sample_cols_enabled:
p = self._sample_cols_p
if p < 100:
def skip_col(i, p=p):
return i > 3 and rstate.uniform(0, 100) > p
if self._sample_rows_enabled:
p = self._sample_rows_p
if p < 100:
def skip_row(i, p=p):
return i > 3 and rstate.uniform(0, 100) > p
header_rows = self._header_rows_count
header_rows_indices = []
if header_rows == 0:
header_rows = None
elif header_rows == 1:
header_rows = 0
header_rows_indices = [0]
else:
header_rows = list(range(header_rows))
header_rows_indices = header_rows
header_cols = self._header_cols_count
header_cols_indices = []
if header_cols == 0:
header_cols = None
elif header_cols == 1:
header_cols = 0
header_cols_indices = [0]
else:
header_cols = list(range(header_cols))
header_cols_indices = header_cols
if transpose:
_skip_row, _skip_col = skip_col, skip_row
else:
_skip_col, _skip_row = skip_col, skip_row
_userows = _usecols = None
userows_mask = usecols_mask = None
if _skip_col is not None:
ncols = pd.read_csv(path,
sep=separator_from_filename(path),
index_col=None,
nrows=1).shape[1]
usecols_mask = np.array([
not _skip_col(i) or i in header_cols_indices
for i in range(ncols)
],
dtype=bool)
_usecols = np.flatnonzero(usecols_mask)
if _skip_row is not None:
userows_mask = [] # record the used rows
def _skip_row(i, test=_skip_row):
r = test(i)
userows_mask.append(r)
return r
meta_df_index = None
row_annot_header = 0
row_annot_columns = None
col_annot_header = 0
col_annot_columns = None
if os.path.splitext(path)[1] == ".mtx":
# 10x cellranger output
X = scipy.io.mmread(path)
assert isinstance(X, scipy.sparse.coo_matrix)
if transpose:
X = X.T
if _skip_row is not None:
userows_mask = np.array(
[not _skip_row(i) for i in range(X.shape[0])])
X = X.tocsr()[np.flatnonzero(userows_mask)]
if _skip_col is not None:
usecols_mask = np.array(
[not _skip_col(i) for i in range(X.shape[1])])
X = X.tocsc()[:, np.flatnonzero(usecols_mask)]
X = X.todense(order="F")
if userows_mask is not None:
meta_df = pd.DataFrame({}, index=np.flatnonzero(userows_mask))
else:
meta_df = pd.DataFrame({}, index=pd.RangeIndex(X.shape[0]))
meta_df_index = meta_df.index
row_annot_header = None
row_annot_columns = ["Barcodes"]
col_annot_header = None
col_annot_columns = ["Id", "Gene"]
leading_cols = leading_rows = 0
else:
df = pd.read_csv(path,
sep=separator_from_filename(path),
index_col=header_cols,
header=header_rows,
skiprows=_skip_row,
usecols=_usecols)
if _skip_row is not None:
userows_mask = np.array(userows_mask, dtype=bool)
if transpose:
df = df.transpose()
userows_mask, usecols_mask = usecols_mask, userows_mask
leading_rows = len(header_cols_indices)
leading_cols = len(header_rows_indices)
else:
leading_rows = len(header_rows_indices)
leading_cols = len(header_cols_indices)
X = df.values
attrs = [ContinuousVariable.make(str(g)) for g in df.columns]
meta_df = df.iloc[:, :0] # Take the index # type: pd.DataFrame
meta_df_index = df.index
meta_parts = (meta_df, )
self.Error.row_annotation_mismatch.clear()
self.Error.col_annotation_mismatch.clear()
if row_annot is not None:
row_annot_df = pd.read_csv(row_annot,
sep=separator_from_filename(row_annot),
header=row_annot_header,
names=row_annot_columns,
index_col=None)
if userows_mask is not None:
# NOTE: we account for column header/ row index
expected = len(userows_mask) - leading_rows
else:
expected = X.shape[0]
if len(row_annot_df) != expected:
self.Error.row_annotation_mismatch(expected, len(row_annot_df))
row_annot_df = None
if row_annot_df is not None and userows_mask is not None:
# use the same sample indices
indices = np.flatnonzero(userows_mask[leading_rows:])
row_annot_df = row_annot_df.iloc[indices]
# if path.endswith(".count") and row_annot.endswith('.meta'):
# assert np.all(row_annot_df.iloc[:, 0] == df.index)
if row_annot_df is not None and meta_df_index is not None:
# Try to match the leading columns with the meta_df_index.
# If found then drop the columns (or index if the level does
# not have a name but the annotation col does)
drop_cols = []
drop_index_level = []
for i in range(meta_df_index.nlevels):
meta_df_level = meta_df_index.get_level_values(i)
if np.all(row_annot_df.iloc[:, i] == meta_df_level):
if meta_df_level.name is None:
drop_index_level.append(i)
elif meta_df_level.name == row_annot_df.columns[
i].name:
drop_cols.append(i)
if drop_cols:
row_annot_df = row_annot_df.drop(columns=drop_cols)
if drop_index_level:
for i in reversed(drop_index_level):
if isinstance(meta_df.index, pd.MultiIndex):
meta_df_index = meta_df_index.droplevel(i)
else:
assert i == 0
meta_df_index = pd.RangeIndex(meta_df_index.size)
meta_df = pd.DataFrame({}, index=meta_df_index)
if row_annot_df is not None:
meta_parts = (meta_df, row_annot_df)
if col_annot is not None:
col_annot_df = pd.read_csv(col_annot,
sep=separator_from_filename(col_annot),
header=col_annot_header,
names=col_annot_columns,
index_col=None)
if usecols_mask is not None:
expected = len(usecols_mask) - leading_cols
else:
expected = X.shape[1]
if len(col_annot_df) != expected:
self.Error.col_annotation_mismatch(expected, len(col_annot_df))
col_annot_df = None
if col_annot_df is not None and usecols_mask is not None:
indices = np.flatnonzero(usecols_mask[leading_cols:])
col_annot_df = col_annot_df.iloc[indices]
if col_annot_df is not None:
assert len(col_annot_df) == X.shape[1]
if not attrs and X.shape[1]: # No column names yet
attrs = [
ContinuousVariable.make(str(v))
for v in col_annot_df.iloc[:, 0]
]
names = [str(c) for c in col_annot_df.columns]
for var, values in zip(attrs, col_annot_df.values):
var.attributes.update(
{n: v
for n, v in zip(names, values)})
if meta_parts:
meta_parts = [
df_.reset_index() if not df_.index.is_integer() else df_
for df_ in meta_parts
]
metas = [
StringVariable.make(name)
for name in chain(*(_.columns for _ in meta_parts))
]
M = np.hstack(tuple(df_.values for df_ in meta_parts))
else:
metas = None
M = None
if not attrs and X.shape[1]:
attrs = Orange.data.Domain.from_numpy(X).attributes
domain = Orange.data.Domain(attrs, metas=metas)
d = Orange.data.Table.from_numpy(domain, X, None, M)
self.Outputs.data.send(d)
self.set_modified(False)
def onDeleteWidget(self):
super().onDeleteWidget()
def _saveState(self):
maxitems = 15
def dataiter(model, role=Qt.UserRole):
return (model.data(model.index(i, 0), role)
for i in range(model.rowCount()))
def recent_paths(model):
return [
el.abspath for el in dataiter(model)
if isinstance(el, RecentPath)
][:maxitems]
self._recent = recent_paths(self.recent_model)
self._recent_row_annotations = recent_paths(
self.row_annotations_combo.model())
self._recent_col_annotations = recent_paths(
self.col_annotations_combo.model())
self._last_path = self._current_path
def saveSettings(self):
self._saveState()
super().saveSettings()
class FileLoader(QWidget):
activated = pyqtSignal()
file_loaded = pyqtSignal()
def __init__(self):
super().__init__()
self.recent_paths = []
self.file_combo = QComboBox()
self.file_combo.setMinimumWidth(80)
self.file_combo.activated.connect(self._activate)
self.browse_btn = QPushButton("...")
icon = self.style().standardIcon(QStyle.SP_DirOpenIcon)
self.browse_btn.setIcon(icon)
self.browse_btn.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Fixed)
self.browse_btn.clicked.connect(self.browse)
self.load_btn = QPushButton("")
icon = self.style().standardIcon(QStyle.SP_BrowserReload)
self.load_btn.setIcon(icon)
self.load_btn.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Fixed)
self.load_btn.setAutoDefault(True)
self.load_btn.clicked.connect(self.file_loaded)
def browse(self):
start_file = self.last_path() or os.path.expanduser("~/")
formats = ["Text files (*.txt)", "All files (*)"]
file_name, _ = QFileDialog.getOpenFileName(None, "Open...", start_file,
";;".join(formats),
formats[0])
if not file_name:
return
self.add_path(file_name)
self._activate()
def _activate(self):
self.activated.emit()
self.file_loaded.emit()
def set_current_file(self, path: str):
if path:
self.add_path(path)
self.file_combo.setCurrentText(path)
else:
self.file_combo.setCurrentText("(none)")
def get_current_file(self) -> Optional[RecentPath]:
index = self.file_combo.currentIndex()
if index >= len(self.recent_paths) or index < 0:
return None
path = self.recent_paths[index]
return path if isinstance(path, RecentPath) else None
def add_path(self, filename: str):
recent = RecentPath.create(filename, [])
if recent in self.recent_paths:
self.recent_paths.remove(recent)
self.recent_paths.insert(0, recent)
self.set_file_list()
def set_file_list(self):
self.file_combo.clear()
for i, recent in enumerate(self.recent_paths):
self.file_combo.addItem(recent.basename)
self.file_combo.model().item(i).setToolTip(recent.abspath)
if not os.path.exists(recent.abspath):
self.file_combo.setItemData(i, QBrush(Qt.red),
Qt.TextColorRole)
self.file_combo.addItem(_DEFAULT_NONE)
def last_path(self) -> Optional[str]:
return self.recent_paths[0].abspath if self.recent_paths else None
class OWLoadData(widget.OWWidget):
name = ""
icon = "icons/LoadData.svg"
priority = 110
class Outputs:
data = widget.Output("Data", Table)
class Information(widget.OWWidget.Information):
modified = widget.Msg(
"Uncommited changes\nPress 'Load data' to submit changes")
class Warning(widget.OWWidget.Warning):
sampling_in_effect = widget.Msg("Sampling is in effect.")
class Error(widget.OWWidget.Error):
row_annotation_mismatch = widget.Msg("Row annotation length mismatch\n"
"Expected {} rows got {}")
col_annotation_mismatch = widget.Msg(
"Column annotation length mismatch\n"
"Expected {} rows got {}")
inadequate_headers = widget.Msg("Headers and Row Labels error")
reading_error = widget.Msg("Cannot read data using given parameters.")
_recent = settings.Setting([]) # type: List[RecentPath]
_recent_row_annotations = settings.Setting([]) # type: List[RecentPath]
_recent_col_annotations = settings.Setting([]) # type: List[RecentPath]
_cells_in_rows = settings.Setting(False)
_col_annotations_enabled = settings.Setting(False)
_row_annotations_enabled = settings.Setting(False)
_last_path = settings.Setting("") # type: str
_header_rows_count = settings.Setting(1) # type: int
_header_cols_count = settings.Setting(1) # type: int
_sample_rows_enabled = settings.Setting(False) # type: bool
_sample_cols_enabled = settings.Setting(False) # type: bool
_sample_cols_p = settings.Setting(10.0) # type: bool
_sample_rows_p = settings.Setting(10.0) # type: bool
settingsHandler = RunaroundSettingsHandler()
want_main_area = False
resizing_enabled = False
cells_in_rows_changed = Signal()
def __init__(self):
super().__init__()
self._current_path = ""
self._data_loader = Loader()
icon_open_dir = self.style().standardIcon(QStyle.SP_DirOpenIcon)
# Top grid with file selection combo box
self.file_layout = grid = QGridLayout()
lb = QLabel("File:")
lb.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
self.recent_combo = cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=20,
toolTip="Select a recent file")
self.recent_model = cb.model() # type: QStandardItemModel
self.recent_combo.activated[int].connect(self._select_recent)
browse = QPushButton("...",
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse)
# reload = QPushButton("Reload", autoDefault=False, icon=icon_reload)
grid.addWidget(lb, 0, 0, Qt.AlignVCenter)
grid.addWidget(cb, 0, 1)
grid.addWidget(browse, 0, 2)
# grid.addWidget(reload, 0, 3)
self.summary_label = label = QLabel("", self)
label.ensurePolished()
f = label.font()
if f.pointSizeF() != -1:
f.setPointSizeF(f.pointSizeF() * 5 / 6)
else:
f.setPixelSize(f.pixelSize() * 5 / 6)
label.setFont(f)
grid.addWidget(label, 1, 1, 1, 3)
self.controlArea.layout().addLayout(grid)
box = gui.widgetBox(self.controlArea,
"Headers and Row Labels",
spacing=-1)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_rows_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_rows_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_rows_count)
hl.addWidget(QLabel("Data starts with", box))
hl.addWidget(self.header_rows_spin)
hl.addWidget(QLabel("header row(s)", box))
hl.addStretch(10)
box.layout().addLayout(hl)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_cols_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_cols_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_cols_count)
hl.addWidget(QLabel("First", box))
hl.addWidget(self.header_cols_spin)
hl.addWidget(QLabel("column(s) are row labels", box))
hl.addStretch(10)
box.layout().addLayout(hl)
self.data_struct_box = box = gui.widgetBox(self.controlArea,
"Input Data Structure")
gui.radioButtons(box,
self,
"_cells_in_rows", [
"Genes in rows, cells in columns",
"Cells in rows, genes in columns"
],
callback=self._cells_in_rows_changed)
box = gui.widgetBox(self.controlArea, "Sample Data", spacing=-1)
grid = QGridLayout()
grid.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(grid)
self.sample_rows_cb = cb = QCheckBox(checked=self._sample_rows_enabled)
self.sample_rows_p_spin = spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_rows_p)
spin.valueChanged.connect(self.set_sample_rows_p)
suffix = QLabel("% of cells")
cb.toggled.connect(self.set_sample_rows_enabled)
grid.addWidget(cb, 0, 0)
grid.addWidget(spin, 0, 1)
grid.addWidget(suffix, 0, 2)
self.sample_cols_cb = cb = QCheckBox(checked=self._sample_cols_enabled)
self.sample_cols_p_spin = spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_cols_p)
spin.valueChanged.connect(self.set_sample_cols_p)
suffix = QLabel("% of genes")
cb.toggled.connect(self.set_sample_cols_enabled)
grid.addWidget(cb, 1, 0)
grid.addWidget(spin, 1, 1)
grid.addWidget(suffix, 1, 2)
grid.setColumnStretch(3, 10)
self.annotation_files_box = box = gui.widgetBox(
self.controlArea, "Cell && Gene Annotation Files")
form = QFormLayout(
formAlignment=Qt.AlignLeft,
rowWrapPolicy=QFormLayout.WrapAllRows,
)
box.layout().addLayout(form)
self.row_annotations_cb = cb = QCheckBox(
"Cell annotations", checked=self._row_annotations_enabled)
self._row_annotations_w = w = QWidget(
enabled=self._row_annotations_enabled)
cb.toggled.connect(self.set_row_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.row_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.row_annotations_combo.activated.connect(
self._row_annotations_combo_changed)
hl.addWidget(self.row_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_row_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.col_annotations_cb = cb = QCheckBox(
"Gene annotations", checked=self._col_annotations_enabled)
self._col_annotations_w = w = QWidget(
enabled=self._col_annotations_enabled)
cb.toggled.connect(self.set_col_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.col_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.col_annotations_combo.activated.connect(
self._col_annotations_combo_changed)
hl.addWidget(self.col_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_col_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.controlArea.layout().addStretch(10)
self.load_data_button = button = VariableTextPushButton(
"Load data",
autoDefault=True,
textChoiceList=["Load data", "Reload"])
self.load_data_button.setAutoDefault(True)
button.clicked.connect(self.commit, Qt.QueuedConnection)
self.controlArea.layout().addWidget(button, alignment=Qt.AlignRight)
init_recent_paths_model(
self.recent_model,
[self.resolve_path(p) for p in self._recent],
)
init_recent_paths_model(
self.row_annotations_combo.model(),
[self.resolve_path(p) for p in self._recent_row_annotations])
init_recent_paths_model(
self.col_annotations_combo.model(),
[self.resolve_path(p) for p in self._recent_col_annotations])
self._update_summary()
self._update_warning()
if self._last_path != "" and os.path.exists(self._last_path):
QTimer.singleShot(0,
lambda: self.set_current_path(self._last_path))
else:
self.recent_combo.setCurrentIndex(-1)
def resolve_path(self, path):
basedir = self.workflowEnv().get("basedir", None)
if not basedir or not path:
return path
return path.resolve([("basedir", basedir)]) or path
def _cells_in_rows_changed(self):
self._data_loader.transposed = not self._cells_in_rows
self._invalidate()
self.cells_in_rows_changed.emit()
def _row_annotations_combo_changed(self):
path = self.row_annotations_combo.currentData(Qt.UserRole)
if isinstance(path, RecentPath) and os.path.exists(path.abspath):
self._data_loader.row_annotation_file = path # type: RecentPath
else:
self._data_loader.row_annotation_file = None
self._invalidate()
def _col_annotations_combo_changed(self):
path = self.col_annotations_combo.currentData(Qt.UserRole)
if isinstance(path, RecentPath) and os.path.exists(path.abspath):
self._data_loader.col_annotation_file = path # type: RecentPath
else:
self._data_loader.col_annotation_file = None
self._invalidate()
def _update_warning(self):
if (self._sample_rows_enabled and self._sample_rows_p < 100) or \
(self._sample_cols_enabled and self._sample_cols_p < 100):
self.Warning.sampling_in_effect()
else:
self.Warning.sampling_in_effect.clear()
def set_sample_rows_enabled(self, enabled, commit=True):
if self._sample_rows_enabled != enabled:
self._sample_rows_enabled = enabled
self.sample_rows_cb.setChecked(enabled)
self._update_warning()
self._data_loader.sample_rows_enabled = enabled
if commit:
self._invalidate()
def set_sample_cols_enabled(self, enabled, commit=True):
if self._sample_cols_enabled != enabled:
self._sample_cols_enabled = enabled
self.sample_cols_cb.setChecked(enabled)
self._update_warning()
self._data_loader.sample_cols_enabled = enabled
if commit:
self._invalidate()
def set_sample_rows_p(self, p, commit=True):
if self._sample_rows_p != p:
self._sample_rows_p = p
self._update_warning()
self.sample_rows_p_spin.setValue(p)
self._data_loader.sample_rows_p = p
if commit:
self._invalidate()
def set_sample_cols_p(self, p, commit=True):
if self._sample_cols_p != p:
self._sample_cols_p = p
self._update_warning()
self.sample_cols_p_spin.setValue(p)
self._data_loader.sample_cols_p = p
if commit:
self._invalidate()
def set_header_rows_count(self, n, commit=True):
if self._header_rows_count != n:
self._header_rows_count = n
self.header_rows_spin.setValue(n)
self._data_loader.header_rows_count = n
if commit:
self._invalidate()
def set_header_cols_count(self, n, commit=True):
if self._header_cols_count != n:
self._header_cols_count = n
self.header_cols_spin.setValue(n)
self._data_loader.header_cols_count = n
if commit:
self._invalidate()
def set_row_annotations_enabled(self, enabled, commit=True):
if self._row_annotations_enabled != enabled:
self._row_annotations_enabled = enabled
self.row_annotations_cb.setChecked(enabled)
self._data_loader.row_annotations_enabled = enabled
if commit:
self._invalidate()
def set_col_annotations_enabled(self, enabled, commit=True):
if self._col_annotations_enabled != enabled:
self._col_annotations_enabled = enabled
self.col_annotations_cb.setChecked(enabled)
self._data_loader.col_annotations_enabled = enabled
if commit:
self._invalidate()
def set_current_path(self, path):
if samepath(self._current_path, path):
return
model = self.recent_model
index = -1
pathitem = None
for i in range(model.rowCount()):
item = model.item(i)
data = item.data(Qt.UserRole) if item is not None else None
if isinstance(data, RecentPath) and samepath(path, data.abspath):
index, pathitem = i, data
break
rpaths = []
if pathitem is None:
assert index == -1
pathitem = RecentPath.create(path, rpaths)
if index != -1:
item = model.takeRow(index)
else:
item = RecentPath_asqstandarditem(pathitem)
model.insertRow(0, item)
self._current_path = path
self.recent_combo.setCurrentIndex(0)
self._data_loader = get_data_loader(path)
self._update_summary()
self.setup_gui()
self._invalidate()
def setup_gui(self):
""" Use loader predefined values. If the value is None, set
loader's parameter to widget's setting value.
"""
loader = self._data_loader
if loader.header_rows_count is not None:
self.set_header_rows_count(loader.header_rows_count, False)
else:
loader.header_rows_count = self._header_rows_count
if loader.header_cols_count is not None:
self.set_header_cols_count(loader.header_cols_count, False)
else:
loader.header_cols_count = self._header_cols_count
if loader.transposed is not None:
self._cells_in_rows = not loader.transposed
else:
loader.transposed = not self._cells_in_rows
if loader.sample_rows_enabled is not None:
self.set_sample_rows_enabled(loader.sample_rows_enabled, False)
else:
loader.sample_rows_enabled = self._sample_rows_enabled
if loader.sample_cols_enabled is not None:
self.set_sample_cols_enabled(loader.sample_cols_enabled, False)
else:
loader.sample_cols_enabled = self._sample_cols_enabled
if loader.sample_rows_p is not None:
self.set_sample_rows_p(loader.sample_rows_p, False)
else:
loader.sample_rows_p = self._sample_rows_p
if loader.sample_cols_p is not None:
self.set_sample_cols_p(loader.sample_cols_p, False)
else:
loader.sample_cols_p = self._sample_cols_p
if loader.row_annotation_file is not None:
index = insert_recent_path(
self.row_annotations_combo.model(),
self.resolve_path(loader.row_annotation_file))
self.row_annotations_combo.setCurrentIndex(index)
self.set_row_annotations_enabled(loader.row_annotations_enabled,
False)
else:
self.row_annotations_combo.setCurrentIndex(-1)
self.set_row_annotations_enabled(False, False)
if loader.col_annotation_file is not None:
index = insert_recent_path(
self.col_annotations_combo.model(),
self.resolve_path(loader.col_annotation_file))
self.col_annotations_combo.setCurrentIndex(index)
self.set_col_annotations_enabled(loader.col_annotations_enabled,
False)
else:
self.col_annotations_combo.setCurrentIndex(-1)
self.set_col_annotations_enabled(False, False)
self.header_rows_spin.setEnabled(loader.FIXED_FORMAT)
self.header_cols_spin.setEnabled(loader.FIXED_FORMAT)
self.data_struct_box.setEnabled(loader.FIXED_FORMAT)
self.annotation_files_box.setEnabled(loader.ENABLE_ANNOTATIONS)
def _update_summary(self):
size = self._data_loader.file_size
ncols = self._data_loader.n_cols
nrows = self._data_loader.n_rows
text = []
if size is not None:
text += [sizeformat(size)]
if nrows is not None:
text += ["{:n} rows".format(nrows)]
if nrows is not None:
text += ["{:n} columns".format(ncols)]
self.summary_label.setText(", ".join(text))
def current_path(self):
return self._current_path
def _select_recent(self, index):
# type: (int) -> None
# select a file from the recent list (entered via combo box `activate`)
assert 0 <= index < self.recent_model.rowCount()
item = self.recent_model.item(index)
pathitem = item.data(Qt.UserRole)
assert isinstance(pathitem, RecentPath)
self.set_current_path(pathitem.abspath)
@Slot()
def browse(self):
dlg = QFileDialog(self)
dlg.setAcceptMode(QFileDialog.AcceptOpen)
dlg.setFileMode(QFileDialog.ExistingFile)
filters = Formats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
filename = dlg.selectedFiles()[0]
self.set_current_path(filename)
@Slot()
def browse_row_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
filename = dlg.selectedFiles()[0]
m = self.row_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.row_annotations_combo.setCurrentIndex(index)
self._invalidate()
@Slot()
def browse_col_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
filename = dlg.selectedFiles()[0]
m = self.col_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.col_annotations_combo.setCurrentIndex(index)
self._invalidate()
def _invalidate(self):
self.set_modified(True)
def set_modified(self, modified):
if modified:
text = "Load data"
else:
text = "Reload"
self.load_data_button.setText(text)
self.load_data_button.setAutoDefault(modified)
# Setting autoDefault once also sets default which persists even after
# settings autoDefault back to False??
self.load_data_button.setDefault(modified)
self.Information.modified(shown=modified)
def commit(self):
path = self._current_path
if not path:
return
self.Outputs.data.send(self._data_loader())
self.show_error_messages()
self.set_modified(False)
def show_error_messages(self):
self.Error.row_annotation_mismatch.clear()
self.Error.col_annotation_mismatch.clear()
self.Error.inadequate_headers.clear()
errors = self._data_loader.errors
if len(errors["row_annot_mismatch"]):
self.Error.row_annotation_mismatch(*errors["row_annot_mismatch"])
if len(errors["col_annot_mismatch"]):
self.Error.col_annotation_mismatch(*errors["col_annot_mismatch"])
if len(errors["inadequate_headers"]):
self.Error.inadequate_headers(*errors["inadequate_headers"])
if len(errors["reading_error"]):
self.Error.reading_error()
def onDeleteWidget(self):
super().onDeleteWidget()
def _saveState(self):
maxitems = 15
def dataiter(model, role=Qt.UserRole):
return (model.data(model.index(i, 0), role)
for i in range(model.rowCount()))
def recent_paths(model):
return [
self.relocate_path(el) for el in dataiter(model)
if isinstance(el, RecentPath)
][:maxitems]
self._recent = recent_paths(self.recent_model)
self._recent_row_annotations = recent_paths(
self.row_annotations_combo.model())
self._recent_col_annotations = recent_paths(
self.col_annotations_combo.model())
self._last_path = self._current_path
def relocate_path(self, path):
basedir = self.workflowEnv().get("basedir", None)
if not basedir or not path:
return path
return RecentPath.create(path.abspath, [("basedir", basedir)])
def saveSettings(self):
self._saveState()
super().saveSettings()
class ColorGradientSelection(QWidget):
activated = Signal(int)
currentIndexChanged = Signal(int)
thresholdsChanged = Signal(float, float)
centerChanged = Signal(float)
def __init__(self, *args, thresholds=(0.0, 1.0), center=None, **kwargs):
super().__init__(*args, **kwargs)
low = round(clip(thresholds[0], 0., 1.), 2)
high = round(clip(thresholds[1], 0., 1.), 2)
high = max(low, high)
self.__threshold_low, self.__threshold_high = low, high
self.__center = center
form = QFormLayout(formAlignment=Qt.AlignLeft,
labelAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow)
form.setContentsMargins(0, 0, 0, 0)
self.gradient_cb = QComboBox(
None,
objectName="gradient-combo-box",
)
self.gradient_cb.setAttribute(Qt.WA_LayoutUsesWidgetRect)
icsize = self.style().pixelMetric(QStyle.PM_SmallIconSize, None,
self.gradient_cb)
self.gradient_cb.setIconSize(QSize(64, icsize))
model = itemmodels.ContinuousPalettesModel()
model.setParent(self)
self.gradient_cb.setModel(model)
self.gradient_cb.activated[int].connect(self.activated)
self.gradient_cb.currentIndexChanged.connect(self.currentIndexChanged)
if center is not None:
def __on_center_changed():
self.__center = float(self.center_edit.text() or "0")
self.centerChanged.emit(self.__center)
self.center_box = QWidget()
center_layout = QHBoxLayout()
self.center_box.setLayout(center_layout)
width = QFontMetrics(self.font()).boundingRect("9999999").width()
self.center_edit = QLineEdit(text=f"{self.__center}",
maximumWidth=width,
placeholderText="0",
alignment=Qt.AlignRight)
self.center_edit.setValidator(QDoubleValidator())
self.center_edit.editingFinished.connect(__on_center_changed)
center_layout.setContentsMargins(0, 0, 0, 0)
center_layout.addStretch(1)
center_layout.addWidget(QLabel("Centered at"))
center_layout.addWidget(self.center_edit)
self.gradient_cb.currentIndexChanged.connect(
self.__update_center_visibility)
else:
self.center_box = None
slider_low = Slider(objectName="threshold-low-slider",
minimum=0,
maximum=100,
value=int(low * 100),
orientation=Qt.Horizontal,
tickPosition=QSlider.TicksBelow,
pageStep=10,
toolTip=self.tr("Low gradient threshold"),
whatsThis=self.tr(
"Applying a low threshold will squeeze the "
"gradient from the lower end"))
slider_high = Slider(objectName="threshold-low-slider",
minimum=0,
maximum=100,
value=int(high * 100),
orientation=Qt.Horizontal,
tickPosition=QSlider.TicksAbove,
pageStep=10,
toolTip=self.tr("High gradient threshold"),
whatsThis=self.tr(
"Applying a high threshold will squeeze the "
"gradient from the higher end"))
form.setWidget(0, QFormLayout.SpanningRole, self.gradient_cb)
if self.center_box:
form.setWidget(1, QFormLayout.SpanningRole, self.center_box)
form.addRow(self.tr("Low:"), slider_low)
form.addRow(self.tr("High:"), slider_high)
self.slider_low = slider_low
self.slider_high = slider_high
self.slider_low.valueChanged.connect(self.__on_slider_low_moved)
self.slider_high.valueChanged.connect(self.__on_slider_high_moved)
self.setLayout(form)
def setModel(self, model: QAbstractItemModel) -> None:
self.gradient_cb.setModel(model)
def model(self) -> QAbstractItemModel:
return self.gradient_cb.model()
def findData(self, data: Any, role: Qt.ItemDataRole) -> int:
return self.gradient_cb.findData(data, role)
def setCurrentIndex(self, index: int) -> None:
self.gradient_cb.setCurrentIndex(index)
self.__update_center_visibility()
def currentIndex(self) -> int:
return self.gradient_cb.currentIndex()
currentIndex_ = Property(int,
currentIndex,
setCurrentIndex,
notify=currentIndexChanged)
def currentData(self, role=Qt.UserRole) -> Any:
return self.gradient_cb.currentData(role)
def thresholds(self) -> Tuple[float, float]:
return self.__threshold_low, self.__threshold_high
thresholds_ = Property(object, thresholds, notify=thresholdsChanged)
def thresholdLow(self) -> float:
return self.__threshold_low
def setThresholdLow(self, low: float) -> None:
self.setThresholds(low, max(self.__threshold_high, low))
thresholdLow_ = Property(float,
thresholdLow,
setThresholdLow,
notify=thresholdsChanged)
def thresholdHigh(self) -> float:
return self.__threshold_high
def setThresholdHigh(self, high: float) -> None:
self.setThresholds(min(self.__threshold_low, high), high)
def center(self) -> float:
return self.__center
def setCenter(self, center: float) -> None:
self.__center = center
self.center_edit.setText(f"{center}")
self.centerChanged.emit(center)
thresholdHigh_ = Property(float,
thresholdLow,
setThresholdLow,
notify=thresholdsChanged)
def __on_slider_low_moved(self, value: int) -> None:
high = self.slider_high
old = self.__threshold_low, self.__threshold_high
self.__threshold_low = value / 100.
if value >= high.value():
self.__threshold_high = value / 100.
high.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def __on_slider_high_moved(self, value: int) -> None:
low = self.slider_low
old = self.__threshold_low, self.__threshold_high
self.__threshold_high = value / 100.
if low.value() >= value:
self.__threshold_low = value / 100
low.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def setThresholds(self, low: float, high: float) -> None:
low = round(clip(low, 0., 1.), 2)
high = round(clip(high, 0., 1.), 2)
if low > high:
high = low
if self.__threshold_low != low or self.__threshold_high != high:
self.__threshold_high = high
self.__threshold_low = low
self.slider_low.setSliderPosition(low * 100)
self.slider_high.setSliderPosition(high * 100)
self.thresholdsChanged.emit(high, low)
def __update_center_visibility(self):
if self.center_box is None:
return
palette = self.currentData()
self.center_box.setVisible(
isinstance(palette, colorpalettes.Palette)
and palette.flags & palette.Flags.Diverging != 0)
class ColorGradientSelection(QWidget):
activated = Signal(int)
currentIndexChanged = Signal(int)
thresholdsChanged = Signal(float, float)
def __init__(self, *args, thresholds=(0.0, 1.0), **kwargs):
super().__init__(*args, **kwargs)
low = round(clip(thresholds[0], 0., 1.), 2)
high = round(clip(thresholds[1], 0., 1.), 2)
high = max(low, high)
self.__threshold_low, self.__threshold_high = low, high
form = QFormLayout(
formAlignment=Qt.AlignLeft,
labelAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow
)
form.setContentsMargins(0, 0, 0, 0)
self.gradient_cb = QComboBox(
None, objectName="gradient-combo-box",
)
self.gradient_cb.setAttribute(Qt.WA_LayoutUsesWidgetRect)
icsize = self.style().pixelMetric(
QStyle.PM_SmallIconSize, None, self.gradient_cb
)
self.gradient_cb.setIconSize(QSize(64, icsize))
model = itemmodels.ContinuousPalettesModel()
model.setParent(self)
self.gradient_cb.setModel(model)
self.gradient_cb.activated[int].connect(self.activated)
self.gradient_cb.currentIndexChanged.connect(self.currentIndexChanged)
slider_low = QSlider(
objectName="threshold-low-slider", minimum=0, maximum=100,
value=int(low * 100), orientation=Qt.Horizontal,
tickPosition=QSlider.TicksBelow, pageStep=10,
toolTip=self.tr("Low gradient threshold"),
whatsThis=self.tr("Applying a low threshold will squeeze the "
"gradient from the lower end")
)
slider_high = QSlider(
objectName="threshold-low-slider", minimum=0, maximum=100,
value=int(high * 100), orientation=Qt.Horizontal,
tickPosition=QSlider.TicksAbove, pageStep=10,
toolTip=self.tr("High gradient threshold"),
whatsThis=self.tr("Applying a high threshold will squeeze the "
"gradient from the higher end")
)
form.setWidget(0, QFormLayout.SpanningRole, self.gradient_cb)
form.addRow(self.tr("Low:"), slider_low)
form.addRow(self.tr("High:"), slider_high)
self.slider_low = slider_low
self.slider_high = slider_high
self.slider_low.valueChanged.connect(self.__on_slider_low_moved)
self.slider_high.valueChanged.connect(self.__on_slider_high_moved)
self.setLayout(form)
def setModel(self, model: QAbstractItemModel) -> None:
self.gradient_cb.setModel(model)
def model(self) -> QAbstractItemModel:
return self.gradient_cb.model()
def findData(self, data: Any, role: Qt.ItemDataRole) -> int:
return self.gradient_cb.findData(data, role)
def setCurrentIndex(self, index: int) -> None:
self.gradient_cb.setCurrentIndex(index)
def currentIndex(self) -> int:
return self.gradient_cb.currentIndex()
currentIndex_ = Property(
int, currentIndex, setCurrentIndex, notify=currentIndexChanged)
def currentData(self, role=Qt.UserRole) -> Any:
return self.gradient_cb.currentData(role)
def thresholds(self) -> Tuple[float, float]:
return self.__threshold_low, self.__threshold_high
thresholds_ = Property(object, thresholds, notify=thresholdsChanged)
def thresholdLow(self) -> float:
return self.__threshold_low
def setThresholdLow(self, low: float) -> None:
self.setThresholds(low, max(self.__threshold_high, low))
thresholdLow_ = Property(
float, thresholdLow, setThresholdLow, notify=thresholdsChanged)
def thresholdHigh(self) -> float:
return self.__threshold_high
def setThresholdHigh(self, high: float) -> None:
self.setThresholds(min(self.__threshold_low, high), high)
thresholdHigh_ = Property(
float, thresholdLow, setThresholdLow, notify=thresholdsChanged)
def __on_slider_low_moved(self, value: int) -> None:
high = self.slider_high
old = self.__threshold_low, self.__threshold_high
self.__threshold_low = value / 100.
if value >= high.value():
self.__threshold_high = value / 100.
high.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def __on_slider_high_moved(self, value: int) -> None:
low = self.slider_low
old = self.__threshold_low, self.__threshold_high
self.__threshold_high = value / 100.
if low.value() >= value:
self.__threshold_low = value / 100
low.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def setThresholds(self, low: float, high: float) -> None:
low = round(clip(low, 0., 1.), 2)
high = round(clip(high, 0., 1.), 2)
if low > high:
high = low
if self.__threshold_low != low or self.__threshold_high != high:
self.__threshold_high = high
self.__threshold_low = low
self.slider_low.setSliderPosition(low * 100)
self.slider_high.setSliderPosition(high * 100)
self.thresholdsChanged.emit(high, low)
class OWImportImages(widget.OWWidget):
name = "Import Images"
description = "Import images from a directory(s)"
icon = "icons/ImportImages.svg"
priority = 110
outputs = [("Data", Orange.data.Table)]
#: list of recent paths
recent_paths = settings.Setting([]) # type: List[RecentPath]
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
# Modality = Qt.WindowModal
MaxRecentItems = 20
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.data = None
self._n_image_categories = 0
self._n_image_data = 0
self._n_skipped = 0
self.__invalidated = False
self.__pendingTask = None
vbox = gui.vBox(self.controlArea)
hbox = gui.hBox(vbox)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True
)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
icons = standard_icons(self)
browseaction = QAction(
"Open/Load Images", self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=icons.dir_open_icon,
toolTip="Select a directory from which to load the images"
)
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction(
"Reload", self,
icon=icons.reload_icon,
toolTip="Reload current image set"
)
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(
browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger
)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=True,
)
hbox.layout().addWidget(self.recent_cb)
hbox.layout().addWidget(browsebutton)
hbox.layout().addWidget(reloadbutton)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled())
)
box = gui.vBox(vbox, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(
text="No image set selected",
wordWrap=True
)
self.progress_widget = QProgressBar(
minimum=0, maximum=0
)
self.cancel_button = QPushButton(
"Cancel", icon=icons.cancel_icon,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def __initRecentItemsModel(self):
self._relocate_recent_files()
recent_paths = []
for item in self.recent_paths:
recent_paths.append(item)
recent_paths = recent_paths[:OWImportImages.MaxRecentItems]
recent_model = self.recent_cb.model()
recent_model.clear()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.recent_paths and os.path.isdir(self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
self.__actions.reload.setEnabled(True)
else:
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(False)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
if OWImportImages.Modality == Qt.WindowModal:
dlg = QFileDialog(
self, "Select Top Level Directory", startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, "Select Top Level Directory", startdir
)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No image set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self._n_image_data
ncategories = self._n_image_categories
n_skipped = self._n_skipped
if ncategories < 2:
text = "{} image{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} images / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root image path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.recent_paths and path is not None and \
os.path.isdir(self.recent_paths[0].abspath) and os.path.isdir(path) \
and os.path.samefile(os.path.isdir(self.recent_paths[0].abspath), path):
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a directory".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
self.__actions.reload.setEnabled(len(self.recent_paths) > 0)
if self.__state == State.Processing:
self.cancel()
return success
def _search_paths(self):
basedir = self.workflowEnv().get("basedir", None)
if basedir is None:
return []
return [("basedir", basedir)]
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath.create(path, self._search_paths())
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [State.Done,
State.NoState,
State.Error,
State.Cancelled]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the image scan task
"""
if self.__state == State.Processing:
self.cancel()
self.data = None
self.start()
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if not self.recent_paths:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')"
.format(self.__pendingTask.startdir))
self.cancel()
startdir = self.recent_paths[0].abspath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(
self, "__onReportProgress", (object,))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
else:
self._n_image_data, self._n_image_categories = 0, 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettyfypath(arg.lastpath))
def commit(self):
"""
Commit a Table from the collected image meta data.
"""
self.send("Data", self.data)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if path.endswith("/"):
path = path[:-1] # remove last /
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
def _relocate_recent_files(self):
search_paths = self._search_paths()
rec = []
for recent in self.recent_paths:
kwargs = dict(
title=recent.title, sheet=recent.sheet,
file_format=recent.file_format)
resolved = recent.resolve(search_paths)
if resolved is not None:
rec.append(
RecentPath.create(resolved.abspath, search_paths, **kwargs))
else:
rec.append(recent)
# change the list in-place for the case the widgets wraps this list
self.recent_paths[:] = rec
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.__initRecentItemsModel()
class OWImportDocuments(widget.OWWidget):
name = "Import Documents"
description = "Import text documents from folders."
icon = "icons/ImportDocuments.svg"
priority = 110
class Outputs:
data = Output("Corpus", Corpus)
skipped_documents = Output("Skipped documents", Table)
LOCAL_FILE, URL = range(2)
source = settings.Setting(LOCAL_FILE)
#: list of recent paths
recent_paths: List[RecentPath] = settings.Setting([])
currentPath: Optional[str] = settings.Setting(None)
recent_urls: List[str] = settings.Setting([])
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
MaxRecentItems = 20
class Warning(widget.OWWidget.Warning):
read_error = widget.Msg("{} couldn't be read.")
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.corpus = None
self.n_text_categories = 0
self.n_text_data = 0
self.skipped_documents = []
self.__invalidated = False
self.__pendingTask = None
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='Source')
source_box = gui.radioButtons(None, self, "source", box=True,
callback=self.start, addToLayout=False)
rb_button = gui.appendRadioButton(source_box, "Folder:",
addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.Fixed)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True
)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
browseaction = QAction(
"Open/Load Documents", self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=self.style().standardIcon(QStyle.SP_DirOpenIcon),
toolTip="Select a folder from which to load the documents"
)
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction(
"Reload", self,
icon=self.style().standardIcon(QStyle.SP_BrowserReload),
toolTip="Reload current document set"
)
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(
browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger,
default=False,
autoDefault=False,
)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=False,
autoDefault=False,
)
box.layout().addWidget(self.recent_cb)
layout.addWidget(box, 0, 1)
layout.addWidget(browsebutton, 0, 2)
layout.addWidget(reloadbutton, 0, 3)
rb_button = gui.appendRadioButton(source_box, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = PyListModel()
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 1, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled())
)
box = gui.vBox(self.controlArea, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(
text="No document set selected",
wordWrap=True
)
self.progress_widget = QProgressBar(
minimum=0, maximum=100
)
self.cancel_button = QPushButton(
"Cancel",
icon=self.style().standardIcon(QStyle.SP_DialogCancelButton),
default=False,
autoDefault=False,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = "http://" + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.start()
def __initRecentItemsModel(self):
if self.currentPath is not None and \
not os.path.isdir(self.currentPath):
self.currentPath = None
recent_paths = []
for item in self.recent_paths:
if os.path.isdir(item.abspath):
recent_paths.append(item)
recent_paths = recent_paths[:OWImportDocuments.MaxRecentItems]
recent_model = self.recent_cb.model()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.currentPath is not None and \
os.path.isdir(self.currentPath) and self.recent_paths and \
os.path.samefile(self.currentPath, self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
else:
self.currentPath = None
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(self.currentPath is not None)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
caption = "Select Top Level Folder"
if OWImportDocuments.Modality == Qt.WindowModal:
dlg = QFileDialog(
self, caption, startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, caption, startdir
)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No document set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self.n_text_data
ncategories = self.n_text_categories
n_skipped = len(self.skipped_documents)
if ncategories < 2:
text = "{} document{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} documents / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root text path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.currentPath is not None and path is not None and \
os.path.isdir(self.currentPath) and os.path.isdir(path) and \
os.path.samefile(self.currentPath, path) and \
self.source == self.LOCAL_FILE:
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a folder".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
if newindex >= 0:
self.currentPath = path
else:
self.currentPath = None
else:
self.currentPath = None
self.__actions.reload.setEnabled(self.currentPath is not None)
if self.__state == State.Processing:
self.cancel()
self.source = self.LOCAL_FILE
return success
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath(path, None, None)
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [State.Done,
State.NoState,
State.Error,
State.Cancelled]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the text scan task
"""
if self.__state == State.Processing:
self.cancel()
self.source = self.LOCAL_FILE
self.corpus = None
self.start()
def start(self):
"""
Start/execute the text indexing operation
"""
self.error()
self.Warning.clear()
self.progress_widget.setValue(0)
self.__invalidated = False
startdir = self.currentPath if self.source == self.LOCAL_FILE \
else self.url_combo.currentText().strip()
if not startdir:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')"
.format(self.__pendingTask.startdir))
self.cancel()
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(
self, "__onReportProgress", (object,))
task = ImportDocuments(startdir, self.source == self.URL,
report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=0)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_text_scan_task_interupt():
try:
return task.run()
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_text_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
corpus, errors = None, []
try:
corpus, errors = task.future.result()
except NoDocumentsException:
state = State.Error
self.error("Folder contains no readable files.")
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if corpus:
self.n_text_data = len(corpus)
self.n_text_categories = len(corpus.domain.class_var.values)\
if corpus.domain.class_var else 0
self.corpus = corpus
if self.corpus:
self.corpus.name = "Documents"
self.skipped_documents = errors
if len(errors):
self.Warning.read_error(
"Some files" if len(errors) > 1 else "One file"
)
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettifypath(arg.lastpath))
self.progress_widget.setValue(int(100 * arg.progress))
def commit(self):
"""
Create and commit a Corpus from the collected text meta data.
"""
self.Outputs.data.send(self.corpus)
if self.skipped_documents:
skipped_table = (
Table.from_list(
SKIPPED_DOMAIN,
[[x, os.path.join(self.currentPath, x)]
for x in self.skipped_documents]
)
)
skipped_table.name = "Skipped documents"
else:
skipped_table = None
self.Outputs.skipped_documents.send(skipped_table)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
def send_report(self):
if not self.currentPath:
return
items = [('Path', self.currentPath),
('Number of documents', self.n_text_data)]
if self.n_text_categories:
items += [('Categories', self.n_text_categories)]
if self.skipped_documents:
items += [('Number of skipped', len(self.skipped_documents))]
self.report_items(items, )
