class OWEditDomain(widget.OWWidget):
name = "Edit Domain"
description = "Rename features and their values."
icon = "icons/EditDomain.svg"
priority = 3125
inputs = [("Data", Orange.data.Table, "set_data")]
outputs = [("Data", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
domain_change_hints = settings.ContextSetting({})
selected_index = settings.ContextSetting({})
autocommit = settings.Setting(False)
def __init__(self):
super().__init__()
self.data = None
self.input_vars = ()
self._invalidated = False
box = gui.vBox(self.controlArea, "Domain Features")
self.domain_model = itemmodels.VariableListModel()
self.domain_view = QListView(
selectionMode=QListView.SingleSelection
)
self.domain_view.setModel(self.domain_model)
self.domain_view.selectionModel().selectionChanged.connect(
self._on_selection_changed)
box.layout().addWidget(self.domain_view)
box = gui.hBox(self.controlArea)
gui.button(box, self, "Reset Selected", callback=self.reset_selected)
gui.button(box, self, "Reset All", callback=self.reset_all)
gui.auto_commit(self.controlArea, self, "autocommit", "Apply")
box = gui.vBox(self.mainArea, "Edit")
self.editor_stack = QStackedWidget()
self.editor_stack.addWidget(DiscreteVariableEditor())
self.editor_stack.addWidget(ContinuousVariableEditor())
self.editor_stack.addWidget(VariableEditor())
box.layout().addWidget(self.editor_stack)
@check_sql_input
def set_data(self, data):
"""Set input data set."""
self.closeContext()
self.clear()
self.data = data
if self.data is not None:
self._initialize()
self.openContext(self.data)
self._restore()
self.unconditional_commit()
def clear(self):
"""Clear the widget state."""
self.data = None
self.domain_model[:] = []
self.input_vars = []
self.domain_change_hints = {}
self.selected_index = -1
def reset_selected(self):
"""Reset the currently selected variable to its original state."""
ind = self.selected_var_index()
if ind >= 0:
var = self.input_vars[ind]
desc = variable_description(var, skip_attributes=True)
if desc in self.domain_change_hints:
del self.domain_change_hints[desc]
self.domain_model[ind] = var
self.editor_stack.currentWidget().set_data(var)
self._invalidate()
def reset_all(self):
"""Reset all variables to their original state."""
self.domain_change_hints = {}
if self.data is not None:
# To invalidate stored hints
self.domain_model[:] = self.input_vars
itemmodels.select_row(self.domain_view, self.selected_index)
self._invalidate()
def selected_var_index(self):
"""Return the selected row in 'Domain Features' view."""
rows = self.domain_view.selectedIndexes()
assert len(rows) <= 1
return rows[0].row() if rows else -1
def _initialize(self):
domain = self.data.domain
self.input_vars = tuple(domain) + domain.metas
self.domain_model[:] = list(self.input_vars)
def _restore(self):
# Restore the variable states from saved settings.
def transform(var):
vdesc = variable_description(var, skip_attributes=True)
if vdesc in self.domain_change_hints:
return variable_from_description(
self.domain_change_hints[vdesc],
compute_value=Orange.preprocess.transformation.Identity(var))
else:
return var
self.domain_model[:] = map(transform, self.input_vars)
# Restore the variable selection if possible
index = self.selected_index
if index >= len(self.input_vars):
index = 0 if len(self.input_vars) else -1
if index >= 0:
itemmodels.select_row(self.domain_view, index)
def _on_selection_changed(self):
self.selected_index = self.selected_var_index()
self.open_editor(self.selected_index)
def open_editor(self, index):
self.clear_editor()
if index < 0:
return
var = self.domain_model[index]
editor_index = 2
if var.is_discrete:
editor_index = 0
elif var.is_continuous:
editor_index = 1
editor = self.editor_stack.widget(editor_index)
self.editor_stack.setCurrentWidget(editor)
editor.set_data(var)
editor.variable_changed.connect(self._on_variable_changed)
def clear_editor(self):
current = self.editor_stack.currentWidget()
try:
current.variable_changed.disconnect(self._on_variable_changed)
except Exception:
pass
current.set_data(None)
def _on_variable_changed(self):
"""User edited the current variable in editor."""
assert 0 <= self.selected_index <= len(self.domain_model)
editor = self.editor_stack.currentWidget()
# Replace the variable in the 'Domain Features' view/model
old_var = self.input_vars[self.selected_index]
new_var = editor.get_data().copy(compute_value=Orange.preprocess.transformation.Identity(old_var))
self.domain_model[self.selected_index] = new_var
# Store the transformation hint.
old_var_desc = variable_description(old_var, skip_attributes=True)
self.domain_change_hints[old_var_desc] = variable_description(new_var)
self._invalidate()
def _invalidate(self):
self.commit()
def commit(self):
"""Send the changed data to output."""
new_data = None
if self.data is not None:
input_domain = self.data.domain
n_attrs = len(input_domain.attributes)
n_vars = len(input_domain.variables)
n_class_vars = len(input_domain.class_vars)
all_new_vars = list(self.domain_model)
attrs = all_new_vars[: n_attrs]
class_vars = all_new_vars[n_attrs: n_attrs + n_class_vars]
new_metas = all_new_vars[n_attrs + n_class_vars:]
new_domain = Orange.data.Domain(attrs, class_vars, new_metas)
new_data = self.data.from_table(new_domain, self.data)
self.send("Data", new_data)
def sizeHint(self):
sh = super().sizeHint()
return sh.expandedTo(QSize(660, 550))
def send_report(self):
if self.data is not None:
self.report_raw("", EditDomainReport(
old_domain=chain(self.data.domain.variables, self.data.domain.metas),
new_domain=self.domain_model).to_html())
else:
self.report_data(None)
class OWFeatureConstructor(OWWidget):
name = "Feature Constructor"
description = "Construct new features (data columns) from a set of " \
"existing features in the input data set."
icon = "icons/FeatureConstructor.svg"
inputs = [("Data", Orange.data.Table, "setData")]
outputs = [("Data", Orange.data.Table)]
want_main_area = False
settingsHandler = FeatureConstructorSettingsHandler()
descriptors = ContextSetting([])
currentIndex = ContextSetting(-1)
EDITORS = [
(ContinuousDescriptor, ContinuousFeatureEditor),
(DiscreteDescriptor, DiscreteFeatureEditor),
(StringDescriptor, StringFeatureEditor)
]
class Error(OWWidget.Error):
more_values_needed = Msg("Discrete feature {} needs more values.")
invalid_expressions = Msg("Invalid expressions: {}.")
def __init__(self):
super().__init__()
self.data = None
self.editors = {}
box = gui.vBox(self.controlArea, "Variable Definitions")
toplayout = QHBoxLayout()
toplayout.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(toplayout)
self.editorstack = QStackedWidget(
sizePolicy=QSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding)
)
for descclass, editorclass in self.EDITORS:
editor = editorclass()
editor.featureChanged.connect(self._on_modified)
self.editors[descclass] = editor
self.editorstack.addWidget(editor)
self.editorstack.setEnabled(False)
buttonlayout = QVBoxLayout(spacing=10)
buttonlayout.setContentsMargins(0, 0, 0, 0)
self.addbutton = QPushButton(
"New", toolTip="Create a new variable",
minimumWidth=120,
shortcut=QKeySequence.New
)
def unique_name(fmt, reserved):
candidates = (fmt.format(i) for i in count(1))
return next(c for c in candidates if c not in reserved)
def reserved_names():
varnames = []
if self.data is not None:
varnames = [var.name for var in
self.data.domain.variables + self.data.domain.metas]
varnames += [desc.name for desc in self.featuremodel]
return set(varnames)
def generate_newname(fmt):
return unique_name(fmt, reserved_names())
menu = QMenu(self.addbutton)
cont = menu.addAction("Continuous")
cont.triggered.connect(
lambda: self.addFeature(
ContinuousDescriptor(generate_newname("X{}"), "", 3))
)
disc = menu.addAction("Discrete")
disc.triggered.connect(
lambda: self.addFeature(
DiscreteDescriptor(generate_newname("D{}"), "",
("A", "B"), -1, False))
)
string = menu.addAction("String")
string.triggered.connect(
lambda: self.addFeature(
StringDescriptor(generate_newname("S{}"), ""))
)
menu.addSeparator()
self.duplicateaction = menu.addAction("Duplicate Selected Variable")
self.duplicateaction.triggered.connect(self.duplicateFeature)
self.duplicateaction.setEnabled(False)
self.addbutton.setMenu(menu)
self.removebutton = QPushButton(
"Remove", toolTip="Remove selected variable",
minimumWidth=120,
shortcut=QKeySequence.Delete
)
self.removebutton.clicked.connect(self.removeSelectedFeature)
buttonlayout.addWidget(self.addbutton)
buttonlayout.addWidget(self.removebutton)
buttonlayout.addStretch(10)
toplayout.addLayout(buttonlayout, 0)
toplayout.addWidget(self.editorstack, 10)
# Layout for the list view
layout = QVBoxLayout(spacing=1, margin=0)
self.featuremodel = DescriptorModel(parent=self)
self.featureview = QListView(
minimumWidth=200,
sizePolicy=QSizePolicy(QSizePolicy.Minimum,
QSizePolicy.MinimumExpanding)
)
self.featureview.setItemDelegate(FeatureItemDelegate(self))
self.featureview.setModel(self.featuremodel)
self.featureview.selectionModel().selectionChanged.connect(
self._on_selectedVariableChanged
)
layout.addWidget(self.featureview)
box.layout().addLayout(layout, 1)
box = gui.hBox(self.controlArea)
box.layout().addWidget(self.report_button)
self.report_button.setMinimumWidth(180)
gui.rubber(box)
commit = gui.button(box, self, "Send", callback=self.apply,
default=True)
commit.setMinimumWidth(180)
def setCurrentIndex(self, index):
index = min(index, len(self.featuremodel) - 1)
self.currentIndex = index
if index >= 0:
itemmodels.select_row(self.featureview, index)
desc = self.featuremodel[min(index, len(self.featuremodel) - 1)]
editor = self.editors[type(desc)]
self.editorstack.setCurrentWidget(editor)
editor.setEditorData(desc, self.data.domain if self.data else None)
self.editorstack.setEnabled(index >= 0)
self.duplicateaction.setEnabled(index >= 0)
self.removebutton.setEnabled(index >= 0)
def _on_selectedVariableChanged(self, selected, *_):
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
else:
self.setCurrentIndex(-1)
def _on_modified(self):
if self.currentIndex >= 0:
editor = self.editorstack.currentWidget()
self.featuremodel[self.currentIndex] = editor.editorData()
self.descriptors = list(self.featuremodel)
def setDescriptors(self, descriptors):
"""
Set a list of variable descriptors to edit.
"""
self.descriptors = descriptors
self.featuremodel[:] = list(self.descriptors)
@check_sql_input
def setData(self, data=None):
"""Set the input dataset."""
self.closeContext()
self.data = data
if self.data is not None:
descriptors = list(self.descriptors)
currindex = self.currentIndex
self.descriptors = []
self.currentIndex = -1
self.openContext(data)
if descriptors != self.descriptors or \
self.currentIndex != currindex:
# disconnect from the selection model while reseting the model
selmodel = self.featureview.selectionModel()
selmodel.selectionChanged.disconnect(
self._on_selectedVariableChanged)
self.featuremodel[:] = list(self.descriptors)
self.setCurrentIndex(self.currentIndex)
selmodel.selectionChanged.connect(
self._on_selectedVariableChanged)
self.editorstack.setEnabled(self.currentIndex >= 0)
def handleNewSignals(self):
if self.data is not None:
self.apply()
else:
self.send("Data", None)
def addFeature(self, descriptor):
self.featuremodel.append(descriptor)
self.setCurrentIndex(len(self.featuremodel) - 1)
editor = self.editorstack.currentWidget()
editor.nameedit.setFocus()
editor.nameedit.selectAll()
def removeFeature(self, index):
del self.featuremodel[index]
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
elif index is None and len(self.featuremodel) > 0:
# Deleting the last item clears selection
self.setCurrentIndex(len(self.featuremodel) - 1)
def removeSelectedFeature(self):
if self.currentIndex >= 0:
self.removeFeature(self.currentIndex)
def duplicateFeature(self):
desc = self.featuremodel[self.currentIndex]
self.addFeature(copy.deepcopy(desc))
def check_attrs_values(self, attr, data):
for i in range(len(data)):
for var in attr:
if not math.isnan(data[i, var]) \
and int(data[i, var]) >= len(var.values):
return var.name
return None
def _validate_descriptors(self, desc):
def validate(source):
try:
return validate_exp(ast.parse(source, mode="eval"))
except Exception:
return False
final = []
invalid = []
for d in desc:
if validate(d.expression):
final.append(d)
else:
final.append(d._replace(expression=""))
invalid.append(d)
if invalid:
self.Error.invalid_expressions(", ".join(s.name for s in invalid))
return final
def apply(self):
self.Error.clear()
if self.data is None:
return
desc = list(self.featuremodel)
desc = self._validate_descriptors(desc)
source_vars = tuple(self.data.domain) + self.data.domain.metas
new_variables = construct_variables(desc, source_vars)
attrs = [var for var in new_variables if var.is_primitive()]
metas = [var for var in new_variables if not var.is_primitive()]
new_domain = Orange.data.Domain(
self.data.domain.attributes + tuple(attrs),
self.data.domain.class_vars,
metas=self.data.domain.metas + tuple(metas)
)
try:
data = Orange.data.Table(new_domain, self.data)
except Exception as err:
log = logging.getLogger(__name__)
log.error("", exc_info=True)
self.error("".join(format_exception_only(type(err), err)).rstrip())
return
disc_attrs_not_ok = self.check_attrs_values(
[var for var in attrs if var.is_discrete], data)
if disc_attrs_not_ok:
self.Error.more_values_needed(disc_attrs_not_ok)
return
self.send("Data", data)
def send_report(self):
items = OrderedDict()
for feature in self.featuremodel:
if isinstance(feature, DiscreteDescriptor):
items[feature.name] = "{} (discrete with values {}{})".format(
feature.expression, feature.values,
"; ordered" * feature.ordered)
elif isinstance(feature, ContinuousDescriptor):
items[feature.name] = "{} (numeric)".format(feature.expression)
else:
items[feature.name] = "{} (text)".format(feature.expression)
self.report_items(
report.plural("Constructed feature{s}", len(items)), items)
class OWRank(OWWidget):
name = "Rank"
description = "Rank and filter data features by their relevance."
icon = "icons/Rank.svg"
priority = 1102
buttons_area_orientation = Qt.Vertical
inputs = [("Data", Table, "setData"),
("Scorer", score.Scorer, "set_learner", widget.Multiple)]
outputs = [("Reduced Data", Table, widget.Default), ("Scores", Table)]
SelectNone, SelectAll, SelectManual, SelectNBest = range(4)
cls_default_selected = Setting({"Gain Ratio", "Gini Decrease"})
reg_default_selected = Setting({"Univariate Linear Regression", "RReliefF"})
selectMethod = Setting(SelectNBest)
nSelected = Setting(5)
auto_apply = Setting(True)
# Header state for discrete/continuous/no_class scores
headerState = Setting([None, None, None])
settings_version = 1
settingsHandler = DomainContextHandler()
selected_rows = ContextSetting([])
gain = inf_gain = gini = anova = chi2 = ulr = relief = rrelief = fcbc = True
_score_vars = ["gain", "inf_gain", "gini", "anova", "chi2", "relief",
"fcbc", "ulr", "rrelief"]
class Warning(OWWidget.Warning):
no_target_var = Msg("Data does not have a target variable")
class Error(OWWidget.Error):
invalid_type = Msg("Cannot handle target variable type {}")
inadequate_learner = Msg("{}")
def __init__(self):
super().__init__()
self.measure_scores = None
self.update_scores = True
self.usefulAttributes = []
self.learners = {}
self.labels = []
self.out_domain_desc = None
self.all_measures = SCORES
self.selectedMeasures = dict([(m.name, True) for m
in self.all_measures])
# Discrete (0) or continuous (1) class mode
self.rankMode = 0
self.data = None
self.discMeasures = [m for m in self.all_measures if
issubclass(DiscreteVariable, m.score.class_type)]
self.contMeasures = [m for m in self.all_measures if
issubclass(ContinuousVariable, m.score.class_type)]
self.score_checks = []
self.cls_scoring_box = gui.vBox(None, "Scoring for Classification")
self.reg_scoring_box = gui.vBox(None, "Scoring for Regression")
boxes = [self.cls_scoring_box] * 7 + [self.reg_scoring_box] * 2
for _score, var, box in zip(SCORES, self._score_vars, boxes):
check = gui.checkBox(
box, self, var, label=_score.name,
callback=lambda val=_score: self.measuresSelectionChanged(val))
self.score_checks.append(check)
self.score_stack = QStackedWidget(self)
self.score_stack.addWidget(self.cls_scoring_box)
self.score_stack.addWidget(self.reg_scoring_box)
self.score_stack.addWidget(QWidget())
self.controlArea.layout().addWidget(self.score_stack)
gui.rubber(self.controlArea)
selMethBox = gui.vBox(
self.controlArea, "Select Attributes", addSpace=True)
grid = QGridLayout()
grid.setContentsMargins(6, 0, 6, 0)
self.selectButtons = QButtonGroup()
self.selectButtons.buttonClicked[int].connect(self.setSelectMethod)
def button(text, buttonid, toolTip=None):
b = QRadioButton(text)
self.selectButtons.addButton(b, buttonid)
if toolTip is not None:
b.setToolTip(toolTip)
return b
b1 = button(self.tr("None"), OWRank.SelectNone)
b2 = button(self.tr("All"), OWRank.SelectAll)
b3 = button(self.tr("Manual"), OWRank.SelectManual)
b4 = button(self.tr("Best ranked:"), OWRank.SelectNBest)
s = gui.spin(selMethBox, self, "nSelected", 1, 100,
callback=self.nSelectedChanged)
grid.addWidget(b1, 0, 0)
grid.addWidget(b2, 1, 0)
grid.addWidget(b3, 2, 0)
grid.addWidget(b4, 3, 0)
grid.addWidget(s, 3, 1)
self.selectButtons.button(self.selectMethod).setChecked(True)
selMethBox.layout().addLayout(grid)
gui.auto_commit(selMethBox, self, "auto_apply", "Send", box=False)
# Discrete, continuous and no_class table views are stacked
self.ranksViewStack = QStackedLayout()
self.mainArea.layout().addLayout(self.ranksViewStack)
self.discRanksView = QTableView()
self.ranksViewStack.addWidget(self.discRanksView)
self.discRanksView.setSelectionBehavior(QTableView.SelectRows)
self.discRanksView.setSelectionMode(QTableView.MultiSelection)
self.discRanksView.setSortingEnabled(True)
self.discRanksLabels = ["#"] + [m.shortname for m in self.discMeasures]
self.discRanksModel = QStandardItemModel(self)
self.discRanksModel.setHorizontalHeaderLabels(self.discRanksLabels)
self.discRanksProxyModel = MySortProxyModel(self)
self.discRanksProxyModel.setSourceModel(self.discRanksModel)
self.discRanksView.setModel(self.discRanksProxyModel)
self.discRanksView.setColumnWidth(0, 20)
self.discRanksView.selectionModel().selectionChanged.connect(
self.commit
)
self.discRanksView.pressed.connect(self.onSelectItem)
self.discRanksView.horizontalHeader().sectionClicked.connect(
self.headerClick
)
self.discRanksView.verticalHeader().sectionClicked.connect(
self.onSelectItem
)
if self.headerState[0] is not None:
self.discRanksView.horizontalHeader().restoreState(
self.headerState[0])
self.contRanksView = QTableView()
self.ranksViewStack.addWidget(self.contRanksView)
self.contRanksView.setSelectionBehavior(QTableView.SelectRows)
self.contRanksView.setSelectionMode(QTableView.MultiSelection)
self.contRanksView.setSortingEnabled(True)
self.contRanksLabels = ["#"] + [m.shortname for m in self.contMeasures]
self.contRanksModel = QStandardItemModel(self)
self.contRanksModel.setHorizontalHeaderLabels(self.contRanksLabels)
self.contRanksProxyModel = MySortProxyModel(self)
self.contRanksProxyModel.setSourceModel(self.contRanksModel)
self.contRanksView.setModel(self.contRanksProxyModel)
self.contRanksView.setColumnWidth(0, 20)
self.contRanksView.selectionModel().selectionChanged.connect(
self.commit
)
self.contRanksView.pressed.connect(self.onSelectItem)
self.contRanksView.horizontalHeader().sectionClicked.connect(
self.headerClick
)
self.contRanksView.verticalHeader().sectionClicked.connect(
self.onSelectItem
)
if self.headerState[1] is not None:
self.contRanksView.horizontalHeader().restoreState(
self.headerState[1])
self.noClassRanksView = QTableView()
self.ranksViewStack.addWidget(self.noClassRanksView)
self.noClassRanksView.setSelectionBehavior(QTableView.SelectRows)
self.noClassRanksView.setSelectionMode(QTableView.MultiSelection)
self.noClassRanksView.setSortingEnabled(True)
self.noClassRanksLabels = ["#"]
self.noClassRanksModel = QStandardItemModel(self)
self.noClassRanksModel.setHorizontalHeaderLabels(self.noClassRanksLabels)
self.noClassRanksProxyModel = MySortProxyModel(self)
self.noClassRanksProxyModel.setSourceModel(self.noClassRanksModel)
self.noClassRanksView.setModel(self.noClassRanksProxyModel)
self.noClassRanksView.setColumnWidth(0, 20)
self.noClassRanksView.selectionModel().selectionChanged.connect(
self.commit
)
self.noClassRanksView.pressed.connect(self.onSelectItem)
self.noClassRanksView.horizontalHeader().sectionClicked.connect(
self.headerClick
)
self.noClassRanksView.verticalHeader().sectionClicked.connect(
self.onSelectItem
)
if self.headerState[2] is not None:
self.noClassRanksView.horizontalHeader().restoreState(
self.headerState[2])
# Switch the current view to Discrete
self.switchRanksMode(0)
self.resetInternals()
self.updateDelegates()
self.updateVisibleScoreColumns()
self.resize(690, 500)
self.measure_scores = table((len(self.measures), 0), None)
def switchRanksMode(self, index):
"""
Switch between discrete/continuous/no_class mode
"""
self.rankMode = index
self.ranksViewStack.setCurrentIndex(index)
if index == 0:
self.ranksView = self.discRanksView
self.ranksModel = self.discRanksModel
self.ranksProxyModel = self.discRanksProxyModel
self.measures = self.discMeasures
self.selected_checks = self.cls_default_selected
self.reg_scoring_box.setSizePolicy(QSizePolicy.Ignored,
QSizePolicy.Ignored)
self.cls_scoring_box.setSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Expanding)
elif index == 1:
self.ranksView = self.contRanksView
self.ranksModel = self.contRanksModel
self.ranksProxyModel = self.contRanksProxyModel
self.measures = self.contMeasures
self.selected_checks = self.reg_default_selected
self.cls_scoring_box.setSizePolicy(QSizePolicy.Ignored,
QSizePolicy.Ignored)
self.reg_scoring_box.setSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Expanding)
else:
self.ranksView = self.noClassRanksView
self.ranksModel = self.noClassRanksModel
self.ranksProxyModel = self.noClassRanksProxyModel
self.measures = []
self.selected_checks = set()
self.reg_scoring_box.setSizePolicy(QSizePolicy.Ignored,
QSizePolicy.Ignored)
self.cls_scoring_box.setSizePolicy(QSizePolicy.Ignored,
QSizePolicy.Ignored)
shape = (len(self.measures) + len(self.learners), 0)
self.measure_scores = table(shape, None)
self.update_scores = False
for check, score in zip(self.score_checks, SCORES):
check.setChecked(score.name in self.selected_checks)
self.update_scores = True
self.score_stack.setCurrentIndex(index)
self.updateVisibleScoreColumns()
@check_sql_input
def setData(self, data):
self.closeContext()
self.clear_messages()
self.resetInternals()
self.data = data
self.switchRanksMode(0)
if self.data is not None:
domain = self.data.domain
attrs = domain.attributes
self.usefulAttributes = [attr for attr in attrs
if attr.is_discrete or attr.is_continuous]
if domain.has_continuous_class:
self.switchRanksMode(1)
elif not domain.class_var:
self.Warning.no_target_var()
self.switchRanksMode(2)
elif not domain.has_discrete_class:
self.Error.invalid_type(type(domain.class_var).__name__)
if issparse(self.data.X): # keep only measures supporting sparse data
self.measures = [m for m in self.measures
if m.score.supports_sparse_data]
self.ranksModel.setRowCount(len(attrs))
for i, a in enumerate(attrs):
if a.is_discrete:
v = len(a.values)
else:
v = "C"
item = ScoreValueItem()
item.setData(v, Qt.DisplayRole)
self.ranksModel.setItem(i, 0, item)
item = QStandardItem(a.name)
item.setData(gui.attributeIconDict[a], Qt.DecorationRole)
self.ranksModel.setVerticalHeaderItem(i, item)
shape = (len(self.measures) + len(self.learners), len(attrs))
self.measure_scores = table(shape, None)
self.updateScores()
else:
self.send("Scores", None)
self.selected_rows = []
self.openContext(data)
self.selectMethodChanged()
self.commit()
def get_selection(self):
selection = self.ranksView.selectionModel().selection()
return list(set(ind.row() for ind in selection.indexes()))
def set_learner(self, learner, lid=None):
if learner is None and lid is not None:
del self.learners[lid]
elif learner is not None:
self.learners[lid] = score_meta(
learner.name,
learner.name,
learner
)
attrs_len = 0 if not self.data else len(self.data.domain.attributes)
shape = (len(self.learners), attrs_len)
self.measure_scores = self.measure_scores[:len(self.measures)]
self.measure_scores += table(shape, None)
self.contRanksModel.setHorizontalHeaderLabels(self.contRanksLabels)
self.discRanksModel.setHorizontalHeaderLabels(self.discRanksLabels)
self.noClassRanksModel.setHorizontalHeaderLabels(
self.noClassRanksLabels)
measures_mask = [False] * len(self.measures)
measures_mask += [True for _ in self.learners]
self.updateScores(measures_mask)
self.commit()
def updateScores(self, measuresMask=None):
"""
Update the current computed scores.
If `measuresMask` is given it must be an list of bool values
indicating what measures should be recomputed.
"""
if not self.data:
return
if self.data.has_missing():
self.information("Missing values have been imputed.")
measures = self.measures + [v for k, v in self.learners.items()]
if measuresMask is None:
# Update all selected measures
measuresMask = [self.selectedMeasures.get(m.name)
for m in self.measures]
measuresMask = measuresMask + [v.name for k, v in
self.learners.items()]
data = self.data
learner_col = len(self.measures)
if len(measuresMask) <= len(self.measures) or \
measuresMask[len(self.measures)]:
self.labels = []
self.Error.inadequate_learner.clear()
self.setStatusMessage("Running")
with self.progressBar():
n_measure_update = len([x for x in measuresMask if x is not False])
count = 0
for index, (meas, mask) in enumerate(zip(measures, measuresMask)):
if not mask:
continue
self.progressBarSet(90 * count / n_measure_update)
count += 1
if index < len(self.measures):
estimator = meas.score()
try:
self.measure_scores[index] = estimator(data)
except ValueError:
self.measure_scores[index] = []
for attr in data.domain.attributes:
try:
self.measure_scores[index].append(
estimator(data, attr))
except ValueError:
self.measure_scores[index].append(None)
else:
learner = meas.score
if isinstance(learner, Learner) and \
not learner.check_learner_adequacy(self.data.domain):
self.Error.inadequate_learner(
learner.learner_adequacy_err_msg)
scores = table((1, len(data.domain.attributes)))
else:
scores = meas.score.score_data(data)
for i, row in enumerate(scores):
self.labels.append(meas.shortname + str(i + 1))
if len(self.measure_scores) > learner_col:
self.measure_scores[learner_col] = row
else:
self.measure_scores.append(row)
learner_col += 1
self.progressBarSet(90)
self.contRanksModel.setHorizontalHeaderLabels(
self.contRanksLabels + self.labels
)
self.discRanksModel.setHorizontalHeaderLabels(
self.discRanksLabels + self.labels
)
self.noClassRanksModel.setHorizontalHeaderLabels(
self.noClassRanksLabels + self.labels
)
self.updateRankModel(measuresMask)
self.ranksProxyModel.invalidate()
self.selectMethodChanged()
self.send("Scores", self.create_scores_table(self.labels))
self.setStatusMessage("")
def updateRankModel(self, measuresMask):
"""
Update the rankModel.
"""
values = []
diff = len(self.measure_scores) - len(measuresMask)
if len(measuresMask):
measuresMask += [measuresMask[-1]] * diff
for i in range(self.ranksModel.columnCount() - 1,
len(self.measure_scores), -1):
self.ranksModel.removeColumn(i)
for i, (scores, m) in enumerate(zip(self.measure_scores, measuresMask)):
if not m and self.ranksModel.item(0, i + 1):
values.append([])
continue
values_one = []
for j, _score in enumerate(scores):
values_one.append(_score)
item = self.ranksModel.item(j, i + 1)
if not item:
item = ScoreValueItem()
self.ranksModel.setItem(j, i + 1, item)
item.setData(_score, Qt.DisplayRole)
values.append(values_one)
for i, (vals, m) in enumerate(zip(values, measuresMask)):
if not m:
continue
valid_vals = [v for v in vals if v is not None]
if valid_vals:
vmin, vmax = min(valid_vals), max(valid_vals)
for j, v in enumerate(vals):
if v is not None:
# Set the bar ratio role for i-th measure.
ratio = float((v - vmin) / ((vmax - vmin) or 1))
item = self.ranksModel.item(j, i + 1)
item.setData(ratio, gui.BarRatioRole)
self.ranksView.setColumnWidth(0, 20)
self.ranksView.resizeRowsToContents()
def resetInternals(self):
self.data = None
self.usefulAttributes = []
self.ranksModel.setRowCount(0)
def onSelectItem(self, index):
"""
Called when the user selects/unselects an item in the table view.
"""
self.selectMethod = OWRank.SelectManual # Manual
self.selectButtons.button(self.selectMethod).setChecked(True)
self.commit()
def setSelectMethod(self, method):
if self.selectMethod != method:
self.selectMethod = method
self.selectButtons.button(method).setChecked(True)
self.selectMethodChanged()
def selectMethodChanged(self):
self.autoSelection()
self.ranksView.setFocus()
def nSelectedChanged(self):
self.selectMethod = OWRank.SelectNBest
self.selectButtons.button(self.selectMethod).setChecked(True)
self.selectMethodChanged()
def autoSelection(self):
selModel = self.ranksView.selectionModel()
rowCount = self.ranksModel.rowCount()
columnCount = self.ranksModel.columnCount()
model = self.ranksProxyModel
if self.selectMethod == OWRank.SelectNone:
selection = QItemSelection()
elif self.selectMethod == OWRank.SelectAll:
selection = QItemSelection(
model.index(0, 0),
model.index(rowCount - 1, columnCount - 1)
)
elif self.selectMethod == OWRank.SelectNBest:
nSelected = min(self.nSelected, rowCount)
selection = QItemSelection(
model.index(0, 0),
model.index(nSelected - 1, columnCount - 1)
)
else:
selection = QItemSelection()
if len(self.selected_rows):
selection = QItemSelection()
for row in self.selected_rows:
selection.append(QItemSelectionRange(
model.index(row, 0), model.index(row, columnCount - 1)))
selModel.select(selection, QItemSelectionModel.ClearAndSelect)
def headerClick(self, index):
if index >= 1 and self.selectMethod == OWRank.SelectNBest:
# Reselect the top ranked attributes
self.autoSelection()
# Store the header states
disc = bytes(self.discRanksView.horizontalHeader().saveState())
cont = bytes(self.contRanksView.horizontalHeader().saveState())
no_class = bytes(self.noClassRanksView.horizontalHeader().saveState())
self.headerState = [disc, cont, no_class]
def measuresSelectionChanged(self, measure):
"""Measure selection has changed. Update column visibility.
"""
checked = self.selectedMeasures[measure.name]
self.selectedMeasures[measure.name] = not checked
if not checked:
self.selected_checks.add(measure.name)
elif measure.name in self.selected_checks:
self.selected_checks.remove(measure.name)
measures_mask = [False] * len(self.measures)
measures_mask += [False for _ in self.learners]
# Update scores for shown column if they are not yet computed.
if measure in self.measures and self.measure_scores:
index = self.measures.index(measure)
if all(s is None for s in self.measure_scores[index]):
measures_mask[index] = True
if self.update_scores:
self.updateScores(measures_mask)
self.updateVisibleScoreColumns()
def updateVisibleScoreColumns(self):
"""
Update the visible columns of the scores view.
"""
for i, measure in enumerate(self.measures):
shown = self.selectedMeasures.get(measure.name)
self.ranksView.setColumnHidden(i + 1, not shown)
self.ranksView.setColumnWidth(i + 1, 100)
index = self.ranksView.horizontalHeader().sortIndicatorSection()
if self.ranksView.isColumnHidden(index):
self.headerState[self.rankMode] = None
if self.headerState[self.rankMode] is None:
def get_sort_by_col(measures, selected_measures):
cols = [i + 1 for i, m in enumerate(measures) if
m.name in selected_measures]
return cols[0] if cols else len(measures) + 1
col = get_sort_by_col(self.measures, self.selected_checks)
self.ranksView.sortByColumn(col, Qt.DescendingOrder)
self.autoSelection()
def updateDelegates(self):
self.contRanksView.setItemDelegate(gui.ColoredBarItemDelegate(self))
self.discRanksView.setItemDelegate(gui.ColoredBarItemDelegate(self))
self.noClassRanksView.setItemDelegate(gui.ColoredBarItemDelegate(self))
def send_report(self):
if not self.data:
return
self.report_domain("Input", self.data.domain)
self.report_table("Ranks", self.ranksView, num_format="{:.3f}")
if self.out_domain_desc is not None:
self.report_items("Output", self.out_domain_desc)
def commit(self):
self.selected_rows = self.get_selection()
if self.data and len(self.data.domain.attributes) == len(
self.selected_rows):
self.selectMethod = OWRank.SelectAll
self.selectButtons.button(self.selectMethod).setChecked(True)
selected = self.selectedAttrs()
if not self.data or not selected:
self.send("Reduced Data", None)
self.out_domain_desc = None
else:
data = Table(Domain(selected, self.data.domain.class_var,
self.data.domain.metas), self.data)
self.send("Reduced Data", data)
self.out_domain_desc = report.describe_domain(data.domain)
def selectedAttrs(self):
if self.data:
inds = self.ranksView.selectionModel().selectedRows(0)
source = self.ranksProxyModel.mapToSource
inds = map(source, inds)
inds = [ind.row() for ind in inds]
return [self.data.domain.attributes[i] for i in inds]
else:
return []
def create_scores_table(self, labels):
indices = [i for i, m in enumerate(self.measures)
if self.selectedMeasures.get(m.name, False)]
measures = [s.name for s in self.measures if
self.selectedMeasures.get(s.name, False)]
measures += [label for label in labels]
if not measures:
return None
features = [ContinuousVariable(s) for s in measures]
metas = [StringVariable("Feature name")]
domain = Domain(features, metas=metas)
scores = np.array([row for i, row in enumerate(self.measure_scores)
if i in indices or i >= len(self.measures)]).T
feature_names = np.array([a.name for a in self.data.domain.attributes])
# Reshape to 2d array as Table does not like 1d arrays
feature_names = feature_names[:, None]
new_table = Table(domain, scores, metas=feature_names)
new_table.name = "Feature Scores"
return new_table
@classmethod
def migrate_settings(cls, settings, version):
if not version:
# Before fc5caa1e1d716607f1f5c4e0b0be265c23280fa0
# headerState had length 2
headerState = settings.get("headerState", None)
if headerState is not None and \
isinstance(headerState, tuple) and \
len(headerState) < 3:
headerState = (list(headerState) + [None] * 3)[:3]
settings["headerState"] = headerState
class OWEditDomain(widget.OWWidget):
name = "Edit Domain"
description = "Rename features and their values."
icon = "icons/EditDomain.svg"
priority = 3125
class Inputs:
data = Input("Data", Orange.data.Table)
class Outputs:
data = Output("Data", Orange.data.Table)
settingsHandler = settings.DomainContextHandler()
domain_change_hints = settings.ContextSetting({})
selected_index = settings.ContextSetting({})
autocommit = settings.Setting(True)
def __init__(self):
super().__init__()
self.data = None
self.input_vars = ()
self._invalidated = False
box = gui.vBox(self.controlArea, "Domain Features")
self.domain_model = itemmodels.VariableListModel()
self.domain_view = QListView(selectionMode=QListView.SingleSelection,
uniformItemSizes=True)
self.domain_view.setModel(self.domain_model)
self.domain_view.selectionModel().selectionChanged.connect(
self._on_selection_changed)
box.layout().addWidget(self.domain_view)
box = gui.hBox(self.controlArea)
gui.button(box, self, "Reset Selected", callback=self.reset_selected)
gui.button(box, self, "Reset All", callback=self.reset_all)
gui.auto_commit(self.controlArea, self, "autocommit", "Apply")
box = gui.vBox(self.mainArea, "Edit")
self.editor_stack = QStackedWidget()
self.editor_stack.addWidget(DiscreteVariableEditor())
self.editor_stack.addWidget(ContinuousVariableEditor())
self.editor_stack.addWidget(VariableEditor())
box.layout().addWidget(self.editor_stack)
self.Error.add_message("duplicate_var_name",
"A variable name is duplicated.")
@Inputs.data
@check_sql_input
def set_data(self, data):
"""Set input dataset."""
self.closeContext()
self.clear()
self.data = data
if self.data is not None:
self._initialize()
self.openContext(self.data)
self._restore()
self.unconditional_commit()
def clear(self):
"""Clear the widget state."""
self.data = None
self.domain_model[:] = []
self.input_vars = []
self.domain_change_hints = {}
self.selected_index = -1
def reset_selected(self):
"""Reset the currently selected variable to its original state."""
ind = self.selected_var_index()
if ind >= 0:
var = self.input_vars[ind]
desc = variable_description(var, skip_attributes=True)
if desc in self.domain_change_hints:
del self.domain_change_hints[desc]
self.domain_model[ind] = var
self.editor_stack.currentWidget().set_data(var)
self._invalidate()
def reset_all(self):
"""Reset all variables to their original state."""
self.domain_change_hints = {}
if self.data is not None:
# To invalidate stored hints
self.domain_model[:] = self.input_vars
itemmodels.select_row(self.domain_view, self.selected_index)
self._invalidate()
def selected_var_index(self):
"""Return the selected row in 'Domain Features' view."""
rows = self.domain_view.selectedIndexes()
assert len(rows) <= 1
return rows[0].row() if rows else -1
def _initialize(self):
domain = self.data.domain
self.input_vars = domain.variables + domain.metas
self.domain_model[:] = list(self.input_vars)
def _restore(self):
# Restore the variable states from saved settings.
def transform(var):
vdesc = variable_description(var, skip_attributes=True)
if vdesc in self.domain_change_hints:
return variable_from_description(
self.domain_change_hints[vdesc],
compute_value=Orange.preprocess.transformation.Identity(
var),
)
else:
return var
self.domain_model[:] = map(transform, self.input_vars)
# Restore the variable selection if possible
index = self.selected_index
if index >= len(self.input_vars):
index = 0 if len(self.input_vars) else -1
if index >= 0:
itemmodels.select_row(self.domain_view, index)
def _on_selection_changed(self):
self.selected_index = self.selected_var_index()
self.open_editor(self.selected_index)
def open_editor(self, index):
self.clear_editor()
if index < 0:
return
var = self.domain_model[index]
editor_index = 2
if var.is_discrete:
editor_index = 0
elif var.is_continuous:
editor_index = 1
editor = self.editor_stack.widget(editor_index)
self.editor_stack.setCurrentWidget(editor)
editor.set_data(var)
editor.variable_changed.connect(self._on_variable_changed)
def clear_editor(self):
current = self.editor_stack.currentWidget()
try:
current.variable_changed.disconnect(self._on_variable_changed)
except Exception:
pass
current.set_data(None)
def _on_variable_changed(self):
"""User edited the current variable in editor."""
assert 0 <= self.selected_index <= len(self.domain_model)
editor = self.editor_stack.currentWidget()
# Replace the variable in the 'Domain Features' view/model
old_var = self.input_vars[self.selected_index]
new_var = editor.get_data().copy(
compute_value=Orange.preprocess.transformation.Identity(old_var))
self.domain_model[self.selected_index] = new_var
# Store the transformation hint.
old_var_desc = variable_description(old_var, skip_attributes=True)
self.domain_change_hints[old_var_desc] = variable_description(new_var)
self._invalidate()
def _invalidate(self):
self.commit()
def commit(self):
"""Send the changed data to output."""
new_data = None
var_names = [vn.name for vn in self.domain_model]
self.Error.duplicate_var_name.clear()
if self.data is not None:
if len(var_names) == len(set(var_names)):
input_domain = self.data.domain
n_attrs = len(input_domain.attributes)
n_class_vars = len(input_domain.class_vars)
all_new_vars = list(self.domain_model)
attrs = all_new_vars[:n_attrs]
class_vars = all_new_vars[n_attrs:n_attrs + n_class_vars]
new_metas = all_new_vars[n_attrs + n_class_vars:]
new_domain = Orange.data.Domain(attrs, class_vars, new_metas)
new_data = self.data.transform(new_domain)
else:
self.Error.duplicate_var_name()
self.Outputs.data.send(new_data)
def sizeHint(self):
sh = super().sizeHint()
return sh.expandedTo(QSize(660, 550))
def send_report(self):
if self.data is not None:
self.report_raw(
"",
EditDomainReport(
old_domain=chain(self.data.domain.variables,
self.data.domain.metas),
new_domain=self.domain_model,
).to_html(),
)
else:
self.report_data(None)
class OWImportImages(widget.OWWidget):
name = "Import Images"
description = "Import images from a directory(s)"
icon = "icons/ImportImages.svg"
priority = 110
outputs = [("Data", Orange.data.Table)]
#: list of recent paths
recent_paths = settings.Setting([]) # type: List[RecentPath]
currentPath = settings.Setting(None)
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
# Modality = Qt.WindowModal
MaxRecentItems = 20
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.data = None
self._n_image_categories = 0
self._n_image_data = 0
self._n_skipped = 0
self.__invalidated = False
self.__pendingTask = None
vbox = gui.vBox(self.controlArea)
hbox = gui.hBox(vbox)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
icons = standard_icons(self)
browseaction = QAction(
"Open/Load Images",
self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=icons.dir_open_icon,
toolTip="Select a directory from which to load the images")
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction("Reload",
self,
icon=icons.reload_icon,
toolTip="Reload current image set")
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=True,
)
hbox.layout().addWidget(self.recent_cb)
hbox.layout().addWidget(browsebutton)
hbox.layout().addWidget(reloadbutton)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled()))
box = gui.vBox(vbox, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(text="No image set selected", wordWrap=True)
self.progress_widget = QProgressBar(minimum=0, maximum=0)
self.cancel_button = QPushButton(
"Cancel",
icon=icons.cancel_icon,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def __initRecentItemsModel(self):
if self.currentPath is not None and \
not os.path.isdir(self.currentPath):
self.currentPath = None
recent_paths = []
for item in self.recent_paths:
if os.path.isdir(item.abspath):
recent_paths.append(item)
recent_paths = recent_paths[:OWImportImages.MaxRecentItems]
recent_model = self.recent_cb.model()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.currentPath is not None and \
os.path.isdir(self.currentPath) and self.recent_paths and \
os.path.samefile(self.currentPath, self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
else:
self.currentPath = None
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(self.currentPath is not None)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
if OWImportImages.Modality == Qt.WindowModal:
dlg = QFileDialog(
self,
"Select Top Level Directory",
startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, "Select Top Level Directory", startdir)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No image set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self._n_image_data
ncategories = self._n_image_categories
n_skipped = self._n_skipped
if ncategories < 2:
text = "{} image{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} images / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root image path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.currentPath is not None and path is not None and \
os.path.isdir(self.currentPath) and os.path.isdir(path) and \
os.path.samefile(self.currentPath, path):
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a directory".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
if newindex >= 0:
self.currentPath = path
else:
self.currentPath = None
else:
self.currentPath = None
self.__actions.reload.setEnabled(self.currentPath is not None)
if self.__state == State.Processing:
self.cancel()
return success
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath(path, None, None)
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [
State.Done, State.NoState, State.Error, State.Cancelled
]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the image scan task
"""
if self.__state == State.Processing:
self.cancel()
self.data = None
self.start()
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if self.currentPath is None:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')".format(
self.__pendingTask.startdir))
self.cancel()
startdir = self.currentPath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(self, "__onReportProgress", (object, ))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettyfypath(arg.lastpath))
def commit(self):
"""
Commit a Table from the collected image meta data.
"""
self.send("Data", self.data)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
class OWFeatureConstructor(OWWidget):
name = "Feature Constructor"
description = "Construct new features (data columns) from a set of " \
"existing features in the input data set."
icon = "icons/FeatureConstructor.svg"
class Inputs:
data = Input("Data", Orange.data.Table)
class Outputs:
data = Output("Data", Orange.data.Table)
want_main_area = False
settingsHandler = FeatureConstructorHandler()
descriptors = ContextSetting([])
currentIndex = ContextSetting(-1)
EDITORS = [
(ContinuousDescriptor, ContinuousFeatureEditor),
(DiscreteDescriptor, DiscreteFeatureEditor),
(StringDescriptor, StringFeatureEditor)
]
class Error(OWWidget.Error):
more_values_needed = Msg("Categorical feature {} needs more values.")
invalid_expressions = Msg("Invalid expressions: {}.")
def __init__(self):
super().__init__()
self.data = None
self.editors = {}
box = gui.vBox(self.controlArea, "Variable Definitions")
toplayout = QHBoxLayout()
toplayout.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(toplayout)
self.editorstack = QStackedWidget(
sizePolicy=QSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding)
)
for descclass, editorclass in self.EDITORS:
editor = editorclass()
editor.featureChanged.connect(self._on_modified)
self.editors[descclass] = editor
self.editorstack.addWidget(editor)
self.editorstack.setEnabled(False)
buttonlayout = QVBoxLayout(spacing=10)
buttonlayout.setContentsMargins(0, 0, 0, 0)
self.addbutton = QPushButton(
"New", toolTip="Create a new variable",
minimumWidth=120,
shortcut=QKeySequence.New
)
def unique_name(fmt, reserved):
candidates = (fmt.format(i) for i in count(1))
return next(c for c in candidates if c not in reserved)
def reserved_names():
varnames = []
if self.data is not None:
varnames = [var.name for var in
self.data.domain.variables + self.data.domain.metas]
varnames += [desc.name for desc in self.featuremodel]
return set(varnames)
def generate_newname(fmt):
return unique_name(fmt, reserved_names())
menu = QMenu(self.addbutton)
cont = menu.addAction("Numeric")
cont.triggered.connect(
lambda: self.addFeature(
ContinuousDescriptor(generate_newname("X{}"), "", 3))
)
disc = menu.addAction("Categorical")
disc.triggered.connect(
lambda: self.addFeature(
DiscreteDescriptor(generate_newname("D{}"), "",
("A", "B"), -1, False))
)
string = menu.addAction("Text")
string.triggered.connect(
lambda: self.addFeature(
StringDescriptor(generate_newname("S{}"), ""))
)
menu.addSeparator()
self.duplicateaction = menu.addAction("Duplicate Selected Variable")
self.duplicateaction.triggered.connect(self.duplicateFeature)
self.duplicateaction.setEnabled(False)
self.addbutton.setMenu(menu)
self.removebutton = QPushButton(
"Remove", toolTip="Remove selected variable",
minimumWidth=120,
shortcut=QKeySequence.Delete
)
self.removebutton.clicked.connect(self.removeSelectedFeature)
buttonlayout.addWidget(self.addbutton)
buttonlayout.addWidget(self.removebutton)
buttonlayout.addStretch(10)
toplayout.addLayout(buttonlayout, 0)
toplayout.addWidget(self.editorstack, 10)
# Layout for the list view
layout = QVBoxLayout(spacing=1, margin=0)
self.featuremodel = DescriptorModel(parent=self)
self.featureview = QListView(
minimumWidth=200,
sizePolicy=QSizePolicy(QSizePolicy.Minimum,
QSizePolicy.MinimumExpanding)
)
self.featureview.setItemDelegate(FeatureItemDelegate(self))
self.featureview.setModel(self.featuremodel)
self.featureview.selectionModel().selectionChanged.connect(
self._on_selectedVariableChanged
)
layout.addWidget(self.featureview)
box.layout().addLayout(layout, 1)
box = gui.hBox(self.controlArea)
gui.rubber(box)
commit = gui.button(box, self, "Send", callback=self.apply,
default=True)
commit.setMinimumWidth(180)
def setCurrentIndex(self, index):
index = min(index, len(self.featuremodel) - 1)
self.currentIndex = index
if index >= 0:
itemmodels.select_row(self.featureview, index)
desc = self.featuremodel[min(index, len(self.featuremodel) - 1)]
editor = self.editors[type(desc)]
self.editorstack.setCurrentWidget(editor)
editor.setEditorData(desc, self.data.domain if self.data else None)
self.editorstack.setEnabled(index >= 0)
self.duplicateaction.setEnabled(index >= 0)
self.removebutton.setEnabled(index >= 0)
def _on_selectedVariableChanged(self, selected, *_):
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
else:
self.setCurrentIndex(-1)
def _on_modified(self):
if self.currentIndex >= 0:
editor = self.editorstack.currentWidget()
self.featuremodel[self.currentIndex] = editor.editorData()
self.descriptors = list(self.featuremodel)
def setDescriptors(self, descriptors):
"""
Set a list of variable descriptors to edit.
"""
self.descriptors = descriptors
self.featuremodel[:] = list(self.descriptors)
@Inputs.data
@check_sql_input
def setData(self, data=None):
"""Set the input dataset."""
self.closeContext()
self.data = data
if self.data is not None:
descriptors = list(self.descriptors)
currindex = self.currentIndex
self.descriptors = []
self.currentIndex = -1
self.openContext(data)
if descriptors != self.descriptors or \
self.currentIndex != currindex:
# disconnect from the selection model while reseting the model
selmodel = self.featureview.selectionModel()
selmodel.selectionChanged.disconnect(
self._on_selectedVariableChanged)
self.featuremodel[:] = list(self.descriptors)
self.setCurrentIndex(self.currentIndex)
selmodel.selectionChanged.connect(
self._on_selectedVariableChanged)
self.editorstack.setEnabled(self.currentIndex >= 0)
def handleNewSignals(self):
if self.data is not None:
self.apply()
else:
self.Outputs.data.send(None)
def addFeature(self, descriptor):
self.featuremodel.append(descriptor)
self.setCurrentIndex(len(self.featuremodel) - 1)
editor = self.editorstack.currentWidget()
editor.nameedit.setFocus()
editor.nameedit.selectAll()
def removeFeature(self, index):
del self.featuremodel[index]
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
elif index is None and len(self.featuremodel) > 0:
# Deleting the last item clears selection
self.setCurrentIndex(len(self.featuremodel) - 1)
def removeSelectedFeature(self):
if self.currentIndex >= 0:
self.removeFeature(self.currentIndex)
def duplicateFeature(self):
desc = self.featuremodel[self.currentIndex]
self.addFeature(copy.deepcopy(desc))
def check_attrs_values(self, attr, data):
for i in range(len(data)):
for var in attr:
if not math.isnan(data[i, var]) \
and int(data[i, var]) >= len(var.values):
return var.name
return None
def _validate_descriptors(self, desc):
def validate(source):
try:
return validate_exp(ast.parse(source, mode="eval"))
except Exception:
return False
final = []
invalid = []
for d in desc:
if validate(d.expression):
final.append(d)
else:
final.append(d._replace(expression=""))
invalid.append(d)
if invalid:
self.Error.invalid_expressions(", ".join(s.name for s in invalid))
return final
def apply(self):
self.Error.clear()
if self.data is None:
return
desc = list(self.featuremodel)
desc = self._validate_descriptors(desc)
source_vars = self.data.domain.variables + self.data.domain.metas
new_variables = construct_variables(desc, source_vars)
attrs = [var for var in new_variables if var.is_primitive()]
metas = [var for var in new_variables if not var.is_primitive()]
new_domain = Orange.data.Domain(
self.data.domain.attributes + tuple(attrs),
self.data.domain.class_vars,
metas=self.data.domain.metas + tuple(metas)
)
try:
data = self.data.transform(new_domain)
except Exception as err:
log = logging.getLogger(__name__)
log.error("", exc_info=True)
self.error("".join(format_exception_only(type(err), err)).rstrip())
return
disc_attrs_not_ok = self.check_attrs_values(
[var for var in attrs if var.is_discrete], data)
if disc_attrs_not_ok:
self.Error.more_values_needed(disc_attrs_not_ok)
return
self.Outputs.data.send(data)
def send_report(self):
items = OrderedDict()
for feature in self.featuremodel:
if isinstance(feature, DiscreteDescriptor):
items[feature.name] = "{} (categorical with values {}{})".format(
feature.expression, feature.values,
"; ordered" * feature.ordered)
elif isinstance(feature, ContinuousDescriptor):
items[feature.name] = "{} (numeric)".format(feature.expression)
else:
items[feature.name] = "{} (text)".format(feature.expression)
self.report_items(
report.plural("Constructed feature{s}", len(items)), items)
class OWEditDomain(widget.OWWidget):
name = "Edit Domain"
description = "Rename variables, edit categories and variable annotations."
icon = "icons/EditDomain.svg"
priority = 3125
keywords = []
class Inputs:
data = Input("Data", Orange.data.Table)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(widget.OWWidget.Error):
duplicate_var_name = widget.Msg("A variable name is duplicated.")
settingsHandler = settings.DomainContextHandler()
settings_version = 2
_domain_change_store = settings.ContextSetting({})
_selected_item = settings.ContextSetting(None) # type: Optional[str]
want_control_area = False
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
#: The current selected variable index
self.selected_index = -1
self._invalidated = False
mainlayout = self.mainArea.layout()
assert isinstance(mainlayout, QVBoxLayout)
layout = QHBoxLayout()
mainlayout.addLayout(layout)
box = QGroupBox("Variables")
box.setLayout(QVBoxLayout())
layout.addWidget(box)
self.variables_model = VariableListModel(parent=self)
self.variables_view = self.domain_view = QListView(
selectionMode=QListView.SingleSelection,
uniformItemSizes=True,
)
self.variables_view.setItemDelegate(VariableEditDelegate(self))
self.variables_view.setModel(self.variables_model)
self.variables_view.selectionModel().selectionChanged.connect(
self._on_selection_changed)
box.layout().addWidget(self.variables_view)
box = QGroupBox("Edit", )
box.setLayout(QVBoxLayout(margin=4))
layout.addWidget(box)
self.editor_stack = QStackedWidget()
self.editor_stack.addWidget(DiscreteVariableEditor())
self.editor_stack.addWidget(ContinuousVariableEditor())
self.editor_stack.addWidget(TimeVariableEditor())
self.editor_stack.addWidget(VariableEditor())
box.layout().addWidget(self.editor_stack)
bbox = QDialogButtonBox()
bbox.setStyleSheet("button-layout: {:d};".format(
QDialogButtonBox.MacLayout))
bapply = QPushButton(
"Apply",
objectName="button-apply",
toolTip="Apply changes and commit data on output.",
default=True,
autoDefault=False)
bapply.clicked.connect(self.commit)
breset = QPushButton(
"Reset Selected",
objectName="button-reset",
toolTip="Rest selected variable to its input state.",
autoDefault=False)
breset.clicked.connect(self.reset_selected)
breset_all = QPushButton(
"Reset All",
objectName="button-reset-all",
toolTip="Reset all variables to their input state.",
autoDefault=False)
breset_all.clicked.connect(self.reset_all)
bbox.addButton(bapply, QDialogButtonBox.AcceptRole)
bbox.addButton(breset, QDialogButtonBox.ResetRole)
bbox.addButton(breset_all, QDialogButtonBox.ResetRole)
mainlayout.addWidget(bbox)
self.variables_view.setFocus(Qt.NoFocusReason) # initial focus
@Inputs.data
def set_data(self, data):
"""Set input dataset."""
self.closeContext()
self.clear()
self.data = data
if self.data is not None:
self.set_domain(data.domain)
self.openContext(self.data)
self._restore()
self.commit()
def clear(self):
"""Clear the widget state."""
self.data = None
self.variables_model.clear()
assert self.selected_index == -1
self.selected_index = -1
self._selected_item = None
self._domain_change_store = {}
def reset_selected(self):
"""Reset the currently selected variable to its original state."""
ind = self.selected_var_index()
if ind >= 0:
model = self.variables_model
midx = model.index(ind)
var = midx.data(Qt.EditRole)
tr = midx.data(TransformRole)
if not tr:
return # nothing to reset
editor = self.editor_stack.currentWidget()
with disconnected(editor.variable_changed,
self._on_variable_changed):
model.setData(midx, [], TransformRole)
editor.set_data(var, [])
self._invalidate()
def reset_all(self):
"""Reset all variables to their original state."""
self._domain_change_store = {}
if self.data is not None:
model = self.variables_model
for i in range(model.rowCount()):
midx = model.index(i)
model.setData(midx, [], TransformRole)
index = self.selected_var_index()
if index >= 0:
self.open_editor(index)
self._invalidate()
def selected_var_index(self):
"""Return the current selected variable index."""
rows = self.variables_view.selectedIndexes()
assert len(rows) <= 1
return rows[0].row() if rows else -1
def set_domain(self, domain):
# type: (Orange.data.Domain) -> None
self.variables_model[:] = [
abstract(v) for v in domain.variables + domain.metas
]
def _restore(self, ):
"""
Restore the edit transform from saved state.
"""
model = self.variables_model
for i in range(model.rowCount()):
midx = model.index(i, 0)
var = model.data(midx, Qt.EditRole)
tr = self._restore_transform(var)
if tr:
model.setData(midx, tr, TransformRole)
# Restore the current variable selection
i = -1
if self._selected_item is not None:
for i, var in enumerate(model):
if var.name == self._selected_item:
break
if i == -1 and model.rowCount():
i = 0
if i != -1:
itemmodels.select_row(self.variables_view, i)
def _on_selection_changed(self):
self.selected_index = self.selected_var_index()
if self.selected_index != -1:
self._selected_item = self.variables_model[
self.selected_index].name
else:
self._selected_item = None
self.open_editor(self.selected_index)
def open_editor(self, index):
# type: (int) -> None
self.clear_editor()
model = self.variables_model
if not 0 <= index < model.rowCount():
return
idx = model.index(index, 0)
var = model.data(idx, Qt.EditRole)
tr = model.data(idx, TransformRole)
if tr is None:
tr = []
editors = {Categorical: 0, Real: 1, Time: 2, String: 3}
editor_index = editors.get(type(var), 3)
editor = self.editor_stack.widget(editor_index)
self.editor_stack.setCurrentWidget(editor)
editor.set_data(var, tr)
editor.variable_changed.connect(self._on_variable_changed,
Qt.UniqueConnection)
def clear_editor(self):
current = self.editor_stack.currentWidget()
try:
current.variable_changed.disconnect(self._on_variable_changed)
except TypeError:
pass
current.set_data(None)
@Slot()
def _on_variable_changed(self):
"""User edited the current variable in editor."""
assert 0 <= self.selected_index <= len(self.variables_model)
editor = self.editor_stack.currentWidget()
var, transform = editor.get_data()
model = self.variables_model
midx = model.index(self.selected_index, 0)
model.setData(midx, transform, TransformRole)
self._store_transform(var, transform)
self._invalidate()
def _store_transform(self, var, transform):
# type: (Variable, List[Transform]) -> None
self._domain_change_store[deconstruct(var)] = [
deconstruct(t) for t in transform
]
def _restore_transform(self, var):
# type: (Variable) -> List[Transform]
tr_ = self._domain_change_store.get(deconstruct(var), [])
tr = []
for t in tr_:
try:
tr.append(reconstruct(*t))
except (NameError, TypeError) as err:
warnings.warn("Failed to restore transform: {}, {!r}".format(
t, err),
UserWarning,
stacklevel=2)
return tr
def _invalidate(self):
self._set_modified(True)
def _set_modified(self, state):
self._invalidated = state
b = self.findChild(QPushButton, "button-apply")
if isinstance(b, QPushButton):
f = b.font()
f.setItalic(state)
b.setFont(f)
def commit(self):
"""
Apply the changes to the input data and send the changed data to output.
"""
self._set_modified(False)
self.Error.duplicate_var_name.clear()
data = self.data
if data is None:
self.Outputs.data.send(None)
return
model = self.variables_model
def state(i):
# type: (int) -> Tuple[Variable, List[Transform]]
midx = self.variables_model.index(i, 0)
return (model.data(midx,
Qt.EditRole), model.data(midx, TransformRole))
state = [state(i) for i in range(model.rowCount())]
if all(tr is None or not tr for _, tr in state):
self.Outputs.data.send(data)
return
output_vars = []
input_vars = data.domain.variables + data.domain.metas
assert all(v_.name == v.name for v, (v_, _) in zip(input_vars, state))
for (_, tr), v in zip(state, input_vars):
if tr is not None and len(tr) > 0:
var = apply_transform(v, tr)
else:
var = v
output_vars.append(var)
if len(output_vars) != len({v.name for v in output_vars}):
self.Error.duplicate_var_name()
self.Outputs.data.send(None)
return
domain = data.domain
nx = len(domain.attributes)
ny = len(domain.class_vars)
domain = Orange.data.Domain(output_vars[:nx], output_vars[nx:nx + ny],
output_vars[nx + ny:])
new_data = data.transform(domain)
# print(new_data)
self.Outputs.data.send(new_data)
def sizeHint(self):
sh = super().sizeHint()
return sh.expandedTo(QSize(660, 550))
def send_report(self):
if self.data is not None:
model = self.variables_model
state = ((model.data(midx,
Qt.EditRole), model.data(midx, TransformRole))
for i in range(model.rowCount())
for midx in [model.index(i)])
parts = []
for var, trs in state:
if trs:
parts.append(report_transform(var, trs))
if parts:
html = ("<ul>" + "".join(map("<li>{}</li>".format, parts)) +
"</ul>")
else:
html = "No changes"
self.report_raw("", html)
else:
self.report_data(None)
@classmethod
def migrate_context(cls, context, version):
# pylint: disable=bad-continuation
if version is None or version <= 1:
hints_ = context.values.get("domain_change_hints", ({}, -2))[0]
store = []
ns = "Orange.data.variable"
mapping = {
"DiscreteVariable":
lambda name, args, attrs:
("Categorical", (name, tuple(args[0][1]), None, ())),
"TimeVariable":
lambda name, _, attrs: ("Time", (name, ())),
"ContinuousVariable":
lambda name, _, attrs: ("Real", (name, (3, "f"), ())),
"StringVariable":
lambda name, _, attrs: ("String", (name, ())),
}
for (module, class_name, *rest), target in hints_.items():
if module != ns:
continue
f = mapping.get(class_name)
if f is None:
continue
trs = []
key_mapped = f(*rest)
item_mapped = f(*target[2:])
src = reconstruct(*key_mapped) # type: Variable
dst = reconstruct(*item_mapped) # type: Variable
if src.name != dst.name:
trs.append(Rename(dst.name))
if src.annotations != dst.annotations:
trs.append(Annotate(dst.annotations))
if isinstance(src, Categorical):
if src.categories != dst.categories:
assert len(src.categories) == len(dst.categories)
trs.append(
CategoriesMapping(
list(zip(src.categories, dst.categories))))
store.append(
(deconstruct(src), [deconstruct(tr) for tr in trs]))
context.values["_domain_change_store"] = (dict(store), -2)
class OWEditDomain(widget.OWWidget):
name = "Edit Domain"
description = "Rename variables, edit categories and variable annotations."
icon = "icons/EditDomain.svg"
priority = 3125
keywords = []
class Inputs:
data = Input("Data", Orange.data.Table)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(widget.OWWidget.Error):
duplicate_var_name = widget.Msg("A variable name is duplicated.")
settingsHandler = settings.DomainContextHandler()
settings_version = 2
_domain_change_store = settings.ContextSetting({})
_selected_item = settings.ContextSetting(None) # type: Optional[str]
want_control_area = False
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
#: The current selected variable index
self.selected_index = -1
self._invalidated = False
mainlayout = self.mainArea.layout()
assert isinstance(mainlayout, QVBoxLayout)
layout = QHBoxLayout()
mainlayout.addLayout(layout)
box = QGroupBox("Variables")
box.setLayout(QVBoxLayout())
layout.addWidget(box)
self.variables_model = VariableListModel(parent=self)
self.variables_view = self.domain_view = QListView(
selectionMode=QListView.SingleSelection,
uniformItemSizes=True,
)
self.variables_view.setItemDelegate(VariableEditDelegate(self))
self.variables_view.setModel(self.variables_model)
self.variables_view.selectionModel().selectionChanged.connect(
self._on_selection_changed
)
box.layout().addWidget(self.variables_view)
box = QGroupBox("Edit", )
box.setLayout(QVBoxLayout(margin=4))
layout.addWidget(box)
self.editor_stack = QStackedWidget()
self.editor_stack.addWidget(DiscreteVariableEditor())
self.editor_stack.addWidget(ContinuousVariableEditor())
self.editor_stack.addWidget(TimeVariableEditor())
self.editor_stack.addWidget(VariableEditor())
box.layout().addWidget(self.editor_stack)
bbox = QDialogButtonBox()
bbox.setStyleSheet(
"button-layout: {:d};".format(QDialogButtonBox.MacLayout))
bapply = QPushButton(
"Apply",
objectName="button-apply",
toolTip="Apply changes and commit data on output.",
default=True,
autoDefault=False
)
bapply.clicked.connect(self.commit)
breset = QPushButton(
"Reset Selected",
objectName="button-reset",
toolTip="Rest selected variable to its input state.",
autoDefault=False
)
breset.clicked.connect(self.reset_selected)
breset_all = QPushButton(
"Reset All",
objectName="button-reset-all",
toolTip="Reset all variables to their input state.",
autoDefault=False
)
breset_all.clicked.connect(self.reset_all)
bbox.addButton(bapply, QDialogButtonBox.AcceptRole)
bbox.addButton(breset, QDialogButtonBox.ResetRole)
bbox.addButton(breset_all, QDialogButtonBox.ResetRole)
mainlayout.addWidget(bbox)
self.variables_view.setFocus(Qt.NoFocusReason) # initial focus
@Inputs.data
def set_data(self, data):
"""Set input dataset."""
self.closeContext()
self.clear()
self.data = data
if self.data is not None:
self.set_domain(data.domain)
self.openContext(self.data)
self._restore()
self.commit()
def clear(self):
"""Clear the widget state."""
self.data = None
self.variables_model.clear()
assert self.selected_index == -1
self.selected_index = -1
self._selected_item = None
self._domain_change_store = {}
def reset_selected(self):
"""Reset the currently selected variable to its original state."""
ind = self.selected_var_index()
if ind >= 0:
model = self.variables_model
midx = model.index(ind)
var = midx.data(Qt.EditRole)
tr = midx.data(TransformRole)
if not tr:
return # nothing to reset
editor = self.editor_stack.currentWidget()
with disconnected(editor.variable_changed,
self._on_variable_changed):
model.setData(midx, [], TransformRole)
editor.set_data(var, [])
self._invalidate()
def reset_all(self):
"""Reset all variables to their original state."""
self._domain_change_store = {}
if self.data is not None:
model = self.variables_model
for i in range(model.rowCount()):
midx = model.index(i)
model.setData(midx, [], TransformRole)
index = self.selected_var_index()
if index >= 0:
self.open_editor(index)
self._invalidate()
def selected_var_index(self):
"""Return the current selected variable index."""
rows = self.variables_view.selectedIndexes()
assert len(rows) <= 1
return rows[0].row() if rows else -1
def set_domain(self, domain):
# type: (Orange.data.Domain) -> None
self.variables_model[:] = [abstract(v)
for v in domain.variables + domain.metas]
def _restore(self, ):
"""
Restore the edit transform from saved state.
"""
model = self.variables_model
for i in range(model.rowCount()):
midx = model.index(i, 0)
var = model.data(midx, Qt.EditRole)
tr = self._restore_transform(var)
if tr:
model.setData(midx, tr, TransformRole)
# Restore the current variable selection
i = -1
if self._selected_item is not None:
for i, var in enumerate(model):
if var.name == self._selected_item:
break
if i == -1 and model.rowCount():
i = 0
if i != -1:
itemmodels.select_row(self.variables_view, i)
def _on_selection_changed(self):
self.selected_index = self.selected_var_index()
if self.selected_index != -1:
self._selected_item = self.variables_model[self.selected_index].name
else:
self._selected_item = None
self.open_editor(self.selected_index)
def open_editor(self, index):
# type: (int) -> None
self.clear_editor()
model = self.variables_model
if not 0 <= index < model.rowCount():
return
idx = model.index(index, 0)
var = model.data(idx, Qt.EditRole)
tr = model.data(idx, TransformRole)
if tr is None:
tr = []
editors = {
Categorical: 0,
Real: 1,
Time: 2,
String: 3
}
editor_index = editors.get(type(var), 3)
editor = self.editor_stack.widget(editor_index)
self.editor_stack.setCurrentWidget(editor)
editor.set_data(var, tr)
editor.variable_changed.connect(
self._on_variable_changed, Qt.UniqueConnection
)
def clear_editor(self):
current = self.editor_stack.currentWidget()
try:
current.variable_changed.disconnect(self._on_variable_changed)
except TypeError:
pass
current.set_data(None)
@Slot()
def _on_variable_changed(self):
"""User edited the current variable in editor."""
assert 0 <= self.selected_index <= len(self.variables_model)
editor = self.editor_stack.currentWidget()
var, transform = editor.get_data()
model = self.variables_model
midx = model.index(self.selected_index, 0)
model.setData(midx, transform, TransformRole)
self._store_transform(var, transform)
self._invalidate()
def _store_transform(self, var, transform):
# type: (Variable, List[Transform]) -> None
self._domain_change_store[deconstruct(var)] = [deconstruct(t) for t in transform]
def _restore_transform(self, var):
# type: (Variable) -> List[Transform]
tr_ = self._domain_change_store.get(deconstruct(var), [])
tr = []
for t in tr_:
try:
tr.append(reconstruct(*t))
except (NameError, TypeError) as err:
warnings.warn(
"Failed to restore transform: {}, {!r}".format(t, err),
UserWarning, stacklevel=2
)
return tr
def _invalidate(self):
self._set_modified(True)
def _set_modified(self, state):
self._invalidated = state
b = self.findChild(QPushButton, "button-apply")
if isinstance(b, QPushButton):
f = b.font()
f.setItalic(state)
b.setFont(f)
def commit(self):
"""
Apply the changes to the input data and send the changed data to output.
"""
self._set_modified(False)
self.Error.duplicate_var_name.clear()
data = self.data
if data is None:
self.Outputs.data.send(None)
return
model = self.variables_model
def state(i):
# type: (int) -> Tuple[Variable, List[Transform]]
midx = self.variables_model.index(i, 0)
return (model.data(midx, Qt.EditRole),
model.data(midx, TransformRole))
state = [state(i) for i in range(model.rowCount())]
if all(tr is None or not tr for _, tr in state):
self.Outputs.data.send(data)
return
output_vars = []
input_vars = data.domain.variables + data.domain.metas
assert all(v_.name == v.name
for v, (v_, _) in zip(input_vars, state))
for (_, tr), v in zip(state, input_vars):
if tr is not None and len(tr) > 0:
var = apply_transform(v, tr)
else:
var = v
output_vars.append(var)
if len(output_vars) != len({v.name for v in output_vars}):
self.Error.duplicate_var_name()
self.Outputs.data.send(None)
return
domain = data.domain
nx = len(domain.attributes)
ny = len(domain.class_vars)
domain = Orange.data.Domain(
output_vars[:nx], output_vars[nx: nx + ny], output_vars[nx + ny:]
)
new_data = data.transform(domain)
# print(new_data)
self.Outputs.data.send(new_data)
def sizeHint(self):
sh = super().sizeHint()
return sh.expandedTo(QSize(660, 550))
def send_report(self):
if self.data is not None:
model = self.variables_model
state = ((model.data(midx, Qt.EditRole),
model.data(midx, TransformRole))
for i in range(model.rowCount())
for midx in [model.index(i)])
parts = []
for var, trs in state:
if trs:
parts.append(report_transform(var, trs))
if parts:
html = ("<ul>" +
"".join(map("<li>{}</li>".format, parts)) +
"</ul>")
else:
html = "No changes"
self.report_raw("", html)
else:
self.report_data(None)
@classmethod
def migrate_context(cls, context, version):
# pylint: disable=bad-continuation
if version is None or version <= 1:
hints_ = context.values.get("domain_change_hints", ({}, -2))[0]
store = []
ns = "Orange.data.variable"
mapping = {
"DiscreteVariable":
lambda name, args, attrs:
("Categorical", (name, tuple(args[0][1]), None, ())),
"TimeVariable":
lambda name, _, attrs:
("Time", (name, ())),
"ContinuousVariable":
lambda name, _, attrs:
("Real", (name, (3, "f"), ())),
"StringVariable":
lambda name, _, attrs:
("String", (name, ())),
}
for (module, class_name, *rest), target in hints_.items():
if module != ns:
continue
f = mapping.get(class_name)
if f is None:
continue
trs = []
key_mapped = f(*rest)
item_mapped = f(*target[2:])
src = reconstruct(*key_mapped) # type: Variable
dst = reconstruct(*item_mapped) # type: Variable
if src.name != dst.name:
trs.append(Rename(dst.name))
if src.annotations != dst.annotations:
trs.append(Annotate(dst.annotations))
if isinstance(src, Categorical):
if src.categories != dst.categories:
assert len(src.categories) == len(dst.categories)
trs.append(CategoriesMapping(
list(zip(src.categories, dst.categories))))
store.append((deconstruct(src), [deconstruct(tr) for tr in trs]))
context.values["_domain_change_store"] = (dict(store), -2)
class OWImportImages(widget.OWWidget):
name = "Import Images"
description = "Import images from a directory(s)"
icon = "icons/ImportImages.svg"
priority = 110
outputs = [("Data", Orange.data.Table)]
#: list of recent paths
recent_paths = settings.Setting([]) # type: List[RecentPath]
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
# Modality = Qt.WindowModal
MaxRecentItems = 20
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.data = None
self._n_image_categories = 0
self._n_image_data = 0
self._n_skipped = 0
self.__invalidated = False
self.__pendingTask = None
vbox = gui.vBox(self.controlArea)
hbox = gui.hBox(vbox)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True
)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
icons = standard_icons(self)
browseaction = QAction(
"Open/Load Images", self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=icons.dir_open_icon,
toolTip="Select a directory from which to load the images"
)
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction(
"Reload", self,
icon=icons.reload_icon,
toolTip="Reload current image set"
)
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(
browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger
)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=True,
)
hbox.layout().addWidget(self.recent_cb)
hbox.layout().addWidget(browsebutton)
hbox.layout().addWidget(reloadbutton)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled())
)
box = gui.vBox(vbox, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(
text="No image set selected",
wordWrap=True
)
self.progress_widget = QProgressBar(
minimum=0, maximum=0
)
self.cancel_button = QPushButton(
"Cancel", icon=icons.cancel_icon,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def __initRecentItemsModel(self):
self._relocate_recent_files()
recent_paths = []
for item in self.recent_paths:
recent_paths.append(item)
recent_paths = recent_paths[:OWImportImages.MaxRecentItems]
recent_model = self.recent_cb.model()
recent_model.clear()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.recent_paths and os.path.isdir(self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
self.__actions.reload.setEnabled(True)
else:
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(False)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
if OWImportImages.Modality == Qt.WindowModal:
dlg = QFileDialog(
self, "Select Top Level Directory", startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, "Select Top Level Directory", startdir
)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No image set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self._n_image_data
ncategories = self._n_image_categories
n_skipped = self._n_skipped
if ncategories < 2:
text = "{} image{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} images / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root image path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.recent_paths and path is not None and \
os.path.isdir(self.recent_paths[0].abspath) and os.path.isdir(path) \
and os.path.samefile(os.path.isdir(self.recent_paths[0].abspath), path):
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a directory".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
self.__actions.reload.setEnabled(len(self.recent_paths) > 0)
if self.__state == State.Processing:
self.cancel()
return success
def _search_paths(self):
basedir = self.workflowEnv().get("basedir", None)
if basedir is None:
return []
return [("basedir", basedir)]
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath.create(path, self._search_paths())
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [State.Done,
State.NoState,
State.Error,
State.Cancelled]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the image scan task
"""
if self.__state == State.Processing:
self.cancel()
self.data = None
self.start()
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if not self.recent_paths:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')"
.format(self.__pendingTask.startdir))
self.cancel()
startdir = self.recent_paths[0].abspath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(
self, "__onReportProgress", (object,))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
else:
self._n_image_data, self._n_image_categories = 0, 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettyfypath(arg.lastpath))
def commit(self):
"""
Commit a Table from the collected image meta data.
"""
self.send("Data", self.data)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if path.endswith("/"):
path = path[:-1] # remove last /
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
def _relocate_recent_files(self):
search_paths = self._search_paths()
rec = []
for recent in self.recent_paths:
kwargs = dict(
title=recent.title, sheet=recent.sheet,
file_format=recent.file_format)
resolved = recent.resolve(search_paths)
if resolved is not None:
rec.append(
RecentPath.create(resolved.abspath, search_paths, **kwargs))
else:
rec.append(recent)
# change the list in-place for the case the widgets wraps this list
self.recent_paths[:] = rec
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.__initRecentItemsModel()
class OWFeatureConstructor(OWWidget):
name = "特征构造器(Feature Constructor)"
description = "用输入数据集中的现有特征构造新特征。"
icon = "icons/FeatureConstructor.svg"
keywords = ['tezheng', 'gouzao', 'tezhenggouzao']
category = "数据(Data)"
icon = "icons/FeatureConstructor.svg"
class Inputs:
data = Input("数据(Data)", Orange.data.Table, replaces=['Data'])
class Outputs:
data = Output("数据(Data)", Orange.data.Table, replaces=['Data'])
want_main_area = False
settingsHandler = FeatureConstructorHandler()
descriptors = ContextSetting([])
currentIndex = ContextSetting(-1)
expressions_with_values = ContextSetting(False)
settings_version = 2
EDITORS = [(ContinuousDescriptor, ContinuousFeatureEditor),
(DateTimeDescriptor, DateTimeFeatureEditor),
(DiscreteDescriptor, DiscreteFeatureEditor),
(StringDescriptor, StringFeatureEditor)]
class Error(OWWidget.Error):
more_values_needed = Msg("Categorical feature {} needs more values.")
invalid_expressions = Msg("Invalid expressions: {}.")
class Warning(OWWidget.Warning):
renamed_var = Msg("Recently added variable has been renamed, "
"to avoid duplicates.\n")
def __init__(self):
super().__init__()
self.data = None
self.editors = {}
box = gui.vBox(self.controlArea, "变量定义")
toplayout = QHBoxLayout()
toplayout.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(toplayout)
self.editorstack = QStackedWidget(sizePolicy=QSizePolicy(
QSizePolicy.MinimumExpanding, QSizePolicy.MinimumExpanding))
for descclass, editorclass in self.EDITORS:
editor = editorclass()
editor.featureChanged.connect(self._on_modified)
self.editors[descclass] = editor
self.editorstack.addWidget(editor)
self.editorstack.setEnabled(False)
buttonlayout = QVBoxLayout(spacing=10)
buttonlayout.setContentsMargins(0, 0, 0, 0)
self.addbutton = QPushButton("新建",
toolTip="Create a new variable",
minimumWidth=120,
shortcut=QKeySequence.New)
def unique_name(fmt, reserved):
candidates = (fmt.format(i) for i in count(1))
return next(c for c in candidates if c not in reserved)
def generate_newname(fmt):
return unique_name(fmt, self.reserved_names())
menu = QMenu(self.addbutton)
cont = menu.addAction("数值数据")
cont.triggered.connect(lambda: self.addFeature(
ContinuousDescriptor(generate_newname("X{}"), "", 3)))
disc = menu.addAction("分类数据")
disc.triggered.connect(lambda: self.addFeature(
DiscreteDescriptor(generate_newname("D{}"), "", (), False)))
string = menu.addAction("文本")
string.triggered.connect(lambda: self.addFeature(
StringDescriptor(generate_newname("S{}"), "")))
datetime = menu.addAction("日期/时间")
datetime.triggered.connect(lambda: self.addFeature(
DateTimeDescriptor(generate_newname("T{}"), "")))
menu.addSeparator()
self.duplicateaction = menu.addAction("复制选中变量")
self.duplicateaction.triggered.connect(self.duplicateFeature)
self.duplicateaction.setEnabled(False)
self.addbutton.setMenu(menu)
self.removebutton = QPushButton("删除",
toolTip="删除选中变量",
minimumWidth=120,
shortcut=QKeySequence.Delete)
self.removebutton.clicked.connect(self.removeSelectedFeature)
buttonlayout.addWidget(self.addbutton)
buttonlayout.addWidget(self.removebutton)
buttonlayout.addStretch(10)
toplayout.addLayout(buttonlayout, 0)
toplayout.addWidget(self.editorstack, 10)
# Layout for the list view
layout = QVBoxLayout(spacing=1, margin=0)
self.featuremodel = DescriptorModel(parent=self)
self.featureview = QListView(minimumWidth=200,
minimumHeight=50,
sizePolicy=QSizePolicy(
QSizePolicy.Minimum,
QSizePolicy.MinimumExpanding))
self.featureview.setItemDelegate(FeatureItemDelegate(self))
self.featureview.setModel(self.featuremodel)
self.featureview.selectionModel().selectionChanged.connect(
self._on_selectedVariableChanged)
layout.addWidget(self.featureview)
box.layout().addLayout(layout, 1)
self.fix_button = gui.button(self.buttonsArea,
self,
"Upgrade Expressions",
callback=self.fix_expressions)
self.fix_button.setHidden(True)
gui.button(self.buttonsArea,
self,
"Send",
callback=self.apply,
default=True)
def setCurrentIndex(self, index):
index = min(index, len(self.featuremodel) - 1)
self.currentIndex = index
if index >= 0:
itemmodels.select_row(self.featureview, index)
desc = self.featuremodel[min(index, len(self.featuremodel) - 1)]
editor = self.editors[type(desc)]
self.editorstack.setCurrentWidget(editor)
editor.setEditorData(desc, self.data.domain if self.data else None)
self.editorstack.setEnabled(index >= 0)
self.duplicateaction.setEnabled(index >= 0)
self.removebutton.setEnabled(index >= 0)
def _on_selectedVariableChanged(self, selected, *_):
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
else:
self.setCurrentIndex(-1)
def _on_modified(self):
if self.currentIndex >= 0:
self.Warning.clear()
editor = self.editorstack.currentWidget()
proposed = editor.editorData().name
uniq = get_unique_names(self.reserved_names(self.currentIndex),
proposed)
feature = editor.editorData()
if editor.editorData().name != uniq:
self.Warning.renamed_var()
feature = feature.__class__(uniq, *feature[1:])
self.featuremodel[self.currentIndex] = feature
self.descriptors = list(self.featuremodel)
def setDescriptors(self, descriptors):
"""
Set a list of variable descriptors to edit.
"""
self.descriptors = descriptors
self.featuremodel[:] = list(self.descriptors)
def reserved_names(self, idx_=None):
varnames = []
if self.data is not None:
varnames = [
var.name
for var in self.data.domain.variables + self.data.domain.metas
]
varnames += [
desc.name for idx, desc in enumerate(self.featuremodel)
if idx != idx_
]
return set(varnames)
@Inputs.data
@check_sql_input
def setData(self, data=None):
"""Set the input dataset."""
self.closeContext()
self.data = data
self.expressions_with_values = False
self.descriptors = []
self.currentIndex = -1
if self.data is not None:
self.openContext(data)
# disconnect from the selection model while reseting the model
selmodel = self.featureview.selectionModel()
selmodel.selectionChanged.disconnect(self._on_selectedVariableChanged)
self.featuremodel[:] = list(self.descriptors)
self.setCurrentIndex(self.currentIndex)
selmodel.selectionChanged.connect(self._on_selectedVariableChanged)
self.fix_button.setHidden(not self.expressions_with_values)
self.editorstack.setEnabled(self.currentIndex >= 0)
def handleNewSignals(self):
if self.data is not None:
self.apply()
else:
self.Outputs.data.send(None)
self.fix_button.setHidden(True)
def addFeature(self, descriptor):
self.featuremodel.append(descriptor)
self.setCurrentIndex(len(self.featuremodel) - 1)
editor = self.editorstack.currentWidget()
editor.nameedit.setFocus()
editor.nameedit.selectAll()
def removeFeature(self, index):
del self.featuremodel[index]
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
elif index is None and self.featuremodel.rowCount():
# Deleting the last item clears selection
self.setCurrentIndex(self.featuremodel.rowCount() - 1)
def removeSelectedFeature(self):
if self.currentIndex >= 0:
self.removeFeature(self.currentIndex)
def duplicateFeature(self):
desc = self.featuremodel[self.currentIndex]
self.addFeature(copy.deepcopy(desc))
@staticmethod
def check_attrs_values(attr, data):
for i in range(len(data)):
for var in attr:
if not math.isnan(data[i, var]) \
and int(data[i, var]) >= len(var.values):
return var.name
return None
def _validate_descriptors(self, desc):
def validate(source):
try:
return validate_exp(ast.parse(source, mode="eval"))
# ast.parse can return arbitrary errors, not only SyntaxError
# pylint: disable=broad-except
except Exception:
return False
final = []
invalid = []
for d in desc:
if validate(d.expression):
final.append(d)
else:
final.append(d._replace(expression=""))
invalid.append(d)
if invalid:
self.Error.invalid_expressions(", ".join(s.name for s in invalid))
return final
def apply(self):
def report_error(err):
log = logging.getLogger(__name__)
log.error("", exc_info=True)
self.error("".join(format_exception_only(type(err), err)).rstrip())
self.Error.clear()
if self.data is None:
return
desc = list(self.featuremodel)
desc = self._validate_descriptors(desc)
try:
new_variables = construct_variables(desc, self.data,
self.expressions_with_values)
# user's expression can contain arbitrary errors
except Exception as err: # pylint: disable=broad-except
report_error(err)
return
attrs = [var for var in new_variables if var.is_primitive()]
metas = [var for var in new_variables if not var.is_primitive()]
new_domain = Orange.data.Domain(
self.data.domain.attributes + tuple(attrs),
self.data.domain.class_vars,
metas=self.data.domain.metas + tuple(metas))
try:
for variable in new_variables:
variable.compute_value.mask_exceptions = False
data = self.data.transform(new_domain)
# user's expression can contain arbitrary errors
# pylint: disable=broad-except
except Exception as err:
report_error(err)
return
finally:
for variable in new_variables:
variable.compute_value.mask_exceptions = True
disc_attrs_not_ok = self.check_attrs_values(
[var for var in attrs if var.is_discrete], data)
if disc_attrs_not_ok:
self.Error.more_values_needed(disc_attrs_not_ok)
return
self.Outputs.data.send(data)
def send_report(self):
items = OrderedDict()
for feature in self.featuremodel:
if isinstance(feature, DiscreteDescriptor):
desc = "categorical"
if feature.values:
desc += " with values " \
+ ", ".join(f"'{val}'" for val in feature.values)
if feature.ordered:
desc += "; ordered"
elif isinstance(feature, ContinuousDescriptor):
desc = "numeric"
elif isinstance(feature, DateTimeDescriptor):
desc = "date/time"
else:
desc = "text"
items[feature.name] = f"{feature.expression} ({desc})"
self.report_items(report.plural("Constructed feature{s}", len(items)),
items)
def fix_expressions(self):
dlg = QMessageBox(
QMessageBox.Question, "Fix Expressions",
"This widget's behaviour has changed. Values of categorical "
"variables are now inserted as their textual representations "
"(strings); previously they appeared as integer numbers, with an "
"attribute '.value' that contained the text.\n\n"
"The widget currently runs in compatibility mode. After "
"expressions are updated, manually check for their correctness.")
dlg.addButton("Update", QMessageBox.ApplyRole)
dlg.addButton("Cancel", QMessageBox.RejectRole)
if dlg.exec() == QMessageBox.RejectRole:
return
def fixer(mo):
var = domain[mo.group(2)]
if mo.group(3) == ".value": # uses string; remove `.value`
return "".join(mo.group(1, 2, 4))
# Uses ints: get them by indexing
return mo.group(1) + "{" + \
", ".join(f"'{val}': {i}"
for i, val in enumerate(var.values)) + \
f"}}[{var.name}]" + mo.group(4)
domain = self.data.domain
disc_vars = "|".join(f"{var.name}"
for var in chain(domain.variables, domain.metas)
if var.is_discrete)
expr = re.compile(r"(^|\W)(" + disc_vars + r")(\.value)?(\W|$)")
self.descriptors[:] = [
descriptor._replace(
expression=expr.sub(fixer, descriptor.expression))
for descriptor in self.descriptors
]
self.expressions_with_values = False
self.fix_button.hide()
index = self.currentIndex
self.featuremodel[:] = list(self.descriptors)
self.setCurrentIndex(index)
self.apply()
@classmethod
def migrate_context(cls, context, version):
if version is None or version < 2:
used_vars = set(
chain(*(
freevars(ast.parse(descriptor.expression, mode="eval"), [])
for descriptor in context.values["descriptors"]
if descriptor.expression)))
disc_vars = {
name
for (name, vtype) in chain(context.attributes.items(),
context.metas.items()) if vtype == 1
}
if used_vars & disc_vars:
context.values["expressions_with_values"] = True
class OWImportDocuments(widget.OWWidget):
name = "Import Documents"
description = "Import text documents from folders."
icon = "icons/ImportDocuments.svg"
priority = 110
class Outputs:
data = Output("Corpus", Corpus)
skipped_documents = Output("Skipped documents", Table)
LOCAL_FILE, URL = range(2)
source = settings.Setting(LOCAL_FILE)
#: list of recent paths
recent_paths: List[RecentPath] = settings.Setting([])
currentPath: Optional[str] = settings.Setting(None)
recent_urls: List[str] = settings.Setting([])
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
MaxRecentItems = 20
class Warning(widget.OWWidget.Warning):
read_error = widget.Msg("{} couldn't be read.")
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.corpus = None
self.n_text_categories = 0
self.n_text_data = 0
self.skipped_documents = []
self.__invalidated = False
self.__pendingTask = None
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='Source')
source_box = gui.radioButtons(None, self, "source", box=True,
callback=self.start, addToLayout=False)
rb_button = gui.appendRadioButton(source_box, "Folder:",
addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.Fixed)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True
)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
browseaction = QAction(
"Open/Load Documents", self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=self.style().standardIcon(QStyle.SP_DirOpenIcon),
toolTip="Select a folder from which to load the documents"
)
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction(
"Reload", self,
icon=self.style().standardIcon(QStyle.SP_BrowserReload),
toolTip="Reload current document set"
)
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(
browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger,
default=False,
autoDefault=False,
)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=False,
autoDefault=False,
)
box.layout().addWidget(self.recent_cb)
layout.addWidget(box, 0, 1)
layout.addWidget(browsebutton, 0, 2)
layout.addWidget(reloadbutton, 0, 3)
rb_button = gui.appendRadioButton(source_box, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = PyListModel()
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 1, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled())
)
box = gui.vBox(self.controlArea, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(
text="No document set selected",
wordWrap=True
)
self.progress_widget = QProgressBar(
minimum=0, maximum=100
)
self.cancel_button = QPushButton(
"Cancel",
icon=self.style().standardIcon(QStyle.SP_DialogCancelButton),
default=False,
autoDefault=False,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = "http://" + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.start()
def __initRecentItemsModel(self):
if self.currentPath is not None and \
not os.path.isdir(self.currentPath):
self.currentPath = None
recent_paths = []
for item in self.recent_paths:
if os.path.isdir(item.abspath):
recent_paths.append(item)
recent_paths = recent_paths[:OWImportDocuments.MaxRecentItems]
recent_model = self.recent_cb.model()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.currentPath is not None and \
os.path.isdir(self.currentPath) and self.recent_paths and \
os.path.samefile(self.currentPath, self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
else:
self.currentPath = None
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(self.currentPath is not None)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
caption = "Select Top Level Folder"
if OWImportDocuments.Modality == Qt.WindowModal:
dlg = QFileDialog(
self, caption, startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, caption, startdir
)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No document set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self.n_text_data
ncategories = self.n_text_categories
n_skipped = len(self.skipped_documents)
if ncategories < 2:
text = "{} document{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} documents / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root text path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.currentPath is not None and path is not None and \
os.path.isdir(self.currentPath) and os.path.isdir(path) and \
os.path.samefile(self.currentPath, path) and \
self.source == self.LOCAL_FILE:
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a folder".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
if newindex >= 0:
self.currentPath = path
else:
self.currentPath = None
else:
self.currentPath = None
self.__actions.reload.setEnabled(self.currentPath is not None)
if self.__state == State.Processing:
self.cancel()
self.source = self.LOCAL_FILE
return success
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath(path, None, None)
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [State.Done,
State.NoState,
State.Error,
State.Cancelled]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the text scan task
"""
if self.__state == State.Processing:
self.cancel()
self.source = self.LOCAL_FILE
self.corpus = None
self.start()
def start(self):
"""
Start/execute the text indexing operation
"""
self.error()
self.Warning.clear()
self.progress_widget.setValue(0)
self.__invalidated = False
startdir = self.currentPath if self.source == self.LOCAL_FILE \
else self.url_combo.currentText().strip()
if not startdir:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')"
.format(self.__pendingTask.startdir))
self.cancel()
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(
self, "__onReportProgress", (object,))
task = ImportDocuments(startdir, self.source == self.URL,
report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=0)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_text_scan_task_interupt():
try:
return task.run()
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_text_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
corpus, errors = None, []
try:
corpus, errors = task.future.result()
except NoDocumentsException:
state = State.Error
self.error("Folder contains no readable files.")
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if corpus:
self.n_text_data = len(corpus)
self.n_text_categories = len(corpus.domain.class_var.values)\
if corpus.domain.class_var else 0
self.corpus = corpus
if self.corpus:
self.corpus.name = "Documents"
self.skipped_documents = errors
if len(errors):
self.Warning.read_error(
"Some files" if len(errors) > 1 else "One file"
)
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettifypath(arg.lastpath))
self.progress_widget.setValue(int(100 * arg.progress))
def commit(self):
"""
Create and commit a Corpus from the collected text meta data.
"""
self.Outputs.data.send(self.corpus)
if self.skipped_documents:
skipped_table = (
Table.from_list(
SKIPPED_DOMAIN,
[[x, os.path.join(self.currentPath, x)]
for x in self.skipped_documents]
)
)
skipped_table.name = "Skipped documents"
else:
skipped_table = None
self.Outputs.skipped_documents.send(skipped_table)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
def send_report(self):
if not self.currentPath:
return
items = [('Path', self.currentPath),
('Number of documents', self.n_text_data)]
if self.n_text_categories:
items += [('Categories', self.n_text_categories)]
if self.skipped_documents:
items += [('Number of skipped', len(self.skipped_documents))]
self.report_items(items, )
class OWFeatureConstructor(OWWidget):
name = "Feature Constructor"
description = "Construct new features (data columns) from a set of " \
"existing features in the input dataset."
icon = "icons/FeatureConstructor.svg"
keywords = ['function', 'lambda']
class Inputs:
data = Input("Data", Orange.data.Table)
class Outputs:
data = Output("Data", Orange.data.Table)
want_main_area = False
settingsHandler = FeatureConstructorHandler()
descriptors = ContextSetting([])
currentIndex = ContextSetting(-1)
EDITORS = [(ContinuousDescriptor, ContinuousFeatureEditor),
(DateTimeDescriptor, DateTimeFeatureEditor),
(DiscreteDescriptor, DiscreteFeatureEditor),
(StringDescriptor, StringFeatureEditor)]
class Error(OWWidget.Error):
more_values_needed = Msg("Categorical feature {} needs more values.")
invalid_expressions = Msg("Invalid expressions: {}.")
class Warning(OWWidget.Warning):
renamed_var = Msg("Recently added variable has been renamed, "
"to avoid duplicates.\n")
def __init__(self):
super().__init__()
self.data = None
self.editors = {}
box = gui.vBox(self.controlArea, "Variable Definitions")
toplayout = QHBoxLayout()
toplayout.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(toplayout)
self.editorstack = QStackedWidget(sizePolicy=QSizePolicy(
QSizePolicy.MinimumExpanding, QSizePolicy.MinimumExpanding))
for descclass, editorclass in self.EDITORS:
editor = editorclass()
editor.featureChanged.connect(self._on_modified)
self.editors[descclass] = editor
self.editorstack.addWidget(editor)
self.editorstack.setEnabled(False)
buttonlayout = QVBoxLayout(spacing=10)
buttonlayout.setContentsMargins(0, 0, 0, 0)
self.addbutton = QPushButton("New",
toolTip="Create a new variable",
minimumWidth=120,
shortcut=QKeySequence.New)
def unique_name(fmt, reserved):
candidates = (fmt.format(i) for i in count(1))
return next(c for c in candidates if c not in reserved)
def generate_newname(fmt):
return unique_name(fmt, self.reserved_names())
menu = QMenu(self.addbutton)
cont = menu.addAction("Numeric")
cont.triggered.connect(lambda: self.addFeature(
ContinuousDescriptor(generate_newname("X{}"), "", 3)))
disc = menu.addAction("Categorical")
disc.triggered.connect(lambda: self.addFeature(
DiscreteDescriptor(generate_newname("D{}"), "", (), False)))
string = menu.addAction("Text")
string.triggered.connect(lambda: self.addFeature(
StringDescriptor(generate_newname("S{}"), "")))
datetime = menu.addAction("Date/Time")
datetime.triggered.connect(lambda: self.addFeature(
DateTimeDescriptor(generate_newname("T{}"), "")))
menu.addSeparator()
self.duplicateaction = menu.addAction("Duplicate Selected Variable")
self.duplicateaction.triggered.connect(self.duplicateFeature)
self.duplicateaction.setEnabled(False)
self.addbutton.setMenu(menu)
self.removebutton = QPushButton("Remove",
toolTip="Remove selected variable",
minimumWidth=120,
shortcut=QKeySequence.Delete)
self.removebutton.clicked.connect(self.removeSelectedFeature)
buttonlayout.addWidget(self.addbutton)
buttonlayout.addWidget(self.removebutton)
buttonlayout.addStretch(10)
toplayout.addLayout(buttonlayout, 0)
toplayout.addWidget(self.editorstack, 10)
# Layout for the list view
layout = QVBoxLayout(spacing=1, margin=0)
self.featuremodel = DescriptorModel(parent=self)
self.featureview = QListView(minimumWidth=200,
minimumHeight=50,
sizePolicy=QSizePolicy(
QSizePolicy.Minimum,
QSizePolicy.MinimumExpanding))
self.featureview.setItemDelegate(FeatureItemDelegate(self))
self.featureview.setModel(self.featuremodel)
self.featureview.selectionModel().selectionChanged.connect(
self._on_selectedVariableChanged)
self.info.set_input_summary(self.info.NoInput)
self.info.set_output_summary(self.info.NoOutput)
layout.addWidget(self.featureview)
box.layout().addLayout(layout, 1)
gui.button(self.buttonsArea,
self,
"Send",
callback=self.apply,
default=True)
def setCurrentIndex(self, index):
index = min(index, len(self.featuremodel) - 1)
self.currentIndex = index
if index >= 0:
itemmodels.select_row(self.featureview, index)
desc = self.featuremodel[min(index, len(self.featuremodel) - 1)]
editor = self.editors[type(desc)]
self.editorstack.setCurrentWidget(editor)
editor.setEditorData(desc, self.data.domain if self.data else None)
self.editorstack.setEnabled(index >= 0)
self.duplicateaction.setEnabled(index >= 0)
self.removebutton.setEnabled(index >= 0)
def _on_selectedVariableChanged(self, selected, *_):
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
else:
self.setCurrentIndex(-1)
def _on_modified(self):
if self.currentIndex >= 0:
self.Warning.clear()
editor = self.editorstack.currentWidget()
proposed = editor.editorData().name
unique = get_unique_names(self.reserved_names(self.currentIndex),
proposed)
feature = editor.editorData()
if editor.editorData().name != unique:
self.Warning.renamed_var()
feature = feature.__class__(unique, *feature[1:])
self.featuremodel[self.currentIndex] = feature
self.descriptors = list(self.featuremodel)
def setDescriptors(self, descriptors):
"""
Set a list of variable descriptors to edit.
"""
self.descriptors = descriptors
self.featuremodel[:] = list(self.descriptors)
def reserved_names(self, idx_=None):
varnames = []
if self.data is not None:
varnames = [
var.name
for var in self.data.domain.variables + self.data.domain.metas
]
varnames += [
desc.name for idx, desc in enumerate(self.featuremodel)
if idx != idx_
]
return set(varnames)
@Inputs.data
@check_sql_input
def setData(self, data=None):
"""Set the input dataset."""
self.closeContext()
self.data = data
self.info.set_input_summary(self.info.NoInput)
if self.data is not None:
descriptors = list(self.descriptors)
currindex = self.currentIndex
self.descriptors = []
self.currentIndex = -1
self.openContext(data)
self.info.set_input_summary(len(data),
format_summary_details(data))
if descriptors != self.descriptors or \
self.currentIndex != currindex:
# disconnect from the selection model while reseting the model
selmodel = self.featureview.selectionModel()
selmodel.selectionChanged.disconnect(
self._on_selectedVariableChanged)
self.featuremodel[:] = list(self.descriptors)
self.setCurrentIndex(self.currentIndex)
selmodel.selectionChanged.connect(
self._on_selectedVariableChanged)
self.editorstack.setEnabled(self.currentIndex >= 0)
def handleNewSignals(self):
if self.data is not None:
self.apply()
else:
self.info.set_output_summary(self.info.NoOutput)
self.Outputs.data.send(None)
def addFeature(self, descriptor):
self.featuremodel.append(descriptor)
self.setCurrentIndex(len(self.featuremodel) - 1)
editor = self.editorstack.currentWidget()
editor.nameedit.setFocus()
editor.nameedit.selectAll()
def removeFeature(self, index):
del self.featuremodel[index]
index = selected_row(self.featureview)
if index is not None:
self.setCurrentIndex(index)
elif index is None and self.featuremodel.rowCount():
# Deleting the last item clears selection
self.setCurrentIndex(self.featuremodel.rowCount() - 1)
def removeSelectedFeature(self):
if self.currentIndex >= 0:
self.removeFeature(self.currentIndex)
def duplicateFeature(self):
desc = self.featuremodel[self.currentIndex]
self.addFeature(copy.deepcopy(desc))
@staticmethod
def check_attrs_values(attr, data):
for i in range(len(data)):
for var in attr:
if not math.isnan(data[i, var]) \
and int(data[i, var]) >= len(var.values):
return var.name
return None
def _validate_descriptors(self, desc):
def validate(source):
try:
return validate_exp(ast.parse(source, mode="eval"))
# ast.parse can return arbitrary errors, not only SyntaxError
# pylint: disable=broad-except
except Exception:
return False
final = []
invalid = []
for d in desc:
if validate(d.expression):
final.append(d)
else:
final.append(d._replace(expression=""))
invalid.append(d)
if invalid:
self.Error.invalid_expressions(", ".join(s.name for s in invalid))
return final
def apply(self):
def report_error(err):
log = logging.getLogger(__name__)
log.error("", exc_info=True)
self.error("".join(format_exception_only(type(err), err)).rstrip())
self.Error.clear()
if self.data is None:
return
desc = list(self.featuremodel)
desc = self._validate_descriptors(desc)
try:
new_variables = construct_variables(desc, self.data)
# user's expression can contain arbitrary errors
except Exception as err: # pylint: disable=broad-except
report_error(err)
return
attrs = [var for var in new_variables if var.is_primitive()]
metas = [var for var in new_variables if not var.is_primitive()]
new_domain = Orange.data.Domain(
self.data.domain.attributes + tuple(attrs),
self.data.domain.class_vars,
metas=self.data.domain.metas + tuple(metas))
try:
for variable in new_variables:
variable.compute_value.mask_exceptions = False
data = self.data.transform(new_domain)
# user's expression can contain arbitrary errors
# pylint: disable=broad-except
except Exception as err:
report_error(err)
return
finally:
for variable in new_variables:
variable.compute_value.mask_exceptions = True
disc_attrs_not_ok = self.check_attrs_values(
[var for var in attrs if var.is_discrete], data)
if disc_attrs_not_ok:
self.Error.more_values_needed(disc_attrs_not_ok)
return
self.info.set_output_summary(len(data), format_summary_details(data))
self.Outputs.data.send(data)
def send_report(self):
items = OrderedDict()
for feature in self.featuremodel:
if isinstance(feature, DiscreteDescriptor):
items[
feature.name] = "{} (categorical with values {}{})".format(
feature.expression, feature.values,
"; ordered" * feature.ordered)
elif isinstance(feature, ContinuousDescriptor):
items[feature.name] = "{} (numeric)".format(feature.expression)
elif isinstance(feature, DateTimeDescriptor):
items[feature.name] = "{} (date/time)".format(
feature.expression)
else:
items[feature.name] = "{} (text)".format(feature.expression)
self.report_items(report.plural("Constructed feature{s}", len(items)),
items)