def set_new_operators(self,
attr_combo,
adding_all,
selected_index=None,
selected_values=None):
old_combo = self.cond_list.cellWidget(attr_combo.row, 1)
prev_text = old_combo.currentText() if old_combo else ""
oper_combo = QComboBox()
oper_combo.row = attr_combo.row
oper_combo.attr_combo = attr_combo
attr_name = attr_combo.currentText()
if attr_name in self.AllTypes:
oper_combo.addItems(self.operator_names[attr_name])
else:
var = self.data.domain[attr_name]
oper_combo.addItems(self.operator_names[type(var)])
if selected_index is None:
selected_index = oper_combo.findText(prev_text)
if selected_index == -1:
selected_index = 0
oper_combo.setCurrentIndex(selected_index)
self.cond_list.setCellWidget(oper_combo.row, 1, oper_combo)
self.set_new_values(oper_combo, adding_all, selected_values)
oper_combo.currentIndexChanged.connect(
lambda _: self.set_new_values(oper_combo, False))
def add_row(self, attr=None, condition_type=None, condition_value=None):
model = self.cond_list.model()
row = model.rowCount()
model.insertRow(row)
attr_combo = QComboBox(
minimumContentsLength=12,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon)
attr_combo.row = row
for var in filter_visible(
chain(self.data.domain.variables, self.data.domain.metas)):
attr_combo.addItem(*gui.attributeItem(var))
attr_combo.setCurrentIndex(attr or 0)
self.cond_list.setCellWidget(row, 0, attr_combo)
index = QPersistentModelIndex(model.index(row, 3))
temp_button = QPushButton(
'×',
self,
flat=True,
styleSheet='* {font-size: 16pt; color: silver}'
'*:hover {color: black}')
temp_button.clicked.connect(lambda: self.remove_one(index.row()))
self.cond_list.setCellWidget(row, 3, temp_button)
self.remove_all_button.setDisabled(False)
self.set_new_operators(attr_combo, attr is not None, condition_type,
condition_value)
attr_combo.currentIndexChanged.connect(
lambda _: self.set_new_operators(attr_combo, False))
self.cond_list.resizeRowToContents(row)
def _(combo: QComboBox, value: str):
model: QAbstractItemModel = combo.model()
values = [
model.data(model.index(i, 0), role=Qt.EditRole)
for i in range(model.rowCount())
]
combo.setCurrentIndex(values.index(value))
class DiscreteVariableEditor(VariableEditor):
def __init__(self, parent: QWidget, items: Tuple[str], callback: Callable):
super().__init__(parent, callback)
self._combo = QComboBox(
parent,
maximumWidth=180,
sizePolicy=QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
)
self._combo.addItems(items + ("?",))
self._combo.currentIndexChanged.connect(self.__on_index_changed)
self.layout().addWidget(self._combo)
@property
def value(self) -> Union[int, float]:
return self._map_to_var_values()
@value.setter
def value(self, value: float):
if np.isnan(value):
value = self._combo.model().rowCount() - 1
assert value == int(value)
self._combo.setCurrentIndex(int(value))
def __on_index_changed(self):
self.valueChanged.emit(self._map_to_var_values())
def _map_to_var_values(self) -> Union[int, float]:
n_values = self._combo.model().rowCount() - 1
current = self._combo.currentIndex()
return current if current < n_values else np.nan
def add_row(self, attr=None, condition_type=None, condition_value=None):
model = self.cond_list.model()
row = model.rowCount()
model.insertRow(row)
attr_combo = QComboBox(
minimumContentsLength=12,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon)
attr_combo.row = row
for var in self._visible_variables(self.data.domain):
attr_combo.addItem(*gui.attributeItem(var))
attr_combo.setCurrentIndex(attr or 0)
self.cond_list.setCellWidget(row, 0, attr_combo)
index = QPersistentModelIndex(model.index(row, 3))
temp_button = QPushButton('×', self, flat=True,
styleSheet='* {font-size: 16pt; color: silver}'
'*:hover {color: black}')
temp_button.clicked.connect(lambda: self.remove_one(index.row()))
self.cond_list.setCellWidget(row, 3, temp_button)
self.remove_all_button.setDisabled(False)
self.set_new_operators(attr_combo, attr is not None,
condition_type, condition_value)
attr_combo.currentIndexChanged.connect(
lambda _: self.set_new_operators(attr_combo, False))
self.cond_list.resizeRowToContents(row)
def _(values: FontList, value: str, key: KeyType, signal: Callable) \
-> QComboBox:
class FontModel(QStringListModel):
def data(self, index, role=Qt.DisplayRole):
if role == Qt.AccessibleDescriptionRole \
and super().data(index, Qt.DisplayRole) == "":
return "separator"
value = super().data(index, role)
if role == Qt.DisplayRole and value.startswith("."):
value = value[1:]
return value
def flags(self, index):
if index.data(Qt.DisplayRole) == "separator":
return Qt.NoItemFlags
else:
return super().flags(index)
combo = QComboBox()
model = FontModel(values)
combo.setModel(model)
combo.setCurrentIndex(values.index(value))
combo.currentIndexChanged.connect(lambda i: signal.emit(key, values[i]))
combo.setItemDelegate(_ComboBoxListDelegate())
return combo
def palette_combo_box(initial_palette):
from Orange.widgets.utils import itemmodels
cb = QComboBox()
model = itemmodels.ContinuousPalettesModel()
cb.setModel(model)
cb.setCurrentIndex(model.indexOf(initial_palette))
cb.setIconSize(QSize(64, 16))
return cb
class AddTracUI(QWidget):
def __init__(self, parent):
super(AddTracUI, self).__init__()
self.title = 'Create TracControl Element'
self.setLayout(QVBoxLayout(self))
self.parent = parent
self.trac = Trac(name='', outputPin=0, enabled=False, icon=Config.icon('tracControl', 'rearDiff'), momentary=False)
self._nameControl = LineEdit('Name', self)
self._nameControl.kb.connect(self.showOSK)
self._outputPinControlLabel = QLabel('Output Pin', self)
self._outputPinControl = QComboBox(self)
for _pin in self.parent.availablePins():
self._outputPinControl.addItem(str(_pin))
self._outputPinControl.setCurrentIndex(self._outputPinControl.findText(str(self.trac.outputPin)))
self._enabledControl = QCheckBox('Enabled', self)
self._iconControlLabel = QLabel('Icon', self)
self._iconControl = QComboBox(self)
for _key in Config.icons['tracControl'].keys():
icon = Config.icon('tracControl', _key)
self._iconControl.addItem(icon['name'], _key)
self._iconControl.setItemIcon(self._iconControl.count() - 1, QIcon(icon['path']))
self._addTracBtn = QPushButton('Add TracControl Element', self)
self._addTracBtn.clicked.connect(self.__createTracBtnAction)
self._cancelBtn = QPushButton('Cancel', self)
self._cancelBtn.clicked.connect(self.__cancel)
_layout = [
['_nameControl'],
['_outputPinControlLabel', '_outputPinControl'],
['_enabledControl'],
['_iconControlLabel', '_iconControl'],
['_addTracBtn', '_cancelBtn']
]
for _list in _layout:
_panel = QWidget(self)
_panel.setLayout(QHBoxLayout(_panel))
_panel.layout().setAlignment(Qt.AlignCenter)
for _control in _list:
_panel.layout().addWidget(eval('self.%s' % _control))
self.layout().addWidget(_panel)
def __createTracBtnAction(self):
self.trac.name = self._nameControl.text()
self.trac.outputPin = int(self._outputPinControl.currentText())
self.trac.enabled = self._enabledControl.isChecked()
self.trac.icon = self._iconControl.currentData()
self.parent.tracs.addTrac(self.trac)
self.parent.tracs.save()
self.parent.loadUI('config_trac')
self.parent.enableConfigButtons()
def __cancel(self):
self.parent.loadUI('config_trac')
self.parent.enableConfigButtons()
def showOSK(self):
self.window().dock.show()
self.window().osk.rWidget = self._nameControl
class DropDownRadioBooleanFilter(QWidget, Control):
"""Container for multiple boolean filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QHBoxLayout())
self.cb = QComboBox(self)
self.layout().addWidget(self.cb)
rblayout = QVBoxLayout()
self.radioButtons = [QRadioButton("Only", self),
QRadioButton("Excluded", self)
]
for b in self.radioButtons:
rblayout.addWidget(b)
self.radioButtons[0].setChecked(True)
self.layout().addLayout(rblayout)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
for op, rb in zip(self.options[0].subelements_top("Option"),
self.radioButtons):
rb.setText(op.displayName)
rb.setChecked(getattr(op, "default", "false") == "true")
def value(self):
return {"excluded": "0" if self.radioButtons[0].isChecked() else "1"}
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
for i, option in enumerate(self.options):
if option.internalName == name:
self.cb.setCurrentIndex(i)
if value == "Only":
self.radioButtons[0].setChecked(True)
class DropDownRadioBooleanFilter(QWidget, Control):
"""Container for multiple boolean filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QHBoxLayout())
self.cb = QComboBox(self)
self.layout().addWidget(self.cb)
rblayout = QVBoxLayout()
self.radioButtons = [QRadioButton("Only", self),
QRadioButton("Excluded", self)
]
for b in self.radioButtons:
rblayout.addWidget(b)
self.radioButtons[0].setChecked(True)
self.layout().addLayout(rblayout)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
for op, rb in zip(self.options[0].subelements_top("Option"),
self.radioButtons):
rb.setText(op.displayName)
rb.setChecked(getattr(op, "default", "false") == "true")
def value(self):
return {"excluded": "0" if self.radioButtons[0].isChecked() else "1"}
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
for i, option in enumerate(self.options):
if option.internalName == name:
self.cb.setCurrentIndex(i)
if value == "Only":
self.radioButtons[0].setChecked(True)
def set_new_operators(self, attr_combo, adding_all,
selected_index=None, selected_values=None):
oper_combo = QComboBox()
oper_combo.row = attr_combo.row
oper_combo.attr_combo = attr_combo
var = self.data.domain[attr_combo.currentText()]
oper_combo.addItems(self.operator_names[type(var)])
oper_combo.setCurrentIndex(selected_index or 0)
self.cond_list.setCellWidget(oper_combo.row, 1, oper_combo)
self.set_new_values(oper_combo, adding_all, selected_values)
oper_combo.currentIndexChanged.connect(
lambda _: self.set_new_values(oper_combo, False))
def set_new_operators(self, attr_combo, adding_all,
selected_index=None, selected_values=None):
oper_combo = QComboBox()
oper_combo.row = attr_combo.row
oper_combo.attr_combo = attr_combo
var = self.data.domain[attr_combo.currentText()]
oper_combo.addItems(self.operator_names[type(var)])
oper_combo.setCurrentIndex(selected_index or 0)
self.cond_list.setCellWidget(oper_combo.row, 1, oper_combo)
self.set_new_values(oper_combo, adding_all, selected_values)
oper_combo.currentIndexChanged.connect(
lambda _: self.set_new_values(oper_combo, False))
def cbselect(cb: QComboBox, value, role: Qt.ItemDataRole = Qt.EditRole) -> None:
"""
Find and select the `value` in the `cb` QComboBox.
Parameters
----------
cb: QComboBox
value: Any
role: Qt.ItemDataRole
The data role in the combo box model to match value against
"""
cb.setCurrentIndex(cb.findData(value, role))
class Scale(BaseEditor):
NoCentering, CenterMean, CenterMedian = 0, 1, 2
NoScaling, ScaleBySD, ScaleBySpan = 0, 1, 2
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__centercb = QComboBox()
self.__centercb.addItems(
["No Centering", "Center by Mean", "Center by Median"])
self.__scalecb = QComboBox()
self.__scalecb.addItems(["No scaling", "Scale by SD", "Scale by span"])
form.addRow("Center:", self.__centercb)
form.addRow("Scale:", self.__scalecb)
self.layout().addLayout(form)
self.__centercb.currentIndexChanged.connect(self.changed)
self.__scalecb.currentIndexChanged.connect(self.changed)
self.__centercb.activated.connect(self.edited)
self.__scalecb.activated.connect(self.edited)
def setParameters(self, params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
self.__centercb.setCurrentIndex(
_enum_to_index(_Scale.CenteringType, center))
self.__scalecb.setCurrentIndex(
_enum_to_index(_Scale.ScalingType, scale))
def parameters(self):
return {
"center":
_index_to_enum(_Scale.CenteringType,
self.__centercb.currentIndex()),
"scale":
_index_to_enum(_Scale.ScalingType, self.__scalecb.currentIndex())
}
@staticmethod
def createinstance(params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
return _Scale(center=center, scale=scale)
def __repr__(self):
return "{}, {}".format(self.__centercb.currentText(),
self.__scalecb.currentText())
class BinarizeEditor(ScBaseEditor):
DEFAULT_CONDITION = Binarize.GreaterOrEqual
DEFAULT_THRESHOLD = 1
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self._threshold = self.DEFAULT_THRESHOLD
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.cond_cb = QComboBox()
self.cond_cb.addItems(["Greater or Equal", "Greater"])
self.cond_cb.currentIndexChanged.connect(self.changed)
self.cond_cb.activated.connect(self.edited)
self.thr_spin = QDoubleSpinBox(minimum=0,
singleStep=0.5,
decimals=1,
value=self._threshold)
self.thr_spin.valueChanged[float].connect(self._set_threshold)
self.thr_spin.editingFinished.connect(self.edited)
form.addRow("Condition:", self.cond_cb)
form.addRow("Threshold:", self.thr_spin)
self.layout().addLayout(form)
def _set_threshold(self, t):
if self._threshold != t:
self._threshold = t
self.thr_spin.setValue(t)
self.changed.emit()
def setParameters(self, params):
cond = params.get("condition", self.DEFAULT_CONDITION)
self.cond_cb.setCurrentIndex(enum_to_index(Binarize.Condition, cond))
self._set_threshold(params.get("threshold", self.DEFAULT_THRESHOLD))
def parameters(self):
cond = index_to_enum(Binarize.Condition, self.cond_cb.currentIndex())
return {"condition": cond, "threshold": self._threshold}
@staticmethod
def createinstance(params):
condition = params.get("condition", BinarizeEditor.DEFAULT_CONDITION)
threshold = params.get("threshold", BinarizeEditor.DEFAULT_THRESHOLD)
return Binarize(condition, threshold)
def __repr__(self):
return "Condition: {}, Threshold: {}".format(
self.cond_cb.currentText(), self.thr_spin.value())
class Scale(BaseEditor):
NoCentering, CenterMean, CenterMedian = 0, 1, 2
NoScaling, ScaleBySD, ScaleBySpan = 0, 1, 2
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__centercb = QComboBox()
self.__centercb.addItems(["No Centering", "Center by Mean",
"Center by Median"])
self.__scalecb = QComboBox()
self.__scalecb.addItems(["No scaling", "Scale by SD",
"Scale by span"])
form.addRow("Center:", self.__centercb)
form.addRow("Scale:", self.__scalecb)
self.layout().addLayout(form)
self.__centercb.currentIndexChanged.connect(self.changed)
self.__scalecb.currentIndexChanged.connect(self.changed)
self.__centercb.activated.connect(self.edited)
self.__scalecb.activated.connect(self.edited)
def setParameters(self, params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
self.__centercb.setCurrentIndex(
enum_to_index(_Scale.CenteringType, center))
self.__scalecb.setCurrentIndex(
enum_to_index(_Scale.ScalingType, scale))
def parameters(self):
return {"center": index_to_enum(_Scale.CenteringType,
self.__centercb.currentIndex()),
"scale": index_to_enum(_Scale.ScalingType,
self.__scalecb.currentIndex())}
@staticmethod
def createinstance(params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
return _Scale(center=center, scale=scale)
def __repr__(self):
return "{}, {}".format(self.__centercb.currentText(),
self.__scalecb.currentText())
class Randomize(BaseEditor):
RandomizeClasses, RandomizeAttributes, RandomizeMetas = _Randomize.Type
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__rand_type_cb = QComboBox()
self.__rand_type_cb.addItems(["Classes", "Features", "Meta data"])
self.__rand_type_cb.currentIndexChanged.connect(self.changed)
self.__rand_type_cb.activated.connect(self.edited)
self.__rand_seed_ch = QCheckBox()
self.__rand_seed_ch.clicked.connect(self.edited)
form.addRow("Randomize:", self.__rand_type_cb)
form.addRow("Replicable shuffling:", self.__rand_seed_ch)
self.layout().addLayout(form)
def setParameters(self, params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
self.__rand_type_cb.setCurrentIndex(
_enum_to_index(_Randomize.Type, rand_type))
self.__rand_seed_ch.setChecked(params.get("rand_seed", 1) or 0)
def parameters(self):
return {
"rand_type":
_index_to_enum(_Randomize.Type,
self.__rand_type_cb.currentIndex()),
"rand_seed":
1 if self.__rand_seed_ch.isChecked() else None,
}
@staticmethod
def createinstance(params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
rand_seed = params.get("rand_seed", 1)
return _Randomize(rand_type=rand_type, rand_seed=rand_seed)
def __repr__(self):
return "{}, {}".format(
self.__rand_type_cb.currentText(),
"Replicable"
if self.__rand_seed_ch.isChecked() else "Not replicable",
)
class DropDownIdListFilter(QWidget, Control):
"""Container for multiple id list filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QVBoxLayout())
self.setContentsMargins(0, 0, 0, 0)
self.cb = QComboBox()
self.idsEdit = QPlainTextEdit()
self.layout().addWidget(self.cb)
self.layout().addWidget(self.idsEdit)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
def value(self):
return str(self.idsEdit.toPlainText()).split()
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
if isinstance(value, list):
value = "\n".join(value)
for i, op in enumerate(self.options):
if name == op.internalName:
self.cb.setCurrentIndex(i)
self.idsEdit.setPlainText(value)
class DropDownIdListFilter(QWidget, Control):
"""Container for multiple id list filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QVBoxLayout())
self.setContentsMargins(0, 0, 0, 0)
self.cb = QComboBox()
self.idsEdit = QPlainTextEdit()
self.layout().addWidget(self.cb)
self.layout().addWidget(self.idsEdit)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
def value(self):
return str(self.idsEdit.toPlainText()).split()
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
if isinstance(value, list):
value = "\n".join(value)
for i, op in enumerate(self.options):
if name == op.internalName:
self.cb.setCurrentIndex(i)
self.idsEdit.setPlainText(value)
class Randomize(BaseEditor):
RandomizeClasses, RandomizeAttributes, RandomizeMetas = _Randomize.Type
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__rand_type_cb = QComboBox()
self.__rand_type_cb.addItems(["Classes",
"Features",
"Meta data"])
self.__rand_type_cb.currentIndexChanged.connect(self.changed)
self.__rand_type_cb.activated.connect(self.edited)
self.__rand_seed_ch = QCheckBox()
self.__rand_seed_ch.clicked.connect(self.edited)
form.addRow("Randomize:", self.__rand_type_cb)
form.addRow("Replicable shuffling:", self.__rand_seed_ch)
self.layout().addLayout(form)
def setParameters(self, params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
self.__rand_type_cb.setCurrentIndex(
enum_to_index(_Randomize.Type, rand_type))
self.__rand_seed_ch.setChecked(params.get("rand_seed", 1) or 0)
def parameters(self):
return {"rand_type": index_to_enum(_Randomize.Type,
self.__rand_type_cb.currentIndex()),
"rand_seed": 1 if self.__rand_seed_ch.isChecked() else None}
@staticmethod
def createinstance(params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
rand_seed = params.get("rand_seed", 1)
return _Randomize(rand_type=rand_type, rand_seed=rand_seed)
def __repr__(self):
return "{}, {}".format(self.__rand_type_cb.currentText(),
"Replicable" if self.__rand_seed_ch.isChecked()
else "Not replicable")
class DiscreteVariableEditor(VariableEditor):
valueChanged = Signal(int)
def __init__(self, parent: QWidget, items: List[str], callback: Callable):
super().__init__(parent, callback)
self._combo = QComboBox(parent,
maximumWidth=180,
sizePolicy=QSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Fixed))
self._combo.addItems(items)
self._combo.currentIndexChanged.connect(self.valueChanged)
self.layout().addWidget(self._combo)
@property
def value(self) -> int:
return self._combo.currentIndex()
@value.setter
def value(self, value: float):
assert value == int(value)
self._combo.setCurrentIndex(int(value))
def add_row(self, attr=None, condition_type=None, condition_value=None):
model = self.cond_list.model()
row = model.rowCount()
model.insertRow(row)
attr_combo = QComboBox(
minimumContentsLength=12,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon)
attr_combo.row = row
for var in filter_visible(chain(self.data.domain.variables, self.data.domain.metas)):
attr_combo.addItem(*gui.attributeItem(var))
attr_combo.setCurrentIndex(attr or 0)
self.cond_list.setCellWidget(row, 0, attr_combo)
self.remove_all_button.setDisabled(False)
self.set_new_operators(attr_combo, attr is not None,
condition_type, condition_value)
attr_combo.currentIndexChanged.connect(
lambda _: self.set_new_operators(attr_combo, False))
self.cond_list.resizeRowToContents(row)
def add_row(self, attr=None, condition_type=None, condition_value=None):
model = self.cond_list.model()
row = model.rowCount()
model.insertRow(row)
attr_combo = QComboBox(
minimumContentsLength=12,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon)
attr_combo.row = row
for var in filter_visible(
chain(self.data.domain.variables, self.data.domain.metas)):
attr_combo.addItem(*gui.attributeItem(var))
attr_combo.setCurrentIndex(attr or 0)
self.cond_list.setCellWidget(row, 0, attr_combo)
self.remove_all_button.setDisabled(False)
self.set_new_operators(attr_combo, attr is not None, condition_type,
condition_value)
attr_combo.currentIndexChanged.connect(
lambda _: self.set_new_operators(attr_combo, False))
self.cond_list.resizeRowToContents(row)
class LogarithmicScaleEditor(ScBaseEditor):
DEFAULT_BASE = LogarithmicScale.BinaryLog
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.base_cb = QComboBox()
self.base_cb.addItems([
"2 (Binary Logarithm)", "e (Natural Logarithm)",
"10 (Common Logarithm)"
])
self.base_cb.currentIndexChanged.connect(self.changed)
self.base_cb.activated.connect(self.edited)
form.addRow("Logarithm Base:", self.base_cb)
self.layout().addLayout(form)
def setParameters(self, params):
base = params.get("base", self.DEFAULT_BASE)
self.base_cb.setCurrentIndex(enum_to_index(LogarithmicScale.Base,
base))
def parameters(self):
return {
"base":
index_to_enum(LogarithmicScale.Base, self.base_cb.currentIndex())
}
@staticmethod
def createinstance(params):
base = params.get("base", LogarithmicScaleEditor.DEFAULT_BASE)
return LogarithmicScale(base)
def __repr__(self):
return "Base: {}".format(self.base_cb.currentText())
def set_new_values(self, oper_combo, adding_all, selected_values=None):
# def remove_children():
# for child in box.children()[1:]:
# box.layout().removeWidget(child)
# child.setParent(None)
def add_textual(contents):
le = gui.lineEdit(box, self, None,
sizePolicy=QSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Expanding))
if contents:
le.setText(contents)
le.setAlignment(Qt.AlignRight)
le.editingFinished.connect(self.conditions_changed)
return le
def add_numeric(contents):
le = add_textual(contents)
le.setValidator(OWSelectRows.QDoubleValidatorEmpty())
return le
def add_datetime(contents):
le = add_textual(contents)
le.setValidator(QRegExpValidator(QRegExp(TimeVariable.REGEX)))
return le
var = self.data.domain[oper_combo.attr_combo.currentText()]
box = self.cond_list.cellWidget(oper_combo.row, 2)
if selected_values is not None:
lc = list(selected_values) + ["", ""]
lc = [str(x) for x in lc[:2]]
else:
lc = ["", ""]
if box and vartype(var) == box.var_type:
lc = self._get_lineedit_contents(box) + lc
oper = oper_combo.currentIndex()
if oper_combo.currentText() == "is defined":
label = QLabel()
label.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, label)
elif var.is_discrete:
if oper_combo.currentText() == "is one of":
if selected_values:
lc = [x for x in list(selected_values)]
button = DropDownToolButton(self, var, lc)
button.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, button)
else:
combo = QComboBox()
combo.addItems([""] + var.values)
if lc[0]:
combo.setCurrentIndex(int(lc[0]))
else:
combo.setCurrentIndex(0)
combo.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, combo)
combo.currentIndexChanged.connect(self.conditions_changed)
else:
box = gui.hBox(self, addToLayout=False)
box.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, box)
if var.is_continuous:
validator = add_datetime if isinstance(var, TimeVariable) else add_numeric
box.controls = [validator(lc[0])]
if oper > 5:
gui.widgetLabel(box, " and ")
box.controls.append(validator(lc[1]))
elif var.is_string:
box.controls = [add_textual(lc[0])]
if oper in [6, 7]:
gui.widgetLabel(box, " and ")
box.controls.append(add_textual(lc[1]))
else:
box.controls = []
if not adding_all:
self.conditions_changed()
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
keywords = ["substitute", "missing"]
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = \
Msg("Model based imputer does not work for sparse data")
class Warning(OWWidget.Warning):
cant_handle_var = Msg("Default method can not handle '{}'")
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(int(Method.Leave)) # type: int
# Per-variable imputation state (synced in storeSpecificSettings)
_variable_imputation_state = settings.ContextSetting(
{}) # type: VariableState
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
self.learner = None # type: Optional[Learner]
self.default_learner = SimpleTreeLearner(min_instances=10,
max_depth=10)
self.modified = False
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QGridLayout(box)
box_layout.setContentsMargins(5, 0, 0, 0)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for i, (method, _) in enumerate(list(METHODS.items())[1:-1]):
imputer = self.create_imputer(method)
button = QRadioButton(imputer.name)
button.setChecked(method == self.default_method_index)
button_group.addButton(button, method)
box_layout.addWidget(button, i % 3, i // 3)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = ListViewSearch(
selectionMode=QListView.ExtendedSelection, uniformItemSizes=True)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for method in Method:
imputer = self.create_imputer(method)
button = QRadioButton(text=imputer.name)
button_group.addButton(button, method)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000.,
maximum=1000.,
singleStep=.1,
decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection)
method_layout.addStretch(2)
reset_button = QPushButton("Restore All to Default",
checked=False,
checkable=False,
clicked=self.reset_variable_state,
default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_apply(self.controlArea, self, "autocommit")
box.button.setFixedWidth(180)
box.layout().insertStretch(0)
self.info.set_input_summary(self.info.NoInput)
self.info.set_output_summary(self.info.NoOutput)
def create_imputer(self, method, *args):
# type: (Method, ...) -> impute.BaseImputeMethod
if method == Method.Model:
if self.learner is not None:
return impute.Model(self.learner)
else:
return impute.Model(self.default_learner)
elif method == Method.AsAboveSoBelow:
assert self.default_method_index != Method.AsAboveSoBelow
default = self.create_imputer(Method(self.default_method_index))
m = AsDefault()
m.method = default
return m
else:
return METHODS[method](*args)
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
assert index != Method.AsAboveSoBelow
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
# update variable view
self.update_varview()
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.cancel()
self.closeContext()
self.varmodel[:] = []
self._variable_imputation_state = {} # type: VariableState
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
# restore per variable imputation state
self._restore_state(self._variable_imputation_state)
summary = len(data) if data else self.info.NoInput
details = format_summary_details(data) if data else ""
self.info.set_input_summary(summary, details)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.cancel()
self.learner = learner or self.default_learner
imputer = self.create_imputer(Method.Model)
button = self.default_button_group.button(Method.Model)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(Method.Model)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = Method.Model
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
# type: (int) -> impute.BaseImputeMethod
"""
Return the imputation method for column by its index.
"""
assert 0 <= column_index < len(self.varmodel)
idx = self.varmodel.index(column_index, 0)
state = idx.data(StateRole)
if state is None:
state = (Method.AsAboveSoBelow, ())
return self.create_imputer(state[0], *state[1])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
summary = len(self.data) if self.data else self.info.NoOutput
detail = format_summary_details(self.data) if self.data else ""
self.info.set_output_summary(summary, detail)
if not self.data or not self.varmodel.rowCount():
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [(i, var, self.get_method_for_column(i))
for i, var in enumerate(self.varmodel)]
# normalize to the effective method bypasing AsDefault
impute_state = [(i, var, m.method if isinstance(m, AsDefault) else m)
for i, var, m in impute_state]
def impute_one(method, var, data):
# Readability counts, pylint: disable=no-else-raise
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(impute_one, copy.deepcopy(method), var,
data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit()
self.setInvalidated(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
def get_variable(variable, future, drop_mask) \
-> Optional[List[Orange.data.Variable]]:
# Returns a (potentially empty) list of variables,
# or None on failure that should interrupt the imputation
assert future.done()
try:
res = future.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
return [] # None?
except VariableNotSupported:
self.Warning.cant_handle_var(variable.name)
return []
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", variable.name, exc_info=True)
self.Error.imputation_failed(variable.name)
return None
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = variable
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
return newvar
def create_data(attributes, class_vars):
domain = Orange.data.Domain(attributes, class_vars,
self.data.domain.metas)
try:
return self.data.from_table(domain, self.data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
return None
self.__task = None
self.setInvalidated(False)
self.progressBarFinished()
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
newvar = get_variable(var, fut, drop_mask)
if newvar is None:
data = None
break
if i < len(self.data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
else:
data = create_data(attributes, class_vars)
self.Outputs.data.send(data)
self.modified = False
summary = len(data) if data else self.info.NoOutput
details = format_summary_details(data) if data else ""
self.info.set_output_summary(summary, details)
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
self.__cancel(wait=False)
def __cancel(self, wait=False):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
if wait:
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setInvalidated(False)
def onDeleteWidget(self):
self.__cancel(wait=True)
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.get_method_for_column(i)
if not isinstance(method, AsDefault):
specific.append("{} ({})".format(var.name, str(method)))
default = self.create_imputer(Method.AsAboveSoBelow)
if specific:
self.report_items((("Default method", default.name),
("Specific imputers", ", ".join(specific))))
else:
self.report_items((("Method", default.name), ))
def _on_var_selection_changed(self):
# Method is well documented, splitting it is not needed for readability,
# thus pylint: disable=too-many-branches
indexes = self.selection.selectedIndexes()
defmethod = (Method.AsAboveSoBelow, ())
methods = [index.data(StateRole) for index in indexes]
methods = [m if m is not None else defmethod for m in methods]
methods = set(methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for m in Method:
if method_type == m:
self.variable_button_group.button(m).setChecked(True)
if method_type == Method.Default:
(fixed_value, ) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
# Update variable methods GUI enabled state based on selection.
for method in Method:
# use a default constructed imputer to query support
imputer = self.create_imputer(method)
enabled = all(
imputer.supports_variable(var) for var in selected_vars)
button = self.variable_button_group.button(method)
button.setEnabled(enabled)
# Update the "Value" edit GUI.
if not has_discrete:
# no discrete variables -> allow mass edit for all (continuous vars)
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
# single discrete var -> enable and fill the values combo
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
# mixed type selection -> disable
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(Method.Default).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
# set current value
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
# type: (Method) -> None
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
# type: (List[QModelIndex], Method) -> None
if method_index == Method.AsAboveSoBelow:
for index in indexes:
self.varmodel.setData(index, None, StateRole)
elif method_index == Method.Default:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
state = (int(Method.Default), (value, ))
self.varmodel.setData(index, state, StateRole)
else:
state = (int(method_index), ())
for index in indexes:
self.varmodel.setData(index, state, StateRole)
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(index,
self.get_method_for_column(index.row()),
DisplayMethodRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(Method.Default).setChecked(True)
self.set_method_for_current_selection(Method.Default)
def reset_variable_state(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, Method.AsAboveSoBelow)
self.variable_button_group.button(
Method.AsAboveSoBelow).setChecked(True)
def _store_state(self):
# type: () -> VariableState
"""
Save the current variable imputation state
"""
state = {} # type: VariableState
for i, var in enumerate(self.varmodel):
index = self.varmodel.index(i)
m = index.data(StateRole)
if m is not None:
state[var_key(var)] = m
return state
def _restore_state(self, state):
# type: (VariableState) -> None
"""
Restore the variable imputation state from the saved state
"""
def check(state):
# check if state is a proper State
if isinstance(state, tuple) and len(state) == 2:
m, p = state
if isinstance(m, int) and isinstance(p, tuple) and \
0 <= m < len(Method):
return True
return False
for i, var in enumerate(self.varmodel):
m = state.get(var_key(var), None)
if check(m):
self.varmodel.setData(self.varmodel.index(i), m, StateRole)
def storeSpecificSettings(self):
self._variable_imputation_state = self._store_state()
super().storeSpecificSettings()
class FeatureEditor(QFrame):
FUNCTIONS = dict(chain([(key, val) for key, val in math.__dict__.items()
if not key.startswith("_")],
[(key, val) for key, val in builtins.__dict__.items()
if key in {"str", "float", "int", "len",
"abs", "max", "min"}]))
featureChanged = Signal()
featureEdited = Signal()
modifiedChanged = Signal(bool)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
layout = QFormLayout(
fieldGrowthPolicy=QFormLayout.ExpandingFieldsGrow
)
layout.setContentsMargins(0, 0, 0, 0)
self.nameedit = QLineEdit(
placeholderText="Name...",
sizePolicy=QSizePolicy(QSizePolicy.Minimum,
QSizePolicy.Fixed)
)
self.expressionedit = QLineEdit(
placeholderText="Expression..."
)
self.attrs_model = itemmodels.VariableListModel(
["Select Feature"], parent=self)
self.attributescb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
)
self.attributescb.setModel(self.attrs_model)
sorted_funcs = sorted(self.FUNCTIONS)
self.funcs_model = itemmodels.PyListModelTooltip()
self.funcs_model.setParent(self)
self.funcs_model[:] = chain(["Select Function"], sorted_funcs)
self.funcs_model.tooltips[:] = chain(
[''],
[self.FUNCTIONS[func].__doc__ for func in sorted_funcs])
self.functionscb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
self.functionscb.setModel(self.funcs_model)
hbox = QHBoxLayout()
hbox.addWidget(self.attributescb)
hbox.addWidget(self.functionscb)
layout.addRow(self.nameedit, self.expressionedit)
layout.addRow(self.tr(""), hbox)
self.setLayout(layout)
self.nameedit.editingFinished.connect(self._invalidate)
self.expressionedit.textChanged.connect(self._invalidate)
self.attributescb.currentIndexChanged.connect(self.on_attrs_changed)
self.functionscb.currentIndexChanged.connect(self.on_funcs_changed)
self._modified = False
def setModified(self, modified):
if not type(modified) is bool:
raise TypeError
if self._modified != modified:
self._modified = modified
self.modifiedChanged.emit(modified)
def modified(self):
return self._modified
modified = Property(bool, modified, setModified,
notify=modifiedChanged)
def setEditorData(self, data, domain):
self.nameedit.setText(data.name)
self.expressionedit.setText(data.expression)
self.setModified(False)
self.featureChanged.emit()
self.attrs_model[:] = ["Select Feature"]
if domain is not None and (domain or domain.metas):
self.attrs_model[:] += chain(domain.attributes,
domain.class_vars,
domain.metas)
def editorData(self):
return FeatureDescriptor(name=self.nameedit.text(),
expression=self.nameedit.text())
def _invalidate(self):
self.setModified(True)
self.featureEdited.emit()
self.featureChanged.emit()
def on_attrs_changed(self):
index = self.attributescb.currentIndex()
if index > 0:
attr = sanitized_name(self.attrs_model[index].name)
self.insert_into_expression(attr)
self.attributescb.setCurrentIndex(0)
def on_funcs_changed(self):
index = self.functionscb.currentIndex()
if index > 0:
func = self.funcs_model[index]
if func in ["atan2", "fmod", "ldexp", "log",
"pow", "copysign", "hypot"]:
self.insert_into_expression(func + "(,)")
self.expressionedit.cursorBackward(False, 2)
elif func in ["e", "pi"]:
self.insert_into_expression(func)
else:
self.insert_into_expression(func + "()")
self.expressionedit.cursorBackward(False)
self.functionscb.setCurrentIndex(0)
def insert_into_expression(self, what):
cp = self.expressionedit.cursorPosition()
ct = self.expressionedit.text()
text = ct[:cp] + what + ct[cp:]
self.expressionedit.setText(text)
self.expressionedit.setFocus()
class OWImportImages(widget.OWWidget):
name = "Import Images"
description = "Import images from a directory(s)"
icon = "icons/ImportImages.svg"
priority = 110
outputs = [("Data", Orange.data.Table)]
#: list of recent paths
recent_paths = settings.Setting([]) # type: List[RecentPath]
currentPath = settings.Setting(None)
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
# Modality = Qt.WindowModal
MaxRecentItems = 20
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.data = None
self._n_image_categories = 0
self._n_image_data = 0
self._n_skipped = 0
self.__invalidated = False
self.__pendingTask = None
vbox = gui.vBox(self.controlArea)
hbox = gui.hBox(vbox)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
icons = standard_icons(self)
browseaction = QAction(
"Open/Load Images",
self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=icons.dir_open_icon,
toolTip="Select a directory from which to load the images")
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction("Reload",
self,
icon=icons.reload_icon,
toolTip="Reload current image set")
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=True,
)
hbox.layout().addWidget(self.recent_cb)
hbox.layout().addWidget(browsebutton)
hbox.layout().addWidget(reloadbutton)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled()))
box = gui.vBox(vbox, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(text="No image set selected", wordWrap=True)
self.progress_widget = QProgressBar(minimum=0, maximum=0)
self.cancel_button = QPushButton(
"Cancel",
icon=icons.cancel_icon,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def __initRecentItemsModel(self):
if self.currentPath is not None and \
not os.path.isdir(self.currentPath):
self.currentPath = None
recent_paths = []
for item in self.recent_paths:
if os.path.isdir(item.abspath):
recent_paths.append(item)
recent_paths = recent_paths[:OWImportImages.MaxRecentItems]
recent_model = self.recent_cb.model()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.currentPath is not None and \
os.path.isdir(self.currentPath) and self.recent_paths and \
os.path.samefile(self.currentPath, self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
else:
self.currentPath = None
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(self.currentPath is not None)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
if OWImportImages.Modality == Qt.WindowModal:
dlg = QFileDialog(
self,
"Select Top Level Directory",
startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, "Select Top Level Directory", startdir)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No image set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self._n_image_data
ncategories = self._n_image_categories
n_skipped = self._n_skipped
if ncategories < 2:
text = "{} image{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} images / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root image path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.currentPath is not None and path is not None and \
os.path.isdir(self.currentPath) and os.path.isdir(path) and \
os.path.samefile(self.currentPath, path):
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a directory".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
if newindex >= 0:
self.currentPath = path
else:
self.currentPath = None
else:
self.currentPath = None
self.__actions.reload.setEnabled(self.currentPath is not None)
if self.__state == State.Processing:
self.cancel()
return success
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath(path, None, None)
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [
State.Done, State.NoState, State.Error, State.Cancelled
]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the image scan task
"""
if self.__state == State.Processing:
self.cancel()
self.data = None
self.start()
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if self.currentPath is None:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')".format(
self.__pendingTask.startdir))
self.cancel()
startdir = self.currentPath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(self, "__onReportProgress", (object, ))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettyfypath(arg.lastpath))
def commit(self):
"""
Commit a Table from the collected image meta data.
"""
self.send("Data", self.data)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
class OWImportImages(widget.OWWidget):
name = "Import Images"
description = "Import images from a directory(s)"
icon = "icons/ImportImages.svg"
priority = 110
outputs = [("Data", Orange.data.Table)]
#: list of recent paths
recent_paths = settings.Setting([]) # type: List[RecentPath]
want_main_area = False
resizing_enabled = False
Modality = Qt.ApplicationModal
# Modality = Qt.WindowModal
MaxRecentItems = 20
def __init__(self):
super().__init__()
#: widget's runtime state
self.__state = State.NoState
self.data = None
self._n_image_categories = 0
self._n_image_data = 0
self._n_skipped = 0
self.__invalidated = False
self.__pendingTask = None
vbox = gui.vBox(self.controlArea)
hbox = gui.hBox(vbox)
self.recent_cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=16,
acceptDrops=True
)
self.recent_cb.installEventFilter(self)
self.recent_cb.activated[int].connect(self.__onRecentActivated)
icons = standard_icons(self)
browseaction = QAction(
"Open/Load Images", self,
iconText="\N{HORIZONTAL ELLIPSIS}",
icon=icons.dir_open_icon,
toolTip="Select a directory from which to load the images"
)
browseaction.triggered.connect(self.__runOpenDialog)
reloadaction = QAction(
"Reload", self,
icon=icons.reload_icon,
toolTip="Reload current image set"
)
reloadaction.triggered.connect(self.reload)
self.__actions = namespace(
browse=browseaction,
reload=reloadaction,
)
browsebutton = QPushButton(
browseaction.iconText(),
icon=browseaction.icon(),
toolTip=browseaction.toolTip(),
clicked=browseaction.trigger
)
reloadbutton = QPushButton(
reloadaction.iconText(),
icon=reloadaction.icon(),
clicked=reloadaction.trigger,
default=True,
)
hbox.layout().addWidget(self.recent_cb)
hbox.layout().addWidget(browsebutton)
hbox.layout().addWidget(reloadbutton)
self.addActions([browseaction, reloadaction])
reloadaction.changed.connect(
lambda: reloadbutton.setEnabled(reloadaction.isEnabled())
)
box = gui.vBox(vbox, "Info")
self.infostack = QStackedWidget()
self.info_area = QLabel(
text="No image set selected",
wordWrap=True
)
self.progress_widget = QProgressBar(
minimum=0, maximum=0
)
self.cancel_button = QPushButton(
"Cancel", icon=icons.cancel_icon,
)
self.cancel_button.clicked.connect(self.cancel)
w = QWidget()
vlayout = QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
hlayout = QHBoxLayout()
hlayout.setContentsMargins(0, 0, 0, 0)
hlayout.addWidget(self.progress_widget)
hlayout.addWidget(self.cancel_button)
vlayout.addLayout(hlayout)
self.pathlabel = TextLabel()
self.pathlabel.setTextElideMode(Qt.ElideMiddle)
self.pathlabel.setAttribute(Qt.WA_MacSmallSize)
vlayout.addWidget(self.pathlabel)
w.setLayout(vlayout)
self.infostack.addWidget(self.info_area)
self.infostack.addWidget(w)
box.layout().addWidget(self.infostack)
self.__initRecentItemsModel()
self.__invalidated = True
self.__executor = ThreadExecutor(self)
QApplication.postEvent(self, QEvent(RuntimeEvent.Init))
def __initRecentItemsModel(self):
self._relocate_recent_files()
recent_paths = []
for item in self.recent_paths:
recent_paths.append(item)
recent_paths = recent_paths[:OWImportImages.MaxRecentItems]
recent_model = self.recent_cb.model()
recent_model.clear()
for pathitem in recent_paths:
item = RecentPath_asqstandarditem(pathitem)
recent_model.appendRow(item)
self.recent_paths = recent_paths
if self.recent_paths and os.path.isdir(self.recent_paths[0].abspath):
self.recent_cb.setCurrentIndex(0)
self.__actions.reload.setEnabled(True)
else:
self.recent_cb.setCurrentIndex(-1)
self.__actions.reload.setEnabled(False)
def customEvent(self, event):
"""Reimplemented."""
if event.type() == RuntimeEvent.Init:
if self.__invalidated:
try:
self.start()
finally:
self.__invalidated = False
super().customEvent(event)
def __runOpenDialog(self):
startdir = os.path.expanduser("~/")
if self.recent_paths:
startdir = os.path.dirname(self.recent_paths[0].abspath)
if OWImportImages.Modality == Qt.WindowModal:
dlg = QFileDialog(
self, "Select Top Level Directory", startdir,
acceptMode=QFileDialog.AcceptOpen,
modal=True,
)
dlg.setFileMode(QFileDialog.Directory)
dlg.setOption(QFileDialog.ShowDirsOnly)
dlg.setDirectory(startdir)
dlg.setAttribute(Qt.WA_DeleteOnClose)
@dlg.accepted.connect
def on_accepted():
dirpath = dlg.selectedFiles()
if dirpath:
self.setCurrentPath(dirpath[0])
self.start()
dlg.open()
else:
dirpath = QFileDialog.getExistingDirectory(
self, "Select Top Level Directory", startdir
)
if dirpath:
self.setCurrentPath(dirpath)
self.start()
def __onRecentActivated(self, index):
item = self.recent_cb.itemData(index)
if item is None:
return
assert isinstance(item, RecentPath)
self.setCurrentPath(item.abspath)
self.start()
def __updateInfo(self):
if self.__state == State.NoState:
text = "No image set selected"
elif self.__state == State.Processing:
text = "Processing"
elif self.__state == State.Done:
nvalid = self._n_image_data
ncategories = self._n_image_categories
n_skipped = self._n_skipped
if ncategories < 2:
text = "{} image{}".format(nvalid, "s" if nvalid != 1 else "")
else:
text = "{} images / {} categories".format(nvalid, ncategories)
if n_skipped > 0:
text = text + ", {} skipped".format(n_skipped)
elif self.__state == State.Cancelled:
text = "Cancelled"
elif self.__state == State.Error:
text = "Error state"
else:
assert False
self.info_area.setText(text)
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
def setCurrentPath(self, path):
"""
Set the current root image path to path
If the path does not exists or is not a directory the current path
is left unchanged
Parameters
----------
path : str
New root import path.
Returns
-------
status : bool
True if the current root import path was successfully
changed to path.
"""
if self.recent_paths and path is not None and \
os.path.isdir(self.recent_paths[0].abspath) and os.path.isdir(path) \
and os.path.samefile(os.path.isdir(self.recent_paths[0].abspath), path):
return True
success = True
error = None
if path is not None:
if not os.path.exists(path):
error = "'{}' does not exist".format(path)
path = None
success = False
elif not os.path.isdir(path):
error = "'{}' is not a directory".format(path)
path = None
success = False
if error is not None:
self.error(error)
warnings.warn(error, UserWarning, stacklevel=3)
else:
self.error()
if path is not None:
newindex = self.addRecentPath(path)
self.recent_cb.setCurrentIndex(newindex)
self.__actions.reload.setEnabled(len(self.recent_paths) > 0)
if self.__state == State.Processing:
self.cancel()
return success
def _search_paths(self):
basedir = self.workflowEnv().get("basedir", None)
if basedir is None:
return []
return [("basedir", basedir)]
def addRecentPath(self, path):
"""
Prepend a path entry to the list of recent paths
If an entry with the same path already exists in the recent path
list it is moved to the first place
Parameters
----------
path : str
"""
existing = None
for pathitem in self.recent_paths:
try:
if os.path.samefile(pathitem.abspath, path):
existing = pathitem
break
except FileNotFoundError:
# file not found if the `pathitem.abspath` no longer exists
pass
model = self.recent_cb.model()
if existing is not None:
selected_index = self.recent_paths.index(existing)
assert model.item(selected_index).data(Qt.UserRole) is existing
self.recent_paths.remove(existing)
row = model.takeRow(selected_index)
self.recent_paths.insert(0, existing)
model.insertRow(0, row)
else:
item = RecentPath.create(path, self._search_paths())
self.recent_paths.insert(0, item)
model.insertRow(0, RecentPath_asqstandarditem(item))
return 0
def __setRuntimeState(self, state):
assert state in State
self.setBlocking(state == State.Processing)
message = ""
if state == State.Processing:
assert self.__state in [State.Done,
State.NoState,
State.Error,
State.Cancelled]
message = "Processing"
elif state == State.Done:
assert self.__state == State.Processing
elif state == State.Cancelled:
assert self.__state == State.Processing
message = "Cancelled"
elif state == State.Error:
message = "Error during processing"
elif state == State.NoState:
message = ""
else:
assert False
self.__state = state
if self.__state == State.Processing:
self.infostack.setCurrentIndex(1)
else:
self.infostack.setCurrentIndex(0)
self.setStatusMessage(message)
self.__updateInfo()
def reload(self):
"""
Restart the image scan task
"""
if self.__state == State.Processing:
self.cancel()
self.data = None
self.start()
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if not self.recent_paths:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')"
.format(self.__pendingTask.startdir))
self.cancel()
startdir = self.recent_paths[0].abspath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(
self, "__onReportProgress", (object,))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
@Slot()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
else:
self._n_image_data, self._n_image_categories = 0, 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def cancel(self):
"""
Cancel current pending task (if any).
"""
if self.__state == State.Processing:
assert self.__pendingTask is not None
self.__pendingTask.cancel()
self.__pendingTask = None
self.__setRuntimeState(State.Cancelled)
@Slot(object)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettyfypath(arg.lastpath))
def commit(self):
"""
Commit a Table from the collected image meta data.
"""
self.send("Data", self.data)
def onDeleteWidget(self):
self.cancel()
self.__executor.shutdown(wait=True)
self.__invalidated = False
def eventFilter(self, receiver, event):
# re-implemented from QWidget
# intercept and process drag drop events on the recent directory
# selection combo box
def dirpath(event):
# type: (QDropEvent) -> Optional[str]
"""Return the directory from a QDropEvent."""
data = event.mimeData()
urls = data.urls()
if len(urls) == 1:
url = urls[0]
path = url.toLocalFile()
if path.endswith("/"):
path = path[:-1] # remove last /
if os.path.isdir(path):
return path
return None
if receiver is self.recent_cb and \
event.type() in {QEvent.DragEnter, QEvent.DragMove,
QEvent.Drop}:
assert isinstance(event, QDropEvent)
path = dirpath(event)
if path is not None and event.possibleActions() & Qt.LinkAction:
event.setDropAction(Qt.LinkAction)
event.accept()
if event.type() == QEvent.Drop:
self.setCurrentPath(path)
self.start()
else:
event.ignore()
return True
return super().eventFilter(receiver, event)
def _relocate_recent_files(self):
search_paths = self._search_paths()
rec = []
for recent in self.recent_paths:
kwargs = dict(
title=recent.title, sheet=recent.sheet,
file_format=recent.file_format)
resolved = recent.resolve(search_paths)
if resolved is not None:
rec.append(
RecentPath.create(resolved.abspath, search_paths, **kwargs))
else:
rec.append(recent)
# change the list in-place for the case the widgets wraps this list
self.recent_paths[:] = rec
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.__initRecentItemsModel()
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = Msg("Model based imputer does not work for sparse data")
DEFAULT_LEARNER = SimpleTreeLearner()
METHODS = [AsDefault(), impute.DoNotImpute(), impute.Average(),
impute.AsValue(), impute.Model(DEFAULT_LEARNER), impute.Random(),
impute.DropInstances(), impute.Default()]
DEFAULT, DO_NOT_IMPUTE, MODEL_BASED_IMPUTER, AS_INPUT = 0, 1, 4, 7
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(DO_NOT_IMPUTE)
variable_methods = settings.ContextSetting({})
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
# copy METHODS (some are modified by the widget)
self.methods = copy.deepcopy(OWImpute.METHODS)
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QVBoxLayout(box)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for i, method in enumerate(self.methods):
if not method.columns_only:
button = QRadioButton(method.name)
button.setChecked(i == self.default_method_index)
button_group.addButton(button, i)
box_layout.addWidget(button)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = QListView(
selectionMode=QListView.ExtendedSelection
)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed
)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for i, method in enumerate(self.methods):
button = QRadioButton(text=method.name)
button_group.addButton(button, i)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected
)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000., maximum=1000., singleStep=.1, decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection
)
method_layout.addStretch(2)
reset_button = QPushButton(
"Restore All to Default", checked=False, checkable=False,
clicked=self.reset_variable_methods, default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_commit(
self.controlArea, self, "autocommit", "Apply",
orientation=Qt.Horizontal, checkbox_label="Apply automatically")
box.layout().insertSpacing(0, 80)
box.layout().insertWidget(0, self.report_button)
self.data = None
self.learner = None
self.modified = False
self.default_method = self.methods[self.default_method_index]
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
self.default_method = self.methods[self.default_method_index]
self.methods[self.DEFAULT].method = self.default_method
# update variable view
for index in map(self.varmodel.index, range(len(self.varmodel))):
method = self.variable_methods.get(
index.row(), self.methods[self.DEFAULT])
self.varmodel.setData(index, method, Qt.UserRole)
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.closeContext()
self.varmodel[:] = []
self.variable_methods = {}
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.learner = learner or self.DEFAULT_LEARNER
imputer = self.methods[self.MODEL_BASED_IMPUTER]
imputer.learner = self.learner
button = self.default_button_group.button(self.MODEL_BASED_IMPUTER)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(self.MODEL_BASED_IMPUTER)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = self.MODEL_BASED_IMPUTER
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
"""Returns the imputation method for column by its index.
"""
if not isinstance(column_index, int):
column_index = column_index.row()
return self.variable_methods.get(column_index,
self.methods[self.DEFAULT])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
if self.data is None or len(self.data) == 0 or len(self.varmodel) == 0:
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [
(i, var, self.variable_methods.get(i, self.default_method))
for i, var in enumerate(self.varmodel)
]
def impute_one(method, var, data):
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(
impute_one, copy.deepcopy(method), var, data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
data = self.data
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
assert fut.done()
newvar = []
try:
res = fut.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
# ?? break
except VariableNotSupported:
self.warning("Default method can not handle '{}'".
format(var.name))
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", var, exc_info=True)
self.Error.imputation_failed(var.name)
attributes = class_vars = None
break
else:
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = var
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
if i < len(data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
if attributes is None:
data = None
else:
domain = Orange.data.Domain(attributes, class_vars,
data.domain.metas)
try:
data = self.data.from_table(domain, data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
data = None
self.Outputs.data.send(data)
self.modified = False
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setBlocking(False)
def onDeleteWidget(self):
self.cancel()
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.variable_methods.get(i, None)
if method is not None:
specific.append("{} ({})".format(var.name, str(method)))
default = self.default_method.name
if specific:
self.report_items((
("Default method", default),
("Specific imputers", ", ".join(specific))
))
else:
self.report_items((("Method", default),))
def _on_var_selection_changed(self):
indexes = self.selection.selectedIndexes()
methods = [self.get_method_for_column(i.row()) for i in indexes]
def method_key(method):
"""
Decompose method into its type and parameters.
"""
# The return value should be hashable and __eq__ comparable
if isinstance(method, AsDefault):
return AsDefault, (method.method,)
elif isinstance(method, impute.Model):
return impute.Model, (method.learner,)
elif isinstance(method, impute.Default):
return impute.Default, (method.default,)
else:
return type(method), None
methods = set(method_key(m) for m in methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for i, m in enumerate(self.methods):
if method_type == type(m):
self.variable_button_group.button(i).setChecked(True)
if method_type is impute.Default:
(fixed_value,) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
for method, button in zip(self.methods,
self.variable_button_group.buttons()):
enabled = all(method.supports_variable(var) for var in
selected_vars)
button.setEnabled(enabled)
if not has_discrete:
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(self.AS_INPUT).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
if method_index == self.DEFAULT:
for index in indexes:
self.variable_methods.pop(index.row(), None)
elif method_index == OWImpute.AS_INPUT:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
method = impute.Default(default=value)
self.variable_methods[index.row()] = method
else:
method = self.methods[method_index]
for index in indexes:
self.variable_methods[index.row()] = method
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(index, self.get_method_for_column(index.row()), Qt.UserRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(self.AS_INPUT).setChecked(True)
self.set_method_for_current_selection(self.AS_INPUT)
def reset_variable_methods(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, self.DEFAULT)
self.variable_button_group.button(self.DEFAULT).setChecked(True)
class ControlCombo(ControlBase, QWidget):
"""This class represents a wrapper to the combo box"""
def __init__(self, label='', default=None, helptext=None):
QWidget.__init__(self)
ControlBase.__init__(self, label, default, helptext)
##########################################################################
############ Functions ###################################################
##########################################################################
def init_form(self):
self._layout = QHBoxLayout()
self._combo = QComboBox(self.form)
if self._label is not None:
self._combolabel = QLabel(self.form)
self._layout.addWidget(self._combolabel)
self._combolabel.setAccessibleName('ControlCombo-label')
self.label = self._label
else:
self._combolabel = None
self._layout.addWidget(self._combo)
self.form.setLayout(self._layout)
self._combo.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
self._layout.setContentsMargins(0, 0, 0, 0)
self.form.setContentsMargins(0, 0, 0, 0)
self.form.setMinimumHeight(38)
self.form.setMaximumHeight(38)
self.form.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed)
self._combo.currentIndexChanged.connect(self._currentIndexChanged)
self._combo.activated.connect(self._activated)
self._combo.highlighted.connect(self._highlighted)
self._combo.editTextChanged.connect(self._editTextChanged)
self._items = {}
self._addingItem = False
def clear(self):
self._items = {}
self._value = None
self._combo.clear()
def add_item(self, label, value=None):
self._addingItem = True
if value is not None:
if not (value in self._items.values()):
self._combo.addItem(label)
else:
if not (label in self._items.keys()):
self._combo.addItem(label)
firstValue = False
if self._items == {}:
firstValue = True
if value is None:
self._items[str(label)] = label
else:
self._items[str(label)] = value
self._addingItem = False
if firstValue:
self.value = self._items[label]
def __add__(self, val):
if isinstance(val, tuple):
self.add_item(val[0], val[1])
else:
self.add_item(val)
return self
def get_item_index_by_name(self, item_name):
"""
Returns the index of the item containing the given name
:param item_name: item name in combo box
:type item_name: string
"""
return self._combo.findText(item_name)
def count(self):
return self._combo.count()
def show(self):
"""
Show the control
"""
QWidget.show(self)
def hide(self):
"""
Hide the control
"""
QWidget.hide(self)
##########################################################################
############ Events ######################################################
##########################################################################
def current_index_changed_event(self, index):
"""Called when the user chooses an item in the combobox and
the selected choice is different from the last one selected.
@index: item's index
"""
pass
def activated_event(self, index):
"""Called when the user chooses an item in the combobox.
Note that this signal happens even when the choice is not changed
@index: item's index
"""
pass
def highlighted_event(self, index):
pass
def edittext_changed_event(self, text):
pass
##########################################################################
############ PROPERTIES ##################################################
##########################################################################
@property
def form(self):
return self
@property
def current_index(self):
return self._combo.currentIndex()
@current_index.setter
def current_index(self, value):
self._combo.setCurrentIndex(value)
@property
def items(self):
return self._items.items()
@property
def value(self):
return self._value
@value.setter
def value(self, value):
for key, val in self.items:
if value == val:
index = self._combo.findText(key)
self._combo.setCurrentIndex(index)
if self._value != value:
self.changed_event()
self._value = val
@property
def text(self):
return str(self._combo.currentText())
@text.setter
def text(self, value):
for key, val in self.items:
if value == key:
self.value = val
break
@property
def label(self):
if self._combolabel:
return self._combolabel.text()
else:
return None
@label.setter
def label(self, value):
"""
Label of the control, if applies
@type value: string
"""
if self._combolabel:
self._combolabel.setText(value)
##########################################################################
############ Private functions ###########################################
##########################################################################
def _activated(self, index):
if not self._addingItem:
item = self._combo.currentText()
if len(item) >= 1:
ControlBase.value.fset(self, self._items[str(item)])
self.activated_event(index)
def _highlighted(self, index):
"""Called when an item in the combobox popup
list is highlighted by the user.
@index: item's index
"""
self.highlighted_event(index)
def _editTextChanged(self, text):
self.edittext_changed_event(text)
def _currentIndexChanged(self, index):
if not self._addingItem:
item = self._combo.currentText()
if len(item) >= 1:
ControlBase.value.fset(self, self._items[str(item)])
self.current_index_changed_event(index)
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = Msg("Model based imputer does not work for sparse data")
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(int(Method.Leave)) # type: int
# Per-variable imputation state (synced in storeSpecificSettings)
_variable_imputation_state = settings.ContextSetting({}) # type: VariableState
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
self.learner = None # type: Optional[Learner]
self.default_learner = SimpleTreeLearner()
self.modified = False
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QVBoxLayout(box)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for method, _ in list(METHODS.items())[1:-1]:
imputer = self.create_imputer(method)
button = QRadioButton(imputer.name)
button.setChecked(method == self.default_method_index)
button_group.addButton(button, method)
box_layout.addWidget(button)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = QListView(
selectionMode=QListView.ExtendedSelection,
uniformItemSizes=True
)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed
)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for method in Method:
imputer = self.create_imputer(method)
button = QRadioButton(text=imputer.name)
button_group.addButton(button, method)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected
)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000., maximum=1000., singleStep=.1, decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection
)
method_layout.addStretch(2)
reset_button = QPushButton(
"Restore All to Default", checked=False, checkable=False,
clicked=self.reset_variable_state, default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_commit(
self.controlArea, self, "autocommit", "Apply",
orientation=Qt.Horizontal,
checkbox_label="Apply automatically")
box.button.setFixedWidth(180)
box.layout().insertStretch(0)
def create_imputer(self, method, *args):
# type: (Method, ...) -> impute.BaseImputeMethod
if method == Method.Model:
if self.learner is not None:
return impute.Model(self.learner)
else:
return impute.Model(self.default_learner)
elif method == Method.AsAboveSoBelow:
assert self.default_method_index != Method.AsAboveSoBelow
default = self.create_imputer(Method(self.default_method_index))
m = AsDefault()
m.method = default
return m
else:
return METHODS[method](*args)
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
assert index != Method.AsAboveSoBelow
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
# update variable view
self.update_varview()
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.closeContext()
self.varmodel[:] = []
self._variable_imputation_state = {} # type: VariableState
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
# restore per variable imputation state
self._restore_state(self._variable_imputation_state)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.learner = learner or self.default_learner
imputer = self.create_imputer(Method.Model)
button = self.default_button_group.button(Method.Model)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(Method.Model)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = Method.Model
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
# type: (int) -> impute.BaseImputeMethod
"""
Return the imputation method for column by its index.
"""
assert 0 <= column_index < len(self.varmodel)
idx = self.varmodel.index(column_index, 0)
state = idx.data(StateRole)
if state is None:
state = (Method.AsAboveSoBelow, ())
return self.create_imputer(state[0], *state[1])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
if self.data is None or len(self.data) == 0 or len(self.varmodel) == 0:
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [
(i, var, self.get_method_for_column(i))
for i, var in enumerate(self.varmodel)
]
# normalize to the effective method bypasing AsDefault
impute_state = [
(i, var, m.method if isinstance(m, AsDefault) else m)
for i, var, m in impute_state
]
def impute_one(method, var, data):
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(
impute_one, copy.deepcopy(method), var, data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
data = self.data
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
assert fut.done()
newvar = []
try:
res = fut.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
# ?? break
except VariableNotSupported:
self.warning("Default method can not handle '{}'".
format(var.name))
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", var, exc_info=True)
self.Error.imputation_failed(var.name)
attributes = class_vars = None
break
else:
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = var
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
if i < len(data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
if attributes is None:
data = None
else:
domain = Orange.data.Domain(attributes, class_vars,
data.domain.metas)
try:
data = self.data.from_table(domain, data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
data = None
self.Outputs.data.send(data)
self.modified = False
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setBlocking(False)
def onDeleteWidget(self):
self.cancel()
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.get_method_for_column(i)
if not isinstance(method, AsDefault):
specific.append("{} ({})".format(var.name, str(method)))
default = self.create_imputer(Method.AsAboveSoBelow)
if specific:
self.report_items((
("Default method", default.name),
("Specific imputers", ", ".join(specific))
))
else:
self.report_items((("Method", default.name),))
def _on_var_selection_changed(self):
indexes = self.selection.selectedIndexes()
defmethod = (Method.AsAboveSoBelow, ())
methods = [index.data(StateRole) for index in indexes]
methods = [m if m is not None else defmethod for m in methods]
methods = set(methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for m in Method:
if method_type == m:
self.variable_button_group.button(m).setChecked(True)
if method_type == Method.Default:
(fixed_value,) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
# Update variable methods GUI enabled state based on selection.
for method in Method:
# use a default constructed imputer to query support
imputer = self.create_imputer(method)
enabled = all(imputer.supports_variable(var) for var in
selected_vars)
button = self.variable_button_group.button(method)
button.setEnabled(enabled)
# Update the "Value" edit GUI.
if not has_discrete:
# no discrete variables -> allow mass edit for all (continuous vars)
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
# single discrete var -> enable and fill the values combo
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
# mixed type selection -> disable
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(Method.Default).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
# set current value
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
# type: (Method) -> None
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
# type: (List[QModelIndex], Method) -> None
if method_index == Method.AsAboveSoBelow:
for index in indexes:
self.varmodel.setData(index, None, StateRole)
elif method_index == Method.Default:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
state = (int(Method.Default), (value,))
self.varmodel.setData(index, state, StateRole)
else:
state = (int(method_index), ())
for index in indexes:
self.varmodel.setData(index, state, StateRole)
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(
index, self.get_method_for_column(index.row()),
DisplayMethodRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(Method.Default).setChecked(True)
self.set_method_for_current_selection(Method.Default)
def reset_variable_state(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, Method.AsAboveSoBelow)
self.variable_button_group.button(Method.AsAboveSoBelow).setChecked(True)
def _store_state(self):
# type: () -> VariableState
"""
Save the current variable imputation state
"""
state = {} # type: VariableState
for i, var in enumerate(self.varmodel):
index = self.varmodel.index(i)
m = index.data(StateRole)
if m is not None:
state[var_key(var)] = m
return state
def _restore_state(self, state):
# type: (VariableState) -> None
"""
Restore the variable imputation state from the saved state
"""
def check(state):
# check if state is a proper State
if isinstance(state, tuple) and len(state) == 2:
m, p = state
if isinstance(m, int) and isinstance(p, tuple) and \
0 <= m < len(Method):
return True
return False
for i, var in enumerate(self.varmodel):
m = state.get(var_key(var), None)
if check(m):
self.varmodel.setData(self.varmodel.index(i), m, StateRole)
def storeSpecificSettings(self):
self._variable_imputation_state = self._store_state()
super().storeSpecificSettings()
class OWSql(OWWidget):
name = "SQL表"
id = "orange.widgets.data.sql"
description = "从SQL加载数据集。"
icon = "icons/SQLTable.svg"
priority = 30
category = "Data"
keywords = ["load"]
class Outputs:
data = Output("Data", Table, doc="Attribute-valued dataset read from the input file.")
settings_version = 2
want_main_area = False
resizing_enabled = False
host = Setting(None)
port = Setting(None)
database = Setting(None)
schema = Setting(None)
username = ""
password = ""
table = Setting(None)
sql = Setting("")
guess_values = Setting(True)
download = Setting(False)
materialize = Setting(False)
materialize_table_name = Setting("")
class Information(OWWidget.Information):
data_sampled = Msg("Data description was generated from a sample.")
class Error(OWWidget.Error):
connection = Msg("{}")
no_backends = Msg("Please install a backend to use this widget")
class Warning(OWWidget.Warning):
missing_extension = Msg("Database is missing extensions: {}")
def __init__(self):
super().__init__()
self.backend = None
self.data_desc_table = None
self.database_desc = None
vbox = gui.vBox(self.controlArea, "Server", addSpace=True)
box = gui.vBox(vbox)
self.backends = BackendModel(Backend.available_backends())
self.backendcombo = QComboBox(box)
if len(self.backends):
self.backendcombo.setModel(self.backends)
else:
self.Error.no_backends()
box.setEnabled(False)
box.layout().addWidget(self.backendcombo)
self.servertext = QLineEdit(box)
self.servertext.setPlaceholderText('Server')
self.servertext.setToolTip('Server')
self.servertext.editingFinished.connect(self._load_credentials)
if self.host:
self.servertext.setText(self.host if not self.port else
'{}:{}'.format(self.host, self.port))
box.layout().addWidget(self.servertext)
self.databasetext = QLineEdit(box)
self.databasetext.setPlaceholderText('Database[/Schema]')
self.databasetext.setToolTip('Database or optionally Database/Schema')
if self.database:
self.databasetext.setText(
self.database if not self.schema else
'{}/{}'.format(self.database, self.schema))
box.layout().addWidget(self.databasetext)
self.usernametext = QLineEdit(box)
self.usernametext.setPlaceholderText('Username')
self.usernametext.setToolTip('Username')
box.layout().addWidget(self.usernametext)
self.passwordtext = QLineEdit(box)
self.passwordtext.setPlaceholderText('Password')
self.passwordtext.setToolTip('Password')
self.passwordtext.setEchoMode(QLineEdit.Password)
box.layout().addWidget(self.passwordtext)
self._load_credentials()
self.tables = TableModel()
tables = gui.hBox(box)
self.tablecombo = QComboBox(
minimumContentsLength=35,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength
)
self.tablecombo.setModel(self.tables)
self.tablecombo.setToolTip('table')
tables.layout().addWidget(self.tablecombo)
index = self.tablecombo.findText(str(self.table))
if index != -1:
self.tablecombo.setCurrentIndex(index)
# set up the callback to select_table in case of selection change
self.tablecombo.activated[int].connect(self.select_table)
self.connectbutton = gui.button(
tables, self, '↻', callback=self.connect)
self.connectbutton.setSizePolicy(
QSizePolicy.Fixed, QSizePolicy.Fixed)
tables.layout().addWidget(self.connectbutton)
self.custom_sql = gui.vBox(box)
self.custom_sql.setVisible(False)
self.sqltext = QTextEdit(self.custom_sql)
self.sqltext.setPlainText(self.sql)
self.custom_sql.layout().addWidget(self.sqltext)
mt = gui.hBox(self.custom_sql)
cb = gui.checkBox(mt, self, 'materialize', 'Materialize to table ')
cb.setToolTip('Save results of the query in a table')
le = gui.lineEdit(mt, self, 'materialize_table_name')
le.setToolTip('Save results of the query in a table')
self.executebtn = gui.button(
self.custom_sql, self, 'Execute', callback=self.open_table)
box.layout().addWidget(self.custom_sql)
gui.checkBox(box, self, "guess_values",
"Auto-discover categorical variables",
callback=self.open_table)
self.downloadcb = gui.checkBox(box, self, "download",
"Download data to local memory",
callback=self.open_table)
gui.rubber(self.buttonsArea)
self.connect()
QTimer.singleShot(0, self.select_table)
def _load_credentials(self):
self._parse_host_port()
cm = self._credential_manager(self.host, self.port)
self.username = cm.username
self.password = cm.password
if self.username:
self.usernametext.setText(self.username)
if self.password:
self.passwordtext.setText(self.password)
def _save_credentials(self):
cm = self._credential_manager(self.host, self.port)
cm.username = self.username or ''
cm.password = self.password or ''
def _credential_manager(self, host, port):
return CredentialManager("SQL Table: {}:{}".format(host, port))
def error(self, id=0, text=""):
super().error(id, text)
err_style = 'QLineEdit {border: 2px solid red;}'
if 'server' in text or 'host' in text:
self.servertext.setStyleSheet(err_style)
else:
self.servertext.setStyleSheet('')
if 'role' in text:
self.usernametext.setStyleSheet(err_style)
else:
self.usernametext.setStyleSheet('')
if 'database' in text:
self.databasetext.setStyleSheet(err_style)
else:
self.databasetext.setStyleSheet('')
def _parse_host_port(self):
hostport = self.servertext.text().split(':')
self.host = hostport[0]
self.port = hostport[1] if len(hostport) == 2 else None
def connect(self):
self._parse_host_port()
self.database, _, self.schema = self.databasetext.text().partition('/')
self.username = self.usernametext.text() or None
self.password = self.passwordtext.text() or None
self.Warning.missing_extension.clear()
self.downloadcb.setEnabled(True)
try:
if self.backendcombo.currentIndex() < 0:
return
backend = self.backends[self.backendcombo.currentIndex()]
self.backend = backend(dict(
host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password
))
self.Error.connection.clear()
if getattr(self.backend, 'missing_extension', False):
self.Warning.missing_extension(
", ".join(self.backend.missing_extension))
self.download = True
self.downloadcb.setEnabled(False)
if not is_postgres(self.backend):
self.download = True
self.downloadcb.setEnabled(False)
self._save_credentials()
self.database_desc = OrderedDict((
("Host", self.host), ("Port", self.port),
("Database", self.database), ("User name", self.username)
))
self.refresh_tables()
except BackendError as err:
error = str(err).split('\n')[0]
self.Error.connection(error)
self.database_desc = self.data_desc_table = None
self.tablecombo.clear()
def refresh_tables(self):
self.tables.clear()
if self.backend is None:
self.data_desc_table = None
return
self.tables.append("Select a table")
self.tables.append("Custom SQL")
self.tables.extend(self.backend.list_tables(self.schema))
# Called on tablecombo selection change:
def select_table(self):
curIdx = self.tablecombo.currentIndex()
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.custom_sql.setVisible(False)
return self.open_table()
else:
self.custom_sql.setVisible(True)
self.data_desc_table = None
self.database_desc["Table"] = "(None)"
self.table = None
if len(str(self.sql)) > 14:
return self.open_table()
#self.Error.missing_extension(
# 's' if len(missing) > 1 else '',
# ', '.join(missing),
# shown=missing)
def open_table(self):
table = self.get_table()
self.data_desc_table = table
self.Outputs.data.send(table)
def get_table(self):
curIdx = self.tablecombo.currentIndex()
if curIdx <= 0:
if self.database_desc:
self.database_desc["Table"] = "(None)"
self.data_desc_table = None
return
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.table = self.tables[self.tablecombo.currentIndex()]
self.database_desc["Table"] = self.table
if "Query" in self.database_desc:
del self.database_desc["Query"]
what = self.table
else:
what = self.sql = self.sqltext.toPlainText()
self.table = "Custom SQL"
if self.materialize:
import psycopg2
if not self.materialize_table_name:
self.Error.connection(
"Specify a table name to materialize the query")
return
try:
with self.backend.execute_sql_query("DROP TABLE IF EXISTS " +
self.materialize_table_name):
pass
with self.backend.execute_sql_query("CREATE TABLE " +
self.materialize_table_name +
" AS " + self.sql):
pass
with self.backend.execute_sql_query("ANALYZE " + self.materialize_table_name):
pass
except (psycopg2.ProgrammingError, BackendError) as ex:
self.Error.connection(str(ex))
return
try:
table = SqlTable(dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password),
what,
backend=type(self.backend),
inspect_values=False)
except BackendError as ex:
self.Error.connection(str(ex))
return
self.Error.connection.clear()
sample = False
if table.approx_len() > LARGE_TABLE and self.guess_values:
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Attribute discovery might take "
"a long time on large tables.\n"
"Do you want to auto discover attributes?")
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
if is_postgres(self.backend):
sample_button = confirm.addButton("Yes, on a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
self.guess_values = False
elif is_postgres(self.backend) and \
confirm.clickedButton() == sample_button:
sample = True
self.Information.clear()
if self.guess_values:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if sample:
s = table.sample_time(1)
domain = s.get_domain(inspect_values=True)
self.Information.data_sampled()
else:
domain = table.get_domain(inspect_values=True)
QApplication.restoreOverrideCursor()
table.domain = domain
if self.download:
if table.approx_len() > AUTO_DL_LIMIT:
if is_postgres(self.backend):
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Data appears to be big. Do you really "
"want to download it to local memory?")
if table.approx_len() <= MAX_DL_LIMIT:
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
sample_button = confirm.addButton("Yes, a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
return
elif confirm.clickedButton() == sample_button:
table = table.sample_percentage(
AUTO_DL_LIMIT / table.approx_len() * 100)
else:
if table.approx_len() > MAX_DL_LIMIT:
QMessageBox.warning(
self, 'Warning', "Data is too big to download.\n")
return
else:
confirm = QMessageBox.question(
self, 'Question',
"Data appears to be big. Do you really "
"want to download it to local memory?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if confirm == QMessageBox.No:
return
table.download_data(MAX_DL_LIMIT)
table = Table(table)
return table
def send_report(self):
if not self.database_desc:
self.report_paragraph("No database connection.")
return
self.report_items("Database", self.database_desc)
if self.data_desc_table:
self.report_items("Data",
report.describe_data(self.data_desc_table))
@classmethod
def migrate_settings(cls, settings, version):
if version < 2:
# Until Orange version 3.4.4 username and password had been stored
# in Settings.
cm = cls._credential_manager(settings["host"], settings["port"])
cm.username = settings["username"]
cm.password = settings["password"]
class EditTracUI(QWidget):
def __init__(self, trac, parent):
super(EditTracUI, self).__init__()
self.title = 'Edit TracControl Element'
self.setLayout(QVBoxLayout(self))
self.layout().setAlignment(Qt.AlignCenter)
self.parent = parent
self.trac = trac
# Init controls
self._nameControl = LineEdit('Name', self)
self._nameControl.setText(self.trac.name)
self._nameControl.kb.connect(self.showOSK)
self._outputPinControlLabel = QLabel('Output Pin', self)
self._outputPinControl = QComboBox(self)
for _pins in self.parent.availablePins():
self._outputPinControl.addItem(str(_pins))
for _i in range(self._outputPinControl.count()):
if self._outputPinControl.itemText(_i) == str(self.trac.outputPin):
self._outputPinControl.setCurrentIndex(_i)
break
self._enabledControl = QCheckBox('Enabled', self)
self._enabledControl.setChecked(self.trac.enabled)
self._iconControlLabel = QLabel('Icon Path', self)
self._iconControl = QComboBox(self)
for _key in Config.icons['tracControl'].keys():
icon = Config.icon('tracControl', _key)
self._iconControl.addItem(icon['name'], _key)
self._iconControl.setItemIcon(self._iconControl.count() - 1,
QIcon(icon['path']))
# Set combobox selection to icon variable
for iconIdx in range(self._iconControl.count()):
if self.trac.icon is not None and self._iconControl.itemData(
iconIdx) == self.trac.icon:
self._iconControl.setCurrentIndex(iconIdx)
break
self._saveBtn = QPushButton('Save', self)
self._saveBtn.clicked.connect(self.__saveBtnAction)
self._cancelBtn = QPushButton('Cancel', self)
self._cancelBtn.clicked.connect(self.__cancel)
_layout = [['_nameControl'],
['_outputPinControlLabel', '_outputPinControl'],
['_enabledControl'], ['_iconControlLabel', '_iconControl'],
['_saveBtn', '_cancelBtn']]
for _list in _layout:
_panel = QWidget(self)
_panel.setLayout(QHBoxLayout(_panel))
_panel.layout().setAlignment(Qt.AlignCenter)
for _ctrl in _list:
_panel.layout().addWidget(eval('self.%s' % _ctrl))
self.layout().addWidget(_panel)
def __saveBtnAction(self):
self.trac.name = self._nameControl.text()
self.trac.outputPin = int(self._outputPinControl.currentText())
self.trac.enabled = self._enabledControl.isChecked()
self.trac.icon = self._iconControl.currentData()
self.parent.tracs.save()
self.parent.loadUI('config_trac')
def __cancel(self):
self.parent.loadUI('config_trac')
def showOSK(self):
self.window().dock.show()
self.window().osk.rWidget = self._nameControl
class OWLoadData(widget.OWWidget):
name = "Load Data"
icon = "icons/upload.svg"
priority = 10
class Outputs:
data = widget.Output("Data", Orange.data.Table)
class Information(widget.OWWidget.Information):
modified = widget.Msg(
"Uncommited changes\nPress 'Load data' to submit changes")
class Warning(widget.OWWidget.Warning):
sampling_in_effect = widget.Msg("Sampling is in effect.")
class Error(widget.OWWidget.Error):
row_annotation_mismatch = widget.Msg("Row annotation length mismatch\n"
"Expected {} rows got {}")
col_annotation_mismatch = widget.Msg(
"Column annotation length mismatch\n"
"Expected {} rows got {}")
_recent = settings.Setting([]) # type: List[str]
_recent_row_annotations = settings.Setting([]) # type: List[str]
_recent_col_annotations = settings.Setting([]) # type: List[str]
_cells_in_rows = settings.Setting(False)
_col_annotations_enabled = settings.Setting(False)
_row_annotations_enabled = settings.Setting(False)
_last_path = settings.Setting("") # type: str
_header_rows_count = settings.Setting(1) # type: int
_header_cols_count = settings.Setting(1) # type: int
_sample_rows_enabled = settings.Setting(False) # type: bool
_sample_cols_enabled = settings.Setting(False) # type: bool
_sample_cols_p = settings.Setting(10.0) # type: bool
_sample_rows_p = settings.Setting(10.0) # type: bool
settingsHandler = RunaroundSettingsHandler()
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
self._current_path = ""
icon_open_dir = self.style().standardIcon(QStyle.SP_DirOpenIcon)
# Top grid with file selection combo box
grid = QGridLayout()
lb = QLabel("File:")
lb.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
self.recent_combo = cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=20,
toolTip="Select a recent file")
self.recent_model = cb.model() # type: QStandardItemModel
self.recent_combo.activated[int].connect(self._select_recent)
browse = QPushButton("...",
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse)
# reload = QPushButton("Reload", autoDefault=False, icon=icon_reload)
grid.addWidget(lb, 0, 0, Qt.AlignVCenter)
grid.addWidget(cb, 0, 1)
grid.addWidget(browse, 0, 2)
# grid.addWidget(reload, 0, 3)
self.summary_label = label = QLabel("", self)
label.ensurePolished()
f = label.font()
if f.pointSizeF() != -1:
f.setPointSizeF(f.pointSizeF() * 5 / 6)
else:
f.setPixelSize(f.pixelSize() * 5 / 6)
label.setFont(f)
grid.addWidget(label, 1, 1, 1, 3)
self.controlArea.layout().addLayout(grid)
box = gui.widgetBox(self.controlArea,
"Headers and Row Labels",
spacing=-1)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_rows_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_rows_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_rows_count)
hl.addWidget(QLabel("Data starts with", box))
hl.addWidget(self.header_rows_spin)
hl.addWidget(QLabel("header row(s)", box))
hl.addStretch(10)
box.layout().addLayout(hl)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_cols_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_cols_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_cols_count)
hl.addWidget(QLabel("First", box))
hl.addWidget(self.header_cols_spin)
hl.addWidget(QLabel("column(s) are row labels", box))
hl.addStretch(10)
box.layout().addLayout(hl)
self.data_struct_box = box = gui.widgetBox(self.controlArea,
"Input Data Structure")
gui.radioButtons(box,
self,
"_cells_in_rows", [
"Genes in rows, samples in columns",
"Samples in rows, genes in columns"
],
callback=self._invalidate)
box = gui.widgetBox(self.controlArea, "Sample Data", spacing=-1)
grid = QGridLayout()
grid.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(grid)
self.sample_rows_cb = cb = QCheckBox(checked=self._sample_rows_enabled)
spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_rows_p,
enabled=self._sample_rows_enabled)
spin.valueChanged.connect(self.set_sample_rows_p)
suffix = QLabel("% of Samples", enabled=self._sample_rows_enabled)
cb.toggled.connect(self.set_sample_rows_enabled)
cb.toggled.connect(spin.setEnabled)
cb.toggled.connect(suffix.setEnabled)
grid.addWidget(cb, 0, 0)
grid.addWidget(spin, 0, 1)
grid.addWidget(suffix, 0, 2)
self.sample_cols_cb = cb = QCheckBox(checked=self._sample_cols_enabled)
spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_cols_p,
enabled=self._sample_cols_enabled)
spin.valueChanged.connect(self.set_sample_cols_p)
suffix = QLabel("% of genes", enabled=self._sample_cols_enabled)
cb.toggled.connect(self.set_sample_cols_enabled)
cb.toggled.connect(spin.setEnabled)
cb.toggled.connect(suffix.setEnabled)
grid.addWidget(cb, 1, 0)
grid.addWidget(spin, 1, 1)
grid.addWidget(suffix, 1, 2)
grid.setColumnStretch(3, 10)
self.annotation_files_box = box = gui.widgetBox(
self.controlArea, "Cell && Gene Annotation Files")
form = QFormLayout(
formAlignment=Qt.AlignLeft,
rowWrapPolicy=QFormLayout.WrapAllRows,
)
box.layout().addLayout(form)
self.row_annotations_cb = cb = QCheckBox(
"Cell annotations", checked=self._row_annotations_enabled)
self._row_annotations_w = w = QWidget(
enabled=self._row_annotations_enabled)
cb.toggled.connect(self.set_row_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.row_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.row_annotations_combo.activated.connect(self._invalidate)
hl.addWidget(self.row_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_row_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.col_annotations_cb = cb = QCheckBox(
"Gene annotations", checked=self._col_annotations_enabled)
self._col_annotations_w = w = QWidget(
enabled=self._col_annotations_enabled)
cb.toggled.connect(self.set_col_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.col_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.col_annotations_combo.activated.connect(self._invalidate)
hl.addWidget(self.col_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_col_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.controlArea.layout().addStretch(10)
self.load_data_button = button = VariableTextPushButton(
"Load data",
autoDefault=True,
textChoiceList=["Load data", "Reload"])
self.load_data_button.setAutoDefault(True)
button.clicked.connect(self.commit, Qt.QueuedConnection)
self.controlArea.layout().addWidget(button, alignment=Qt.AlignRight)
init_recent_paths_model(
self.recent_model,
[RecentPath.create(p, []) for p in self._recent],
)
init_recent_paths_model(
self.row_annotations_combo.model(),
[RecentPath.create(p, []) for p in self._recent_row_annotations])
init_recent_paths_model(
self.col_annotations_combo.model(),
[RecentPath.create(p, []) for p in self._recent_col_annotations])
self._update_summary()
self._update_warning()
if self._last_path != "" and os.path.exists(self._last_path):
self.set_current_path(self._last_path)
else:
self.recent_combo.setCurrentIndex(-1)
def _update_warning(self):
if (self._sample_rows_enabled and self._sample_rows_p < 100) or \
(self._sample_cols_enabled and self._sample_cols_p < 100):
self.Warning.sampling_in_effect()
else:
self.Warning.sampling_in_effect.clear()
def set_sample_rows_enabled(self, enabled):
if self._sample_rows_enabled != enabled:
self._sample_rows_enabled = enabled
self._update_warning()
self._invalidate()
def set_sample_cols_enabled(self, enabled):
if self._sample_cols_enabled != enabled:
self._sample_cols_enabled = enabled
self._update_warning()
self._invalidate()
def set_sample_rows_p(self, p):
if self._sample_rows_p != p:
self._sample_rows_p = p
self._update_warning()
self._invalidate()
def set_sample_cols_p(self, p):
if self._sample_cols_p != p:
self._sample_cols_p = p
self._update_warning()
self._invalidate()
def set_header_rows_count(self, n):
if self._header_rows_count != n:
self._header_rows_count = n
self.header_rows_spin.setValue(n)
self._invalidate()
def set_header_cols_count(self, n):
if self._header_cols_count != n:
self._header_cols_count = n
self.header_cols_spin.setValue(n)
self._invalidate()
def set_row_annotations_enabled(self, enabled):
if self._row_annotations_enabled != enabled:
self._row_annotations_enabled = enabled
self.row_annotations_cb.setChecked(enabled)
self._invalidate()
def set_col_annotations_enabled(self, enabled):
if self._col_annotations_enabled != enabled:
self._col_annotations_enabled = enabled
self.col_annotations_cb.setChecked(enabled)
self._invalidate()
def set_current_path(self, path):
if samepath(self._current_path, path):
return
model = self.recent_model
index = -1
pathitem = None
for i in range(model.rowCount()):
item = model.item(i)
data = item.data(Qt.UserRole) if item is not None else None
if isinstance(data, RecentPath) and samepath(path, data.abspath):
index, pathitem = i, data
break
rpaths = []
if pathitem is None:
assert index == -1
pathitem = RecentPath.create(path, rpaths)
if index != -1:
item = model.takeRow(index)
else:
item = RecentPath_asqstandarditem(pathitem)
opts = infer_options(path)
if path.endswith(".count"):
self.set_header_rows_count(1)
self.set_header_cols_count(1)
fixed_format = False
elif path.endswith(".mtx"):
self.set_header_rows_count(0)
self.set_header_cols_count(0)
fixed_format = False
self._cells_in_rows = False
else:
fixed_format = True
if opts.transposed is not None:
self._cells_in_rows = not opts.transposed
self.data_struct_box.setEnabled(fixed_format)
self.header_rows_spin.setEnabled(fixed_format)
self.header_cols_spin.setEnabled(fixed_format)
model.insertRow(0, item)
self._current_path = path
self.recent_combo.setCurrentIndex(0)
self._update_summary()
if opts.row_annotation_file is not None:
index = insert_recent_path(
self.row_annotations_combo.model(),
RecentPath.create(opts.row_annotation_file, []))
self.row_annotations_combo.setCurrentIndex(index)
self.set_row_annotations_enabled(True)
else:
self.row_annotations_combo.setCurrentIndex(-1)
if opts.column_annotation_file is not None:
index = insert_recent_path(
self.col_annotations_combo.model(),
RecentPath.create(opts.column_annotation_file, []))
self.col_annotations_combo.setCurrentIndex(index)
self.set_col_annotations_enabled(True)
else:
self.col_annotations_combo.setCurrentIndex(-1)
if path.endswith(".mtx") \
and opts.row_annotation_file is not None \
and opts.column_annotation_file is not None \
and os.path.basename(opts.row_annotation_file) == "barcodes.tsv" \
and os.path.basename(opts.column_annotation_file) == "genes.tsv":
# 10x gene-barcode matrix
# TODO: The genes/barcodes files should be unconditionally loaded
# alongside the mtx. The row/col annotations might be used to
# specify additional sources. For the time being they are put in
# the corresponding comboboxes and made uneditable.
self.annotation_files_box.setEnabled(False)
else:
self.annotation_files_box.setEnabled(True)
self._invalidate()
def _update_summary(self):
path = self._current_path
size = None
ncols = None
nrows = None
try:
st = os.stat(path)
except OSError:
pass
else:
size = st.st_size
if os.path.splitext(path)[1] == ".mtx":
try:
with open(path, "rb") as f:
nrows, ncols = scipy.io.mminfo(f)[:2]
except OSError:
pass
except ValueError:
pass
else:
try:
with open(path, "rt", encoding="latin-1") as f:
sep = separator_from_filename(path)
ncols = len(next(csv.reader(f, delimiter=sep)))
nrows = sum(1 for _ in f)
except OSError:
pass
except StopIteration:
pass
text = []
if size is not None:
text += [sizeformat(size)]
if nrows is not None:
text += ["{:n} rows".format(nrows)]
if nrows is not None:
text += ["{:n} columns".format(ncols)]
self.summary_label.setText(", ".join(text))
def current_path(self):
return self._current_path
def _select_recent(self, index):
# type: (int) -> None
# select a file from the recent list (entered via combo box `activate`)
assert 0 <= index < self.recent_model.rowCount()
item = self.recent_model.item(index)
pathitem = item.data(Qt.UserRole)
assert isinstance(pathitem, RecentPath)
self.set_current_path(pathitem.abspath)
@Slot()
def browse(self):
dlg = QFileDialog(self)
dlg.setAcceptMode(QFileDialog.AcceptOpen)
dlg.setFileMode(QFileDialog.ExistingFile)
filters = Formats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
f = dlg.selectedNameFilter()
filename = dlg.selectedFiles()[0]
self.set_current_path(filename)
@Slot()
def browse_row_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
f = dlg.selectedNameFilter()
filename = dlg.selectedFiles()[0]
m = self.row_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.row_annotations_combo.setCurrentIndex(index)
self._invalidate()
@Slot()
def browse_col_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
f = dlg.selectedNameFilter()
filename = dlg.selectedFiles()[0]
m = self.col_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.col_annotations_combo.setCurrentIndex(index)
self._invalidate()
def _invalidate(self):
self.set_modified(True)
def set_modified(self, modified):
if modified:
text = "Load data"
else:
text = "Reload"
self.load_data_button.setText(text)
self.load_data_button.setAutoDefault(modified)
# Setting autoDefault once also sets default which persists even after
# settings autoDefault back to False??
self.load_data_button.setDefault(modified)
self.Information.modified(shown=modified)
def commit(self):
path = self._current_path
if not path:
return
transpose = not self._cells_in_rows
row_annot = self.row_annotations_combo.currentData(Qt.UserRole)
col_annot = self.col_annotations_combo.currentData(Qt.UserRole)
if self._row_annotations_enabled and \
isinstance(row_annot, RecentPath) and \
os.path.exists(row_annot.abspath):
row_annot = row_annot.abspath # type: str
else:
row_annot = None
if self._col_annotations_enabled and \
isinstance(col_annot, RecentPath) and \
os.path.exists(col_annot.abspath):
col_annot = col_annot.abspath # type: str
else:
col_annot = None
meta_parts = [] # type: List[pd.DataFrame]
attrs = [] # type: List[ContinuousVariable]
metas = [] # type: List[StringVariable]
rstate = np.random.RandomState(0x667)
skip_row = skip_col = None
if self._sample_cols_enabled:
p = self._sample_cols_p
if p < 100:
def skip_col(i, p=p):
return i > 3 and rstate.uniform(0, 100) > p
if self._sample_rows_enabled:
p = self._sample_rows_p
if p < 100:
def skip_row(i, p=p):
return i > 3 and rstate.uniform(0, 100) > p
header_rows = self._header_rows_count
header_rows_indices = []
if header_rows == 0:
header_rows = None
elif header_rows == 1:
header_rows = 0
header_rows_indices = [0]
else:
header_rows = list(range(header_rows))
header_rows_indices = header_rows
header_cols = self._header_cols_count
header_cols_indices = []
if header_cols == 0:
header_cols = None
elif header_cols == 1:
header_cols = 0
header_cols_indices = [0]
else:
header_cols = list(range(header_cols))
header_cols_indices = header_cols
if transpose:
_skip_row, _skip_col = skip_col, skip_row
else:
_skip_col, _skip_row = skip_col, skip_row
_userows = _usecols = None
userows_mask = usecols_mask = None
if _skip_col is not None:
ncols = pd.read_csv(path,
sep=separator_from_filename(path),
index_col=None,
nrows=1).shape[1]
usecols_mask = np.array([
not _skip_col(i) or i in header_cols_indices
for i in range(ncols)
],
dtype=bool)
_usecols = np.flatnonzero(usecols_mask)
if _skip_row is not None:
userows_mask = [] # record the used rows
def _skip_row(i, test=_skip_row):
r = test(i)
userows_mask.append(r)
return r
meta_df_index = None
row_annot_header = 0
row_annot_columns = None
col_annot_header = 0
col_annot_columns = None
if os.path.splitext(path)[1] == ".mtx":
# 10x cellranger output
X = scipy.io.mmread(path)
assert isinstance(X, scipy.sparse.coo_matrix)
if transpose:
X = X.T
if _skip_row is not None:
userows_mask = np.array(
[not _skip_row(i) for i in range(X.shape[0])])
X = X.tocsr()[np.flatnonzero(userows_mask)]
if _skip_col is not None:
usecols_mask = np.array(
[not _skip_col(i) for i in range(X.shape[1])])
X = X.tocsc()[:, np.flatnonzero(usecols_mask)]
X = X.todense(order="F")
if userows_mask is not None:
meta_df = pd.DataFrame({}, index=np.flatnonzero(userows_mask))
else:
meta_df = pd.DataFrame({}, index=pd.RangeIndex(X.shape[0]))
meta_df_index = meta_df.index
row_annot_header = None
row_annot_columns = ["Barcodes"]
col_annot_header = None
col_annot_columns = ["Id", "Gene"]
leading_cols = leading_rows = 0
else:
df = pd.read_csv(path,
sep=separator_from_filename(path),
index_col=header_cols,
header=header_rows,
skiprows=_skip_row,
usecols=_usecols)
if _skip_row is not None:
userows_mask = np.array(userows_mask, dtype=bool)
if transpose:
df = df.transpose()
userows_mask, usecols_mask = usecols_mask, userows_mask
leading_rows = len(header_cols_indices)
leading_cols = len(header_rows_indices)
else:
leading_rows = len(header_rows_indices)
leading_cols = len(header_cols_indices)
X = df.values
attrs = [ContinuousVariable.make(str(g)) for g in df.columns]
meta_df = df.iloc[:, :0] # Take the index # type: pd.DataFrame
meta_df_index = df.index
meta_parts = (meta_df, )
self.Error.row_annotation_mismatch.clear()
self.Error.col_annotation_mismatch.clear()
if row_annot is not None:
row_annot_df = pd.read_csv(row_annot,
sep=separator_from_filename(row_annot),
header=row_annot_header,
names=row_annot_columns,
index_col=None)
if userows_mask is not None:
# NOTE: we account for column header/ row index
expected = len(userows_mask) - leading_rows
else:
expected = X.shape[0]
if len(row_annot_df) != expected:
self.Error.row_annotation_mismatch(expected, len(row_annot_df))
row_annot_df = None
if row_annot_df is not None and userows_mask is not None:
# use the same sample indices
indices = np.flatnonzero(userows_mask[leading_rows:])
row_annot_df = row_annot_df.iloc[indices]
# if path.endswith(".count") and row_annot.endswith('.meta'):
# assert np.all(row_annot_df.iloc[:, 0] == df.index)
if row_annot_df is not None and meta_df_index is not None:
# Try to match the leading columns with the meta_df_index.
# If found then drop the columns (or index if the level does
# not have a name but the annotation col does)
drop_cols = []
drop_index_level = []
for i in range(meta_df_index.nlevels):
meta_df_level = meta_df_index.get_level_values(i)
if np.all(row_annot_df.iloc[:, i] == meta_df_level):
if meta_df_level.name is None:
drop_index_level.append(i)
elif meta_df_level.name == row_annot_df.columns[
i].name:
drop_cols.append(i)
if drop_cols:
row_annot_df = row_annot_df.drop(columns=drop_cols)
if drop_index_level:
for i in reversed(drop_index_level):
if isinstance(meta_df.index, pd.MultiIndex):
meta_df_index = meta_df_index.droplevel(i)
else:
assert i == 0
meta_df_index = pd.RangeIndex(meta_df_index.size)
meta_df = pd.DataFrame({}, index=meta_df_index)
if row_annot_df is not None:
meta_parts = (meta_df, row_annot_df)
if col_annot is not None:
col_annot_df = pd.read_csv(col_annot,
sep=separator_from_filename(col_annot),
header=col_annot_header,
names=col_annot_columns,
index_col=None)
if usecols_mask is not None:
expected = len(usecols_mask) - leading_cols
else:
expected = X.shape[1]
if len(col_annot_df) != expected:
self.Error.col_annotation_mismatch(expected, len(col_annot_df))
col_annot_df = None
if col_annot_df is not None and usecols_mask is not None:
indices = np.flatnonzero(usecols_mask[leading_cols:])
col_annot_df = col_annot_df.iloc[indices]
if col_annot_df is not None:
assert len(col_annot_df) == X.shape[1]
if not attrs and X.shape[1]: # No column names yet
attrs = [
ContinuousVariable.make(str(v))
for v in col_annot_df.iloc[:, 0]
]
names = [str(c) for c in col_annot_df.columns]
for var, values in zip(attrs, col_annot_df.values):
var.attributes.update(
{n: v
for n, v in zip(names, values)})
if meta_parts:
meta_parts = [
df_.reset_index() if not df_.index.is_integer() else df_
for df_ in meta_parts
]
metas = [
StringVariable.make(name)
for name in chain(*(_.columns for _ in meta_parts))
]
M = np.hstack(tuple(df_.values for df_ in meta_parts))
else:
metas = None
M = None
if not attrs and X.shape[1]:
attrs = Orange.data.Domain.from_numpy(X).attributes
domain = Orange.data.Domain(attrs, metas=metas)
d = Orange.data.Table.from_numpy(domain, X, None, M)
self.Outputs.data.send(d)
self.set_modified(False)
def onDeleteWidget(self):
super().onDeleteWidget()
def _saveState(self):
maxitems = 15
def dataiter(model, role=Qt.UserRole):
return (model.data(model.index(i, 0), role)
for i in range(model.rowCount()))
def recent_paths(model):
return [
el.abspath for el in dataiter(model)
if isinstance(el, RecentPath)
][:maxitems]
self._recent = recent_paths(self.recent_model)
self._recent_row_annotations = recent_paths(
self.row_annotations_combo.model())
self._recent_col_annotations = recent_paths(
self.col_annotations_combo.model())
self._last_path = self._current_path
def saveSettings(self):
self._saveState()
super().saveSettings()
class UserPrefUI(QWidget):
def __init__(self, prefs, parent):
super(UserPrefUI, self).__init__()
self.title = 'User Preferences'
self.setLayout(QVBoxLayout(self))
self.parent = parent
self.prefs = prefs
# Init logger
self.logger = Logger('userPrefsUI', 'UI : User Preferences')
# Set up container
container = QWidget()
container.setLayout(QVBoxLayout())
# Set up scroll area
scrollArea = QScrollArea()
scrollArea.setWidget(container)
scrollArea.setWidgetResizable(True)
self.layout().addWidget(scrollArea)
# Permanent controls
self._saveButton = QPushButton('Save', self)
self._saveButton.clicked.connect(self.__save)
self._startMaximized = QCheckBox('Start Maximized', self)
self._allowDuplicatePins = QCheckBox('Allow Duplicate Output Pins',
self)
self._enableOBA = QCheckBox('OnBoard Air', self)
self._enableLighting = QCheckBox('Lighting', self)
self._enableTracControl = QCheckBox('Traction Control', self)
self._enableCamViewer = QCheckBox('Camera Viewer', self)
self._enableGyro = QCheckBox('Inclinometer', self)
self._panelLabel = QLabel(
'<html><center>Enable Modules:<br>\
<em>Restart TacOS for changes to take effect.</em></center></html>',
self)
self._i2cBus = QComboBox(self)
self._i2cBus.addItems(Config.busList)
self._i2cBusLabel = QLabel('I2C Bus', self)
self._i2cAddress = QLineEdit('I2C Address', self)
self._i2cLabel = QLabel(
'<html><center>I2C Parameters:<br>\
<em>Restart TacOS for changes to take effect.</em></center></html>',
self)
self._i2cDebug = QCheckBox('I2C Debug Mode', self)
self._i2cDebugLabel = QLabel(
'<html><center>I2C Debug Mode:<br>\
<em>Disables all I2C comms.<br>\
Restart TacOS for changes to take effect.</em></center></html>',
self)
self._debugLogging = QCheckBox('Enable Debug Logs', self)
self._restartButton = QPushButton('Restart TacOS', self)
self._restartButton.clicked.connect(self.__restart)
# Set initial values
for control in [
'startMaximized', 'allowDuplicatePins', 'enableOBA',
'enableLighting', 'enableTracControl', 'enableCamViewer',
'enableGyro', 'i2cDebug', 'debugLogging'
]:
if control in prefs.keys():
exec('self._%s.setChecked(prefs["%s"])' % (control, control))
else:
if control in [
'enableOBA', 'enableLighting', 'enableTracControl',
'enableCamViewer', 'enableGyro'
]:
exec('self._%s.setChecked(True)' % control)
else:
exec('self._%s.setChecked(False)' % control)
if 'i2cAddress' in prefs.keys():
self._i2cAddress.setText(prefs['i2cAddress'])
else:
self._i2cAddress.setText('0x20')
if 'i2cBus' in prefs.keys():
self._i2cBus.setCurrentText(str(prefs['i2cBus']))
else:
self._i2cBus.setCurrentIndex(0)
layoutList = [[
'_startMaximized', '_allowDuplicatePins', '_debugLogging'
], ['_panelLabel'],
['_enableOBA', '_enableLighting', '_enableTracControl'],
['_enableCamViewer', '_enableGyro'], ['_i2cLabel'],
['_i2cBusLabel', '_i2cBus', '_i2cAddress'],
['_i2cDebugLabel'], ['_i2cDebug'],
['_saveButton', '_restartButton']]
for i in layoutList:
panel = QWidget()
panel.setLayout(QHBoxLayout(panel))
panel.layout().setSpacing(20)
panel.layout().setAlignment(Qt.AlignCenter)
for c in i:
panel.layout().addWidget(eval('self.%s' % c))
container.layout().addWidget(panel)
def refresh(self):
self.__init__(self.window().prefs, self.parent)
def __save(self):
self.parent.prefs = self.__getPrefs()
self.parent.savePrefs()
def __restart(self):
os.system("sudo sh /home/pi/TacOS/launcher.sh")
sys.exit()
def __getPrefs(self):
_updates = {
'startMaximized': self._startMaximized.isChecked(),
'allowDuplicatePins': self._allowDuplicatePins.isChecked(),
'enableOBA': self._enableOBA.isChecked(),
'enableLighting': self._enableLighting.isChecked(),
'enableTracControl': self._enableTracControl.isChecked(),
'enableCamViewer': self._enableCamViewer.isChecked(),
'enableGyro': self._enableGyro.isChecked(),
'i2cBus': self._i2cBus.currentText(),
'i2cAddress': hex(int(self._i2cAddress.text(), 16)),
'i2cDebug': self._i2cDebug.isChecked(),
'debugLogging': self._debugLogging.isChecked()
}
for _key in _updates.keys():
self.prefs[_key] = _updates[_key]
return self.prefs
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = Msg("Model based imputer does not work for sparse data")
DEFAULT_LEARNER = SimpleTreeLearner()
METHODS = [AsDefault(), impute.DoNotImpute(), impute.Average(),
impute.AsValue(), impute.Model(DEFAULT_LEARNER), impute.Random(),
impute.DropInstances(), impute.Default()]
DEFAULT, DO_NOT_IMPUTE, MODEL_BASED_IMPUTER, AS_INPUT = 0, 1, 4, 7
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(DO_NOT_IMPUTE)
variable_methods = settings.ContextSetting({})
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
# copy METHODS (some are modified by the widget)
self.methods = copy.deepcopy(OWImpute.METHODS)
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QVBoxLayout(box)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for i, method in enumerate(self.methods):
if not method.columns_only:
button = QRadioButton(method.name)
button.setChecked(i == self.default_method_index)
button_group.addButton(button, i)
box_layout.addWidget(button)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = QListView(
selectionMode=QListView.ExtendedSelection
)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed
)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for i, method in enumerate(self.methods):
button = QRadioButton(text=method.name)
button_group.addButton(button, i)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected
)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000., maximum=1000., singleStep=.1, decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection
)
method_layout.addStretch(2)
reset_button = QPushButton(
"Restore All to Default", checked=False, checkable=False,
clicked=self.reset_variable_methods, default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_commit(
self.controlArea, self, "autocommit", "Apply",
orientation=Qt.Horizontal,
checkbox_label="Apply automatically")
box.button.setFixedWidth(180)
box.layout().insertStretch(0)
self.data = None
self.learner = None
self.modified = False
self.default_method = self.methods[self.default_method_index]
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
self.default_method = self.methods[self.default_method_index]
self.methods[self.DEFAULT].method = self.default_method
# update variable view
for index in map(self.varmodel.index, range(len(self.varmodel))):
method = self.variable_methods.get(
index.row(), self.methods[self.DEFAULT])
self.varmodel.setData(index, method, Qt.UserRole)
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.closeContext()
self.varmodel[:] = []
self.variable_methods = {}
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.learner = learner or self.DEFAULT_LEARNER
imputer = self.methods[self.MODEL_BASED_IMPUTER]
imputer.learner = self.learner
button = self.default_button_group.button(self.MODEL_BASED_IMPUTER)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(self.MODEL_BASED_IMPUTER)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = self.MODEL_BASED_IMPUTER
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
"""Returns the imputation method for column by its index.
"""
if not isinstance(column_index, int):
column_index = column_index.row()
return self.variable_methods.get(column_index,
self.methods[self.DEFAULT])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
if self.data is None or len(self.data) == 0 or len(self.varmodel) == 0:
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [
(i, var, self.variable_methods.get(i, self.default_method))
for i, var in enumerate(self.varmodel)
]
def impute_one(method, var, data):
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(
impute_one, copy.deepcopy(method), var, data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
data = self.data
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
assert fut.done()
newvar = []
try:
res = fut.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
# ?? break
except VariableNotSupported:
self.warning("Default method can not handle '{}'".
format(var.name))
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", var, exc_info=True)
self.Error.imputation_failed(var.name)
attributes = class_vars = None
break
else:
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = var
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
if i < len(data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
if attributes is None:
data = None
else:
domain = Orange.data.Domain(attributes, class_vars,
data.domain.metas)
try:
data = self.data.from_table(domain, data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
data = None
self.Outputs.data.send(data)
self.modified = False
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setBlocking(False)
def onDeleteWidget(self):
self.cancel()
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.variable_methods.get(i, None)
if method is not None:
specific.append("{} ({})".format(var.name, str(method)))
default = self.default_method.name
if specific:
self.report_items((
("Default method", default),
("Specific imputers", ", ".join(specific))
))
else:
self.report_items((("Method", default),))
def _on_var_selection_changed(self):
indexes = self.selection.selectedIndexes()
methods = [self.get_method_for_column(i.row()) for i in indexes]
def method_key(method):
"""
Decompose method into its type and parameters.
"""
# The return value should be hashable and __eq__ comparable
if isinstance(method, AsDefault):
return AsDefault, (method.method,)
elif isinstance(method, impute.Model):
return impute.Model, (method.learner,)
elif isinstance(method, impute.Default):
return impute.Default, (method.default,)
else:
return type(method), None
methods = set(method_key(m) for m in methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for i, m in enumerate(self.methods):
if method_type == type(m):
self.variable_button_group.button(i).setChecked(True)
if method_type is impute.Default:
(fixed_value,) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
for method, button in zip(self.methods,
self.variable_button_group.buttons()):
enabled = all(method.supports_variable(var) for var in
selected_vars)
button.setEnabled(enabled)
if not has_discrete:
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(self.AS_INPUT).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
if method_index == self.DEFAULT:
for index in indexes:
self.variable_methods.pop(index.row(), None)
elif method_index == OWImpute.AS_INPUT:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
method = impute.Default(default=value)
self.variable_methods[index.row()] = method
else:
method = self.methods[method_index]
for index in indexes:
self.variable_methods[index.row()] = method
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(index, self.get_method_for_column(index.row()), Qt.UserRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(self.AS_INPUT).setChecked(True)
self.set_method_for_current_selection(self.AS_INPUT)
def reset_variable_methods(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, self.DEFAULT)
self.variable_button_group.button(self.DEFAULT).setChecked(True)
def set_new_values(self, oper_combo, adding_all, selected_values=None):
# def remove_children():
# for child in box.children()[1:]:
# box.layout().removeWidget(child)
# child.setParent(None)
def add_textual(contents):
le = gui.lineEdit(box,
self,
None,
sizePolicy=QSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Expanding))
if contents:
le.setText(contents)
le.setAlignment(Qt.AlignRight)
le.editingFinished.connect(self.conditions_changed)
return le
def add_numeric(contents):
le = add_textual(contents)
le.setValidator(OWSelectRows.QDoubleValidatorEmpty())
return le
def add_datetime(contents):
le = add_textual(contents)
le.setValidator(QRegExpValidator(QRegExp(TimeVariable.REGEX)))
return le
var = self.data.domain[oper_combo.attr_combo.currentText()]
box = self.cond_list.cellWidget(oper_combo.row, 2)
if selected_values is not None:
lc = list(selected_values) + ["", ""]
lc = [str(x) for x in lc[:2]]
else:
lc = ["", ""]
if box and vartype(var) == box.var_type:
lc = self._get_lineedit_contents(box) + lc
oper = oper_combo.currentIndex()
if oper_combo.currentText() == "is defined":
label = QLabel()
label.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, label)
elif var.is_discrete:
if oper_combo.currentText() == "is one of":
if selected_values:
lc = [x for x in list(selected_values)]
button = DropDownToolButton(self, var, lc)
button.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, button)
else:
combo = QComboBox()
combo.addItems([""] + var.values)
if lc[0]:
combo.setCurrentIndex(int(lc[0]))
else:
combo.setCurrentIndex(0)
combo.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, combo)
combo.currentIndexChanged.connect(self.conditions_changed)
else:
box = gui.hBox(self, addToLayout=False)
box.var_type = vartype(var)
self.cond_list.setCellWidget(oper_combo.row, 2, box)
if var.is_continuous:
validator = add_datetime if isinstance(
var, TimeVariable) else add_numeric
box.controls = [validator(lc[0])]
if oper > 5:
gui.widgetLabel(box, " and ")
box.controls.append(validator(lc[1]))
elif var.is_string:
box.controls = [add_textual(lc[0])]
if oper in [6, 7]:
gui.widgetLabel(box, " and ")
box.controls.append(add_textual(lc[1]))
else:
box.controls = []
if not adding_all:
self.conditions_changed()
class OWLoadData(widget.OWWidget):
name = ""
icon = "icons/LoadData.svg"
priority = 110
class Outputs:
data = widget.Output("Data", Table)
class Information(widget.OWWidget.Information):
modified = widget.Msg(
"Uncommited changes\nPress 'Load data' to submit changes")
class Warning(widget.OWWidget.Warning):
sampling_in_effect = widget.Msg("Sampling is in effect.")
class Error(widget.OWWidget.Error):
row_annotation_mismatch = widget.Msg("Row annotation length mismatch\n"
"Expected {} rows got {}")
col_annotation_mismatch = widget.Msg(
"Column annotation length mismatch\n"
"Expected {} rows got {}")
inadequate_headers = widget.Msg("Headers and Row Labels error")
reading_error = widget.Msg("Cannot read data using given parameters.")
_recent = settings.Setting([]) # type: List[RecentPath]
_recent_row_annotations = settings.Setting([]) # type: List[RecentPath]
_recent_col_annotations = settings.Setting([]) # type: List[RecentPath]
_cells_in_rows = settings.Setting(False)
_col_annotations_enabled = settings.Setting(False)
_row_annotations_enabled = settings.Setting(False)
_last_path = settings.Setting("") # type: str
_header_rows_count = settings.Setting(1) # type: int
_header_cols_count = settings.Setting(1) # type: int
_sample_rows_enabled = settings.Setting(False) # type: bool
_sample_cols_enabled = settings.Setting(False) # type: bool
_sample_cols_p = settings.Setting(10.0) # type: bool
_sample_rows_p = settings.Setting(10.0) # type: bool
settingsHandler = RunaroundSettingsHandler()
want_main_area = False
resizing_enabled = False
cells_in_rows_changed = Signal()
def __init__(self):
super().__init__()
self._current_path = ""
self._data_loader = Loader()
icon_open_dir = self.style().standardIcon(QStyle.SP_DirOpenIcon)
# Top grid with file selection combo box
self.file_layout = grid = QGridLayout()
lb = QLabel("File:")
lb.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
self.recent_combo = cb = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=20,
toolTip="Select a recent file")
self.recent_model = cb.model() # type: QStandardItemModel
self.recent_combo.activated[int].connect(self._select_recent)
browse = QPushButton("...",
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse)
# reload = QPushButton("Reload", autoDefault=False, icon=icon_reload)
grid.addWidget(lb, 0, 0, Qt.AlignVCenter)
grid.addWidget(cb, 0, 1)
grid.addWidget(browse, 0, 2)
# grid.addWidget(reload, 0, 3)
self.summary_label = label = QLabel("", self)
label.ensurePolished()
f = label.font()
if f.pointSizeF() != -1:
f.setPointSizeF(f.pointSizeF() * 5 / 6)
else:
f.setPixelSize(f.pixelSize() * 5 / 6)
label.setFont(f)
grid.addWidget(label, 1, 1, 1, 3)
self.controlArea.layout().addLayout(grid)
box = gui.widgetBox(self.controlArea,
"Headers and Row Labels",
spacing=-1)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_rows_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_rows_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_rows_count)
hl.addWidget(QLabel("Data starts with", box))
hl.addWidget(self.header_rows_spin)
hl.addWidget(QLabel("header row(s)", box))
hl.addStretch(10)
box.layout().addLayout(hl)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
self.header_cols_spin = spin = QSpinBox(box,
minimum=0,
maximum=3,
value=self._header_cols_count,
keyboardTracking=False)
spin.valueChanged.connect(self.set_header_cols_count)
hl.addWidget(QLabel("First", box))
hl.addWidget(self.header_cols_spin)
hl.addWidget(QLabel("column(s) are row labels", box))
hl.addStretch(10)
box.layout().addLayout(hl)
self.data_struct_box = box = gui.widgetBox(self.controlArea,
"Input Data Structure")
gui.radioButtons(box,
self,
"_cells_in_rows", [
"Genes in rows, cells in columns",
"Cells in rows, genes in columns"
],
callback=self._cells_in_rows_changed)
box = gui.widgetBox(self.controlArea, "Sample Data", spacing=-1)
grid = QGridLayout()
grid.setContentsMargins(0, 0, 0, 0)
box.layout().addLayout(grid)
self.sample_rows_cb = cb = QCheckBox(checked=self._sample_rows_enabled)
self.sample_rows_p_spin = spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_rows_p)
spin.valueChanged.connect(self.set_sample_rows_p)
suffix = QLabel("% of cells")
cb.toggled.connect(self.set_sample_rows_enabled)
grid.addWidget(cb, 0, 0)
grid.addWidget(spin, 0, 1)
grid.addWidget(suffix, 0, 2)
self.sample_cols_cb = cb = QCheckBox(checked=self._sample_cols_enabled)
self.sample_cols_p_spin = spin = QSpinBox(minimum=0,
maximum=100,
value=self._sample_cols_p)
spin.valueChanged.connect(self.set_sample_cols_p)
suffix = QLabel("% of genes")
cb.toggled.connect(self.set_sample_cols_enabled)
grid.addWidget(cb, 1, 0)
grid.addWidget(spin, 1, 1)
grid.addWidget(suffix, 1, 2)
grid.setColumnStretch(3, 10)
self.annotation_files_box = box = gui.widgetBox(
self.controlArea, "Cell && Gene Annotation Files")
form = QFormLayout(
formAlignment=Qt.AlignLeft,
rowWrapPolicy=QFormLayout.WrapAllRows,
)
box.layout().addLayout(form)
self.row_annotations_cb = cb = QCheckBox(
"Cell annotations", checked=self._row_annotations_enabled)
self._row_annotations_w = w = QWidget(
enabled=self._row_annotations_enabled)
cb.toggled.connect(self.set_row_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.row_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.row_annotations_combo.activated.connect(
self._row_annotations_combo_changed)
hl.addWidget(self.row_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_row_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.col_annotations_cb = cb = QCheckBox(
"Gene annotations", checked=self._col_annotations_enabled)
self._col_annotations_w = w = QWidget(
enabled=self._col_annotations_enabled)
cb.toggled.connect(self.set_col_annotations_enabled)
cb.toggled.connect(w.setEnabled)
hl = QHBoxLayout()
hl.setContentsMargins(0, 0, 0, 0)
w.setLayout(hl)
self.col_annotations_combo = QComboBox(
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
minimumContentsLength=18)
self.col_annotations_combo.activated.connect(
self._col_annotations_combo_changed)
hl.addWidget(self.col_annotations_combo)
hl.addWidget(
QPushButton("...",
box,
autoDefault=False,
icon=icon_open_dir,
clicked=self.browse_col_annotations))
# hl.addWidget(QPushButton("Reload", box, autoDefault=False,
# icon=icon_reload))
form.addRow(cb, w)
self.controlArea.layout().addStretch(10)
self.load_data_button = button = VariableTextPushButton(
"Load data",
autoDefault=True,
textChoiceList=["Load data", "Reload"])
self.load_data_button.setAutoDefault(True)
button.clicked.connect(self.commit, Qt.QueuedConnection)
self.controlArea.layout().addWidget(button, alignment=Qt.AlignRight)
init_recent_paths_model(
self.recent_model,
[self.resolve_path(p) for p in self._recent],
)
init_recent_paths_model(
self.row_annotations_combo.model(),
[self.resolve_path(p) for p in self._recent_row_annotations])
init_recent_paths_model(
self.col_annotations_combo.model(),
[self.resolve_path(p) for p in self._recent_col_annotations])
self._update_summary()
self._update_warning()
if self._last_path != "" and os.path.exists(self._last_path):
QTimer.singleShot(0,
lambda: self.set_current_path(self._last_path))
else:
self.recent_combo.setCurrentIndex(-1)
def resolve_path(self, path):
basedir = self.workflowEnv().get("basedir", None)
if not basedir or not path:
return path
return path.resolve([("basedir", basedir)]) or path
def _cells_in_rows_changed(self):
self._data_loader.transposed = not self._cells_in_rows
self._invalidate()
self.cells_in_rows_changed.emit()
def _row_annotations_combo_changed(self):
path = self.row_annotations_combo.currentData(Qt.UserRole)
if isinstance(path, RecentPath) and os.path.exists(path.abspath):
self._data_loader.row_annotation_file = path # type: RecentPath
else:
self._data_loader.row_annotation_file = None
self._invalidate()
def _col_annotations_combo_changed(self):
path = self.col_annotations_combo.currentData(Qt.UserRole)
if isinstance(path, RecentPath) and os.path.exists(path.abspath):
self._data_loader.col_annotation_file = path # type: RecentPath
else:
self._data_loader.col_annotation_file = None
self._invalidate()
def _update_warning(self):
if (self._sample_rows_enabled and self._sample_rows_p < 100) or \
(self._sample_cols_enabled and self._sample_cols_p < 100):
self.Warning.sampling_in_effect()
else:
self.Warning.sampling_in_effect.clear()
def set_sample_rows_enabled(self, enabled, commit=True):
if self._sample_rows_enabled != enabled:
self._sample_rows_enabled = enabled
self.sample_rows_cb.setChecked(enabled)
self._update_warning()
self._data_loader.sample_rows_enabled = enabled
if commit:
self._invalidate()
def set_sample_cols_enabled(self, enabled, commit=True):
if self._sample_cols_enabled != enabled:
self._sample_cols_enabled = enabled
self.sample_cols_cb.setChecked(enabled)
self._update_warning()
self._data_loader.sample_cols_enabled = enabled
if commit:
self._invalidate()
def set_sample_rows_p(self, p, commit=True):
if self._sample_rows_p != p:
self._sample_rows_p = p
self._update_warning()
self.sample_rows_p_spin.setValue(p)
self._data_loader.sample_rows_p = p
if commit:
self._invalidate()
def set_sample_cols_p(self, p, commit=True):
if self._sample_cols_p != p:
self._sample_cols_p = p
self._update_warning()
self.sample_cols_p_spin.setValue(p)
self._data_loader.sample_cols_p = p
if commit:
self._invalidate()
def set_header_rows_count(self, n, commit=True):
if self._header_rows_count != n:
self._header_rows_count = n
self.header_rows_spin.setValue(n)
self._data_loader.header_rows_count = n
if commit:
self._invalidate()
def set_header_cols_count(self, n, commit=True):
if self._header_cols_count != n:
self._header_cols_count = n
self.header_cols_spin.setValue(n)
self._data_loader.header_cols_count = n
if commit:
self._invalidate()
def set_row_annotations_enabled(self, enabled, commit=True):
if self._row_annotations_enabled != enabled:
self._row_annotations_enabled = enabled
self.row_annotations_cb.setChecked(enabled)
self._data_loader.row_annotations_enabled = enabled
if commit:
self._invalidate()
def set_col_annotations_enabled(self, enabled, commit=True):
if self._col_annotations_enabled != enabled:
self._col_annotations_enabled = enabled
self.col_annotations_cb.setChecked(enabled)
self._data_loader.col_annotations_enabled = enabled
if commit:
self._invalidate()
def set_current_path(self, path):
if samepath(self._current_path, path):
return
model = self.recent_model
index = -1
pathitem = None
for i in range(model.rowCount()):
item = model.item(i)
data = item.data(Qt.UserRole) if item is not None else None
if isinstance(data, RecentPath) and samepath(path, data.abspath):
index, pathitem = i, data
break
rpaths = []
if pathitem is None:
assert index == -1
pathitem = RecentPath.create(path, rpaths)
if index != -1:
item = model.takeRow(index)
else:
item = RecentPath_asqstandarditem(pathitem)
model.insertRow(0, item)
self._current_path = path
self.recent_combo.setCurrentIndex(0)
self._data_loader = get_data_loader(path)
self._update_summary()
self.setup_gui()
self._invalidate()
def setup_gui(self):
""" Use loader predefined values. If the value is None, set
loader's parameter to widget's setting value.
"""
loader = self._data_loader
if loader.header_rows_count is not None:
self.set_header_rows_count(loader.header_rows_count, False)
else:
loader.header_rows_count = self._header_rows_count
if loader.header_cols_count is not None:
self.set_header_cols_count(loader.header_cols_count, False)
else:
loader.header_cols_count = self._header_cols_count
if loader.transposed is not None:
self._cells_in_rows = not loader.transposed
else:
loader.transposed = not self._cells_in_rows
if loader.sample_rows_enabled is not None:
self.set_sample_rows_enabled(loader.sample_rows_enabled, False)
else:
loader.sample_rows_enabled = self._sample_rows_enabled
if loader.sample_cols_enabled is not None:
self.set_sample_cols_enabled(loader.sample_cols_enabled, False)
else:
loader.sample_cols_enabled = self._sample_cols_enabled
if loader.sample_rows_p is not None:
self.set_sample_rows_p(loader.sample_rows_p, False)
else:
loader.sample_rows_p = self._sample_rows_p
if loader.sample_cols_p is not None:
self.set_sample_cols_p(loader.sample_cols_p, False)
else:
loader.sample_cols_p = self._sample_cols_p
if loader.row_annotation_file is not None:
index = insert_recent_path(
self.row_annotations_combo.model(),
self.resolve_path(loader.row_annotation_file))
self.row_annotations_combo.setCurrentIndex(index)
self.set_row_annotations_enabled(loader.row_annotations_enabled,
False)
else:
self.row_annotations_combo.setCurrentIndex(-1)
self.set_row_annotations_enabled(False, False)
if loader.col_annotation_file is not None:
index = insert_recent_path(
self.col_annotations_combo.model(),
self.resolve_path(loader.col_annotation_file))
self.col_annotations_combo.setCurrentIndex(index)
self.set_col_annotations_enabled(loader.col_annotations_enabled,
False)
else:
self.col_annotations_combo.setCurrentIndex(-1)
self.set_col_annotations_enabled(False, False)
self.header_rows_spin.setEnabled(loader.FIXED_FORMAT)
self.header_cols_spin.setEnabled(loader.FIXED_FORMAT)
self.data_struct_box.setEnabled(loader.FIXED_FORMAT)
self.annotation_files_box.setEnabled(loader.ENABLE_ANNOTATIONS)
def _update_summary(self):
size = self._data_loader.file_size
ncols = self._data_loader.n_cols
nrows = self._data_loader.n_rows
text = []
if size is not None:
text += [sizeformat(size)]
if nrows is not None:
text += ["{:n} rows".format(nrows)]
if nrows is not None:
text += ["{:n} columns".format(ncols)]
self.summary_label.setText(", ".join(text))
def current_path(self):
return self._current_path
def _select_recent(self, index):
# type: (int) -> None
# select a file from the recent list (entered via combo box `activate`)
assert 0 <= index < self.recent_model.rowCount()
item = self.recent_model.item(index)
pathitem = item.data(Qt.UserRole)
assert isinstance(pathitem, RecentPath)
self.set_current_path(pathitem.abspath)
@Slot()
def browse(self):
dlg = QFileDialog(self)
dlg.setAcceptMode(QFileDialog.AcceptOpen)
dlg.setFileMode(QFileDialog.ExistingFile)
filters = Formats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
filename = dlg.selectedFiles()[0]
self.set_current_path(filename)
@Slot()
def browse_row_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
filename = dlg.selectedFiles()[0]
m = self.row_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.row_annotations_combo.setCurrentIndex(index)
self._invalidate()
@Slot()
def browse_col_annotations(self):
dlg = QFileDialog(self,
acceptMode=QFileDialog.AcceptOpen,
fileMode=QFileDialog.ExistingFile)
filters = AnnotationFormats
dlg.setNameFilters(filters)
if filters:
dlg.selectNameFilter(filters[0])
if dlg.exec_() == QFileDialog.Accepted:
filename = dlg.selectedFiles()[0]
m = self.col_annotations_combo.model() # type: QStandardItemModel
pathitem = RecentPath.create(filename, [])
index = insert_recent_path(m, pathitem)
self.col_annotations_combo.setCurrentIndex(index)
self._invalidate()
def _invalidate(self):
self.set_modified(True)
def set_modified(self, modified):
if modified:
text = "Load data"
else:
text = "Reload"
self.load_data_button.setText(text)
self.load_data_button.setAutoDefault(modified)
# Setting autoDefault once also sets default which persists even after
# settings autoDefault back to False??
self.load_data_button.setDefault(modified)
self.Information.modified(shown=modified)
def commit(self):
path = self._current_path
if not path:
return
self.Outputs.data.send(self._data_loader())
self.show_error_messages()
self.set_modified(False)
def show_error_messages(self):
self.Error.row_annotation_mismatch.clear()
self.Error.col_annotation_mismatch.clear()
self.Error.inadequate_headers.clear()
errors = self._data_loader.errors
if len(errors["row_annot_mismatch"]):
self.Error.row_annotation_mismatch(*errors["row_annot_mismatch"])
if len(errors["col_annot_mismatch"]):
self.Error.col_annotation_mismatch(*errors["col_annot_mismatch"])
if len(errors["inadequate_headers"]):
self.Error.inadequate_headers(*errors["inadequate_headers"])
if len(errors["reading_error"]):
self.Error.reading_error()
def onDeleteWidget(self):
super().onDeleteWidget()
def _saveState(self):
maxitems = 15
def dataiter(model, role=Qt.UserRole):
return (model.data(model.index(i, 0), role)
for i in range(model.rowCount()))
def recent_paths(model):
return [
self.relocate_path(el) for el in dataiter(model)
if isinstance(el, RecentPath)
][:maxitems]
self._recent = recent_paths(self.recent_model)
self._recent_row_annotations = recent_paths(
self.row_annotations_combo.model())
self._recent_col_annotations = recent_paths(
self.col_annotations_combo.model())
self._last_path = self._current_path
def relocate_path(self, path):
basedir = self.workflowEnv().get("basedir", None)
if not basedir or not path:
return path
return RecentPath.create(path.abspath, [("basedir", basedir)])
def saveSettings(self):
self._saveState()
super().saveSettings()
class OWSql(OWBaseSql):
name = "SQL Table"
id = "orange.widgets.data.sql"
description = "Load dataset from SQL."
icon = "icons/SQLTable.svg"
priority = 30
category = "Data"
keywords = ["load"]
class Outputs:
data = Output("Data",
Table,
doc="Attribute-valued dataset read from the input file.")
settings_version = 2
table = Setting(None)
sql = Setting("")
guess_values = Setting(True)
download = Setting(False)
materialize = Setting(False)
materialize_table_name = Setting("")
class Information(OWBaseSql.Information):
data_sampled = Msg("Data description was generated from a sample.")
class Warning(OWBaseSql.Warning):
missing_extension = Msg("Database is missing extensions: {}")
class Error(OWBaseSql.Error):
no_backends = Msg("Please install a backend to use this widget.")
def __init__(self):
# Lint
self.backends = None
self.backendcombo = None
self.tables = None
self.tablecombo = None
self.sqltext = None
self.custom_sql = None
self.downloadcb = None
super().__init__()
def _setup_gui(self):
super()._setup_gui()
self._add_backend_controls()
self._add_tables_controls()
def _add_backend_controls(self):
box = self.serverbox
self.backends = BackendModel(Backend.available_backends())
self.backendcombo = QComboBox(box)
if self.backends:
self.backendcombo.setModel(self.backends)
else:
self.Error.no_backends()
box.setEnabled(False)
box.layout().insertWidget(0, self.backendcombo)
def _add_tables_controls(self):
vbox = gui.vBox(self.controlArea, "Tables", addSpace=True)
box = gui.vBox(vbox)
self.tables = TableModel()
self.tablecombo = QComboBox(
minimumContentsLength=35,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength)
self.tablecombo.setModel(self.tables)
self.tablecombo.setToolTip('table')
self.tablecombo.activated[int].connect(self.select_table)
box.layout().addWidget(self.tablecombo)
self.custom_sql = gui.vBox(box)
self.custom_sql.setVisible(False)
self.sqltext = QTextEdit(self.custom_sql)
self.sqltext.setPlainText(self.sql)
self.custom_sql.layout().addWidget(self.sqltext)
mt = gui.hBox(self.custom_sql)
cb = gui.checkBox(mt, self, 'materialize', 'Materialize to table ')
cb.setToolTip('Save results of the query in a table')
le = gui.lineEdit(mt, self, 'materialize_table_name')
le.setToolTip('Save results of the query in a table')
gui.button(self.custom_sql, self, 'Execute', callback=self.open_table)
box.layout().addWidget(self.custom_sql)
gui.checkBox(box,
self,
"guess_values",
"Auto-discover categorical variables",
callback=self.open_table)
self.downloadcb = gui.checkBox(box,
self,
"download",
"Download data to local memory",
callback=self.open_table)
def highlight_error(self, text=""):
err = ['', 'QLineEdit {border: 2px solid red;}']
self.servertext.setStyleSheet(err['server' in text or 'host' in text])
self.usernametext.setStyleSheet(err['role' in text])
self.databasetext.setStyleSheet(err['database' in text])
def get_backend(self):
if self.backendcombo.currentIndex() < 0:
return None
return self.backends[self.backendcombo.currentIndex()]
def on_connection_success(self):
if getattr(self.backend, 'missing_extension', False):
self.Warning.missing_extension(", ".join(
self.backend.missing_extension))
self.download = True
self.downloadcb.setEnabled(False)
if not is_postgres(self.backend):
self.download = True
self.downloadcb.setEnabled(False)
super().on_connection_success()
self.refresh_tables()
self.select_table()
def on_connection_error(self, err):
super().on_connection_error(err)
self.highlight_error(str(err).split("\n")[0])
def clear(self):
super().clear()
self.Warning.missing_extension.clear()
self.downloadcb.setEnabled(True)
self.highlight_error()
self.tablecombo.clear()
self.tablecombo.repaint()
def refresh_tables(self):
self.tables.clear()
if self.backend is None:
self.data_desc_table = None
return
self.tables.append("Select a table")
self.tables.append("Custom SQL")
self.tables.extend(self.backend.list_tables(self.schema))
index = self.tablecombo.findText(str(self.table))
self.tablecombo.setCurrentIndex(index if index != -1 else 0)
self.tablecombo.repaint()
# Called on tablecombo selection change:
def select_table(self):
curIdx = self.tablecombo.currentIndex()
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.custom_sql.setVisible(False)
return self.open_table()
else:
self.custom_sql.setVisible(True)
self.data_desc_table = None
self.database_desc["Table"] = "(None)"
self.table = None
if len(str(self.sql)) > 14:
return self.open_table()
return None
def get_table(self):
curIdx = self.tablecombo.currentIndex()
if curIdx <= 0:
if self.database_desc:
self.database_desc["Table"] = "(None)"
self.data_desc_table = None
return None
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.table = self.tables[self.tablecombo.currentIndex()]
self.database_desc["Table"] = self.table
if "Query" in self.database_desc:
del self.database_desc["Query"]
what = self.table
else:
what = self.sql = self.sqltext.toPlainText()
self.table = "Custom SQL"
if self.materialize:
if not self.materialize_table_name:
self.Error.connection(
"Specify a table name to materialize the query")
return None
try:
with self.backend.execute_sql_query(
"DROP TABLE IF EXISTS " +
self.materialize_table_name):
pass
with self.backend.execute_sql_query(
"CREATE TABLE " + self.materialize_table_name +
" AS " + self.sql):
pass
with self.backend.execute_sql_query(
"ANALYZE " + self.materialize_table_name):
pass
except BackendError as ex:
self.Error.connection(str(ex))
return None
try:
table = SqlTable(dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password),
what,
backend=type(self.backend),
inspect_values=False)
except BackendError as ex:
self.Error.connection(str(ex))
return None
self.Error.connection.clear()
sample = False
if table.approx_len() > LARGE_TABLE and self.guess_values:
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Attribute discovery might take "
"a long time on large tables.\n"
"Do you want to auto discover attributes?")
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
if is_postgres(self.backend):
sample_button = confirm.addButton("Yes, on a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
self.guess_values = False
elif is_postgres(self.backend) and \
confirm.clickedButton() == sample_button:
sample = True
self.Information.clear()
if self.guess_values:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if sample:
s = table.sample_time(1)
domain = s.get_domain(inspect_values=True)
self.Information.data_sampled()
else:
domain = table.get_domain(inspect_values=True)
QApplication.restoreOverrideCursor()
table.domain = domain
if self.download:
if table.approx_len() > AUTO_DL_LIMIT:
if is_postgres(self.backend):
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Data appears to be big. Do you really "
"want to download it to local memory?")
if table.approx_len() <= MAX_DL_LIMIT:
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
sample_button = confirm.addButton("Yes, a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
return None
elif confirm.clickedButton() == sample_button:
table = table.sample_percentage(
AUTO_DL_LIMIT / table.approx_len() * 100)
else:
if table.approx_len() > MAX_DL_LIMIT:
QMessageBox.warning(self, 'Warning',
"Data is too big to download.\n")
return None
else:
confirm = QMessageBox.question(
self, 'Question',
"Data appears to be big. Do you really "
"want to download it to local memory?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if confirm == QMessageBox.No:
return None
table.download_data(MAX_DL_LIMIT)
table = Table(table)
return table
@classmethod
def migrate_settings(cls, settings, version):
if version < 2:
# Until Orange version 3.4.4 username and password had been stored
# in Settings.
cm = cls._credential_manager(settings["host"], settings["port"])
cm.username = settings["username"]
cm.password = settings["password"]
class OWFile(widget.OWWidget, RecentPathsWComboMixin):
name = "File"
id = "orange.widgets.data.file"
description = "Read data from an input file or network " \
"and send a data table to the output."
icon = "icons/File.svg"
priority = 10
category = "Data"
keywords = ["file", "load", "read", "open"]
class Outputs:
data = Output("Data",
Table,
doc="Attribute-valued dataset read from the input file.")
want_main_area = False
buttons_area_orientation = None
SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())]
SIZE_LIMIT = 1e7
LOCAL_FILE, URL = range(2)
settingsHandler = PerfectDomainContextHandler(
match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL)
# pylint seems to want declarations separated from definitions
recent_paths: List[RecentPath]
recent_urls: List[str]
variables: list
# Overload RecentPathsWidgetMixin.recent_paths to set defaults
recent_paths = Setting([
RecentPath("", "sample-datasets", "iris.tab"),
RecentPath("", "sample-datasets", "titanic.tab"),
RecentPath("", "sample-datasets", "housing.tab"),
RecentPath("", "sample-datasets", "heart_disease.tab"),
RecentPath("", "sample-datasets", "brown-selected.tab"),
RecentPath("", "sample-datasets", "zoo.tab"),
])
recent_urls = Setting([])
source = Setting(LOCAL_FILE)
sheet_names = Setting({})
url = Setting("")
variables = ContextSetting([])
domain_editor = SettingProvider(DomainEditor)
class Information(widget.OWWidget.Information):
no_file_selected = Msg("No file selected.")
class Warning(widget.OWWidget.Warning):
file_too_big = Msg("The file is too large to load automatically."
" Press Reload to load.")
load_warning = Msg("Read warning:\n{}")
performance_warning = Msg(
"Categorical variables with >100 values may decrease performance.")
renamed_vars = Msg("Some variables have been renamed "
"to avoid duplicates.\n{}")
multiple_targets = Msg("Most widgets do not support multiple targets")
class Error(widget.OWWidget.Error):
file_not_found = Msg("File not found.")
missing_reader = Msg("Missing reader.")
sheet_error = Msg("Error listing available sheets.")
unknown = Msg("Read error:\n{}")
UserAdviceMessages = [
widget.Message(
"Use CSV File Import widget for advanced options "
"for comma-separated files", "use-csv-file-import"),
widget.Message(
"This widget loads only tabular data. Use other widgets to load "
"other data types like models, distance matrices and networks.",
"other-data-types")
]
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
readers = [
f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)
]
def group_readers_per_addon_key(w):
# readers from Orange.data.io should go first
def package(w):
package = w.qualified_name().split(".")[:-1]
package = package[:2]
if ".".join(package) == "Orange.data":
return ["0"] # force "Orange" to come first
return package
return package(w), w.DESCRIPTION
self.available_readers = sorted(set(readers),
key=group_readers_per_addon_key)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='Source')
vbox = gui.radioButtons(None,
self,
"source",
box=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = QComboBox()
self.sheet_combo.activated[str].connect(self.select_sheet)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 1, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='File Type')
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.reader_combo = QComboBox(self)
self.reader_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.reader_combo.activated[int].connect(self.select_reader)
box.layout().addWidget(self.reader_combo)
layout.addWidget(box, 0, 1)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(box)
gui.button(box,
self,
"Reset",
callback=self.reset_domain_edit,
autoDefault=False)
gui.rubber(box)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
hBox = gui.hBox(self.controlArea)
gui.rubber(hBox)
gui.button(hBox,
self,
"Browse documentation datasets",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(hBox)
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
@staticmethod
def sizeHint():
return QSize(600, 550)
def select_file(self, n):
assert n < len(self.recent_paths)
super().select_file(n)
if self.recent_paths:
self.source = self.LOCAL_FILE
self.load_data()
self.set_file_list()
def select_sheet(self):
self.recent_paths[0].sheet = self.sheet_combo.currentText()
self.load_data()
def select_reader(self, n):
if self.source != self.LOCAL_FILE:
return # ignore for URL's
if self.recent_paths:
path = self.recent_paths[0]
if n == 0: # default
path.file_format = None
self.load_data()
elif n <= len(self.available_readers):
reader = self.available_readers[n - 1]
path.file_format = reader.qualified_name()
self.load_data()
else: # the rest include just qualified names
path.file_format = self.reader_combo.itemText(n)
self.load_data()
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = 'http://' + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.load_data()
def browse_file(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QMessageBox.information(
None, "File",
"Cannot find the directory with documentation datasets")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
filename, reader, _ = open_filename_dialog(start_file, None,
self.available_readers)
if not filename:
return
self.add_path(filename)
if reader is not None:
self.recent_paths[0].file_format = reader.qualified_name()
self.source = self.LOCAL_FILE
self.load_data()
# Open a file, create data from it and send it over the data channel
def load_data(self):
# We need to catch any exception type since anything can happen in
# file readers
self.closeContext()
self.domain_editor.set_domain(None)
self.apply_button.setEnabled(False)
self.clear_messages()
self.set_file_list()
error = self._try_load()
if error:
error()
self.data = None
self.sheet_box.hide()
self.Outputs.data.send(None)
self.infolabel.setText("No data.")
def _try_load(self):
self._initialize_reader_combo()
# pylint: disable=broad-except
if self.source == self.LOCAL_FILE:
if self.last_path() is None:
return self.Information.no_file_selected
elif not os.path.exists(self.last_path()):
return self.Error.file_not_found
else:
url = self.url_combo.currentText().strip()
if not url:
return self.Information.no_file_selected
def mark_problematic_reader():
self.reader_combo.setItemData(self.reader_combo.currentIndex(),
QBrush(Qt.red), Qt.ForegroundRole)
try:
self.reader = self._get_reader() # also sets current reader index
assert self.reader is not None
except MissingReaderException:
mark_problematic_reader()
return self.Error.missing_reader
except Exception as ex:
mark_problematic_reader()
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
try:
self._update_sheet_combo()
except Exception:
return self.Error.sheet_error
with log_warnings() as warnings:
try:
data = self.reader.read()
except Exception as ex:
mark_problematic_reader()
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
if warnings:
self.Warning.load_warning(warnings[-1].message.args[0])
self.infolabel.setText(self._describe(data))
self.loaded_file = self.last_path()
add_origin(data, self.loaded_file)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
return None
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
self.reader_combo.setEnabled(True)
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
qname_index = {
r.qualified_name(): i
for i, r in enumerate(self.available_readers)
}
if qname in qname_index:
self.reader_combo.setCurrentIndex(qname_index[qname] + 1)
else:
# reader may be accessible, but not in self.available_readers
# (perhaps its code was moved)
self.reader_combo.addItem(qname)
self.reader_combo.setCurrentIndex(
len(self.reader_combo) - 1)
try:
reader_class = class_from_qualified_name(qname)
except Exception as ex:
raise MissingReaderException(
f'Can not find reader "{qname}"') from ex
reader = reader_class(path)
else:
self.reader_combo.setCurrentIndex(0)
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
return UrlReader(url)
def _update_sheet_combo(self):
if len(self.reader.sheets) < 2:
self.sheet_box.hide()
self.reader.select_sheet(None)
return
self.sheet_combo.clear()
self.sheet_combo.addItems(self.reader.sheets)
self._select_active_sheet()
self.sheet_box.show()
def _select_active_sheet(self):
try:
idx = self.reader.sheets.index(self.reader.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.reader.select_sheet(None)
self.sheet_combo.setCurrentIndex(0)
def _initialize_reader_combo(self):
self.reader_combo.clear()
filters = [format_filter(f) for f in self.available_readers]
self.reader_combo.addItems([DEFAULT_READER_TEXT] + filters)
self.reader_combo.setCurrentIndex(0)
self.reader_combo.setDisabled(True)
# additional readers may be added in self._get_reader()
@staticmethod
def _describe(table):
def missing_prop(prop):
if prop:
return f"({prop * 100:.1f}% missing values)"
else:
return "(no missing values)"
domain = table.domain
text = ""
attrs = getattr(table, "attributes", {})
descs = [
attrs[desc] for desc in ("Name", "Description") if desc in attrs
]
if len(descs) == 2:
descs[0] = f"<b>{descs[0]}</b>"
if descs:
text += f"<p>{'<br/>'.join(descs)}</p>"
text += f"<p>{len(table)} instance(s)"
missing_in_attr = missing_prop(table.has_missing_attribute()
and table.get_nan_frequency_attribute())
missing_in_class = missing_prop(table.has_missing_class()
and table.get_nan_frequency_class())
text += f"<br/>{len(domain.attributes)} feature(s) {missing_in_attr}"
if domain.has_continuous_class:
text += f"<br/>Regression; numerical class {missing_in_class}"
elif domain.has_discrete_class:
text += "<br/>Classification; categorical class " \
f"with {len(domain.class_var.values)} values {missing_in_class}"
elif table.domain.class_vars:
text += "<br/>Multi-target; " \
f"{len(table.domain.class_vars)} target variables " \
f"{missing_in_class}"
else:
text += "<br/>Data has no target variable."
text += f"<br/>{len(domain.metas)} meta attribute(s)"
text += "</p>"
if 'Timestamp' in table.domain:
# Google Forms uses this header to timestamp responses
text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \
f"Last entry: {table[-1, 'Timestamp']}</p>"
return text
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def reset_domain_edit(self):
self.domain_editor.reset_domain()
self.apply_domain_edit()
def _inspect_discrete_variables(self, domain):
for var in chain(domain.variables, domain.metas):
if var.is_discrete and len(var.values) > 100:
self.Warning.performance_warning()
def apply_domain_edit(self):
self.Warning.performance_warning.clear()
self.Warning.renamed_vars.clear()
if self.data is None:
table = None
else:
domain, cols, renamed = \
self.domain_editor.get_domain(self.data.domain, self.data,
deduplicate=True)
if not (domain.variables or domain.metas):
table = None
elif domain is self.data.domain:
table = self.data
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
self._inspect_discrete_variables(domain)
if renamed:
self.Warning.renamed_vars(f"Renamed: {', '.join(renamed)}")
self.Warning.multiple_targets(
shown=table is not None and len(table.domain.class_vars) > 1)
self.Outputs.data.send(table)
self.apply_button.setEnabled(False)
def get_widget_name_extension(self):
_, name = os.path.split(self.loaded_file)
return os.path.splitext(name)[0]
def send_report(self):
def get_ext_name(filename):
try:
return FileFormat.names[os.path.splitext(filename)[1]]
except KeyError:
return "unknown"
if self.data is None:
self.report_paragraph("File", "No file.")
return
if self.source == self.LOCAL_FILE:
home = os.path.expanduser("~")
if self.loaded_file.startswith(home):
# os.path.join does not like ~
name = "~" + os.path.sep + \
self.loaded_file[len(home):].lstrip("/").lstrip("\\")
else:
name = self.loaded_file
if self.sheet_combo.isVisible():
name += f" ({self.sheet_combo.currentText()})"
self.report_items("File", [("File name", name),
("Format", get_ext_name(name))])
else:
self.report_items("Data", [("Resource", self.url),
("Format", get_ext_name(self.url))])
self.report_data("Data", self.data)
@staticmethod
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except MissingReaderException:
pass
def dropEvent(self, event):
"""Handle file drops"""
urls = event.mimeData().urls()
if urls:
self.add_path(urls[0].toLocalFile()) # add first file
self.source = self.LOCAL_FILE
self.load_data()
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.update_file_list(key, value, oldvalue)
class FeatureEditor(QFrame):
FUNCTIONS = dict(chain([(key, val) for key, val in math.__dict__.items()
if not key.startswith("_")],
[("str", str)]))
featureChanged = Signal()
featureEdited = Signal()
modifiedChanged = Signal(bool)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
layout = QFormLayout(
fieldGrowthPolicy=QFormLayout.ExpandingFieldsGrow
)
layout.setContentsMargins(0, 0, 0, 0)
self.nameedit = QLineEdit(
placeholderText="Name...",
sizePolicy=QSizePolicy(QSizePolicy.Minimum,
QSizePolicy.Fixed)
)
self.expressionedit = QLineEdit(
placeholderText="Expression..."
)
self.attrs_model = itemmodels.VariableListModel(
["Select Feature"], parent=self)
self.attributescb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
)
self.attributescb.setModel(self.attrs_model)
sorted_funcs = sorted(self.FUNCTIONS)
self.funcs_model = itemmodels.PyListModelTooltip()
self.funcs_model.setParent(self)
self.funcs_model[:] = chain(["Select Function"], sorted_funcs)
self.funcs_model.tooltips[:] = chain(
[''],
[self.FUNCTIONS[func].__doc__ for func in sorted_funcs])
self.functionscb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
self.functionscb.setModel(self.funcs_model)
hbox = QHBoxLayout()
hbox.addWidget(self.attributescb)
hbox.addWidget(self.functionscb)
layout.addRow(self.nameedit, self.expressionedit)
layout.addRow(self.tr(""), hbox)
self.setLayout(layout)
self.nameedit.editingFinished.connect(self._invalidate)
self.expressionedit.textChanged.connect(self._invalidate)
self.attributescb.currentIndexChanged.connect(self.on_attrs_changed)
self.functionscb.currentIndexChanged.connect(self.on_funcs_changed)
self._modified = False
def setModified(self, modified):
if not type(modified) is bool:
raise TypeError
if self._modified != modified:
self._modified = modified
self.modifiedChanged.emit(modified)
def modified(self):
return self._modified
modified = Property(bool, modified, setModified,
notify=modifiedChanged)
def setEditorData(self, data, domain):
self.nameedit.setText(data.name)
self.expressionedit.setText(data.expression)
self.setModified(False)
self.featureChanged.emit()
self.attrs_model[:] = ["Select Feature"]
if domain is not None and (domain or domain.metas):
self.attrs_model[:] += chain(domain.attributes,
domain.class_vars,
domain.metas)
def editorData(self):
return FeatureDescriptor(name=self.nameedit.text(),
expression=self.nameedit.text())
def _invalidate(self):
self.setModified(True)
self.featureEdited.emit()
self.featureChanged.emit()
def on_attrs_changed(self):
index = self.attributescb.currentIndex()
if index > 0:
attr = sanitized_name(self.attrs_model[index].name)
self.insert_into_expression(attr)
self.attributescb.setCurrentIndex(0)
def on_funcs_changed(self):
index = self.functionscb.currentIndex()
if index > 0:
func = self.funcs_model[index]
if func in ["atan2", "fmod", "ldexp", "log",
"pow", "copysign", "hypot"]:
self.insert_into_expression(func + "(,)")
self.expressionedit.cursorBackward(False, 2)
elif func in ["e", "pi"]:
self.insert_into_expression(func)
else:
self.insert_into_expression(func + "()")
self.expressionedit.cursorBackward(False)
self.functionscb.setCurrentIndex(0)
def insert_into_expression(self, what):
cp = self.expressionedit.cursorPosition()
ct = self.expressionedit.text()
text = ct[:cp] + what + ct[cp:]
self.expressionedit.setText(text)
self.expressionedit.setFocus()
class OWSql(OWWidget):
name = "SQL Table"
id = "orange.widgets.data.sql"
description = "Load data set from SQL."
icon = "icons/SQLTable.svg"
priority = 30
category = "Data"
keywords = ["data", "file", "load", "read", "SQL"]
class Outputs:
data = Output("Data", Table, doc="Attribute-valued data set read from the input file.")
settings_version = 2
want_main_area = False
resizing_enabled = False
host = Setting(None)
port = Setting(None)
database = Setting(None)
schema = Setting(None)
username = ""
password = ""
table = Setting(None)
sql = Setting("")
guess_values = Setting(True)
download = Setting(False)
materialize = Setting(False)
materialize_table_name = Setting("")
class Information(OWWidget.Information):
data_sampled = Msg("Data description was generated from a sample.")
class Error(OWWidget.Error):
connection = Msg("{}")
no_backends = Msg("Please install a backend to use this widget")
missing_extension = Msg("Database is missing extension{}: {}")
def __init__(self):
super().__init__()
self.backend = None
self.data_desc_table = None
self.database_desc = None
vbox = gui.vBox(self.controlArea, "Server", addSpace=True)
box = gui.vBox(vbox)
self.backends = BackendModel(Backend.available_backends())
self.backendcombo = QComboBox(box)
if len(self.backends):
self.backendcombo.setModel(self.backends)
else:
self.Error.no_backends()
box.setEnabled(False)
box.layout().addWidget(self.backendcombo)
self.servertext = QLineEdit(box)
self.servertext.setPlaceholderText('Server')
self.servertext.setToolTip('Server')
self.servertext.editingFinished.connect(self._load_credentials)
if self.host:
self.servertext.setText(self.host if not self.port else
'{}:{}'.format(self.host, self.port))
box.layout().addWidget(self.servertext)
self.databasetext = QLineEdit(box)
self.databasetext.setPlaceholderText('Database[/Schema]')
self.databasetext.setToolTip('Database or optionally Database/Schema')
if self.database:
self.databasetext.setText(
self.database if not self.schema else
'{}/{}'.format(self.database, self.schema))
box.layout().addWidget(self.databasetext)
self.usernametext = QLineEdit(box)
self.usernametext.setPlaceholderText('Username')
self.usernametext.setToolTip('Username')
box.layout().addWidget(self.usernametext)
self.passwordtext = QLineEdit(box)
self.passwordtext.setPlaceholderText('Password')
self.passwordtext.setToolTip('Password')
self.passwordtext.setEchoMode(QLineEdit.Password)
box.layout().addWidget(self.passwordtext)
self._load_credentials()
self.tables = TableModel()
tables = gui.hBox(box)
self.tablecombo = QComboBox(
minimumContentsLength=35,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength
)
self.tablecombo.setModel(self.tables)
self.tablecombo.setToolTip('table')
tables.layout().addWidget(self.tablecombo)
self.connect()
index = self.tablecombo.findText(str(self.table))
if index != -1:
self.tablecombo.setCurrentIndex(index)
# set up the callback to select_table in case of selection change
self.tablecombo.activated[int].connect(self.select_table)
self.connectbutton = gui.button(
tables, self, '↻', callback=self.connect)
self.connectbutton.setSizePolicy(
QSizePolicy.Fixed, QSizePolicy.Fixed)
tables.layout().addWidget(self.connectbutton)
self.custom_sql = gui.vBox(box)
self.custom_sql.setVisible(False)
self.sqltext = QTextEdit(self.custom_sql)
self.sqltext.setPlainText(self.sql)
self.custom_sql.layout().addWidget(self.sqltext)
mt = gui.hBox(self.custom_sql)
cb = gui.checkBox(mt, self, 'materialize', 'Materialize to table ')
cb.setToolTip('Save results of the query in a table')
le = gui.lineEdit(mt, self, 'materialize_table_name')
le.setToolTip('Save results of the query in a table')
self.executebtn = gui.button(
self.custom_sql, self, 'Execute', callback=self.open_table)
box.layout().addWidget(self.custom_sql)
gui.checkBox(box, self, "guess_values",
"Auto-discover categorical variables",
callback=self.open_table)
gui.checkBox(box, self, "download",
"Download data to local memory",
callback=self.open_table)
gui.rubber(self.buttonsArea)
QTimer.singleShot(0, self.select_table)
def _load_credentials(self):
self._parse_host_port()
cm = self._credential_manager(self.host, self.port)
self.username = cm.username
self.password = cm.password
if self.username:
self.usernametext.setText(self.username)
if self.password:
self.passwordtext.setText(self.password)
def _save_credentials(self):
cm = self._credential_manager(self.host, self.port)
cm.username = self.username or ''
cm.password = self.password or ''
def _credential_manager(self, host, port):
return CredentialManager("SQL Table: {}:{}".format(host, port))
def error(self, id=0, text=""):
super().error(id, text)
err_style = 'QLineEdit {border: 2px solid red;}'
if 'server' in text or 'host' in text:
self.servertext.setStyleSheet(err_style)
else:
self.servertext.setStyleSheet('')
if 'role' in text:
self.usernametext.setStyleSheet(err_style)
else:
self.usernametext.setStyleSheet('')
if 'database' in text:
self.databasetext.setStyleSheet(err_style)
else:
self.databasetext.setStyleSheet('')
def _parse_host_port(self):
hostport = self.servertext.text().split(':')
self.host = hostport[0]
self.port = hostport[1] if len(hostport) == 2 else None
def connect(self):
self._parse_host_port()
self.database, _, self.schema = self.databasetext.text().partition('/')
self.username = self.usernametext.text() or None
self.password = self.passwordtext.text() or None
try:
if self.backendcombo.currentIndex() < 0:
return
backend = self.backends[self.backendcombo.currentIndex()]
self.backend = backend(dict(
host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password
))
self.Error.connection.clear()
self._save_credentials()
self.database_desc = OrderedDict((
("Host", self.host), ("Port", self.port),
("Database", self.database), ("User name", self.username)
))
self.refresh_tables()
except BackendError as err:
error = str(err).split('\n')[0]
self.Error.connection(error)
self.database_desc = self.data_desc_table = None
self.tablecombo.clear()
def refresh_tables(self):
self.tables.clear()
self.Error.missing_extension.clear()
if self.backend is None:
self.data_desc_table = None
return
self.tables.append("Select a table")
self.tables.append("Custom SQL")
self.tables.extend(self.backend.list_tables(self.schema))
# Called on tablecombo selection change:
def select_table(self):
curIdx = self.tablecombo.currentIndex()
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.custom_sql.setVisible(False)
return self.open_table()
else:
self.custom_sql.setVisible(True)
self.data_desc_table = None
self.database_desc["Table"] = "(None)"
self.table = None
if len(str(self.sql)) > 14:
return self.open_table()
#self.Error.missing_extension(
# 's' if len(missing) > 1 else '',
# ', '.join(missing),
# shown=missing)
def open_table(self):
table = self.get_table()
self.data_desc_table = table
self.Outputs.data.send(table)
def get_table(self):
curIdx = self.tablecombo.currentIndex()
if curIdx <= 0:
if self.database_desc:
self.database_desc["Table"] = "(None)"
self.data_desc_table = None
return
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.table = self.tables[self.tablecombo.currentIndex()]
self.database_desc["Table"] = self.table
if "Query" in self.database_desc:
del self.database_desc["Query"]
what = self.table
else:
what = self.sql = self.sqltext.toPlainText()
self.table = "Custom SQL"
if self.materialize:
import psycopg2
if not self.materialize_table_name:
self.Error.connection(
"Specify a table name to materialize the query")
return
try:
with self.backend.execute_sql_query("DROP TABLE IF EXISTS " +
self.materialize_table_name):
pass
with self.backend.execute_sql_query("CREATE TABLE " +
self.materialize_table_name +
" AS " + self.sql):
pass
with self.backend.execute_sql_query("ANALYZE " + self.materialize_table_name):
pass
except (psycopg2.ProgrammingError, BackendError) as ex:
self.Error.connection(str(ex))
return
try:
table = SqlTable(dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password),
what,
backend=type(self.backend),
inspect_values=False)
except BackendError as ex:
self.Error.connection(str(ex))
return
self.Error.connection.clear()
sample = False
if table.approx_len() > LARGE_TABLE and self.guess_values:
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Attribute discovery might take "
"a long time on large tables.\n"
"Do you want to auto discover attributes?")
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
sample_button = confirm.addButton("Yes, on a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
self.guess_values = False
elif confirm.clickedButton() == sample_button:
sample = True
self.Information.clear()
if self.guess_values:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if sample:
s = table.sample_time(1)
domain = s.get_domain(inspect_values=True)
self.Information.data_sampled()
else:
domain = table.get_domain(inspect_values=True)
QApplication.restoreOverrideCursor()
table.domain = domain
if self.download:
if table.approx_len() > MAX_DL_LIMIT:
QMessageBox.warning(
self, 'Warning', "Data is too big to download.\n"
"Consider using the Data Sampler widget to download "
"a sample instead.")
self.download = False
elif table.approx_len() > AUTO_DL_LIMIT:
confirm = QMessageBox.question(
self, 'Question', "Data appears to be big. Do you really "
"want to download it to local memory?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if confirm == QMessageBox.No:
self.download = False
if self.download:
table.download_data(MAX_DL_LIMIT)
table = Table(table)
return table
def send_report(self):
if not self.database_desc:
self.report_paragraph("No database connection.")
return
self.report_items("Database", self.database_desc)
if self.data_desc_table:
self.report_items("Data",
report.describe_data(self.data_desc_table))
@classmethod
def migrate_settings(cls, settings, version):
if version < 2:
# Until Orange version 3.4.4 username and password had been stored
# in Settings.
cm = cls._credential_manager(settings["host"], settings["port"])
cm.username = settings["username"]
cm.password = settings["password"]
