class Scale(BaseEditor):
NoCentering, CenterMean, CenterMedian = 0, 1, 2
NoScaling, ScaleBySD, ScaleBySpan = 0, 1, 2
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__centercb = QComboBox()
self.__centercb.addItems(
["No Centering", "Center by Mean", "Center by Median"])
self.__scalecb = QComboBox()
self.__scalecb.addItems(["No scaling", "Scale by SD", "Scale by span"])
form.addRow("Center:", self.__centercb)
form.addRow("Scale:", self.__scalecb)
self.layout().addLayout(form)
self.__centercb.currentIndexChanged.connect(self.changed)
self.__scalecb.currentIndexChanged.connect(self.changed)
self.__centercb.activated.connect(self.edited)
self.__scalecb.activated.connect(self.edited)
def setParameters(self, params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
self.__centercb.setCurrentIndex(
_enum_to_index(_Scale.CenteringType, center))
self.__scalecb.setCurrentIndex(
_enum_to_index(_Scale.ScalingType, scale))
def parameters(self):
return {
"center":
_index_to_enum(_Scale.CenteringType,
self.__centercb.currentIndex()),
"scale":
_index_to_enum(_Scale.ScalingType, self.__scalecb.currentIndex())
}
@staticmethod
def createinstance(params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
return _Scale(center=center, scale=scale)
def __repr__(self):
return "{}, {}".format(self.__centercb.currentText(),
self.__scalecb.currentText())
class Scale(BaseEditor):
NoCentering, CenterMean, CenterMedian = 0, 1, 2
NoScaling, ScaleBySD, ScaleBySpan = 0, 1, 2
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__centercb = QComboBox()
self.__centercb.addItems(["No Centering", "Center by Mean",
"Center by Median"])
self.__scalecb = QComboBox()
self.__scalecb.addItems(["No scaling", "Scale by SD",
"Scale by span"])
form.addRow("Center:", self.__centercb)
form.addRow("Scale:", self.__scalecb)
self.layout().addLayout(form)
self.__centercb.currentIndexChanged.connect(self.changed)
self.__scalecb.currentIndexChanged.connect(self.changed)
self.__centercb.activated.connect(self.edited)
self.__scalecb.activated.connect(self.edited)
def setParameters(self, params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
self.__centercb.setCurrentIndex(
enum_to_index(_Scale.CenteringType, center))
self.__scalecb.setCurrentIndex(
enum_to_index(_Scale.ScalingType, scale))
def parameters(self):
return {"center": index_to_enum(_Scale.CenteringType,
self.__centercb.currentIndex()),
"scale": index_to_enum(_Scale.ScalingType,
self.__scalecb.currentIndex())}
@staticmethod
def createinstance(params):
center = params.get("center", _Scale.CenteringType.Mean)
scale = params.get("scale", _Scale.ScalingType.Std)
return _Scale(center=center, scale=scale)
def __repr__(self):
return "{}, {}".format(self.__centercb.currentText(),
self.__scalecb.currentText())
class DiscreteVariableEditor(VariableEditor):
def __init__(self, parent: QWidget, items: Tuple[str], callback: Callable):
super().__init__(parent, callback)
self._combo = QComboBox(
parent,
maximumWidth=180,
sizePolicy=QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
)
self._combo.addItems(items + ("?",))
self._combo.currentIndexChanged.connect(self.__on_index_changed)
self.layout().addWidget(self._combo)
@property
def value(self) -> Union[int, float]:
return self._map_to_var_values()
@value.setter
def value(self, value: float):
if np.isnan(value):
value = self._combo.model().rowCount() - 1
assert value == int(value)
self._combo.setCurrentIndex(int(value))
def __on_index_changed(self):
self.valueChanged.emit(self._map_to_var_values())
def _map_to_var_values(self) -> Union[int, float]:
n_values = self._combo.model().rowCount() - 1
current = self._combo.currentIndex()
return current if current < n_values else np.nan
class DropDownRadioBooleanFilter(QWidget, Control):
"""Container for multiple boolean filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QHBoxLayout())
self.cb = QComboBox(self)
self.layout().addWidget(self.cb)
rblayout = QVBoxLayout()
self.radioButtons = [QRadioButton("Only", self),
QRadioButton("Excluded", self)
]
for b in self.radioButtons:
rblayout.addWidget(b)
self.radioButtons[0].setChecked(True)
self.layout().addLayout(rblayout)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
for op, rb in zip(self.options[0].subelements_top("Option"),
self.radioButtons):
rb.setText(op.displayName)
rb.setChecked(getattr(op, "default", "false") == "true")
def value(self):
return {"excluded": "0" if self.radioButtons[0].isChecked() else "1"}
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
for i, option in enumerate(self.options):
if option.internalName == name:
self.cb.setCurrentIndex(i)
if value == "Only":
self.radioButtons[0].setChecked(True)
class DropDownRadioBooleanFilter(QWidget, Control):
"""Container for multiple boolean filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QHBoxLayout())
self.cb = QComboBox(self)
self.layout().addWidget(self.cb)
rblayout = QVBoxLayout()
self.radioButtons = [QRadioButton("Only", self),
QRadioButton("Excluded", self)
]
for b in self.radioButtons:
rblayout.addWidget(b)
self.radioButtons[0].setChecked(True)
self.layout().addLayout(rblayout)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
for op, rb in zip(self.options[0].subelements_top("Option"),
self.radioButtons):
rb.setText(op.displayName)
rb.setChecked(getattr(op, "default", "false") == "true")
def value(self):
return {"excluded": "0" if self.radioButtons[0].isChecked() else "1"}
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
for i, option in enumerate(self.options):
if option.internalName == name:
self.cb.setCurrentIndex(i)
if value == "Only":
self.radioButtons[0].setChecked(True)
class BinarizeEditor(ScBaseEditor):
DEFAULT_CONDITION = Binarize.GreaterOrEqual
DEFAULT_THRESHOLD = 1
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self._threshold = self.DEFAULT_THRESHOLD
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.cond_cb = QComboBox()
self.cond_cb.addItems(["Greater or Equal", "Greater"])
self.cond_cb.currentIndexChanged.connect(self.changed)
self.cond_cb.activated.connect(self.edited)
self.thr_spin = QDoubleSpinBox(minimum=0,
singleStep=0.5,
decimals=1,
value=self._threshold)
self.thr_spin.valueChanged[float].connect(self._set_threshold)
self.thr_spin.editingFinished.connect(self.edited)
form.addRow("Condition:", self.cond_cb)
form.addRow("Threshold:", self.thr_spin)
self.layout().addLayout(form)
def _set_threshold(self, t):
if self._threshold != t:
self._threshold = t
self.thr_spin.setValue(t)
self.changed.emit()
def setParameters(self, params):
cond = params.get("condition", self.DEFAULT_CONDITION)
self.cond_cb.setCurrentIndex(enum_to_index(Binarize.Condition, cond))
self._set_threshold(params.get("threshold", self.DEFAULT_THRESHOLD))
def parameters(self):
cond = index_to_enum(Binarize.Condition, self.cond_cb.currentIndex())
return {"condition": cond, "threshold": self._threshold}
@staticmethod
def createinstance(params):
condition = params.get("condition", BinarizeEditor.DEFAULT_CONDITION)
threshold = params.get("threshold", BinarizeEditor.DEFAULT_THRESHOLD)
return Binarize(condition, threshold)
def __repr__(self):
return "Condition: {}, Threshold: {}".format(
self.cond_cb.currentText(), self.thr_spin.value())
class Randomize(BaseEditor):
RandomizeClasses, RandomizeAttributes, RandomizeMetas = _Randomize.Type
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__rand_type_cb = QComboBox()
self.__rand_type_cb.addItems(["Classes", "Features", "Meta data"])
self.__rand_type_cb.currentIndexChanged.connect(self.changed)
self.__rand_type_cb.activated.connect(self.edited)
self.__rand_seed_ch = QCheckBox()
self.__rand_seed_ch.clicked.connect(self.edited)
form.addRow("Randomize:", self.__rand_type_cb)
form.addRow("Replicable shuffling:", self.__rand_seed_ch)
self.layout().addLayout(form)
def setParameters(self, params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
self.__rand_type_cb.setCurrentIndex(
_enum_to_index(_Randomize.Type, rand_type))
self.__rand_seed_ch.setChecked(params.get("rand_seed", 1) or 0)
def parameters(self):
return {
"rand_type":
_index_to_enum(_Randomize.Type,
self.__rand_type_cb.currentIndex()),
"rand_seed":
1 if self.__rand_seed_ch.isChecked() else None,
}
@staticmethod
def createinstance(params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
rand_seed = params.get("rand_seed", 1)
return _Randomize(rand_type=rand_type, rand_seed=rand_seed)
def __repr__(self):
return "{}, {}".format(
self.__rand_type_cb.currentText(),
"Replicable"
if self.__rand_seed_ch.isChecked() else "Not replicable",
)
class DropDownIdListFilter(QWidget, Control):
"""Container for multiple id list filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QVBoxLayout())
self.setContentsMargins(0, 0, 0, 0)
self.cb = QComboBox()
self.idsEdit = QPlainTextEdit()
self.layout().addWidget(self.cb)
self.layout().addWidget(self.idsEdit)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
def value(self):
return str(self.idsEdit.toPlainText()).split()
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
if isinstance(value, list):
value = "\n".join(value)
for i, op in enumerate(self.options):
if name == op.internalName:
self.cb.setCurrentIndex(i)
self.idsEdit.setPlainText(value)
class DropDownIdListFilter(QWidget, Control):
"""Container for multiple id list filters
"""
def __init__(self, tree, dataset, master, parent=None):
QWidget.__init__(self, parent)
Control.__init__(self, tree, dataset, master)
self.setLayout(QVBoxLayout())
self.setContentsMargins(0, 0, 0, 0)
self.cb = QComboBox()
self.idsEdit = QPlainTextEdit()
self.layout().addWidget(self.cb)
self.layout().addWidget(self.idsEdit)
self.options = []
self.setOptions(tree.subelements_top("Option"))
def setOptions(self, options):
self.cb.clear()
self.options = []
for option in options:
self.cb.addItem(option.displayName)
self.options.append(option)
def value(self):
return str(self.idsEdit.toPlainText()).split()
def query(self):
filter = self.options[self.cb.currentIndex()]
filter = biomart.FilterDescription(
self.tree.registry, "FilterDescription",
filter.attributes, filter.children)
return [("Filter", filter, self.value())]
def setControlValue(self, name, value):
if isinstance(value, list):
value = "\n".join(value)
for i, op in enumerate(self.options):
if name == op.internalName:
self.cb.setCurrentIndex(i)
self.idsEdit.setPlainText(value)
class Randomize(BaseEditor):
RandomizeClasses, RandomizeAttributes, RandomizeMetas = _Randomize.Type
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__rand_type_cb = QComboBox()
self.__rand_type_cb.addItems(["Classes",
"Features",
"Meta data"])
self.__rand_type_cb.currentIndexChanged.connect(self.changed)
self.__rand_type_cb.activated.connect(self.edited)
self.__rand_seed_ch = QCheckBox()
self.__rand_seed_ch.clicked.connect(self.edited)
form.addRow("Randomize:", self.__rand_type_cb)
form.addRow("Replicable shuffling:", self.__rand_seed_ch)
self.layout().addLayout(form)
def setParameters(self, params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
self.__rand_type_cb.setCurrentIndex(
enum_to_index(_Randomize.Type, rand_type))
self.__rand_seed_ch.setChecked(params.get("rand_seed", 1) or 0)
def parameters(self):
return {"rand_type": index_to_enum(_Randomize.Type,
self.__rand_type_cb.currentIndex()),
"rand_seed": 1 if self.__rand_seed_ch.isChecked() else None}
@staticmethod
def createinstance(params):
rand_type = params.get("rand_type", Randomize.RandomizeClasses)
rand_seed = params.get("rand_seed", 1)
return _Randomize(rand_type=rand_type, rand_seed=rand_seed)
def __repr__(self):
return "{}, {}".format(self.__rand_type_cb.currentText(),
"Replicable" if self.__rand_seed_ch.isChecked()
else "Not replicable")
class DiscreteVariableEditor(VariableEditor):
valueChanged = Signal(int)
def __init__(self, parent: QWidget, items: List[str], callback: Callable):
super().__init__(parent, callback)
self._combo = QComboBox(parent,
maximumWidth=180,
sizePolicy=QSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Fixed))
self._combo.addItems(items)
self._combo.currentIndexChanged.connect(self.valueChanged)
self.layout().addWidget(self._combo)
@property
def value(self) -> int:
return self._combo.currentIndex()
@value.setter
def value(self, value: float):
assert value == int(value)
self._combo.setCurrentIndex(int(value))
class LogarithmicScaleEditor(ScBaseEditor):
DEFAULT_BASE = LogarithmicScale.BinaryLog
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.base_cb = QComboBox()
self.base_cb.addItems([
"2 (Binary Logarithm)", "e (Natural Logarithm)",
"10 (Common Logarithm)"
])
self.base_cb.currentIndexChanged.connect(self.changed)
self.base_cb.activated.connect(self.edited)
form.addRow("Logarithm Base:", self.base_cb)
self.layout().addLayout(form)
def setParameters(self, params):
base = params.get("base", self.DEFAULT_BASE)
self.base_cb.setCurrentIndex(enum_to_index(LogarithmicScale.Base,
base))
def parameters(self):
return {
"base":
index_to_enum(LogarithmicScale.Base, self.base_cb.currentIndex())
}
@staticmethod
def createinstance(params):
base = params.get("base", LogarithmicScaleEditor.DEFAULT_BASE)
return LogarithmicScale(base)
def __repr__(self):
return "Base: {}".format(self.base_cb.currentText())
class FileUploadHelper(QDialog):
# settings
kegg_domain = 'KEGG'
supported_domains = OrderedDict({
'Gene Ontology': gene_ontology_domain,
'Gene Sets': gene_sets_domain
})
supported_organisms = [
common_taxid_to_name(tax_id) for tax_id in common_taxids()
]
hierarchies = {
'GO - Biological Process': ('GO', 'biological_process'),
'GO - Molecular Function': ('GO', 'molecular_function'),
'GO - Cellular Component': ('GO', 'cellular_component'),
'KEGG - Pathways': ('KEGG', 'pathways'),
'KEGG - Orthologs': ('KEGG', 'orthologs')
}
def __init__(self, parent=None):
super(FileUploadHelper, self).__init__(
parent, Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint
| Qt.WindowCloseButtonHint | Qt.WindowMaximizeButtonHint)
self.setAttribute(Qt.WA_DeleteOnClose)
self.setWindowTitle('Add new file')
self.info_state = INFO_FILE_SCHEMA
self.layout = QVBoxLayout(self)
# domain selection combobox
self.domain_selection = QComboBox()
self.domain_selection.addItems(self.supported_domains.keys())
self.domain_selection.currentIndexChanged.connect(
self.__on_domain_selection)
self.__create_selection_row('Domain: ', self.domain_selection)
# domain selection combobox
self.hierarchy_selection = QComboBox()
self.hierarchy_selection.addItems(self.hierarchies.keys())
self.layout.addWidget(self.hierarchy_selection,
alignment=Qt.AlignVCenter)
self.__on_domain_selection()
# select organism
self.organism_selection = QComboBox()
self.organism_selection.addItems(self.supported_organisms)
self.__create_selection_row('Organism: ', self.organism_selection)
# title
self.line_edit_title = QLineEdit()
self.__create_selection_row('Title: ', self.line_edit_title)
# tags
self.line_edit_tags = QLineEdit()
self.__create_selection_row('Tags (comma-separated): ',
self.line_edit_tags)
# file selector
self.file_info = QLabel()
self.file_select_btn = QPushButton('Select File', self)
self.file_select_btn.clicked.connect(self.__handle_file_selector)
self.__create_selection_row(' ', self.file_select_btn)
# add file info section
self.layout.addWidget(self.file_info, alignment=Qt.AlignCenter)
self.layout.addStretch(1)
# Ok and Cancel buttons
self.buttons = QDialogButtonBox(
QDialogButtonBox.Ok | QDialogButtonBox.Cancel, Qt.Horizontal, self)
self.layout.addWidget(self.buttons, alignment=Qt.AlignJustify)
self.buttons.accepted.connect(self.__accept)
self.buttons.rejected.connect(self.__close)
# path to a selected file
self.file_path = None
def __on_domain_selection(self):
selected = self.__get_selected_domain() == gene_sets_domain
self.hierarchy_selection.setVisible(selected)
def __get_selected_domain(self):
domain_label = list(self.supported_domains.keys())[
self.domain_selection.currentIndex()]
return self.supported_domains[domain_label]
def __get_selected_hier(self):
hier_label = list(
self.hierarchies.keys())[self.hierarchy_selection.currentIndex()]
return self.hierarchies[hier_label]
def __create_selection_row(self, label, widget):
self.layout.addWidget(QLabel(label), alignment=Qt.AlignLeft)
self.layout.addWidget(widget, alignment=Qt.AlignVCenter)
def __accept(self):
if self.file_path:
self.info_state = self.__parse_selection()
self.__move_to_serverfiles_folder(self.file_path)
self.parent().initialize_files_view()
self.close()
def __close(self):
self.close()
def closeEvent(self, event):
# clean-up
self.parent()._dialog = None
def __filename(self, domain, organism):
""" Create filename based od domain name and organism.
"""
if domain in self.supported_domains.values(
) and domain == gene_ontology_domain and organism:
return FILENAME_ANNOTATION.format(organism)
elif domain in self.supported_domains.values(
) and domain == gene_sets_domain and organism:
return filename((self.__get_selected_hier()), organism)
def __parse_selection(self):
try:
domain = self.__get_selected_domain()
organism = taxname_to_taxid(self.supported_organisms[
self.organism_selection.currentIndex()])
except KeyError as e:
raise e
return {
'domain': domain,
'organism': organism,
'filename': self.__filename(domain, organism),
'title': self.line_edit_title.text(),
'tags': self.line_edit_tags.text().split(','),
'source': SOURCE_USER
}
def __move_to_serverfiles_folder(self, selected_file_path):
domain_path = serverfiles.localpath(self.info_state['domain'])
file_path = os.path.join(domain_path, self.info_state['filename'])
create_folder(domain_path)
try:
copyfile(selected_file_path, file_path)
except IOError as e:
# TODO: handle error properly
raise e
# if copy successful create .info file
create_info_file(file_path, **self.info_state)
def __handle_file_selector(self):
self.file_path = QFileDialog.getOpenFileName(self, 'Open File')[0]
self.file_info.setText('Selected File: {}'.format(
os.path.basename(self.file_path)))
class ControlCombo(ControlBase, QWidget):
"""This class represents a wrapper to the combo box"""
def __init__(self, label='', default=None, helptext=None):
QWidget.__init__(self)
ControlBase.__init__(self, label, default, helptext)
##########################################################################
############ Functions ###################################################
##########################################################################
def init_form(self):
self._layout = QHBoxLayout()
self._combo = QComboBox(self.form)
if self._label is not None:
self._combolabel = QLabel(self.form)
self._layout.addWidget(self._combolabel)
self._combolabel.setAccessibleName('ControlCombo-label')
self.label = self._label
else:
self._combolabel = None
self._layout.addWidget(self._combo)
self.form.setLayout(self._layout)
self._combo.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
self._layout.setContentsMargins(0, 0, 0, 0)
self.form.setContentsMargins(0, 0, 0, 0)
self.form.setMinimumHeight(38)
self.form.setMaximumHeight(38)
self.form.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed)
self._combo.currentIndexChanged.connect(self._currentIndexChanged)
self._combo.activated.connect(self._activated)
self._combo.highlighted.connect(self._highlighted)
self._combo.editTextChanged.connect(self._editTextChanged)
self._items = {}
self._addingItem = False
def clear(self):
self._items = {}
self._value = None
self._combo.clear()
def add_item(self, label, value=None):
self._addingItem = True
if value is not None:
if not (value in self._items.values()):
self._combo.addItem(label)
else:
if not (label in self._items.keys()):
self._combo.addItem(label)
firstValue = False
if self._items == {}:
firstValue = True
if value is None:
self._items[str(label)] = label
else:
self._items[str(label)] = value
self._addingItem = False
if firstValue:
self.value = self._items[label]
def __add__(self, val):
if isinstance(val, tuple):
self.add_item(val[0], val[1])
else:
self.add_item(val)
return self
def get_item_index_by_name(self, item_name):
"""
Returns the index of the item containing the given name
:param item_name: item name in combo box
:type item_name: string
"""
return self._combo.findText(item_name)
def count(self):
return self._combo.count()
def show(self):
"""
Show the control
"""
QWidget.show(self)
def hide(self):
"""
Hide the control
"""
QWidget.hide(self)
##########################################################################
############ Events ######################################################
##########################################################################
def current_index_changed_event(self, index):
"""Called when the user chooses an item in the combobox and
the selected choice is different from the last one selected.
@index: item's index
"""
pass
def activated_event(self, index):
"""Called when the user chooses an item in the combobox.
Note that this signal happens even when the choice is not changed
@index: item's index
"""
pass
def highlighted_event(self, index):
pass
def edittext_changed_event(self, text):
pass
##########################################################################
############ PROPERTIES ##################################################
##########################################################################
@property
def form(self):
return self
@property
def current_index(self):
return self._combo.currentIndex()
@current_index.setter
def current_index(self, value):
self._combo.setCurrentIndex(value)
@property
def items(self):
return self._items.items()
@property
def value(self):
return self._value
@value.setter
def value(self, value):
for key, val in self.items:
if value == val:
index = self._combo.findText(key)
self._combo.setCurrentIndex(index)
if self._value != value:
self.changed_event()
self._value = val
@property
def text(self):
return str(self._combo.currentText())
@text.setter
def text(self, value):
for key, val in self.items:
if value == key:
self.value = val
break
@property
def label(self):
if self._combolabel:
return self._combolabel.text()
else:
return None
@label.setter
def label(self, value):
"""
Label of the control, if applies
@type value: string
"""
if self._combolabel:
self._combolabel.setText(value)
##########################################################################
############ Private functions ###########################################
##########################################################################
def _activated(self, index):
if not self._addingItem:
item = self._combo.currentText()
if len(item) >= 1:
ControlBase.value.fset(self, self._items[str(item)])
self.activated_event(index)
def _highlighted(self, index):
"""Called when an item in the combobox popup
list is highlighted by the user.
@index: item's index
"""
self.highlighted_event(index)
def _editTextChanged(self, text):
self.edittext_changed_event(text)
def _currentIndexChanged(self, index):
if not self._addingItem:
item = self._combo.currentText()
if len(item) >= 1:
ControlBase.value.fset(self, self._items[str(item)])
self.current_index_changed_event(index)
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = Msg("Model based imputer does not work for sparse data")
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(int(Method.Leave)) # type: int
# Per-variable imputation state (synced in storeSpecificSettings)
_variable_imputation_state = settings.ContextSetting({}) # type: VariableState
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
self.learner = None # type: Optional[Learner]
self.default_learner = SimpleTreeLearner()
self.modified = False
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QVBoxLayout(box)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for method, _ in list(METHODS.items())[1:-1]:
imputer = self.create_imputer(method)
button = QRadioButton(imputer.name)
button.setChecked(method == self.default_method_index)
button_group.addButton(button, method)
box_layout.addWidget(button)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = QListView(
selectionMode=QListView.ExtendedSelection,
uniformItemSizes=True
)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed
)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for method in Method:
imputer = self.create_imputer(method)
button = QRadioButton(text=imputer.name)
button_group.addButton(button, method)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected
)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000., maximum=1000., singleStep=.1, decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection
)
method_layout.addStretch(2)
reset_button = QPushButton(
"Restore All to Default", checked=False, checkable=False,
clicked=self.reset_variable_state, default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_commit(
self.controlArea, self, "autocommit", "Apply",
orientation=Qt.Horizontal,
checkbox_label="Apply automatically")
box.button.setFixedWidth(180)
box.layout().insertStretch(0)
def create_imputer(self, method, *args):
# type: (Method, ...) -> impute.BaseImputeMethod
if method == Method.Model:
if self.learner is not None:
return impute.Model(self.learner)
else:
return impute.Model(self.default_learner)
elif method == Method.AsAboveSoBelow:
assert self.default_method_index != Method.AsAboveSoBelow
default = self.create_imputer(Method(self.default_method_index))
m = AsDefault()
m.method = default
return m
else:
return METHODS[method](*args)
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
assert index != Method.AsAboveSoBelow
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
# update variable view
self.update_varview()
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.closeContext()
self.varmodel[:] = []
self._variable_imputation_state = {} # type: VariableState
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
# restore per variable imputation state
self._restore_state(self._variable_imputation_state)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.learner = learner or self.default_learner
imputer = self.create_imputer(Method.Model)
button = self.default_button_group.button(Method.Model)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(Method.Model)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = Method.Model
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
# type: (int) -> impute.BaseImputeMethod
"""
Return the imputation method for column by its index.
"""
assert 0 <= column_index < len(self.varmodel)
idx = self.varmodel.index(column_index, 0)
state = idx.data(StateRole)
if state is None:
state = (Method.AsAboveSoBelow, ())
return self.create_imputer(state[0], *state[1])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
if self.data is None or len(self.data) == 0 or len(self.varmodel) == 0:
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [
(i, var, self.get_method_for_column(i))
for i, var in enumerate(self.varmodel)
]
# normalize to the effective method bypasing AsDefault
impute_state = [
(i, var, m.method if isinstance(m, AsDefault) else m)
for i, var, m in impute_state
]
def impute_one(method, var, data):
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(
impute_one, copy.deepcopy(method), var, data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
data = self.data
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
assert fut.done()
newvar = []
try:
res = fut.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
# ?? break
except VariableNotSupported:
self.warning("Default method can not handle '{}'".
format(var.name))
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", var, exc_info=True)
self.Error.imputation_failed(var.name)
attributes = class_vars = None
break
else:
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = var
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
if i < len(data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
if attributes is None:
data = None
else:
domain = Orange.data.Domain(attributes, class_vars,
data.domain.metas)
try:
data = self.data.from_table(domain, data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
data = None
self.Outputs.data.send(data)
self.modified = False
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setBlocking(False)
def onDeleteWidget(self):
self.cancel()
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.get_method_for_column(i)
if not isinstance(method, AsDefault):
specific.append("{} ({})".format(var.name, str(method)))
default = self.create_imputer(Method.AsAboveSoBelow)
if specific:
self.report_items((
("Default method", default.name),
("Specific imputers", ", ".join(specific))
))
else:
self.report_items((("Method", default.name),))
def _on_var_selection_changed(self):
indexes = self.selection.selectedIndexes()
defmethod = (Method.AsAboveSoBelow, ())
methods = [index.data(StateRole) for index in indexes]
methods = [m if m is not None else defmethod for m in methods]
methods = set(methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for m in Method:
if method_type == m:
self.variable_button_group.button(m).setChecked(True)
if method_type == Method.Default:
(fixed_value,) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
# Update variable methods GUI enabled state based on selection.
for method in Method:
# use a default constructed imputer to query support
imputer = self.create_imputer(method)
enabled = all(imputer.supports_variable(var) for var in
selected_vars)
button = self.variable_button_group.button(method)
button.setEnabled(enabled)
# Update the "Value" edit GUI.
if not has_discrete:
# no discrete variables -> allow mass edit for all (continuous vars)
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
# single discrete var -> enable and fill the values combo
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
# mixed type selection -> disable
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(Method.Default).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
# set current value
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
# type: (Method) -> None
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
# type: (List[QModelIndex], Method) -> None
if method_index == Method.AsAboveSoBelow:
for index in indexes:
self.varmodel.setData(index, None, StateRole)
elif method_index == Method.Default:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
state = (int(Method.Default), (value,))
self.varmodel.setData(index, state, StateRole)
else:
state = (int(method_index), ())
for index in indexes:
self.varmodel.setData(index, state, StateRole)
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(
index, self.get_method_for_column(index.row()),
DisplayMethodRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(Method.Default).setChecked(True)
self.set_method_for_current_selection(Method.Default)
def reset_variable_state(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, Method.AsAboveSoBelow)
self.variable_button_group.button(Method.AsAboveSoBelow).setChecked(True)
def _store_state(self):
# type: () -> VariableState
"""
Save the current variable imputation state
"""
state = {} # type: VariableState
for i, var in enumerate(self.varmodel):
index = self.varmodel.index(i)
m = index.data(StateRole)
if m is not None:
state[var_key(var)] = m
return state
def _restore_state(self, state):
# type: (VariableState) -> None
"""
Restore the variable imputation state from the saved state
"""
def check(state):
# check if state is a proper State
if isinstance(state, tuple) and len(state) == 2:
m, p = state
if isinstance(m, int) and isinstance(p, tuple) and \
0 <= m < len(Method):
return True
return False
for i, var in enumerate(self.varmodel):
m = state.get(var_key(var), None)
if check(m):
self.varmodel.setData(self.varmodel.index(i), m, StateRole)
def storeSpecificSettings(self):
self._variable_imputation_state = self._store_state()
super().storeSpecificSettings()
class OWSql(OWWidget):
name = "SQL Table"
id = "orange.widgets.data.sql"
description = "Load data set from SQL."
icon = "icons/SQLTable.svg"
priority = 10
category = "Data"
keywords = ["data", "file", "load", "read"]
class Outputs:
data = Output(
"Data",
Table,
doc="Attribute-valued data set read from the input file.")
settings_version = 2
want_main_area = False
resizing_enabled = False
host = Setting(None)
port = Setting(None)
database = Setting(None)
schema = Setting(None)
username = ""
password = ""
table = Setting(None)
sql = Setting("")
guess_values = Setting(True)
download = Setting(False)
materialize = Setting(False)
materialize_table_name = Setting("")
class Information(OWWidget.Information):
data_sampled = Msg("Data description was generated from a sample.")
class Error(OWWidget.Error):
connection = Msg("{}")
no_backends = Msg("Please install a backend to use this widget")
missing_extension = Msg("Database is missing extension{}: {}")
def __init__(self):
super().__init__()
self.backend = None
self.data_desc_table = None
self.database_desc = None
vbox = gui.vBox(self.controlArea, "Server", addSpace=True)
box = gui.vBox(vbox)
self.backendmodel = BackendModel(Backend.available_backends())
self.backendcombo = QComboBox(box)
if len(self.backendmodel):
self.backendcombo.setModel(self.backendmodel)
else:
self.Error.no_backends()
box.setEnabled(False)
box.layout().addWidget(self.backendcombo)
self.servertext = QLineEdit(box)
self.servertext.setPlaceholderText('Server')
self.servertext.setToolTip('Server')
self.servertext.editingFinished.connect(self._load_credentials)
if self.host:
self.servertext.setText(self.host if not self.port else '{}:{}'.
format(self.host, self.port))
box.layout().addWidget(self.servertext)
self.databasetext = QLineEdit(box)
self.databasetext.setPlaceholderText('Database[/Schema]')
self.databasetext.setToolTip('Database or optionally Database/Schema')
if self.database:
self.databasetext.setText(
self.database if not self.schema else '{}/{}'.
format(self.database, self.schema))
box.layout().addWidget(self.databasetext)
self.usernametext = QLineEdit(box)
self.usernametext.setPlaceholderText('Username')
self.usernametext.setToolTip('Username')
box.layout().addWidget(self.usernametext)
self.passwordtext = QLineEdit(box)
self.passwordtext.setPlaceholderText('Password')
self.passwordtext.setToolTip('Password')
self.passwordtext.setEchoMode(QLineEdit.Password)
box.layout().addWidget(self.passwordtext)
self._load_credentials()
tables = gui.hBox(box)
self.tablemodel = TableModel()
self.tablecombo = QComboBox(
minimumContentsLength=35,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength)
self.tablecombo.setModel(self.tablemodel)
self.tablecombo.setToolTip('table')
tables.layout().addWidget(self.tablecombo)
self.tablecombo.activated[int].connect(self.select_table)
self.connectbutton = gui.button(tables,
self,
'↻',
callback=self.connect)
self.connectbutton.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
tables.layout().addWidget(self.connectbutton)
self.custom_sql = gui.vBox(box)
self.custom_sql.setVisible(False)
self.sqltext = QTextEdit(self.custom_sql)
self.sqltext.setPlainText(self.sql)
self.custom_sql.layout().addWidget(self.sqltext)
mt = gui.hBox(self.custom_sql)
cb = gui.checkBox(mt, self, 'materialize', 'Materialize to table ')
cb.setToolTip('Save results of the query in a table')
le = gui.lineEdit(mt, self, 'materialize_table_name')
le.setToolTip('Save results of the query in a table')
self.executebtn = gui.button(self.custom_sql,
self,
'Execute',
callback=self.open_table)
box.layout().addWidget(self.custom_sql)
gui.checkBox(box,
self,
"guess_values",
"Auto-discover discrete variables",
callback=self.open_table)
gui.checkBox(box,
self,
"download",
"Download data to local memory",
callback=self.open_table)
gui.rubber(self.buttonsArea)
QTimer.singleShot(0, self.connect)
def _load_credentials(self):
self._parse_host_port()
cm = self._credential_manager(self.host, self.port)
self.username = cm.username
self.password = cm.password
if self.username:
self.usernametext.setText(self.username)
if self.password:
self.passwordtext.setText(self.password)
def _save_credentials(self):
cm = self._credential_manager(self.host, self.port)
cm.username = self.username
cm.password = self.password
def _credential_manager(self, host, port):
return CredentialManager("SQL Table: {}:{}".format(host, port))
def error(self, id=0, text=""):
super().error(id, text)
err_style = 'QLineEdit {border: 2px solid red;}'
if 'server' in text or 'host' in text:
self.servertext.setStyleSheet(err_style)
else:
self.servertext.setStyleSheet('')
if 'role' in text:
self.usernametext.setStyleSheet(err_style)
else:
self.usernametext.setStyleSheet('')
if 'database' in text:
self.databasetext.setStyleSheet(err_style)
else:
self.databasetext.setStyleSheet('')
def _parse_host_port(self):
hostport = self.servertext.text().split(':')
self.host = hostport[0]
self.port = hostport[1] if len(hostport) == 2 else None
def connect(self):
self._parse_host_port()
self.database, _, self.schema = self.databasetext.text().partition('/')
self.username = self.usernametext.text() or None
self.password = self.passwordtext.text() or None
try:
if self.backendcombo.currentIndex() < 0:
return
backend = self.backendmodel[self.backendcombo.currentIndex()]
self.backend = backend(
dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password))
self.Error.connection.clear()
self._save_credentials()
self.database_desc = OrderedDict(
(("Host", self.host), ("Port", self.port),
("Database", self.database), ("User name", self.username)))
self.refresh_tables()
self.select_table()
except BackendError as err:
error = str(err).split('\n')[0]
self.Error.connection(error)
self.database_desc = self.data_desc_table = None
self.tablecombo.clear()
def refresh_tables(self):
self.tablemodel.clear()
self.Error.missing_extension.clear()
if self.backend is None:
self.data_desc_table = None
return
self.tablemodel.append("Select a table")
self.tablemodel.extend(self.backend.list_tables(self.schema))
self.tablemodel.append("Custom SQL")
def select_table(self):
curIdx = self.tablecombo.currentIndex()
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.custom_sql.setVisible(False)
return self.open_table()
else:
self.custom_sql.setVisible(True)
self.data_desc_table = None
self.database_desc["Table"] = "(None)"
self.table = None
#self.Error.missing_extension(
# 's' if len(missing) > 1 else '',
# ', '.join(missing),
# shown=missing)
def open_table(self):
table = self.get_table()
self.data_desc_table = table
self.Outputs.data.send(table)
def get_table(self):
if self.tablecombo.currentIndex() <= 0:
if self.database_desc:
self.database_desc["Table"] = "(None)"
self.data_desc_table = None
return
if self.tablecombo.currentIndex() < self.tablecombo.count() - 1:
self.table = self.tablemodel[self.tablecombo.currentIndex()]
self.database_desc["Table"] = self.table
if "Query" in self.database_desc:
del self.database_desc["Query"]
else:
self.sql = self.table = self.sqltext.toPlainText()
if self.materialize:
import psycopg2
if not self.materialize_table_name:
self.Error.connection(
"Specify a table name to materialize the query")
return
try:
with self.backend.execute_sql_query(
"DROP TABLE IF EXISTS " +
self.materialize_table_name):
pass
with self.backend.execute_sql_query(
"CREATE TABLE " + self.materialize_table_name +
" AS " + self.table):
pass
with self.backend.execute_sql_query(
"ANALYZE " + self.materialize_table_name):
pass
self.table = self.materialize_table_name
except (psycopg2.ProgrammingError, BackendError) as ex:
self.Error.connection(str(ex))
return
try:
table = SqlTable(dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password),
self.table,
backend=type(self.backend),
inspect_values=False)
except BackendError as ex:
self.Error.connection(str(ex))
return
self.Error.connection.clear()
sample = False
if table.approx_len() > LARGE_TABLE and self.guess_values:
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Attribute discovery might take "
"a long time on large tables.\n"
"Do you want to auto discover attributes?")
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
sample_button = confirm.addButton("Yes, on a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
self.guess_values = False
elif confirm.clickedButton() == sample_button:
sample = True
self.Information.clear()
if self.guess_values:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if sample:
s = table.sample_time(1)
domain = s.get_domain(inspect_values=True)
self.Information.data_sampled()
else:
domain = table.get_domain(inspect_values=True)
QApplication.restoreOverrideCursor()
table.domain = domain
if self.download:
if table.approx_len() > MAX_DL_LIMIT:
QMessageBox.warning(
self, 'Warning', "Data is too big to download.\n"
"Consider using the Data Sampler widget to download "
"a sample instead.")
self.download = False
elif table.approx_len() > AUTO_DL_LIMIT:
confirm = QMessageBox.question(
self, 'Question', "Data appears to be big. Do you really "
"want to download it to local memory?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if confirm == QMessageBox.No:
self.download = False
if self.download:
table.download_data(MAX_DL_LIMIT)
table = Table(table)
return table
def send_report(self):
if not self.database_desc:
self.report_paragraph("No database connection.")
return
self.report_items("Database", self.database_desc)
if self.data_desc_table:
self.report_items("Data",
report.describe_data(self.data_desc_table))
@classmethod
def migrate_settings(cls, settings, version):
if version < 2:
# Until Orange version 3.4.4 username and password had been stored
# in Settings.
cm = cls._credential_manager(settings["host"], settings["port"])
cm.username = settings["username"]
cm.password = settings["password"]
class OWSql(OWWidget):
name = "SQL Table"
id = "orange.widgets.data.sql"
description = "Load data set from SQL."
icon = "icons/SQLTable.svg"
priority = 30
category = "Data"
keywords = ["data", "file", "load", "read", "SQL"]
class Outputs:
data = Output("Data", Table, doc="Attribute-valued data set read from the input file.")
settings_version = 2
want_main_area = False
resizing_enabled = False
host = Setting(None)
port = Setting(None)
database = Setting(None)
schema = Setting(None)
username = ""
password = ""
table = Setting(None)
sql = Setting("")
guess_values = Setting(True)
download = Setting(False)
materialize = Setting(False)
materialize_table_name = Setting("")
class Information(OWWidget.Information):
data_sampled = Msg("Data description was generated from a sample.")
class Error(OWWidget.Error):
connection = Msg("{}")
no_backends = Msg("Please install a backend to use this widget")
missing_extension = Msg("Database is missing extension{}: {}")
def __init__(self):
super().__init__()
self.backend = None
self.data_desc_table = None
self.database_desc = None
vbox = gui.vBox(self.controlArea, "Server", addSpace=True)
box = gui.vBox(vbox)
self.backends = BackendModel(Backend.available_backends())
self.backendcombo = QComboBox(box)
if len(self.backends):
self.backendcombo.setModel(self.backends)
else:
self.Error.no_backends()
box.setEnabled(False)
box.layout().addWidget(self.backendcombo)
self.servertext = QLineEdit(box)
self.servertext.setPlaceholderText('Server')
self.servertext.setToolTip('Server')
self.servertext.editingFinished.connect(self._load_credentials)
if self.host:
self.servertext.setText(self.host if not self.port else
'{}:{}'.format(self.host, self.port))
box.layout().addWidget(self.servertext)
self.databasetext = QLineEdit(box)
self.databasetext.setPlaceholderText('Database[/Schema]')
self.databasetext.setToolTip('Database or optionally Database/Schema')
if self.database:
self.databasetext.setText(
self.database if not self.schema else
'{}/{}'.format(self.database, self.schema))
box.layout().addWidget(self.databasetext)
self.usernametext = QLineEdit(box)
self.usernametext.setPlaceholderText('Username')
self.usernametext.setToolTip('Username')
box.layout().addWidget(self.usernametext)
self.passwordtext = QLineEdit(box)
self.passwordtext.setPlaceholderText('Password')
self.passwordtext.setToolTip('Password')
self.passwordtext.setEchoMode(QLineEdit.Password)
box.layout().addWidget(self.passwordtext)
self._load_credentials()
self.tables = TableModel()
tables = gui.hBox(box)
self.tablecombo = QComboBox(
minimumContentsLength=35,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength
)
self.tablecombo.setModel(self.tables)
self.tablecombo.setToolTip('table')
tables.layout().addWidget(self.tablecombo)
self.connect()
index = self.tablecombo.findText(str(self.table))
if index != -1:
self.tablecombo.setCurrentIndex(index)
# set up the callback to select_table in case of selection change
self.tablecombo.activated[int].connect(self.select_table)
self.connectbutton = gui.button(
tables, self, '↻', callback=self.connect)
self.connectbutton.setSizePolicy(
QSizePolicy.Fixed, QSizePolicy.Fixed)
tables.layout().addWidget(self.connectbutton)
self.custom_sql = gui.vBox(box)
self.custom_sql.setVisible(False)
self.sqltext = QTextEdit(self.custom_sql)
self.sqltext.setPlainText(self.sql)
self.custom_sql.layout().addWidget(self.sqltext)
mt = gui.hBox(self.custom_sql)
cb = gui.checkBox(mt, self, 'materialize', 'Materialize to table ')
cb.setToolTip('Save results of the query in a table')
le = gui.lineEdit(mt, self, 'materialize_table_name')
le.setToolTip('Save results of the query in a table')
self.executebtn = gui.button(
self.custom_sql, self, 'Execute', callback=self.open_table)
box.layout().addWidget(self.custom_sql)
gui.checkBox(box, self, "guess_values",
"Auto-discover categorical variables",
callback=self.open_table)
gui.checkBox(box, self, "download",
"Download data to local memory",
callback=self.open_table)
gui.rubber(self.buttonsArea)
QTimer.singleShot(0, self.select_table)
def _load_credentials(self):
self._parse_host_port()
cm = self._credential_manager(self.host, self.port)
self.username = cm.username
self.password = cm.password
if self.username:
self.usernametext.setText(self.username)
if self.password:
self.passwordtext.setText(self.password)
def _save_credentials(self):
cm = self._credential_manager(self.host, self.port)
cm.username = self.username or ''
cm.password = self.password or ''
def _credential_manager(self, host, port):
return CredentialManager("SQL Table: {}:{}".format(host, port))
def error(self, id=0, text=""):
super().error(id, text)
err_style = 'QLineEdit {border: 2px solid red;}'
if 'server' in text or 'host' in text:
self.servertext.setStyleSheet(err_style)
else:
self.servertext.setStyleSheet('')
if 'role' in text:
self.usernametext.setStyleSheet(err_style)
else:
self.usernametext.setStyleSheet('')
if 'database' in text:
self.databasetext.setStyleSheet(err_style)
else:
self.databasetext.setStyleSheet('')
def _parse_host_port(self):
hostport = self.servertext.text().split(':')
self.host = hostport[0]
self.port = hostport[1] if len(hostport) == 2 else None
def connect(self):
self._parse_host_port()
self.database, _, self.schema = self.databasetext.text().partition('/')
self.username = self.usernametext.text() or None
self.password = self.passwordtext.text() or None
try:
if self.backendcombo.currentIndex() < 0:
return
backend = self.backends[self.backendcombo.currentIndex()]
self.backend = backend(dict(
host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password
))
self.Error.connection.clear()
self._save_credentials()
self.database_desc = OrderedDict((
("Host", self.host), ("Port", self.port),
("Database", self.database), ("User name", self.username)
))
self.refresh_tables()
except BackendError as err:
error = str(err).split('\n')[0]
self.Error.connection(error)
self.database_desc = self.data_desc_table = None
self.tablecombo.clear()
def refresh_tables(self):
self.tables.clear()
self.Error.missing_extension.clear()
if self.backend is None:
self.data_desc_table = None
return
self.tables.append("Select a table")
self.tables.append("Custom SQL")
self.tables.extend(self.backend.list_tables(self.schema))
# Called on tablecombo selection change:
def select_table(self):
curIdx = self.tablecombo.currentIndex()
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.custom_sql.setVisible(False)
return self.open_table()
else:
self.custom_sql.setVisible(True)
self.data_desc_table = None
self.database_desc["Table"] = "(None)"
self.table = None
if len(str(self.sql)) > 14:
return self.open_table()
#self.Error.missing_extension(
# 's' if len(missing) > 1 else '',
# ', '.join(missing),
# shown=missing)
def open_table(self):
table = self.get_table()
self.data_desc_table = table
self.Outputs.data.send(table)
def get_table(self):
curIdx = self.tablecombo.currentIndex()
if curIdx <= 0:
if self.database_desc:
self.database_desc["Table"] = "(None)"
self.data_desc_table = None
return
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.table = self.tables[self.tablecombo.currentIndex()]
self.database_desc["Table"] = self.table
if "Query" in self.database_desc:
del self.database_desc["Query"]
what = self.table
else:
what = self.sql = self.sqltext.toPlainText()
self.table = "Custom SQL"
if self.materialize:
import psycopg2
if not self.materialize_table_name:
self.Error.connection(
"Specify a table name to materialize the query")
return
try:
with self.backend.execute_sql_query("DROP TABLE IF EXISTS " +
self.materialize_table_name):
pass
with self.backend.execute_sql_query("CREATE TABLE " +
self.materialize_table_name +
" AS " + self.sql):
pass
with self.backend.execute_sql_query("ANALYZE " + self.materialize_table_name):
pass
except (psycopg2.ProgrammingError, BackendError) as ex:
self.Error.connection(str(ex))
return
try:
table = SqlTable(dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password),
what,
backend=type(self.backend),
inspect_values=False)
except BackendError as ex:
self.Error.connection(str(ex))
return
self.Error.connection.clear()
sample = False
if table.approx_len() > LARGE_TABLE and self.guess_values:
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Attribute discovery might take "
"a long time on large tables.\n"
"Do you want to auto discover attributes?")
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
sample_button = confirm.addButton("Yes, on a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
self.guess_values = False
elif confirm.clickedButton() == sample_button:
sample = True
self.Information.clear()
if self.guess_values:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if sample:
s = table.sample_time(1)
domain = s.get_domain(inspect_values=True)
self.Information.data_sampled()
else:
domain = table.get_domain(inspect_values=True)
QApplication.restoreOverrideCursor()
table.domain = domain
if self.download:
if table.approx_len() > MAX_DL_LIMIT:
QMessageBox.warning(
self, 'Warning', "Data is too big to download.\n"
"Consider using the Data Sampler widget to download "
"a sample instead.")
self.download = False
elif table.approx_len() > AUTO_DL_LIMIT:
confirm = QMessageBox.question(
self, 'Question', "Data appears to be big. Do you really "
"want to download it to local memory?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if confirm == QMessageBox.No:
self.download = False
if self.download:
table.download_data(MAX_DL_LIMIT)
table = Table(table)
return table
def send_report(self):
if not self.database_desc:
self.report_paragraph("No database connection.")
return
self.report_items("Database", self.database_desc)
if self.data_desc_table:
self.report_items("Data",
report.describe_data(self.data_desc_table))
@classmethod
def migrate_settings(cls, settings, version):
if version < 2:
# Until Orange version 3.4.4 username and password had been stored
# in Settings.
cm = cls._credential_manager(settings["host"], settings["port"])
cm.username = settings["username"]
cm.password = settings["password"]
class ColorGradientSelection(QWidget):
activated = Signal(int)
currentIndexChanged = Signal(int)
thresholdsChanged = Signal(float, float)
def __init__(self, *args, thresholds=(0.0, 1.0), **kwargs):
super().__init__(*args, **kwargs)
low = round(clip(thresholds[0], 0., 1.), 2)
high = round(clip(thresholds[1], 0., 1.), 2)
high = max(low, high)
self.__threshold_low, self.__threshold_high = low, high
form = QFormLayout(
formAlignment=Qt.AlignLeft,
labelAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow
)
form.setContentsMargins(0, 0, 0, 0)
self.gradient_cb = QComboBox(
None, objectName="gradient-combo-box",
)
self.gradient_cb.setAttribute(Qt.WA_LayoutUsesWidgetRect)
icsize = self.style().pixelMetric(
QStyle.PM_SmallIconSize, None, self.gradient_cb
)
self.gradient_cb.setIconSize(QSize(64, icsize))
model = itemmodels.ContinuousPalettesModel()
model.setParent(self)
self.gradient_cb.setModel(model)
self.gradient_cb.activated[int].connect(self.activated)
self.gradient_cb.currentIndexChanged.connect(self.currentIndexChanged)
slider_low = QSlider(
objectName="threshold-low-slider", minimum=0, maximum=100,
value=int(low * 100), orientation=Qt.Horizontal,
tickPosition=QSlider.TicksBelow, pageStep=10,
toolTip=self.tr("Low gradient threshold"),
whatsThis=self.tr("Applying a low threshold will squeeze the "
"gradient from the lower end")
)
slider_high = QSlider(
objectName="threshold-low-slider", minimum=0, maximum=100,
value=int(high * 100), orientation=Qt.Horizontal,
tickPosition=QSlider.TicksAbove, pageStep=10,
toolTip=self.tr("High gradient threshold"),
whatsThis=self.tr("Applying a high threshold will squeeze the "
"gradient from the higher end")
)
form.setWidget(0, QFormLayout.SpanningRole, self.gradient_cb)
form.addRow(self.tr("Low:"), slider_low)
form.addRow(self.tr("High:"), slider_high)
self.slider_low = slider_low
self.slider_high = slider_high
self.slider_low.valueChanged.connect(self.__on_slider_low_moved)
self.slider_high.valueChanged.connect(self.__on_slider_high_moved)
self.setLayout(form)
def setModel(self, model: QAbstractItemModel) -> None:
self.gradient_cb.setModel(model)
def model(self) -> QAbstractItemModel:
return self.gradient_cb.model()
def findData(self, data: Any, role: Qt.ItemDataRole) -> int:
return self.gradient_cb.findData(data, role)
def setCurrentIndex(self, index: int) -> None:
self.gradient_cb.setCurrentIndex(index)
def currentIndex(self) -> int:
return self.gradient_cb.currentIndex()
currentIndex_ = Property(
int, currentIndex, setCurrentIndex, notify=currentIndexChanged)
def currentData(self, role=Qt.UserRole) -> Any:
return self.gradient_cb.currentData(role)
def thresholds(self) -> Tuple[float, float]:
return self.__threshold_low, self.__threshold_high
thresholds_ = Property(object, thresholds, notify=thresholdsChanged)
def thresholdLow(self) -> float:
return self.__threshold_low
def setThresholdLow(self, low: float) -> None:
self.setThresholds(low, max(self.__threshold_high, low))
thresholdLow_ = Property(
float, thresholdLow, setThresholdLow, notify=thresholdsChanged)
def thresholdHigh(self) -> float:
return self.__threshold_high
def setThresholdHigh(self, high: float) -> None:
self.setThresholds(min(self.__threshold_low, high), high)
thresholdHigh_ = Property(
float, thresholdLow, setThresholdLow, notify=thresholdsChanged)
def __on_slider_low_moved(self, value: int) -> None:
high = self.slider_high
old = self.__threshold_low, self.__threshold_high
self.__threshold_low = value / 100.
if value >= high.value():
self.__threshold_high = value / 100.
high.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def __on_slider_high_moved(self, value: int) -> None:
low = self.slider_low
old = self.__threshold_low, self.__threshold_high
self.__threshold_high = value / 100.
if low.value() >= value:
self.__threshold_low = value / 100
low.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def setThresholds(self, low: float, high: float) -> None:
low = round(clip(low, 0., 1.), 2)
high = round(clip(high, 0., 1.), 2)
if low > high:
high = low
if self.__threshold_low != low or self.__threshold_high != high:
self.__threshold_high = high
self.__threshold_low = low
self.slider_low.setSliderPosition(low * 100)
self.slider_high.setSliderPosition(high * 100)
self.thresholdsChanged.emit(high, low)
class LabelSelectionWidget(QWidget):
"""
A widget for selection of label values.
The widget displays the contents of the model with two widgets:
* The top level model items are displayed in a combo box.
* The children of the current selected top level item are
displayed in a subordinate list view.
.. note:: This is not a QAbstractItemView subclass.
"""
# Current group/root index has changed.
groupChanged = Signal(int)
# Selection for the current group/root has changed.
groupSelectionChanged = Signal(int)
def __init__(self):
super().__init__()
self.__model = None
self.__selectionMode = QListView.ExtendedSelection
self.__currentIndex = -1
self.__selections = {}
self.__parent = None
self.targets = []
layout = QVBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
def group_box(title):
box = QGroupBox(title)
box.setFlat(True)
lay = QVBoxLayout()
lay.setContentsMargins(0, 0, 0, 0)
box.setLayout(lay)
return box
self.labels_combo = QComboBox()
self.values_view = QListView(selectionMode=self.__selectionMode)
self.labels_combo.currentIndexChanged.connect(
self.__onCurrentIndexChanged)
l_box = group_box(self.tr("Label"))
v_box = group_box(self.tr("Values"))
l_box.layout().addWidget(self.labels_combo)
v_box.layout().addWidget(self.values_view)
layout.addWidget(l_box)
layout.addWidget(v_box)
self.setLayout(layout)
self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
def set_selection(self):
model = self.model()
# FIX: This assumes fixed target key/values order.
indices = [
model.index(i, 0, model.index(keyind, 0))
for keyind, selected in enumerate(self.__parent.stored_selections)
for i in selected
]
selection = itemselection(indices)
self.setCurrentGroupIndex(self.__parent.current_group_index)
self.setSelection(selection)
def selected_split(self):
group_index = self.currentGroupIndex()
if not (0 <= group_index < len(self.targets)):
return None, []
group = self.targets[group_index]
selection = [
ind.row() for ind in self.currentGroupSelection().indexes()
]
return group, selection
def set_data(self, parent, data):
""" Initialize widget state after receiving new data.
"""
self.__parent = parent
# self.__currentIndex = parent.current_group_index
if data is not None:
column_groups, row_groups = group_candidates(data)
self.targets = column_groups + row_groups
model = QStandardItemModel()
for item in [standarditem_from(desc) for desc in self.targets]:
model.appendRow(item)
self.setModel(model)
else:
self.targets = []
self.setModel(None)
def clear(self):
""" Clear the widget/model (same as ``setModel(None)``).
"""
if self.__model is not None:
self.values_view.selectionModel().clearSelection()
self.values_view.selectionModel().selectionChanged.disconnect(
self.__onSelectionChanged)
self.values_view.setModel(None)
self.labels_combo.setModel(QStandardItemModel(self.labels_combo))
self.__currentIndex = -1
self.__selections = {}
self.__model = None
def setModel(self, model):
"""
Set the source model for display.
The model should be a tree model with depth 2.
Parameters
----------
model : QtCore.QAbstractItemModel
"""
if model is self.__model:
return
self.clear()
if model is None:
return
self.__model = model
self.values_view.setModel(model)
self.values_view.setRootIndex(model.index(0, 0))
self.values_view.selectionModel().selectionChanged.connect(
self.__onSelectionChanged)
# will emit the currentIndexChanged (if the model is not empty)
self.labels_combo.setModel(model)
def model(self):
"""
Return the current model.
Returns
-------
model : QtCore.QAbstractItemModel
"""
return self.__model
def setCurrentGroupIndex(self, index):
"""
Set the current selected group/root index row.
Parameters
----------
index : int
Group index.
"""
self.labels_combo.setCurrentIndex(index)
def currentGroupIndex(self):
"""
Return the current selected group/root index row.
Returns
-------
row : index
Current group row index (-1 if there is no current index)
"""
return self.labels_combo.currentIndex()
def setSelection(self, selection):
"""
Set the model item selection.
Parameters
----------
selection : QtCore.QItemSelection
Item selection.
"""
if self.values_view.selectionModel() is not None:
indices = selection.indexes()
pind = defaultdict(list)
for index in indices:
parent = index.parent()
if parent.isValid():
if parent == self.__model.index(parent.row(),
parent.column()):
pind[parent.row()].append(QPersistentModelIndex(index))
else:
warnings.warn("Die Die Die")
else:
# top level index
pass
self.__selections = pind
self.__restoreSelection()
def selection(self):
"""
Return the item selection.
Returns
-------
selection : QtCore.QItemSelection
"""
selection = QItemSelection()
if self.__model is None:
return selection
for pind in chain(*self.__selections.values()):
ind = self.__model.index(pind.row(), pind.column(), pind.parent())
if ind.isValid():
selection.select(ind, ind)
return selection
def currentGroupSelection(self):
"""
Return the item selection for the current group only.
"""
if self.values_view.selectionModel() is not None:
return self.values_view.selectionModel().selection()
else:
return QItemSelection()
def __onCurrentIndexChanged(self, index):
self.__storeSelection(self.__currentIndex,
self.values_view.selectedIndexes())
self.__currentIndex = index
if self.__model is not None:
root = self.__model.index(index, 0)
self.values_view.setRootIndex(root)
self.__restoreSelection()
self.groupChanged.emit(index)
def __onSelectionChanged(self, old, new):
self.__storeSelection(self.__currentIndex,
self.values_view.selectedIndexes())
self.groupSelectionChanged.emit(self.__currentIndex)
def __storeSelection(self, groupind, indices):
# Store current values selection for the current group
groupind = self.__currentIndex
indices = [
QPersistentModelIndex(ind)
for ind in self.values_view.selectedIndexes()
]
self.__selections[groupind] = indices
def __restoreSelection(self):
# Restore previous selection for root (if available)
assert self.__model is not None
groupind = self.__currentIndex
root = self.__model.index(groupind, 0)
sel = self.__selections.get(groupind, [])
indices = [
self.__model.index(pind.row(), pind.column(), root) for pind in sel
if pind.isValid() and pind.parent() == root
]
selection = QItemSelection()
for ind in indices:
selection.select(ind, ind)
self.values_view.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def sizeHint(self):
"""Reimplemented from QWidget.sizeHint"""
return QSize(100, 200)
class OWSplitInstancesWidget(OWWidget):
"""
Widget for splitting instances.
If we have dataset with "quantity" column we can multiply that row
number of times it is present fo r example:
Name | product | quantity
John | icecream | 3
Lisa | pie | 2
Will be transformed to:
Name | product
John | icecream
John | icecream
John | icecream
Lisa | pie
Lisa | pie
"""
# Widget's name as displayed in the canvas
name = "Split instances"
# Short widget description
description = "Lets the user input a number"
# An icon resource file path for this widget
# (a path relative to the module where this widget is defined)
icon = "../img/icon.svg"
category = "Data"
class Inputs:
"""Input fields definition"""
data = Input("Data", Table)
class Outputs:
"""Output fields definition"""
data = Output("Data", Table)
def __init__(self):
self.data = None
# List of all features (model for ComboBox)
self.features = itemmodels.VariableListModel(
["Select Feature"], parent=self)
box = gui.vBox(self.controlArea, "Select ")
self.features_widget = QComboBox(
self.controlArea,
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
)
self.features_widget.setModel(self.features)
self.features_widget.currentIndexChanged.connect(self.feature_changed)
self.button = QPushButton("Split")
self.button.clicked.connect(self.split)
self.button.setEnabled(False)
box.layout().addWidget(self.features_widget)
box.layout().addWidget(self.button)
@Inputs.data
def set_data(self, data: Table):
"""
Handler for input data
:param data: Input Orange Table
:return: None
"""
self.data = data
self.features[:] = ["Select Feature"]
if self.data \
and self.data.domain is not None \
and not self.data.domain.empty():
self.features[:] += [
var for var in self.data.domain.variables
if self.is_integer_col(var)
] # only integer columns may be quantity columns
def selected_col(self)->Optional(str):
"""
Get selected feature value
:return: Selected feature name r None if Nothing selected
"""
index = self.features_widget.currentIndex()
if index < 1:
return None
return self.features[index]
def feature_changed(self):
"""
Handler for changed selected feature value in ComboBox
:return: None
"""
if self.selected_col():
self.button.setEnabled(True)
else:
self.button.setEnabled(False)
def handleNewSignals(self):
if self.data is None:
self.Outputs.data.send(None)
else:
self.feature_changed()
def split(self):
"""
Function for splitting provided data
:return: None
"""
col = self.selected_col().name
# all columns except quantity
without_count_col = tuple(
var for var in self.data.domain.variables
if var.name != col
)
# domain without quantit column
new_domain = Domain(
without_count_col,
self.data.domain.class_vars,
self.data.domain.metas
)
res = Table.from_domain(
new_domain
)
for row in self.data:
res.append(Instance(new_domain, row))
rows_added = 1
if not np.isnan(row[col]):
while rows_added < row[col]:
res.append(Instance(new_domain, row))
rows_added += 1
self.Outputs.data.send(res)
def is_integer_col(self, name: str)->bool:
"""
Check if provided column is integer
:param name: column name
:return: is column integer or not
"""
if not isinstance(self.data.domain[name], ContinuousVariable):
return False
return all([
np.isnan(row[name]) # we count NaN columns as qty 1
or np.equal(np.mod(row[name], 1), 0) # check that column is integer
for row in self.data
])
class NormalizeEditor(ScBaseEditor):
DEFAULT_GROUP_BY = False
DEFAULT_GROUP_VAR = None
DEFAULT_METHOD = Normalize.CPM
def __init__(self, parent=None, master=None, **kwargs):
super().__init__(parent, **kwargs)
self._group_var = self.DEFAULT_GROUP_VAR
self._master = master
self._master.input_data_changed.connect(self._set_model)
self.setLayout(QVBoxLayout())
form = QFormLayout()
cpm_b = QRadioButton("Counts per million", checked=True)
med_b = QRadioButton("Median")
self.group = QButtonGroup()
self.group.buttonClicked.connect(self._on_button_clicked)
for i, button in enumerate([cpm_b, med_b]):
index = index_to_enum(Normalize.Method, i).value
self.group.addButton(button, index - 1)
form.addRow(button)
self.group_by_check = QCheckBox("Cell Groups: ",
enabled=self.DEFAULT_GROUP_BY)
self.group_by_check.clicked.connect(self.edited)
self.group_by_combo = QComboBox(enabled=self.DEFAULT_GROUP_BY)
self.group_by_model = DomainModel(order=(DomainModel.METAS,
DomainModel.CLASSES),
valid_types=DiscreteVariable,
alphabetical=True)
self.group_by_combo.setModel(self.group_by_model)
self.group_by_combo.currentIndexChanged.connect(self.changed)
self.group_by_combo.activated.connect(self.edited)
form.addRow(self.group_by_check, self.group_by_combo)
self.layout().addLayout(form)
self._set_model()
def _set_model(self):
data = self._master.data
self.group_by_model.set_domain(data and data.domain)
enable = bool(self.group_by_model)
self.group_by_check.setChecked(False)
self.group_by_check.setEnabled(enable)
self.group_by_combo.setEnabled(enable)
if self.group_by_model:
self.group_by_combo.setCurrentIndex(0)
if self._group_var and self._group_var in data.domain:
index = self.group_by_model.indexOf(self._group_var)
self.group_by_combo.setCurrentIndex(index)
else:
self.group_by_combo.setCurrentText(None)
def _on_button_clicked(self):
self.changed.emit()
self.edited.emit()
def setParameters(self, params):
method = params.get("method", self.DEFAULT_METHOD)
index = enum_to_index(Normalize.Method, method)
self.group.buttons()[index].setChecked(True)
self._group_var = params.get("group_var", self.DEFAULT_GROUP_VAR)
group = bool(self._group_var and self.group_by_model)
if group:
index = self.group_by_model.indexOf(self._group_var)
self.group_by_combo.setCurrentIndex(index)
group_by = params.get("group_by", self.DEFAULT_GROUP_BY)
self.group_by_check.setChecked(group_by and group)
def parameters(self):
index = self.group_by_combo.currentIndex()
group_var = self.group_by_model[index] if index > -1 else None
group_by = self.group_by_check.isChecked()
method = index_to_enum(Normalize.Method, self.group.checkedId())
return {"group_var": group_var, "group_by": group_by, "method": method}
@staticmethod
def createinstance(params):
group_var = params.get("group_var")
group_by = params.get("group_by", NormalizeEditor.DEFAULT_GROUP_BY)
method = params.get("method", NormalizeEditor.DEFAULT_METHOD)
return NormalizeGroups(group_var, method) \
if group_by and group_var else NormalizeSamples(method)
def __repr__(self):
method = self.group.button(self.group.checkedId()).text()
index = self.group_by_combo.currentIndex()
group_var = self.group_by_model[index] if index > -1 else None
group_by = self.group_by_check.isChecked()
group_text = ", Grouped by: {}".format(group_var) if group_by else ""
return "Method: {}".format(method) + group_text
class ColorGradientSelection(QWidget):
activated = Signal(int)
currentIndexChanged = Signal(int)
thresholdsChanged = Signal(float, float)
centerChanged = Signal(float)
def __init__(self, *args, thresholds=(0.0, 1.0), center=None, **kwargs):
super().__init__(*args, **kwargs)
low = round(clip(thresholds[0], 0., 1.), 2)
high = round(clip(thresholds[1], 0., 1.), 2)
high = max(low, high)
self.__threshold_low, self.__threshold_high = low, high
self.__center = center
form = QFormLayout(formAlignment=Qt.AlignLeft,
labelAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow)
form.setContentsMargins(0, 0, 0, 0)
self.gradient_cb = QComboBox(
None,
objectName="gradient-combo-box",
)
self.gradient_cb.setAttribute(Qt.WA_LayoutUsesWidgetRect)
icsize = self.style().pixelMetric(QStyle.PM_SmallIconSize, None,
self.gradient_cb)
self.gradient_cb.setIconSize(QSize(64, icsize))
model = itemmodels.ContinuousPalettesModel()
model.setParent(self)
self.gradient_cb.setModel(model)
self.gradient_cb.activated[int].connect(self.activated)
self.gradient_cb.currentIndexChanged.connect(self.currentIndexChanged)
if center is not None:
def __on_center_changed():
self.__center = float(self.center_edit.text() or "0")
self.centerChanged.emit(self.__center)
self.center_box = QWidget()
center_layout = QHBoxLayout()
self.center_box.setLayout(center_layout)
width = QFontMetrics(self.font()).boundingRect("9999999").width()
self.center_edit = QLineEdit(text=f"{self.__center}",
maximumWidth=width,
placeholderText="0",
alignment=Qt.AlignRight)
self.center_edit.setValidator(QDoubleValidator())
self.center_edit.editingFinished.connect(__on_center_changed)
center_layout.setContentsMargins(0, 0, 0, 0)
center_layout.addStretch(1)
center_layout.addWidget(QLabel("Centered at"))
center_layout.addWidget(self.center_edit)
self.gradient_cb.currentIndexChanged.connect(
self.__update_center_visibility)
else:
self.center_box = None
slider_low = Slider(objectName="threshold-low-slider",
minimum=0,
maximum=100,
value=int(low * 100),
orientation=Qt.Horizontal,
tickPosition=QSlider.TicksBelow,
pageStep=10,
toolTip=self.tr("Low gradient threshold"),
whatsThis=self.tr(
"Applying a low threshold will squeeze the "
"gradient from the lower end"))
slider_high = Slider(objectName="threshold-low-slider",
minimum=0,
maximum=100,
value=int(high * 100),
orientation=Qt.Horizontal,
tickPosition=QSlider.TicksAbove,
pageStep=10,
toolTip=self.tr("High gradient threshold"),
whatsThis=self.tr(
"Applying a high threshold will squeeze the "
"gradient from the higher end"))
form.setWidget(0, QFormLayout.SpanningRole, self.gradient_cb)
if self.center_box:
form.setWidget(1, QFormLayout.SpanningRole, self.center_box)
form.addRow(self.tr("Low:"), slider_low)
form.addRow(self.tr("High:"), slider_high)
self.slider_low = slider_low
self.slider_high = slider_high
self.slider_low.valueChanged.connect(self.__on_slider_low_moved)
self.slider_high.valueChanged.connect(self.__on_slider_high_moved)
self.setLayout(form)
def setModel(self, model: QAbstractItemModel) -> None:
self.gradient_cb.setModel(model)
def model(self) -> QAbstractItemModel:
return self.gradient_cb.model()
def findData(self, data: Any, role: Qt.ItemDataRole) -> int:
return self.gradient_cb.findData(data, role)
def setCurrentIndex(self, index: int) -> None:
self.gradient_cb.setCurrentIndex(index)
self.__update_center_visibility()
def currentIndex(self) -> int:
return self.gradient_cb.currentIndex()
currentIndex_ = Property(int,
currentIndex,
setCurrentIndex,
notify=currentIndexChanged)
def currentData(self, role=Qt.UserRole) -> Any:
return self.gradient_cb.currentData(role)
def thresholds(self) -> Tuple[float, float]:
return self.__threshold_low, self.__threshold_high
thresholds_ = Property(object, thresholds, notify=thresholdsChanged)
def thresholdLow(self) -> float:
return self.__threshold_low
def setThresholdLow(self, low: float) -> None:
self.setThresholds(low, max(self.__threshold_high, low))
thresholdLow_ = Property(float,
thresholdLow,
setThresholdLow,
notify=thresholdsChanged)
def thresholdHigh(self) -> float:
return self.__threshold_high
def setThresholdHigh(self, high: float) -> None:
self.setThresholds(min(self.__threshold_low, high), high)
def center(self) -> float:
return self.__center
def setCenter(self, center: float) -> None:
self.__center = center
self.center_edit.setText(f"{center}")
self.centerChanged.emit(center)
thresholdHigh_ = Property(float,
thresholdLow,
setThresholdLow,
notify=thresholdsChanged)
def __on_slider_low_moved(self, value: int) -> None:
high = self.slider_high
old = self.__threshold_low, self.__threshold_high
self.__threshold_low = value / 100.
if value >= high.value():
self.__threshold_high = value / 100.
high.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def __on_slider_high_moved(self, value: int) -> None:
low = self.slider_low
old = self.__threshold_low, self.__threshold_high
self.__threshold_high = value / 100.
if low.value() >= value:
self.__threshold_low = value / 100
low.setSliderPosition(value)
new = self.__threshold_low, self.__threshold_high
if new != old:
self.thresholdsChanged.emit(*new)
def setThresholds(self, low: float, high: float) -> None:
low = round(clip(low, 0., 1.), 2)
high = round(clip(high, 0., 1.), 2)
if low > high:
high = low
if self.__threshold_low != low or self.__threshold_high != high:
self.__threshold_high = high
self.__threshold_low = low
self.slider_low.setSliderPosition(low * 100)
self.slider_high.setSliderPosition(high * 100)
self.thresholdsChanged.emit(high, low)
def __update_center_visibility(self):
if self.center_box is None:
return
palette = self.currentData()
self.center_box.setVisible(
isinstance(palette, colorpalettes.Palette)
and palette.flags & palette.Flags.Diverging != 0)
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = Msg("Model based imputer does not work for sparse data")
DEFAULT_LEARNER = SimpleTreeLearner()
METHODS = [AsDefault(), impute.DoNotImpute(), impute.Average(),
impute.AsValue(), impute.Model(DEFAULT_LEARNER), impute.Random(),
impute.DropInstances(), impute.Default()]
DEFAULT, DO_NOT_IMPUTE, MODEL_BASED_IMPUTER, AS_INPUT = 0, 1, 4, 7
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(DO_NOT_IMPUTE)
variable_methods = settings.ContextSetting({})
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
# copy METHODS (some are modified by the widget)
self.methods = copy.deepcopy(OWImpute.METHODS)
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QVBoxLayout(box)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for i, method in enumerate(self.methods):
if not method.columns_only:
button = QRadioButton(method.name)
button.setChecked(i == self.default_method_index)
button_group.addButton(button, i)
box_layout.addWidget(button)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = QListView(
selectionMode=QListView.ExtendedSelection
)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed
)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for i, method in enumerate(self.methods):
button = QRadioButton(text=method.name)
button_group.addButton(button, i)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected
)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000., maximum=1000., singleStep=.1, decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection
)
method_layout.addStretch(2)
reset_button = QPushButton(
"Restore All to Default", checked=False, checkable=False,
clicked=self.reset_variable_methods, default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_commit(
self.controlArea, self, "autocommit", "Apply",
orientation=Qt.Horizontal, checkbox_label="Apply automatically")
box.layout().insertSpacing(0, 80)
box.layout().insertWidget(0, self.report_button)
self.data = None
self.learner = None
self.modified = False
self.default_method = self.methods[self.default_method_index]
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
self.default_method = self.methods[self.default_method_index]
self.methods[self.DEFAULT].method = self.default_method
# update variable view
for index in map(self.varmodel.index, range(len(self.varmodel))):
method = self.variable_methods.get(
index.row(), self.methods[self.DEFAULT])
self.varmodel.setData(index, method, Qt.UserRole)
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.closeContext()
self.varmodel[:] = []
self.variable_methods = {}
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.learner = learner or self.DEFAULT_LEARNER
imputer = self.methods[self.MODEL_BASED_IMPUTER]
imputer.learner = self.learner
button = self.default_button_group.button(self.MODEL_BASED_IMPUTER)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(self.MODEL_BASED_IMPUTER)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = self.MODEL_BASED_IMPUTER
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
"""Returns the imputation method for column by its index.
"""
if not isinstance(column_index, int):
column_index = column_index.row()
return self.variable_methods.get(column_index,
self.methods[self.DEFAULT])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
if self.data is None or len(self.data) == 0 or len(self.varmodel) == 0:
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [
(i, var, self.variable_methods.get(i, self.default_method))
for i, var in enumerate(self.varmodel)
]
def impute_one(method, var, data):
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(
impute_one, copy.deepcopy(method), var, data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
data = self.data
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
assert fut.done()
newvar = []
try:
res = fut.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
# ?? break
except VariableNotSupported:
self.warning("Default method can not handle '{}'".
format(var.name))
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", var, exc_info=True)
self.Error.imputation_failed(var.name)
attributes = class_vars = None
break
else:
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = var
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
if i < len(data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
if attributes is None:
data = None
else:
domain = Orange.data.Domain(attributes, class_vars,
data.domain.metas)
try:
data = self.data.from_table(domain, data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
data = None
self.Outputs.data.send(data)
self.modified = False
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setBlocking(False)
def onDeleteWidget(self):
self.cancel()
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.variable_methods.get(i, None)
if method is not None:
specific.append("{} ({})".format(var.name, str(method)))
default = self.default_method.name
if specific:
self.report_items((
("Default method", default),
("Specific imputers", ", ".join(specific))
))
else:
self.report_items((("Method", default),))
def _on_var_selection_changed(self):
indexes = self.selection.selectedIndexes()
methods = [self.get_method_for_column(i.row()) for i in indexes]
def method_key(method):
"""
Decompose method into its type and parameters.
"""
# The return value should be hashable and __eq__ comparable
if isinstance(method, AsDefault):
return AsDefault, (method.method,)
elif isinstance(method, impute.Model):
return impute.Model, (method.learner,)
elif isinstance(method, impute.Default):
return impute.Default, (method.default,)
else:
return type(method), None
methods = set(method_key(m) for m in methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for i, m in enumerate(self.methods):
if method_type == type(m):
self.variable_button_group.button(i).setChecked(True)
if method_type is impute.Default:
(fixed_value,) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
for method, button in zip(self.methods,
self.variable_button_group.buttons()):
enabled = all(method.supports_variable(var) for var in
selected_vars)
button.setEnabled(enabled)
if not has_discrete:
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(self.AS_INPUT).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
if method_index == self.DEFAULT:
for index in indexes:
self.variable_methods.pop(index.row(), None)
elif method_index == OWImpute.AS_INPUT:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
method = impute.Default(default=value)
self.variable_methods[index.row()] = method
else:
method = self.methods[method_index]
for index in indexes:
self.variable_methods[index.row()] = method
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(index, self.get_method_for_column(index.row()), Qt.UserRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(self.AS_INPUT).setChecked(True)
self.set_method_for_current_selection(self.AS_INPUT)
def reset_variable_methods(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, self.DEFAULT)
self.variable_button_group.button(self.DEFAULT).setChecked(True)
class FeatureEditor(QFrame):
FUNCTIONS = dict(chain([(key, val) for key, val in math.__dict__.items()
if not key.startswith("_")],
[(key, val) for key, val in builtins.__dict__.items()
if key in {"str", "float", "int", "len",
"abs", "max", "min"}]))
featureChanged = Signal()
featureEdited = Signal()
modifiedChanged = Signal(bool)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
layout = QFormLayout(
fieldGrowthPolicy=QFormLayout.ExpandingFieldsGrow
)
layout.setContentsMargins(0, 0, 0, 0)
self.nameedit = QLineEdit(
placeholderText="Name...",
sizePolicy=QSizePolicy(QSizePolicy.Minimum,
QSizePolicy.Fixed)
)
self.expressionedit = QLineEdit(
placeholderText="Expression..."
)
self.attrs_model = itemmodels.VariableListModel(
["Select Feature"], parent=self)
self.attributescb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
)
self.attributescb.setModel(self.attrs_model)
sorted_funcs = sorted(self.FUNCTIONS)
self.funcs_model = itemmodels.PyListModelTooltip()
self.funcs_model.setParent(self)
self.funcs_model[:] = chain(["Select Function"], sorted_funcs)
self.funcs_model.tooltips[:] = chain(
[''],
[self.FUNCTIONS[func].__doc__ for func in sorted_funcs])
self.functionscb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
self.functionscb.setModel(self.funcs_model)
hbox = QHBoxLayout()
hbox.addWidget(self.attributescb)
hbox.addWidget(self.functionscb)
layout.addRow(self.nameedit, self.expressionedit)
layout.addRow(self.tr(""), hbox)
self.setLayout(layout)
self.nameedit.editingFinished.connect(self._invalidate)
self.expressionedit.textChanged.connect(self._invalidate)
self.attributescb.currentIndexChanged.connect(self.on_attrs_changed)
self.functionscb.currentIndexChanged.connect(self.on_funcs_changed)
self._modified = False
def setModified(self, modified):
if not type(modified) is bool:
raise TypeError
if self._modified != modified:
self._modified = modified
self.modifiedChanged.emit(modified)
def modified(self):
return self._modified
modified = Property(bool, modified, setModified,
notify=modifiedChanged)
def setEditorData(self, data, domain):
self.nameedit.setText(data.name)
self.expressionedit.setText(data.expression)
self.setModified(False)
self.featureChanged.emit()
self.attrs_model[:] = ["Select Feature"]
if domain is not None and (domain or domain.metas):
self.attrs_model[:] += chain(domain.attributes,
domain.class_vars,
domain.metas)
def editorData(self):
return FeatureDescriptor(name=self.nameedit.text(),
expression=self.nameedit.text())
def _invalidate(self):
self.setModified(True)
self.featureEdited.emit()
self.featureChanged.emit()
def on_attrs_changed(self):
index = self.attributescb.currentIndex()
if index > 0:
attr = sanitized_name(self.attrs_model[index].name)
self.insert_into_expression(attr)
self.attributescb.setCurrentIndex(0)
def on_funcs_changed(self):
index = self.functionscb.currentIndex()
if index > 0:
func = self.funcs_model[index]
if func in ["atan2", "fmod", "ldexp", "log",
"pow", "copysign", "hypot"]:
self.insert_into_expression(func + "(,)")
self.expressionedit.cursorBackward(False, 2)
elif func in ["e", "pi"]:
self.insert_into_expression(func)
else:
self.insert_into_expression(func + "()")
self.expressionedit.cursorBackward(False)
self.functionscb.setCurrentIndex(0)
def insert_into_expression(self, what):
cp = self.expressionedit.cursorPosition()
ct = self.expressionedit.text()
text = ct[:cp] + what + ct[cp:]
self.expressionedit.setText(text)
self.expressionedit.setFocus()
class FileLoader(QWidget):
activated = pyqtSignal()
file_loaded = pyqtSignal()
def __init__(self):
super().__init__()
self.recent_paths = []
self.file_combo = QComboBox()
self.file_combo.setMinimumWidth(80)
self.file_combo.activated.connect(self._activate)
self.browse_btn = QPushButton("...")
icon = self.style().standardIcon(QStyle.SP_DirOpenIcon)
self.browse_btn.setIcon(icon)
self.browse_btn.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Fixed)
self.browse_btn.clicked.connect(self.browse)
self.load_btn = QPushButton("")
icon = self.style().standardIcon(QStyle.SP_BrowserReload)
self.load_btn.setIcon(icon)
self.load_btn.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Fixed)
self.load_btn.setAutoDefault(True)
self.load_btn.clicked.connect(self.file_loaded)
def browse(self):
start_file = self.last_path() or os.path.expanduser("~/")
formats = ["Text files (*.txt)", "All files (*)"]
file_name, _ = QFileDialog.getOpenFileName(None, "Open...", start_file,
";;".join(formats),
formats[0])
if not file_name:
return
self.add_path(file_name)
self._activate()
def _activate(self):
self.activated.emit()
self.file_loaded.emit()
def set_current_file(self, path: str):
if path:
self.add_path(path)
self.file_combo.setCurrentText(path)
else:
self.file_combo.setCurrentText("(none)")
def get_current_file(self) -> Optional[RecentPath]:
index = self.file_combo.currentIndex()
if index >= len(self.recent_paths) or index < 0:
return None
path = self.recent_paths[index]
return path if isinstance(path, RecentPath) else None
def add_path(self, filename: str):
recent = RecentPath.create(filename, [])
if recent in self.recent_paths:
self.recent_paths.remove(recent)
self.recent_paths.insert(0, recent)
self.set_file_list()
def set_file_list(self):
self.file_combo.clear()
for i, recent in enumerate(self.recent_paths):
self.file_combo.addItem(recent.basename)
self.file_combo.model().item(i).setToolTip(recent.abspath)
if not os.path.exists(recent.abspath):
self.file_combo.setItemData(i, QBrush(Qt.red),
Qt.TextColorRole)
self.file_combo.addItem(_DEFAULT_NONE)
def last_path(self) -> Optional[str]:
return self.recent_paths[0].abspath if self.recent_paths else None
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
keywords = ["substitute", "missing"]
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = \
Msg("Model based imputer does not work for sparse data")
class Warning(OWWidget.Warning):
cant_handle_var = Msg("Default method can not handle '{}'")
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(int(Method.Leave)) # type: int
# Per-variable imputation state (synced in storeSpecificSettings)
_variable_imputation_state = settings.ContextSetting(
{}) # type: VariableState
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
self.learner = None # type: Optional[Learner]
self.default_learner = SimpleTreeLearner(min_instances=10,
max_depth=10)
self.modified = False
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QGridLayout(box)
box_layout.setContentsMargins(5, 0, 0, 0)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for i, (method, _) in enumerate(list(METHODS.items())[1:-1]):
imputer = self.create_imputer(method)
button = QRadioButton(imputer.name)
button.setChecked(method == self.default_method_index)
button_group.addButton(button, method)
box_layout.addWidget(button, i % 3, i // 3)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = ListViewSearch(
selectionMode=QListView.ExtendedSelection, uniformItemSizes=True)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for method in Method:
imputer = self.create_imputer(method)
button = QRadioButton(text=imputer.name)
button_group.addButton(button, method)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000.,
maximum=1000.,
singleStep=.1,
decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection)
method_layout.addStretch(2)
reset_button = QPushButton("Restore All to Default",
checked=False,
checkable=False,
clicked=self.reset_variable_state,
default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_apply(self.controlArea, self, "autocommit")
box.button.setFixedWidth(180)
box.layout().insertStretch(0)
self.info.set_input_summary(self.info.NoInput)
self.info.set_output_summary(self.info.NoOutput)
def create_imputer(self, method, *args):
# type: (Method, ...) -> impute.BaseImputeMethod
if method == Method.Model:
if self.learner is not None:
return impute.Model(self.learner)
else:
return impute.Model(self.default_learner)
elif method == Method.AsAboveSoBelow:
assert self.default_method_index != Method.AsAboveSoBelow
default = self.create_imputer(Method(self.default_method_index))
m = AsDefault()
m.method = default
return m
else:
return METHODS[method](*args)
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
assert index != Method.AsAboveSoBelow
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
# update variable view
self.update_varview()
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.cancel()
self.closeContext()
self.varmodel[:] = []
self._variable_imputation_state = {} # type: VariableState
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
# restore per variable imputation state
self._restore_state(self._variable_imputation_state)
summary = len(data) if data else self.info.NoInput
details = format_summary_details(data) if data else ""
self.info.set_input_summary(summary, details)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.cancel()
self.learner = learner or self.default_learner
imputer = self.create_imputer(Method.Model)
button = self.default_button_group.button(Method.Model)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(Method.Model)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = Method.Model
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
# type: (int) -> impute.BaseImputeMethod
"""
Return the imputation method for column by its index.
"""
assert 0 <= column_index < len(self.varmodel)
idx = self.varmodel.index(column_index, 0)
state = idx.data(StateRole)
if state is None:
state = (Method.AsAboveSoBelow, ())
return self.create_imputer(state[0], *state[1])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
summary = len(self.data) if self.data else self.info.NoOutput
detail = format_summary_details(self.data) if self.data else ""
self.info.set_output_summary(summary, detail)
if not self.data or not self.varmodel.rowCount():
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [(i, var, self.get_method_for_column(i))
for i, var in enumerate(self.varmodel)]
# normalize to the effective method bypasing AsDefault
impute_state = [(i, var, m.method if isinstance(m, AsDefault) else m)
for i, var, m in impute_state]
def impute_one(method, var, data):
# Readability counts, pylint: disable=no-else-raise
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(impute_one, copy.deepcopy(method), var,
data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit()
self.setInvalidated(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
def get_variable(variable, future, drop_mask) \
-> Optional[List[Orange.data.Variable]]:
# Returns a (potentially empty) list of variables,
# or None on failure that should interrupt the imputation
assert future.done()
try:
res = future.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
return [] # None?
except VariableNotSupported:
self.Warning.cant_handle_var(variable.name)
return []
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", variable.name, exc_info=True)
self.Error.imputation_failed(variable.name)
return None
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = variable
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
return newvar
def create_data(attributes, class_vars):
domain = Orange.data.Domain(attributes, class_vars,
self.data.domain.metas)
try:
return self.data.from_table(domain, self.data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
return None
self.__task = None
self.setInvalidated(False)
self.progressBarFinished()
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
newvar = get_variable(var, fut, drop_mask)
if newvar is None:
data = None
break
if i < len(self.data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
else:
data = create_data(attributes, class_vars)
self.Outputs.data.send(data)
self.modified = False
summary = len(data) if data else self.info.NoOutput
details = format_summary_details(data) if data else ""
self.info.set_output_summary(summary, details)
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
self.__cancel(wait=False)
def __cancel(self, wait=False):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
if wait:
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setInvalidated(False)
def onDeleteWidget(self):
self.__cancel(wait=True)
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.get_method_for_column(i)
if not isinstance(method, AsDefault):
specific.append("{} ({})".format(var.name, str(method)))
default = self.create_imputer(Method.AsAboveSoBelow)
if specific:
self.report_items((("Default method", default.name),
("Specific imputers", ", ".join(specific))))
else:
self.report_items((("Method", default.name), ))
def _on_var_selection_changed(self):
# Method is well documented, splitting it is not needed for readability,
# thus pylint: disable=too-many-branches
indexes = self.selection.selectedIndexes()
defmethod = (Method.AsAboveSoBelow, ())
methods = [index.data(StateRole) for index in indexes]
methods = [m if m is not None else defmethod for m in methods]
methods = set(methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for m in Method:
if method_type == m:
self.variable_button_group.button(m).setChecked(True)
if method_type == Method.Default:
(fixed_value, ) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
# Update variable methods GUI enabled state based on selection.
for method in Method:
# use a default constructed imputer to query support
imputer = self.create_imputer(method)
enabled = all(
imputer.supports_variable(var) for var in selected_vars)
button = self.variable_button_group.button(method)
button.setEnabled(enabled)
# Update the "Value" edit GUI.
if not has_discrete:
# no discrete variables -> allow mass edit for all (continuous vars)
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
# single discrete var -> enable and fill the values combo
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
# mixed type selection -> disable
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(Method.Default).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
# set current value
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
# type: (Method) -> None
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
# type: (List[QModelIndex], Method) -> None
if method_index == Method.AsAboveSoBelow:
for index in indexes:
self.varmodel.setData(index, None, StateRole)
elif method_index == Method.Default:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
state = (int(Method.Default), (value, ))
self.varmodel.setData(index, state, StateRole)
else:
state = (int(method_index), ())
for index in indexes:
self.varmodel.setData(index, state, StateRole)
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(index,
self.get_method_for_column(index.row()),
DisplayMethodRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(Method.Default).setChecked(True)
self.set_method_for_current_selection(Method.Default)
def reset_variable_state(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, Method.AsAboveSoBelow)
self.variable_button_group.button(
Method.AsAboveSoBelow).setChecked(True)
def _store_state(self):
# type: () -> VariableState
"""
Save the current variable imputation state
"""
state = {} # type: VariableState
for i, var in enumerate(self.varmodel):
index = self.varmodel.index(i)
m = index.data(StateRole)
if m is not None:
state[var_key(var)] = m
return state
def _restore_state(self, state):
# type: (VariableState) -> None
"""
Restore the variable imputation state from the saved state
"""
def check(state):
# check if state is a proper State
if isinstance(state, tuple) and len(state) == 2:
m, p = state
if isinstance(m, int) and isinstance(p, tuple) and \
0 <= m < len(Method):
return True
return False
for i, var in enumerate(self.varmodel):
m = state.get(var_key(var), None)
if check(m):
self.varmodel.setData(self.varmodel.index(i), m, StateRole)
def storeSpecificSettings(self):
self._variable_imputation_state = self._store_state()
super().storeSpecificSettings()
class OWImpute(OWWidget):
name = "Impute"
description = "Impute missing values in the data table."
icon = "icons/Impute.svg"
priority = 2130
class Inputs:
data = Input("Data", Orange.data.Table)
learner = Input("Learner", Learner)
class Outputs:
data = Output("Data", Orange.data.Table)
class Error(OWWidget.Error):
imputation_failed = Msg("Imputation failed for '{}'")
model_based_imputer_sparse = Msg("Model based imputer does not work for sparse data")
DEFAULT_LEARNER = SimpleTreeLearner()
METHODS = [AsDefault(), impute.DoNotImpute(), impute.Average(),
impute.AsValue(), impute.Model(DEFAULT_LEARNER), impute.Random(),
impute.DropInstances(), impute.Default()]
DEFAULT, DO_NOT_IMPUTE, MODEL_BASED_IMPUTER, AS_INPUT = 0, 1, 4, 7
settingsHandler = settings.DomainContextHandler()
_default_method_index = settings.Setting(DO_NOT_IMPUTE)
variable_methods = settings.ContextSetting({})
autocommit = settings.Setting(True)
want_main_area = False
resizing_enabled = False
def __init__(self):
super().__init__()
# copy METHODS (some are modified by the widget)
self.methods = copy.deepcopy(OWImpute.METHODS)
main_layout = QVBoxLayout()
main_layout.setContentsMargins(10, 10, 10, 10)
self.controlArea.layout().addLayout(main_layout)
box = QGroupBox(title=self.tr("Default Method"), flat=False)
box_layout = QVBoxLayout(box)
main_layout.addWidget(box)
button_group = QButtonGroup()
button_group.buttonClicked[int].connect(self.set_default_method)
for i, method in enumerate(self.methods):
if not method.columns_only:
button = QRadioButton(method.name)
button.setChecked(i == self.default_method_index)
button_group.addButton(button, i)
box_layout.addWidget(button)
self.default_button_group = button_group
box = QGroupBox(title=self.tr("Individual Attribute Settings"),
flat=False)
main_layout.addWidget(box)
horizontal_layout = QHBoxLayout(box)
main_layout.addWidget(box)
self.varview = QListView(
selectionMode=QListView.ExtendedSelection
)
self.varview.setItemDelegate(DisplayFormatDelegate())
self.varmodel = itemmodels.VariableListModel()
self.varview.setModel(self.varmodel)
self.varview.selectionModel().selectionChanged.connect(
self._on_var_selection_changed
)
self.selection = self.varview.selectionModel()
horizontal_layout.addWidget(self.varview)
method_layout = QVBoxLayout()
horizontal_layout.addLayout(method_layout)
button_group = QButtonGroup()
for i, method in enumerate(self.methods):
button = QRadioButton(text=method.name)
button_group.addButton(button, i)
method_layout.addWidget(button)
self.value_combo = QComboBox(
minimumContentsLength=8,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength,
activated=self._on_value_selected
)
self.value_double = QDoubleSpinBox(
editingFinished=self._on_value_selected,
minimum=-1000., maximum=1000., singleStep=.1, decimals=3,
)
self.value_stack = value_stack = QStackedWidget()
value_stack.addWidget(self.value_combo)
value_stack.addWidget(self.value_double)
method_layout.addWidget(value_stack)
button_group.buttonClicked[int].connect(
self.set_method_for_current_selection
)
method_layout.addStretch(2)
reset_button = QPushButton(
"Restore All to Default", checked=False, checkable=False,
clicked=self.reset_variable_methods, default=False,
autoDefault=False)
method_layout.addWidget(reset_button)
self.variable_button_group = button_group
box = gui.auto_commit(
self.controlArea, self, "autocommit", "Apply",
orientation=Qt.Horizontal,
checkbox_label="Apply automatically")
box.button.setFixedWidth(180)
box.layout().insertStretch(0)
self.data = None
self.learner = None
self.modified = False
self.default_method = self.methods[self.default_method_index]
self.executor = qconcurrent.ThreadExecutor(self)
self.__task = None
@property
def default_method_index(self):
return self._default_method_index
@default_method_index.setter
def default_method_index(self, index):
if self._default_method_index != index:
self._default_method_index = index
self.default_button_group.button(index).setChecked(True)
self.default_method = self.methods[self.default_method_index]
self.methods[self.DEFAULT].method = self.default_method
# update variable view
for index in map(self.varmodel.index, range(len(self.varmodel))):
method = self.variable_methods.get(
index.row(), self.methods[self.DEFAULT])
self.varmodel.setData(index, method, Qt.UserRole)
self._invalidate()
def set_default_method(self, index):
"""Set the current selected default imputation method.
"""
self.default_method_index = index
@Inputs.data
@check_sql_input
def set_data(self, data):
self.closeContext()
self.varmodel[:] = []
self.variable_methods = {}
self.modified = False
self.data = data
if data is not None:
self.varmodel[:] = data.domain.variables
self.openContext(data.domain)
self.update_varview()
self.unconditional_commit()
@Inputs.learner
def set_learner(self, learner):
self.learner = learner or self.DEFAULT_LEARNER
imputer = self.methods[self.MODEL_BASED_IMPUTER]
imputer.learner = self.learner
button = self.default_button_group.button(self.MODEL_BASED_IMPUTER)
button.setText(imputer.name)
variable_button = self.variable_button_group.button(self.MODEL_BASED_IMPUTER)
variable_button.setText(imputer.name)
if learner is not None:
self.default_method_index = self.MODEL_BASED_IMPUTER
self.update_varview()
self.commit()
def get_method_for_column(self, column_index):
"""Returns the imputation method for column by its index.
"""
if not isinstance(column_index, int):
column_index = column_index.row()
return self.variable_methods.get(column_index,
self.methods[self.DEFAULT])
def _invalidate(self):
self.modified = True
if self.__task is not None:
self.cancel()
self.commit()
def commit(self):
self.cancel()
self.warning()
self.Error.imputation_failed.clear()
self.Error.model_based_imputer_sparse.clear()
if self.data is None or len(self.data) == 0 or len(self.varmodel) == 0:
self.Outputs.data.send(self.data)
self.modified = False
return
data = self.data
impute_state = [
(i, var, self.variable_methods.get(i, self.default_method))
for i, var in enumerate(self.varmodel)
]
def impute_one(method, var, data):
# type: (impute.BaseImputeMethod, Variable, Table) -> Any
if isinstance(method, impute.Model) and data.is_sparse():
raise SparseNotSupported()
elif isinstance(method, impute.DropInstances):
return RowMask(method(data, var))
elif not method.supports_variable(var):
raise VariableNotSupported(var)
else:
return method(data, var)
futures = []
for _, var, method in impute_state:
f = self.executor.submit(
impute_one, copy.deepcopy(method), var, data)
futures.append(f)
w = qconcurrent.FutureSetWatcher(futures)
w.doneAll.connect(self.__commit_finish)
w.progressChanged.connect(self.__progress_changed)
self.__task = Task(futures, w)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
@Slot()
def __commit_finish(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
futures = self.__task.futures
assert len(futures) == len(self.varmodel)
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
data = self.data
attributes = []
class_vars = []
drop_mask = np.zeros(len(self.data), bool)
for i, (var, fut) in enumerate(zip(self.varmodel, futures)):
assert fut.done()
newvar = []
try:
res = fut.result()
except SparseNotSupported:
self.Error.model_based_imputer_sparse()
# ?? break
except VariableNotSupported:
self.warning("Default method can not handle '{}'".
format(var.name))
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error for %s", var, exc_info=True)
self.Error.imputation_failed(var.name)
attributes = class_vars = None
break
else:
if isinstance(res, RowMask):
drop_mask |= res.mask
newvar = var
else:
newvar = res
if isinstance(newvar, Orange.data.Variable):
newvar = [newvar]
if i < len(data.domain.attributes):
attributes.extend(newvar)
else:
class_vars.extend(newvar)
if attributes is None:
data = None
else:
domain = Orange.data.Domain(attributes, class_vars,
data.domain.metas)
try:
data = self.data.from_table(domain, data[~drop_mask])
except Exception: # pylint: disable=broad-except
log = logging.getLogger(__name__)
log.info("Error", exc_info=True)
self.Error.imputation_failed("Unknown")
data = None
self.Outputs.data.send(data)
self.modified = False
@Slot(int, int)
def __progress_changed(self, n, d):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
self.progressBarSet(100. * n / d)
def cancel(self):
if self.__task is not None:
task, self.__task = self.__task, None
task.cancel()
task.watcher.doneAll.disconnect(self.__commit_finish)
task.watcher.progressChanged.disconnect(self.__progress_changed)
concurrent.futures.wait(task.futures)
task.watcher.flush()
self.progressBarFinished()
self.setBlocking(False)
def onDeleteWidget(self):
self.cancel()
super().onDeleteWidget()
def send_report(self):
specific = []
for i, var in enumerate(self.varmodel):
method = self.variable_methods.get(i, None)
if method is not None:
specific.append("{} ({})".format(var.name, str(method)))
default = self.default_method.name
if specific:
self.report_items((
("Default method", default),
("Specific imputers", ", ".join(specific))
))
else:
self.report_items((("Method", default),))
def _on_var_selection_changed(self):
indexes = self.selection.selectedIndexes()
methods = [self.get_method_for_column(i.row()) for i in indexes]
def method_key(method):
"""
Decompose method into its type and parameters.
"""
# The return value should be hashable and __eq__ comparable
if isinstance(method, AsDefault):
return AsDefault, (method.method,)
elif isinstance(method, impute.Model):
return impute.Model, (method.learner,)
elif isinstance(method, impute.Default):
return impute.Default, (method.default,)
else:
return type(method), None
methods = set(method_key(m) for m in methods)
selected_vars = [self.varmodel[index.row()] for index in indexes]
has_discrete = any(var.is_discrete for var in selected_vars)
fixed_value = None
value_stack_enabled = False
current_value_widget = None
if len(methods) == 1:
method_type, parameters = methods.pop()
for i, m in enumerate(self.methods):
if method_type == type(m):
self.variable_button_group.button(i).setChecked(True)
if method_type is impute.Default:
(fixed_value,) = parameters
elif self.variable_button_group.checkedButton() is not None:
# Uncheck the current button
self.variable_button_group.setExclusive(False)
self.variable_button_group.checkedButton().setChecked(False)
self.variable_button_group.setExclusive(True)
assert self.variable_button_group.checkedButton() is None
for method, button in zip(self.methods,
self.variable_button_group.buttons()):
enabled = all(method.supports_variable(var) for var in
selected_vars)
button.setEnabled(enabled)
if not has_discrete:
value_stack_enabled = True
current_value_widget = self.value_double
elif len(selected_vars) == 1:
value_stack_enabled = True
current_value_widget = self.value_combo
self.value_combo.clear()
self.value_combo.addItems(selected_vars[0].values)
else:
value_stack_enabled = False
current_value_widget = None
self.variable_button_group.button(self.AS_INPUT).setEnabled(False)
self.value_stack.setEnabled(value_stack_enabled)
if current_value_widget is not None:
self.value_stack.setCurrentWidget(current_value_widget)
if fixed_value is not None:
if current_value_widget is self.value_combo:
self.value_combo.setCurrentIndex(fixed_value)
elif current_value_widget is self.value_double:
self.value_double.setValue(fixed_value)
else:
assert False
def set_method_for_current_selection(self, method_index):
indexes = self.selection.selectedIndexes()
self.set_method_for_indexes(indexes, method_index)
def set_method_for_indexes(self, indexes, method_index):
if method_index == self.DEFAULT:
for index in indexes:
self.variable_methods.pop(index.row(), None)
elif method_index == OWImpute.AS_INPUT:
current = self.value_stack.currentWidget()
if current is self.value_combo:
value = self.value_combo.currentIndex()
else:
value = self.value_double.value()
for index in indexes:
method = impute.Default(default=value)
self.variable_methods[index.row()] = method
else:
method = self.methods[method_index]
for index in indexes:
self.variable_methods[index.row()] = method
self.update_varview(indexes)
self._invalidate()
def update_varview(self, indexes=None):
if indexes is None:
indexes = map(self.varmodel.index, range(len(self.varmodel)))
for index in indexes:
self.varmodel.setData(index, self.get_method_for_column(index.row()), Qt.UserRole)
def _on_value_selected(self):
# The fixed 'Value' in the widget has been changed by the user.
self.variable_button_group.button(self.AS_INPUT).setChecked(True)
self.set_method_for_current_selection(self.AS_INPUT)
def reset_variable_methods(self):
indexes = list(map(self.varmodel.index, range(len(self.varmodel))))
self.set_method_for_indexes(indexes, self.DEFAULT)
self.variable_button_group.button(self.DEFAULT).setChecked(True)
class OWFile(widget.OWWidget, RecentPathsWComboMixin):
name = "File"
id = "orange.widgets.data.file"
description = "Read data from an input file or network " \
"and send a data table to the output."
icon = "icons/File.svg"
priority = 10
category = "Data"
keywords = ["file", "load", "read", "open"]
class Outputs:
data = Output("Data",
Table,
doc="Attribute-valued dataset read from the input file.")
want_main_area = False
buttons_area_orientation = None
SEARCH_PATHS = [("sample-datasets", get_sample_datasets_dir())]
SIZE_LIMIT = 1e7
LOCAL_FILE, URL = range(2)
settingsHandler = PerfectDomainContextHandler(
match_values=PerfectDomainContextHandler.MATCH_VALUES_ALL)
# pylint seems to want declarations separated from definitions
recent_paths: List[RecentPath]
recent_urls: List[str]
variables: list
# Overload RecentPathsWidgetMixin.recent_paths to set defaults
recent_paths = Setting([
RecentPath("", "sample-datasets", "iris.tab"),
RecentPath("", "sample-datasets", "titanic.tab"),
RecentPath("", "sample-datasets", "housing.tab"),
RecentPath("", "sample-datasets", "heart_disease.tab"),
RecentPath("", "sample-datasets", "brown-selected.tab"),
RecentPath("", "sample-datasets", "zoo.tab"),
])
recent_urls = Setting([])
source = Setting(LOCAL_FILE)
sheet_names = Setting({})
url = Setting("")
variables = ContextSetting([])
domain_editor = SettingProvider(DomainEditor)
class Information(widget.OWWidget.Information):
no_file_selected = Msg("No file selected.")
class Warning(widget.OWWidget.Warning):
file_too_big = Msg("The file is too large to load automatically."
" Press Reload to load.")
load_warning = Msg("Read warning:\n{}")
performance_warning = Msg(
"Categorical variables with >100 values may decrease performance.")
renamed_vars = Msg("Some variables have been renamed "
"to avoid duplicates.\n{}")
multiple_targets = Msg("Most widgets do not support multiple targets")
class Error(widget.OWWidget.Error):
file_not_found = Msg("File not found.")
missing_reader = Msg("Missing reader.")
sheet_error = Msg("Error listing available sheets.")
unknown = Msg("Read error:\n{}")
UserAdviceMessages = [
widget.Message(
"Use CSV File Import widget for advanced options "
"for comma-separated files", "use-csv-file-import"),
widget.Message(
"This widget loads only tabular data. Use other widgets to load "
"other data types like models, distance matrices and networks.",
"other-data-types")
]
def __init__(self):
super().__init__()
RecentPathsWComboMixin.__init__(self)
self.domain = None
self.data = None
self.loaded_file = ""
self.reader = None
readers = [
f for f in FileFormat.formats
if getattr(f, 'read', None) and getattr(f, "EXTENSIONS", None)
]
def group_readers_per_addon_key(w):
# readers from Orange.data.io should go first
def package(w):
package = w.qualified_name().split(".")[:-1]
package = package[:2]
if ".".join(package) == "Orange.data":
return ["0"] # force "Orange" to come first
return package
return package(w), w.DESCRIPTION
self.available_readers = sorted(set(readers),
key=group_readers_per_addon_key)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='Source')
vbox = gui.radioButtons(None,
self,
"source",
box=True,
callback=self.load_data,
addToLayout=False)
rb_button = gui.appendRadioButton(vbox, "File:", addToLayout=False)
layout.addWidget(rb_button, 0, 0, Qt.AlignVCenter)
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.file_combo.activated[int].connect(self.select_file)
box.layout().addWidget(self.file_combo)
layout.addWidget(box, 0, 1)
file_button = gui.button(None,
self,
'...',
callback=self.browse_file,
autoDefault=False)
file_button.setIcon(self.style().standardIcon(QStyle.SP_DirOpenIcon))
file_button.setSizePolicy(Policy.Maximum, Policy.Fixed)
layout.addWidget(file_button, 0, 2)
reload_button = gui.button(None,
self,
"Reload",
callback=self.load_data,
autoDefault=False)
reload_button.setIcon(self.style().standardIcon(
QStyle.SP_BrowserReload))
reload_button.setSizePolicy(Policy.Fixed, Policy.Fixed)
layout.addWidget(reload_button, 0, 3)
self.sheet_box = gui.hBox(None, addToLayout=False, margin=0)
self.sheet_combo = QComboBox()
self.sheet_combo.activated[str].connect(self.select_sheet)
self.sheet_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_label = QLabel()
self.sheet_label.setText('Sheet')
self.sheet_label.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.sheet_box.layout().addWidget(self.sheet_label, Qt.AlignLeft)
self.sheet_box.layout().addWidget(self.sheet_combo, Qt.AlignVCenter)
layout.addWidget(self.sheet_box, 2, 1)
self.sheet_box.hide()
rb_button = gui.appendRadioButton(vbox, "URL:", addToLayout=False)
layout.addWidget(rb_button, 3, 0, Qt.AlignVCenter)
self.url_combo = url_combo = QComboBox()
url_model = NamedURLModel(self.sheet_names)
url_model.wrap(self.recent_urls)
url_combo.setLineEdit(LineEditSelectOnFocus())
url_combo.setModel(url_model)
url_combo.setSizePolicy(Policy.Ignored, Policy.Fixed)
url_combo.setEditable(True)
url_combo.setInsertPolicy(url_combo.InsertAtTop)
url_edit = url_combo.lineEdit()
l, t, r, b = url_edit.getTextMargins()
url_edit.setTextMargins(l + 5, t, r, b)
layout.addWidget(url_combo, 3, 1, 1, 3)
url_combo.activated.connect(self._url_set)
# whit completer we set that combo box is case sensitive when
# matching the history
completer = QCompleter()
completer.setCaseSensitivity(Qt.CaseSensitive)
url_combo.setCompleter(completer)
layout = QGridLayout()
layout.setSpacing(4)
gui.widgetBox(self.controlArea, orientation=layout, box='File Type')
box = gui.hBox(None, addToLayout=False, margin=0)
box.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.reader_combo = QComboBox(self)
self.reader_combo.setSizePolicy(Policy.MinimumExpanding, Policy.Fixed)
self.reader_combo.activated[int].connect(self.select_reader)
box.layout().addWidget(self.reader_combo)
layout.addWidget(box, 0, 1)
box = gui.vBox(self.controlArea, "Info")
self.infolabel = gui.widgetLabel(box, 'No data loaded.')
box = gui.widgetBox(self.controlArea, "Columns (Double click to edit)")
self.domain_editor = DomainEditor(self)
self.editor_model = self.domain_editor.model()
box.layout().addWidget(self.domain_editor)
box = gui.hBox(box)
gui.button(box,
self,
"Reset",
callback=self.reset_domain_edit,
autoDefault=False)
gui.rubber(box)
self.apply_button = gui.button(box,
self,
"Apply",
callback=self.apply_domain_edit)
self.apply_button.setEnabled(False)
self.apply_button.setFixedWidth(170)
self.editor_model.dataChanged.connect(
lambda: self.apply_button.setEnabled(True))
hBox = gui.hBox(self.controlArea)
gui.rubber(hBox)
gui.button(hBox,
self,
"Browse documentation datasets",
callback=lambda: self.browse_file(True),
autoDefault=False)
gui.rubber(hBox)
self.set_file_list()
# Must not call open_file from within __init__. open_file
# explicitly re-enters the event loop (by a progress bar)
self.setAcceptDrops(True)
if self.source == self.LOCAL_FILE:
last_path = self.last_path()
if last_path and os.path.exists(last_path) and \
os.path.getsize(last_path) > self.SIZE_LIMIT:
self.Warning.file_too_big()
return
QTimer.singleShot(0, self.load_data)
@staticmethod
def sizeHint():
return QSize(600, 550)
def select_file(self, n):
assert n < len(self.recent_paths)
super().select_file(n)
if self.recent_paths:
self.source = self.LOCAL_FILE
self.load_data()
self.set_file_list()
def select_sheet(self):
self.recent_paths[0].sheet = self.sheet_combo.currentText()
self.load_data()
def select_reader(self, n):
if self.source != self.LOCAL_FILE:
return # ignore for URL's
if self.recent_paths:
path = self.recent_paths[0]
if n == 0: # default
path.file_format = None
self.load_data()
elif n <= len(self.available_readers):
reader = self.available_readers[n - 1]
path.file_format = reader.qualified_name()
self.load_data()
else: # the rest include just qualified names
path.file_format = self.reader_combo.itemText(n)
self.load_data()
def _url_set(self):
url = self.url_combo.currentText()
pos = self.recent_urls.index(url)
url = url.strip()
if not urlparse(url).scheme:
url = 'http://' + url
self.url_combo.setItemText(pos, url)
self.recent_urls[pos] = url
self.source = self.URL
self.load_data()
def browse_file(self, in_demos=False):
if in_demos:
start_file = get_sample_datasets_dir()
if not os.path.exists(start_file):
QMessageBox.information(
None, "File",
"Cannot find the directory with documentation datasets")
return
else:
start_file = self.last_path() or os.path.expanduser("~/")
filename, reader, _ = open_filename_dialog(start_file, None,
self.available_readers)
if not filename:
return
self.add_path(filename)
if reader is not None:
self.recent_paths[0].file_format = reader.qualified_name()
self.source = self.LOCAL_FILE
self.load_data()
# Open a file, create data from it and send it over the data channel
def load_data(self):
# We need to catch any exception type since anything can happen in
# file readers
self.closeContext()
self.domain_editor.set_domain(None)
self.apply_button.setEnabled(False)
self.clear_messages()
self.set_file_list()
error = self._try_load()
if error:
error()
self.data = None
self.sheet_box.hide()
self.Outputs.data.send(None)
self.infolabel.setText("No data.")
def _try_load(self):
self._initialize_reader_combo()
# pylint: disable=broad-except
if self.source == self.LOCAL_FILE:
if self.last_path() is None:
return self.Information.no_file_selected
elif not os.path.exists(self.last_path()):
return self.Error.file_not_found
else:
url = self.url_combo.currentText().strip()
if not url:
return self.Information.no_file_selected
def mark_problematic_reader():
self.reader_combo.setItemData(self.reader_combo.currentIndex(),
QBrush(Qt.red), Qt.ForegroundRole)
try:
self.reader = self._get_reader() # also sets current reader index
assert self.reader is not None
except MissingReaderException:
mark_problematic_reader()
return self.Error.missing_reader
except Exception as ex:
mark_problematic_reader()
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
try:
self._update_sheet_combo()
except Exception:
return self.Error.sheet_error
with log_warnings() as warnings:
try:
data = self.reader.read()
except Exception as ex:
mark_problematic_reader()
log.exception(ex)
return lambda x=ex: self.Error.unknown(str(x))
if warnings:
self.Warning.load_warning(warnings[-1].message.args[0])
self.infolabel.setText(self._describe(data))
self.loaded_file = self.last_path()
add_origin(data, self.loaded_file)
self.data = data
self.openContext(data.domain)
self.apply_domain_edit() # sends data
return None
def _get_reader(self) -> FileFormat:
if self.source == self.LOCAL_FILE:
path = self.last_path()
self.reader_combo.setEnabled(True)
if self.recent_paths and self.recent_paths[0].file_format:
qname = self.recent_paths[0].file_format
qname_index = {
r.qualified_name(): i
for i, r in enumerate(self.available_readers)
}
if qname in qname_index:
self.reader_combo.setCurrentIndex(qname_index[qname] + 1)
else:
# reader may be accessible, but not in self.available_readers
# (perhaps its code was moved)
self.reader_combo.addItem(qname)
self.reader_combo.setCurrentIndex(
len(self.reader_combo) - 1)
try:
reader_class = class_from_qualified_name(qname)
except Exception as ex:
raise MissingReaderException(
f'Can not find reader "{qname}"') from ex
reader = reader_class(path)
else:
self.reader_combo.setCurrentIndex(0)
reader = FileFormat.get_reader(path)
if self.recent_paths and self.recent_paths[0].sheet:
reader.select_sheet(self.recent_paths[0].sheet)
return reader
else:
url = self.url_combo.currentText().strip()
return UrlReader(url)
def _update_sheet_combo(self):
if len(self.reader.sheets) < 2:
self.sheet_box.hide()
self.reader.select_sheet(None)
return
self.sheet_combo.clear()
self.sheet_combo.addItems(self.reader.sheets)
self._select_active_sheet()
self.sheet_box.show()
def _select_active_sheet(self):
try:
idx = self.reader.sheets.index(self.reader.sheet)
self.sheet_combo.setCurrentIndex(idx)
except ValueError:
# Requested sheet does not exist in this file
self.reader.select_sheet(None)
self.sheet_combo.setCurrentIndex(0)
def _initialize_reader_combo(self):
self.reader_combo.clear()
filters = [format_filter(f) for f in self.available_readers]
self.reader_combo.addItems([DEFAULT_READER_TEXT] + filters)
self.reader_combo.setCurrentIndex(0)
self.reader_combo.setDisabled(True)
# additional readers may be added in self._get_reader()
@staticmethod
def _describe(table):
def missing_prop(prop):
if prop:
return f"({prop * 100:.1f}% missing values)"
else:
return "(no missing values)"
domain = table.domain
text = ""
attrs = getattr(table, "attributes", {})
descs = [
attrs[desc] for desc in ("Name", "Description") if desc in attrs
]
if len(descs) == 2:
descs[0] = f"<b>{descs[0]}</b>"
if descs:
text += f"<p>{'<br/>'.join(descs)}</p>"
text += f"<p>{len(table)} instance(s)"
missing_in_attr = missing_prop(table.has_missing_attribute()
and table.get_nan_frequency_attribute())
missing_in_class = missing_prop(table.has_missing_class()
and table.get_nan_frequency_class())
text += f"<br/>{len(domain.attributes)} feature(s) {missing_in_attr}"
if domain.has_continuous_class:
text += f"<br/>Regression; numerical class {missing_in_class}"
elif domain.has_discrete_class:
text += "<br/>Classification; categorical class " \
f"with {len(domain.class_var.values)} values {missing_in_class}"
elif table.domain.class_vars:
text += "<br/>Multi-target; " \
f"{len(table.domain.class_vars)} target variables " \
f"{missing_in_class}"
else:
text += "<br/>Data has no target variable."
text += f"<br/>{len(domain.metas)} meta attribute(s)"
text += "</p>"
if 'Timestamp' in table.domain:
# Google Forms uses this header to timestamp responses
text += f"<p>First entry: {table[0, 'Timestamp']}<br/>" \
f"Last entry: {table[-1, 'Timestamp']}</p>"
return text
def storeSpecificSettings(self):
self.current_context.modified_variables = self.variables[:]
def retrieveSpecificSettings(self):
if hasattr(self.current_context, "modified_variables"):
self.variables[:] = self.current_context.modified_variables
def reset_domain_edit(self):
self.domain_editor.reset_domain()
self.apply_domain_edit()
def _inspect_discrete_variables(self, domain):
for var in chain(domain.variables, domain.metas):
if var.is_discrete and len(var.values) > 100:
self.Warning.performance_warning()
def apply_domain_edit(self):
self.Warning.performance_warning.clear()
self.Warning.renamed_vars.clear()
if self.data is None:
table = None
else:
domain, cols, renamed = \
self.domain_editor.get_domain(self.data.domain, self.data,
deduplicate=True)
if not (domain.variables or domain.metas):
table = None
elif domain is self.data.domain:
table = self.data
else:
X, y, m = cols
table = Table.from_numpy(domain, X, y, m, self.data.W)
table.name = self.data.name
table.ids = np.array(self.data.ids)
table.attributes = getattr(self.data, 'attributes', {})
self._inspect_discrete_variables(domain)
if renamed:
self.Warning.renamed_vars(f"Renamed: {', '.join(renamed)}")
self.Warning.multiple_targets(
shown=table is not None and len(table.domain.class_vars) > 1)
self.Outputs.data.send(table)
self.apply_button.setEnabled(False)
def get_widget_name_extension(self):
_, name = os.path.split(self.loaded_file)
return os.path.splitext(name)[0]
def send_report(self):
def get_ext_name(filename):
try:
return FileFormat.names[os.path.splitext(filename)[1]]
except KeyError:
return "unknown"
if self.data is None:
self.report_paragraph("File", "No file.")
return
if self.source == self.LOCAL_FILE:
home = os.path.expanduser("~")
if self.loaded_file.startswith(home):
# os.path.join does not like ~
name = "~" + os.path.sep + \
self.loaded_file[len(home):].lstrip("/").lstrip("\\")
else:
name = self.loaded_file
if self.sheet_combo.isVisible():
name += f" ({self.sheet_combo.currentText()})"
self.report_items("File", [("File name", name),
("Format", get_ext_name(name))])
else:
self.report_items("Data", [("Resource", self.url),
("Format", get_ext_name(self.url))])
self.report_data("Data", self.data)
@staticmethod
def dragEnterEvent(event):
"""Accept drops of valid file urls"""
urls = event.mimeData().urls()
if urls:
try:
FileFormat.get_reader(urls[0].toLocalFile())
event.acceptProposedAction()
except MissingReaderException:
pass
def dropEvent(self, event):
"""Handle file drops"""
urls = event.mimeData().urls()
if urls:
self.add_path(urls[0].toLocalFile()) # add first file
self.source = self.LOCAL_FILE
self.load_data()
def workflowEnvChanged(self, key, value, oldvalue):
"""
Function called when environment changes (e.g. while saving the scheme)
It make sure that all environment connected values are modified
(e.g. relative file paths are changed)
"""
self.update_file_list(key, value, oldvalue)
class OWSql(OWBaseSql):
name = "SQL Table"
id = "orange.widgets.data.sql"
description = "Load dataset from SQL."
icon = "icons/SQLTable.svg"
priority = 30
category = "Data"
keywords = ["load"]
class Outputs:
data = Output("Data",
Table,
doc="Attribute-valued dataset read from the input file.")
settings_version = 2
table = Setting(None)
sql = Setting("")
guess_values = Setting(True)
download = Setting(False)
materialize = Setting(False)
materialize_table_name = Setting("")
class Information(OWBaseSql.Information):
data_sampled = Msg("Data description was generated from a sample.")
class Warning(OWBaseSql.Warning):
missing_extension = Msg("Database is missing extensions: {}")
class Error(OWBaseSql.Error):
no_backends = Msg("Please install a backend to use this widget.")
def __init__(self):
# Lint
self.backends = None
self.backendcombo = None
self.tables = None
self.tablecombo = None
self.sqltext = None
self.custom_sql = None
self.downloadcb = None
super().__init__()
def _setup_gui(self):
super()._setup_gui()
self._add_backend_controls()
self._add_tables_controls()
def _add_backend_controls(self):
box = self.serverbox
self.backends = BackendModel(Backend.available_backends())
self.backendcombo = QComboBox(box)
if self.backends:
self.backendcombo.setModel(self.backends)
else:
self.Error.no_backends()
box.setEnabled(False)
box.layout().insertWidget(0, self.backendcombo)
def _add_tables_controls(self):
vbox = gui.vBox(self.controlArea, "Tables", addSpace=True)
box = gui.vBox(vbox)
self.tables = TableModel()
self.tablecombo = QComboBox(
minimumContentsLength=35,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength)
self.tablecombo.setModel(self.tables)
self.tablecombo.setToolTip('table')
self.tablecombo.activated[int].connect(self.select_table)
box.layout().addWidget(self.tablecombo)
self.custom_sql = gui.vBox(box)
self.custom_sql.setVisible(False)
self.sqltext = QTextEdit(self.custom_sql)
self.sqltext.setPlainText(self.sql)
self.custom_sql.layout().addWidget(self.sqltext)
mt = gui.hBox(self.custom_sql)
cb = gui.checkBox(mt, self, 'materialize', 'Materialize to table ')
cb.setToolTip('Save results of the query in a table')
le = gui.lineEdit(mt, self, 'materialize_table_name')
le.setToolTip('Save results of the query in a table')
gui.button(self.custom_sql, self, 'Execute', callback=self.open_table)
box.layout().addWidget(self.custom_sql)
gui.checkBox(box,
self,
"guess_values",
"Auto-discover categorical variables",
callback=self.open_table)
self.downloadcb = gui.checkBox(box,
self,
"download",
"Download data to local memory",
callback=self.open_table)
def highlight_error(self, text=""):
err = ['', 'QLineEdit {border: 2px solid red;}']
self.servertext.setStyleSheet(err['server' in text or 'host' in text])
self.usernametext.setStyleSheet(err['role' in text])
self.databasetext.setStyleSheet(err['database' in text])
def get_backend(self):
if self.backendcombo.currentIndex() < 0:
return None
return self.backends[self.backendcombo.currentIndex()]
def on_connection_success(self):
if getattr(self.backend, 'missing_extension', False):
self.Warning.missing_extension(", ".join(
self.backend.missing_extension))
self.download = True
self.downloadcb.setEnabled(False)
if not is_postgres(self.backend):
self.download = True
self.downloadcb.setEnabled(False)
super().on_connection_success()
self.refresh_tables()
self.select_table()
def on_connection_error(self, err):
super().on_connection_error(err)
self.highlight_error(str(err).split("\n")[0])
def clear(self):
super().clear()
self.Warning.missing_extension.clear()
self.downloadcb.setEnabled(True)
self.highlight_error()
self.tablecombo.clear()
self.tablecombo.repaint()
def refresh_tables(self):
self.tables.clear()
if self.backend is None:
self.data_desc_table = None
return
self.tables.append("Select a table")
self.tables.append("Custom SQL")
self.tables.extend(self.backend.list_tables(self.schema))
index = self.tablecombo.findText(str(self.table))
self.tablecombo.setCurrentIndex(index if index != -1 else 0)
self.tablecombo.repaint()
# Called on tablecombo selection change:
def select_table(self):
curIdx = self.tablecombo.currentIndex()
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.custom_sql.setVisible(False)
return self.open_table()
else:
self.custom_sql.setVisible(True)
self.data_desc_table = None
self.database_desc["Table"] = "(None)"
self.table = None
if len(str(self.sql)) > 14:
return self.open_table()
return None
def get_table(self):
curIdx = self.tablecombo.currentIndex()
if curIdx <= 0:
if self.database_desc:
self.database_desc["Table"] = "(None)"
self.data_desc_table = None
return None
if self.tablecombo.itemText(curIdx) != "Custom SQL":
self.table = self.tables[self.tablecombo.currentIndex()]
self.database_desc["Table"] = self.table
if "Query" in self.database_desc:
del self.database_desc["Query"]
what = self.table
else:
what = self.sql = self.sqltext.toPlainText()
self.table = "Custom SQL"
if self.materialize:
if not self.materialize_table_name:
self.Error.connection(
"Specify a table name to materialize the query")
return None
try:
with self.backend.execute_sql_query(
"DROP TABLE IF EXISTS " +
self.materialize_table_name):
pass
with self.backend.execute_sql_query(
"CREATE TABLE " + self.materialize_table_name +
" AS " + self.sql):
pass
with self.backend.execute_sql_query(
"ANALYZE " + self.materialize_table_name):
pass
except BackendError as ex:
self.Error.connection(str(ex))
return None
try:
table = SqlTable(dict(host=self.host,
port=self.port,
database=self.database,
user=self.username,
password=self.password),
what,
backend=type(self.backend),
inspect_values=False)
except BackendError as ex:
self.Error.connection(str(ex))
return None
self.Error.connection.clear()
sample = False
if table.approx_len() > LARGE_TABLE and self.guess_values:
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Attribute discovery might take "
"a long time on large tables.\n"
"Do you want to auto discover attributes?")
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
if is_postgres(self.backend):
sample_button = confirm.addButton("Yes, on a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
self.guess_values = False
elif is_postgres(self.backend) and \
confirm.clickedButton() == sample_button:
sample = True
self.Information.clear()
if self.guess_values:
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
if sample:
s = table.sample_time(1)
domain = s.get_domain(inspect_values=True)
self.Information.data_sampled()
else:
domain = table.get_domain(inspect_values=True)
QApplication.restoreOverrideCursor()
table.domain = domain
if self.download:
if table.approx_len() > AUTO_DL_LIMIT:
if is_postgres(self.backend):
confirm = QMessageBox(self)
confirm.setIcon(QMessageBox.Warning)
confirm.setText("Data appears to be big. Do you really "
"want to download it to local memory?")
if table.approx_len() <= MAX_DL_LIMIT:
confirm.addButton("Yes", QMessageBox.YesRole)
no_button = confirm.addButton("No", QMessageBox.NoRole)
sample_button = confirm.addButton("Yes, a sample",
QMessageBox.YesRole)
confirm.exec()
if confirm.clickedButton() == no_button:
return None
elif confirm.clickedButton() == sample_button:
table = table.sample_percentage(
AUTO_DL_LIMIT / table.approx_len() * 100)
else:
if table.approx_len() > MAX_DL_LIMIT:
QMessageBox.warning(self, 'Warning',
"Data is too big to download.\n")
return None
else:
confirm = QMessageBox.question(
self, 'Question',
"Data appears to be big. Do you really "
"want to download it to local memory?",
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if confirm == QMessageBox.No:
return None
table.download_data(MAX_DL_LIMIT)
table = Table(table)
return table
@classmethod
def migrate_settings(cls, settings, version):
if version < 2:
# Until Orange version 3.4.4 username and password had been stored
# in Settings.
cm = cls._credential_manager(settings["host"], settings["port"])
cm.username = settings["username"]
cm.password = settings["password"]
class FeatureEditor(QFrame):
FUNCTIONS = dict(chain([(key, val) for key, val in math.__dict__.items()
if not key.startswith("_")],
[("str", str)]))
featureChanged = Signal()
featureEdited = Signal()
modifiedChanged = Signal(bool)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
layout = QFormLayout(
fieldGrowthPolicy=QFormLayout.ExpandingFieldsGrow
)
layout.setContentsMargins(0, 0, 0, 0)
self.nameedit = QLineEdit(
placeholderText="Name...",
sizePolicy=QSizePolicy(QSizePolicy.Minimum,
QSizePolicy.Fixed)
)
self.expressionedit = QLineEdit(
placeholderText="Expression..."
)
self.attrs_model = itemmodels.VariableListModel(
["Select Feature"], parent=self)
self.attributescb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
)
self.attributescb.setModel(self.attrs_model)
sorted_funcs = sorted(self.FUNCTIONS)
self.funcs_model = itemmodels.PyListModelTooltip()
self.funcs_model.setParent(self)
self.funcs_model[:] = chain(["Select Function"], sorted_funcs)
self.funcs_model.tooltips[:] = chain(
[''],
[self.FUNCTIONS[func].__doc__ for func in sorted_funcs])
self.functionscb = QComboBox(
minimumContentsLength=16,
sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLengthWithIcon,
sizePolicy=QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
self.functionscb.setModel(self.funcs_model)
hbox = QHBoxLayout()
hbox.addWidget(self.attributescb)
hbox.addWidget(self.functionscb)
layout.addRow(self.nameedit, self.expressionedit)
layout.addRow(self.tr(""), hbox)
self.setLayout(layout)
self.nameedit.editingFinished.connect(self._invalidate)
self.expressionedit.textChanged.connect(self._invalidate)
self.attributescb.currentIndexChanged.connect(self.on_attrs_changed)
self.functionscb.currentIndexChanged.connect(self.on_funcs_changed)
self._modified = False
def setModified(self, modified):
if not type(modified) is bool:
raise TypeError
if self._modified != modified:
self._modified = modified
self.modifiedChanged.emit(modified)
def modified(self):
return self._modified
modified = Property(bool, modified, setModified,
notify=modifiedChanged)
def setEditorData(self, data, domain):
self.nameedit.setText(data.name)
self.expressionedit.setText(data.expression)
self.setModified(False)
self.featureChanged.emit()
self.attrs_model[:] = ["Select Feature"]
if domain is not None and (domain or domain.metas):
self.attrs_model[:] += chain(domain.attributes,
domain.class_vars,
domain.metas)
def editorData(self):
return FeatureDescriptor(name=self.nameedit.text(),
expression=self.nameedit.text())
def _invalidate(self):
self.setModified(True)
self.featureEdited.emit()
self.featureChanged.emit()
def on_attrs_changed(self):
index = self.attributescb.currentIndex()
if index > 0:
attr = sanitized_name(self.attrs_model[index].name)
self.insert_into_expression(attr)
self.attributescb.setCurrentIndex(0)
def on_funcs_changed(self):
index = self.functionscb.currentIndex()
if index > 0:
func = self.funcs_model[index]
if func in ["atan2", "fmod", "ldexp", "log",
"pow", "copysign", "hypot"]:
self.insert_into_expression(func + "(,)")
self.expressionedit.cursorBackward(False, 2)
elif func in ["e", "pi"]:
self.insert_into_expression(func)
else:
self.insert_into_expression(func + "()")
self.expressionedit.cursorBackward(False)
self.functionscb.setCurrentIndex(0)
def insert_into_expression(self, what):
cp = self.expressionedit.cursorPosition()
ct = self.expressionedit.text()
text = ct[:cp] + what + ct[cp:]
self.expressionedit.setText(text)
self.expressionedit.setFocus()
class Scale(BaseEditor):
NoCentering, CenterMean, CenterMedian = 0, 1, 2
NoScaling, ScaleBySD, ScaleBySpan = 0, 1, 2
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.setLayout(QVBoxLayout())
form = QFormLayout()
self.__centercb = QComboBox()
self.__centercb.addItems(
["No Centering", "Center by Mean", "Center by Median"])
self.__scalecb = QComboBox()
self.__scalecb.addItems(["No scaling", "Scale by SD", "Scale by span"])
form.addRow("Center:", self.__centercb)
form.addRow("Scale:", self.__scalecb)
self.layout().addLayout(form)
self.__centercb.currentIndexChanged.connect(self.changed)
self.__scalecb.currentIndexChanged.connect(self.changed)
self.__centercb.activated.connect(self.edited)
self.__scalecb.activated.connect(self.edited)
def setParameters(self, params):
center = params.get("center", Scale.CenterMean)
scale = params.get("scale", Scale.ScaleBySD)
self.__centercb.setCurrentIndex(center)
self.__scalecb.setCurrentIndex(scale)
def parameters(self):
return {
"center": self.__centercb.currentIndex(),
"scale": self.__scalecb.currentIndex()
}
@staticmethod
def createinstance(params):
center = params.get("center", Scale.CenterMean)
scale = params.get("scale", Scale.ScaleBySD)
if center == Scale.NoCentering:
center = None
elif center == Scale.CenterMean:
center = _Scaling.mean
elif center == Scale.CenterMedian:
center = _Scaling.median
else:
assert False
if scale == Scale.NoScaling:
scale = None
elif scale == Scale.ScaleBySD:
scale = _Scaling.std
elif scale == Scale.ScaleBySpan:
scale = _Scaling.span
else:
assert False
return _Scaling(center=center, scale=scale)
def __repr__(self):
return "{}, {}".format(self.__centercb.currentText(),
self.__scalecb.currentText())
