class OWConfusionMatrix(widget.OWWidget):
"""Confusion matrix widget"""
name = "Confusion Matrix"
description = "Display a confusion matrix constructed from " \
"the results of classifier evaluations."
icon = "icons/ConfusionMatrix.svg"
priority = 1001
class Inputs:
evaluation_results = Input("Evaluation Results",
Orange.evaluation.Results)
class Outputs:
selected_data = Output("Selected Data",
Orange.data.Table,
default=True)
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Orange.data.Table)
quantities = [
"Number of instances", "Proportion of predicted",
"Proportion of actual"
]
settings_version = 1
settingsHandler = settings.ClassValuesContextHandler()
selected_learner = settings.Setting([0], schema_only=True)
selection = settings.ContextSetting(set())
selected_quantity = settings.Setting(0)
append_predictions = settings.Setting(True)
append_probabilities = settings.Setting(False)
autocommit = settings.Setting(True)
UserAdviceMessages = [
widget.Message(
"Clicking on cells or in headers outputs the corresponding "
"data instances", "click_cell")
]
class Error(widget.OWWidget.Error):
no_regression = Msg("Confusion Matrix cannot show regression results.")
invalid_values = Msg(
"Evaluation Results input contains invalid values")
def __init__(self):
super().__init__()
self.data = None
self.results = None
self.learners = []
self.headers = []
self.learners_box = gui.listBox(self.controlArea,
self,
"selected_learner",
"learners",
box=True,
callback=self._learner_changed)
self.outputbox = gui.vBox(self.controlArea, "Output")
box = gui.hBox(self.outputbox)
gui.checkBox(box,
self,
"append_predictions",
"Predictions",
callback=self._invalidate)
gui.checkBox(box,
self,
"append_probabilities",
"Probabilities",
callback=self._invalidate)
gui.auto_commit(self.outputbox,
self,
"autocommit",
"Send Selected",
"Send Automatically",
box=False)
self.mainArea.layout().setContentsMargins(0, 0, 0, 0)
box = gui.vBox(self.mainArea, box=True)
sbox = gui.hBox(box)
gui.rubber(sbox)
gui.comboBox(sbox,
self,
"selected_quantity",
items=self.quantities,
label="Show: ",
orientation=Qt.Horizontal,
callback=self._update)
self.tablemodel = QStandardItemModel(self)
view = self.tableview = QTableView(
editTriggers=QTableView.NoEditTriggers)
view.setModel(self.tablemodel)
view.horizontalHeader().hide()
view.verticalHeader().hide()
view.horizontalHeader().setMinimumSectionSize(60)
view.selectionModel().selectionChanged.connect(self._invalidate)
view.setShowGrid(False)
view.setItemDelegate(BorderedItemDelegate(Qt.white))
view.setSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding)
view.clicked.connect(self.cell_clicked)
box.layout().addWidget(view)
selbox = gui.hBox(box)
gui.button(selbox,
self,
"Select Correct",
callback=self.select_correct,
autoDefault=False)
gui.button(selbox,
self,
"Select Misclassified",
callback=self.select_wrong,
autoDefault=False)
gui.button(selbox,
self,
"Clear Selection",
callback=self.select_none,
autoDefault=False)
def sizeHint(self):
"""Initial size"""
return QSize(750, 340)
def _item(self, i, j):
return self.tablemodel.item(i, j) or QStandardItem()
def _set_item(self, i, j, item):
self.tablemodel.setItem(i, j, item)
def _init_table(self, nclasses):
item = self._item(0, 2)
item.setData("Predicted", Qt.DisplayRole)
item.setTextAlignment(Qt.AlignCenter)
item.setFlags(Qt.NoItemFlags)
self._set_item(0, 2, item)
item = self._item(2, 0)
item.setData("Actual", Qt.DisplayRole)
item.setTextAlignment(Qt.AlignHCenter | Qt.AlignBottom)
item.setFlags(Qt.NoItemFlags)
self.tableview.setItemDelegateForColumn(0, gui.VerticalItemDelegate())
self._set_item(2, 0, item)
self.tableview.setSpan(0, 2, 1, nclasses)
self.tableview.setSpan(2, 0, nclasses, 1)
font = self.tablemodel.invisibleRootItem().font()
bold_font = QFont(font)
bold_font.setBold(True)
for i in (0, 1):
for j in (0, 1):
item = self._item(i, j)
item.setFlags(Qt.NoItemFlags)
self._set_item(i, j, item)
for p, label in enumerate(self.headers):
for i, j in ((1, p + 2), (p + 2, 1)):
item = self._item(i, j)
item.setData(label, Qt.DisplayRole)
item.setFont(bold_font)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
if p < len(self.headers) - 1:
item.setData("br"[j == 1], BorderRole)
item.setData(QColor(192, 192, 192), BorderColorRole)
self._set_item(i, j, item)
hor_header = self.tableview.horizontalHeader()
if len(' '.join(self.headers)) < 120:
hor_header.setSectionResizeMode(QHeaderView.ResizeToContents)
else:
hor_header.setDefaultSectionSize(60)
self.tablemodel.setRowCount(nclasses + 3)
self.tablemodel.setColumnCount(nclasses + 3)
@Inputs.evaluation_results
def set_results(self, results):
"""Set the input results."""
prev_sel_learner = self.selected_learner.copy()
self.clear()
self.warning()
self.closeContext()
data = None
if results is not None and results.data is not None:
data = results.data[results.row_indices]
if data is not None and not data.domain.has_discrete_class:
self.Error.no_regression()
data = results = None
else:
self.Error.no_regression.clear()
nan_values = False
if results is not None:
assert isinstance(results, Orange.evaluation.Results)
if np.any(np.isnan(results.actual)) or \
np.any(np.isnan(results.predicted)):
# Error out here (could filter them out with a warning
# instead).
nan_values = True
results = data = None
if nan_values:
self.Error.invalid_values()
else:
self.Error.invalid_values.clear()
self.results = results
self.data = data
if data is not None:
class_values = data.domain.class_var.values
elif results is not None:
raise NotImplementedError
if results is None:
self.report_button.setDisabled(True)
else:
self.report_button.setDisabled(False)
nmodels = results.predicted.shape[0]
self.headers = class_values + \
[unicodedata.lookup("N-ARY SUMMATION")]
# NOTE: The 'learner_names' is set in 'Test Learners' widget.
if hasattr(results, "learner_names"):
self.learners = results.learner_names
else:
self.learners = [
"Learner #{}".format(i + 1) for i in range(nmodels)
]
self._init_table(len(class_values))
self.openContext(data.domain.class_var)
if not prev_sel_learner or prev_sel_learner[0] >= len(
self.learners):
if self.learners:
self.selected_learner[:] = [0]
else:
self.selected_learner[:] = prev_sel_learner
self._update()
self._set_selection()
self.unconditional_commit()
def clear(self):
"""Reset the widget, clear controls"""
self.results = None
self.data = None
self.tablemodel.clear()
self.headers = []
# Clear learners last. This action will invoke `_learner_changed`
self.learners = []
def select_correct(self):
"""Select the diagonal elements of the matrix"""
selection = QItemSelection()
n = self.tablemodel.rowCount()
for i in range(2, n):
index = self.tablemodel.index(i, i)
selection.select(index, index)
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def select_wrong(self):
"""Select the off-diagonal elements of the matrix"""
selection = QItemSelection()
n = self.tablemodel.rowCount()
for i in range(2, n):
for j in range(i + 1, n):
index = self.tablemodel.index(i, j)
selection.select(index, index)
index = self.tablemodel.index(j, i)
selection.select(index, index)
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def select_none(self):
"""Reset selection"""
self.tableview.selectionModel().clear()
def cell_clicked(self, model_index):
"""Handle cell click event"""
i, j = model_index.row(), model_index.column()
if not i or not j:
return
n = self.tablemodel.rowCount()
index = self.tablemodel.index
selection = None
if i == j == 1 or i == j == n - 1:
selection = QItemSelection(index(2, 2), index(n - 1, n - 1))
elif i in (1, n - 1):
selection = QItemSelection(index(2, j), index(n - 1, j))
elif j in (1, n - 1):
selection = QItemSelection(index(i, 2), index(i, n - 1))
if selection is not None:
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def _prepare_data(self):
indices = self.tableview.selectedIndexes()
indices = {(ind.row() - 2, ind.column() - 2) for ind in indices}
actual = self.results.actual
learner_name = self.learners[self.selected_learner[0]]
predicted = self.results.predicted[self.selected_learner[0]]
selected = [
i for i, t in enumerate(zip(actual, predicted)) if t in indices
]
extra = []
class_var = self.data.domain.class_var
metas = self.data.domain.metas
if self.append_predictions:
extra.append(predicted.reshape(-1, 1))
var = Orange.data.DiscreteVariable(
"{}({})".format(class_var.name, learner_name),
class_var.values)
metas = metas + (var, )
if self.append_probabilities and \
self.results.probabilities is not None:
probs = self.results.probabilities[self.selected_learner[0]]
extra.append(np.array(probs, dtype=object))
pvars = [
Orange.data.ContinuousVariable("p({})".format(value))
for value in class_var.values
]
metas = metas + tuple(pvars)
domain = Orange.data.Domain(self.data.domain.attributes,
self.data.domain.class_vars, metas)
data = self.data.transform(domain)
if len(extra):
data.metas[:, len(self.data.domain.metas):] = \
np.hstack(tuple(extra))
data.name = learner_name
if selected:
annotated_data = create_annotated_table(data, selected)
data = data[selected]
else:
annotated_data = create_annotated_table(data, [])
data = None
return data, annotated_data
def commit(self):
"""Output data instances corresponding to selected cells"""
if self.results is not None and self.data is not None \
and self.selected_learner:
data, annotated_data = self._prepare_data()
else:
data = None
annotated_data = None
self.Outputs.selected_data.send(data)
self.Outputs.annotated_data.send(annotated_data)
def _invalidate(self):
indices = self.tableview.selectedIndexes()
self.selection = {(ind.row() - 2, ind.column() - 2) for ind in indices}
self.commit()
def _set_selection(self):
selection = QItemSelection()
index = self.tableview.model().index
for row, col in self.selection:
sel = index(row + 2, col + 2)
selection.select(sel, sel)
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def _learner_changed(self):
self._update()
self._set_selection()
self.commit()
def _update(self):
def _isinvalid(x):
return isnan(x) or isinf(x)
# Update the displayed confusion matrix
if self.results is not None and self.selected_learner:
cmatrix = confusion_matrix(self.results, self.selected_learner[0])
colsum = cmatrix.sum(axis=0)
rowsum = cmatrix.sum(axis=1)
n = len(cmatrix)
diag = np.diag_indices(n)
colors = cmatrix.astype(np.double)
colors[diag] = 0
if self.selected_quantity == 0:
normalized = cmatrix.astype(np.int)
formatstr = "{}"
div = np.array([colors.max()])
else:
if self.selected_quantity == 1:
normalized = 100 * cmatrix / colsum
div = colors.max(axis=0)
else:
normalized = 100 * cmatrix / rowsum[:, np.newaxis]
div = colors.max(axis=1)[:, np.newaxis]
formatstr = "{:2.1f} %"
div[div == 0] = 1
colors /= div
colors[diag] = normalized[diag] / normalized[diag].max()
for i in range(n):
for j in range(n):
val = normalized[i, j]
col_val = colors[i, j]
item = self._item(i + 2, j + 2)
item.setData(
"NA" if _isinvalid(val) else formatstr.format(val),
Qt.DisplayRole)
bkcolor = QColor.fromHsl(
[0, 240][i == j], 160,
255 if _isinvalid(col_val) else int(255 -
30 * col_val))
item.setData(QBrush(bkcolor), Qt.BackgroundRole)
item.setData("trbl", BorderRole)
item.setToolTip("actual: {}\npredicted: {}".format(
self.headers[i], self.headers[j]))
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
self._set_item(i + 2, j + 2, item)
bold_font = self.tablemodel.invisibleRootItem().font()
bold_font.setBold(True)
def _sum_item(value, border=""):
item = QStandardItem()
item.setData(value, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
item.setFont(bold_font)
item.setData(border, BorderRole)
item.setData(QColor(192, 192, 192), BorderColorRole)
return item
for i in range(n):
self._set_item(n + 2, i + 2, _sum_item(int(colsum[i]), "t"))
self._set_item(i + 2, n + 2, _sum_item(int(rowsum[i]), "l"))
self._set_item(n + 2, n + 2, _sum_item(int(rowsum.sum())))
def send_report(self):
"""Send report"""
if self.results is not None and self.selected_learner:
self.report_table(
"Confusion matrix for {} (showing {})".format(
self.learners[self.selected_learner[0]],
self.quantities[self.selected_quantity].lower()),
self.tableview)
@classmethod
def migrate_settings(cls, settings, version):
if not version:
# For some period of time the 'selected_learner' property was
# changed from List[int] -> int
# (commit 4e49bb3fd0e11262f3ebf4b1116a91a4b49cc982) and then back
# again (commit 8a492d79a2e17154a0881e24a05843406c8892c0)
if "selected_learner" in settings and \
isinstance(settings["selected_learner"], int):
settings["selected_learner"] = [settings["selected_learner"]]
class PivotTableView(QTableView):
selection_changed = pyqtSignal()
TOTAL_STRING = "Total"
def __init__(self):
super().__init__(editTriggers=QTableView.NoEditTriggers)
self._n_classesv = None # number of row_feature values
self._n_classesh = None # number of col_feature values
self._n_agg_func = None # number of aggregation functions
self._n_leading_rows = None # number of leading rows
self._n_leading_cols = None # number of leading columns
self.table_model = QStandardItemModel(self)
self.setModel(self.table_model)
self.horizontalHeader().hide()
self.verticalHeader().hide()
self.horizontalHeader().setMinimumSectionSize(60)
self.setShowGrid(False)
self.setSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding)
self.setItemDelegate(BorderedItemDelegate())
self.pressed.connect(self.__cell_clicked)
self.clicked.connect(self.__cell_clicked)
self.entered.connect(self.__cell_entered)
self.__clicked_cell = None
@property
def add_agg_column(self) -> bool:
return self._n_agg_func > 1
def __cell_entered(self, model_index):
if self.__clicked_cell is None:
return
index = self.table_model.index
selection = None
i_end, j_end = model_index.row(), model_index.column()
i_start, j_start = self.__clicked_cell
i_start, i_end = sorted([i_start, i_end])
j_start, j_end = sorted([j_start, j_end])
if i_start >= self._n_leading_rows and j_start >= self._n_leading_cols:
i_start = (i_start - self._n_leading_rows) // self._n_agg_func * \
self._n_agg_func + self._n_leading_rows
i_end = (i_end - self._n_leading_rows) // self._n_agg_func * \
self._n_agg_func + self._n_leading_rows + self._n_agg_func - 1
start, end = index(i_start, j_start), index(i_end, j_end)
selection = QItemSelection(start, end)
if selection is not None:
self.selectionModel().select(selection,
QItemSelectionModel.ClearAndSelect)
self.selection_changed.emit()
def __cell_clicked(self, model_index):
i, j = model_index.row(), model_index.column()
self.__clicked_cell = (i, j)
m, n = self.table_model.rowCount(), self.table_model.columnCount()
index = self.table_model.index
selection = None
if i > m - self._n_agg_func - 1 and j == n - 1:
start_index = index(self._n_leading_rows, self._n_leading_cols)
selection = QItemSelection(start_index, index(m - 1, n - 1))
elif i == self._n_leading_rows - 1 or i > m - self._n_agg_func - 1:
start_index = index(self._n_leading_rows, j)
selection = QItemSelection(start_index, index(m - 1, j))
elif j in (self._n_leading_cols - 1, n - 1, 1):
i_start = (i - self._n_leading_rows) // self._n_agg_func * \
self._n_agg_func + self._n_leading_rows
i_end = i_start + self._n_agg_func - 1
start_index = index(i_start, self._n_leading_cols)
selection = QItemSelection(start_index, index(i_end, n - 1))
elif i >= self._n_leading_rows and j >= self._n_leading_cols:
i_start = (i - self._n_leading_rows) // self._n_agg_func * \
self._n_agg_func + self._n_leading_rows
i_end = i_start + self._n_agg_func - 1
selection = QItemSelection(index(i_start, j), index(i_end, j))
if selection is not None:
self.selectionModel().select(selection,
QItemSelectionModel.ClearAndSelect)
def mouseReleaseEvent(self, e):
super().mouseReleaseEvent(e)
self.selection_changed.emit()
def update_table(self, titleh: str, titlev: str, table: Table,
table_total_h: Table, table_total_v: Table,
table_total: Table):
self.clear()
if not table:
return
self._initialize(table, table_total_h)
self._set_headers(titleh, titlev, table)
self._set_values(table[:, 2:])
self._set_totals(table_total_h[:, 2:], table_total_v, table_total)
self._draw_lines()
self._resize(table)
def _initialize(self, table, table_total_h):
self._n_classesv = int(len(table) / len(table_total_h))
self._n_classesh = table.X.shape[1] - 2
self._n_agg_func = len(table_total_h)
self._n_leading_rows = 2
self._n_leading_cols = 2 + int(len(table_total_h) > 1)
def _set_headers(self, titleh, titlev, table):
self.__set_horizontal_title(titleh)
self.__set_vertical_title(titlev)
self.__set_flags_title()
self.__set_horizontal_headers(table)
self.__set_vertical_headers(table)
def __set_horizontal_title(self, titleh):
item = QStandardItem()
item.setData(titleh, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignCenter)
self.table_model.setItem(0, self._n_leading_cols, item)
self.setSpan(0, self._n_leading_cols, 1, self._n_classesh + 3)
def __set_vertical_title(self, titlev):
item = QStandardItem()
item.setData(titlev, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignHCenter | Qt.AlignBottom)
self.setItemDelegateForColumn(0, gui.VerticalItemDelegate(extend=True))
self.table_model.setItem(self._n_leading_rows, 0, item)
row_span = self._n_classesv * self._n_agg_func + 1
self.setSpan(self._n_leading_rows, 0, row_span, 1)
def __set_flags_title(self):
item = self.table_model.item(0, self._n_leading_cols)
item.setFlags(Qt.NoItemFlags)
item = self.table_model.item(self._n_leading_rows, 0)
item.setFlags(Qt.NoItemFlags)
for i, j in product(range(self._n_leading_rows),
range(self._n_leading_cols)):
item = QStandardItem()
item.setFlags(Qt.NoItemFlags)
self.table_model.setItem(i, j, item)
def __set_horizontal_headers(self, table):
labels = [a.name for a in table.domain[1:]] + [self.TOTAL_STRING]
if not self.add_agg_column:
labels[0] = str(table[0, 1])
for i, label in enumerate(labels, self._n_leading_cols - 1):
self.table_model.setItem(1, i, self._create_header_item(label))
def __set_vertical_headers(self, table):
labels = [(str(row[0]), str(row[1])) for row in table]
i = self._n_leading_rows - 1
for i, (l1, l2) in enumerate(labels, self._n_leading_rows):
l1 = "" if (i - self._n_leading_rows) % self._n_agg_func else l1
self.table_model.setItem(i, 1, self._create_header_item(l1))
if self.add_agg_column:
self.table_model.setItem(i, 2, self._create_header_item(l2))
if self.add_agg_column:
labels = [str(row[1]) for row in table[:self._n_agg_func]]
start = self._n_leading_rows + self._n_agg_func * self._n_classesv
for j, l2 in enumerate(labels, i + 1):
l1 = self.TOTAL_STRING if j == start else ""
self.table_model.setItem(j, 1, self._create_header_item(l1))
self.table_model.setItem(j, 2, self._create_header_item(l2))
else:
item = self._create_header_item(self.TOTAL_STRING)
self.table_model.setItem(i + 1, 1, item)
def _set_values(self, table):
for i, j in product(range(len(table)), range(len(table[0]))):
value = table[i, j]
item = self._create_value_item(str(value))
self.table_model.setItem(i + self._n_leading_rows,
j + self._n_leading_cols, item)
def _set_totals(self, table_total_h, table_total_v, table_total):
def set_total_item(table, get_row, get_col):
for i, j in product(range(len(table)), range(len(table[0]))):
item = self._create_header_item(str(table[i, j]))
self.table_model.setItem(get_row(i), get_col(j), item)
last_row = self._n_leading_rows + self._n_classesv * self._n_agg_func
last_col = self._n_leading_cols + self._n_classesh
set_total_item(table_total_v, lambda x: x + self._n_leading_rows,
lambda x: last_col)
set_total_item(table_total_h, lambda x: x + last_row,
lambda x: x + self._n_leading_cols)
set_total_item(table_total, lambda x: x + last_row, lambda x: last_col)
def _create_header_item(self, text):
bold_font = self.table_model.invisibleRootItem().font()
bold_font.setBold(True)
item = QStandardItem()
item.setData(text, Qt.DisplayRole)
item.setFont(bold_font)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
return item
@staticmethod
def _create_value_item(text):
item = QStandardItem()
item.setData(text, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
return item
def _draw_lines(self):
end_col = self._n_leading_cols + self._n_classesh + 1
total_row = self._n_leading_rows + self._n_classesv * self._n_agg_func
indices = [(total_row, j) for j in range(1, end_col)]
for i in range(self._n_classesv):
inner_row = self._n_agg_func * i + self._n_leading_rows
inner_indices = [(inner_row, j) for j in range(1, end_col)]
indices = indices + inner_indices
if not self.add_agg_column:
break
for i, j in indices:
item = self.table_model.item(i, j)
item.setData("t", BorderRole)
item.setData(QColor(160, 160, 160), BorderColorRole)
def _resize(self, table):
labels = [a.name for a in table.domain[1:]] + [self.TOTAL_STRING]
if len(' '.join(labels)) < 120:
self.horizontalHeader().setSectionResizeMode(
QHeaderView.ResizeToContents)
else:
self.horizontalHeader().setDefaultSectionSize(60)
def get_selection(self) -> Set:
m, n = self._n_leading_rows, self._n_leading_cols
return {(ind.row() - m, ind.column() - n)
for ind in self.selectedIndexes()}
def set_selection(self, indexes: Set):
selection = QItemSelection()
index = self.model().index
for row, col in indexes:
sel = index(row + self._n_leading_rows, col + self._n_leading_cols)
selection.select(sel, sel)
self.selectionModel().select(selection,
QItemSelectionModel.ClearAndSelect)
def clear(self):
self.table_model.clear()
class ContingencyTable(QTableView):
"""
A contingency table widget which can be used wherever ``QTableView`` could be used.
Parameters
----------
parent : Orange.widgets.widget.OWWidget
The containing widget to which the table is connected.
Attributes
----------
classesv : :obj:`list` of :obj:`str`
Vertical class headers.
classesh : :obj:`list` of :obj:`str`
Horizontal class headers.
headerv : :obj:`str`, optional
Vertical top header.
headerh : :obj:`str`, optional
Horizontal top header.
corner_string : str
String that is top right and bottom left corner of the table.
Default is ``unicodedata.lookup("N-ARY SUMMATION")``.
"""
def __init__(self, parent):
super().__init__(editTriggers=QTableView.NoEditTriggers)
self.bold_headers = None
self.circles = False
self.classesv = None
self.classesh = None
self.headerv = None
self.headerh = None
self.parent = parent
self.corner_string = unicodedata.lookup("N-ARY SUMMATION")
self.tablemodel = QStandardItemModel(self)
self.setModel(self.tablemodel)
self.horizontalHeader().hide()
self.verticalHeader().hide()
self.horizontalHeader().setMinimumSectionSize(60)
self.setShowGrid(False)
self.setSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding)
self.clicked.connect(self._cell_clicked)
def mouseReleaseEvent(self, e):
super().mouseReleaseEvent(e)
self.parent._invalidate()
def keyPressEvent(self, event):
super().keyPressEvent(event)
self.parent._invalidate()
def _cell_clicked(self, model_index):
"""Handle cell click event"""
i, j = model_index.row(), model_index.column()
if not i or not j:
return
n = self.tablemodel.rowCount()
m = self.tablemodel.columnCount()
index = self.tablemodel.index
selection = None
if i == j == 1 or not self.circles and i == n - 1 and j == m - 1:
selection = QItemSelection(index(2, 2), index(n - 1, m - 1))
elif i == 1 or not self.circles and i == n - 1:
selection = QItemSelection(index(2, j), index(n - 1, j))
elif j == 1 or not self.circles and j == m - 1:
selection = QItemSelection(index(i, 2), index(i, m - 1))
if selection is not None:
self.selectionModel().select(selection,
QItemSelectionModel.ClearAndSelect)
def _item(self, i, j):
return self.tablemodel.item(i, j) or QStandardItem()
def _set_item(self, i, j, item):
self.tablemodel.setItem(i, j, item)
def set_variables(self, variablev, variableh, **kwargs):
"""
Sets class headers and top headers and initializes table structure.
Parameters
----------
variablev : Orange.data.variable.DiscreteVariable
Class headers are set to ``variablev.values``, top header is set to ``variablev.name``.
variableh : Orange.data.variable.DiscreteVariable
Class headers are set to ``variableh.values``, top header is set to ``variableh.name``.
"""
self.classesv = variablev.values
self.classesh = variableh.values
self.headerv = variablev.name
self.headerh = variableh.name
self.initialize(**kwargs)
def set_headers(self,
classesv,
classesh,
headerv=None,
headerh=None,
**kwargs):
"""
Sets class headers and top headers and initializes table structure.
Parameters
----------
classesv : :obj:`list` of :obj:`str`
Vertical class headers.
classesh : :obj:`list` of :obj:`str`
Horizontal class headers.
headerv : :obj:`str`, optional
Vertical top header.
headerh : :obj:`str`, optional
Horizontal top header.
"""
self.classesv = classesv
self.classesh = classesh
self.headerv = headerv
self.headerh = headerh
self.initialize(**kwargs)
def _style_cells(self):
"""
Style all cells.
"""
if self.circles:
self.setItemDelegate(CircleItemDelegate(Qt.white))
else:
self.setItemDelegate(BorderedItemDelegate(Qt.white))
item = self._item(0, 2)
item.setData(self.headerh, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignCenter)
item.setFlags(Qt.NoItemFlags)
self._set_item(0, 2, item)
item = self._item(2, 0)
item.setData(self.headerv, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignHCenter | Qt.AlignBottom)
item.setFlags(Qt.NoItemFlags)
self.setItemDelegateForColumn(0, gui.VerticalItemDelegate())
self._set_item(2, 0, item)
self.setSpan(0, 2, 1, len(self.classesh) + 1)
self.setSpan(2, 0, len(self.classesv) + 1, 1)
for i in (0, 1):
for j in (0, 1):
item = self._item(i, j)
item.setFlags(Qt.NoItemFlags)
self._set_item(i, j, item)
def _initialize_headers(self):
"""
Fill headers with content and style them.
"""
font = self.tablemodel.invisibleRootItem().font()
bold_font = QFont(font)
bold_font.setBold(True)
for headers, ix in ((self.classesv + [self.corner_string], lambda p:
(p + 2, 1)),
(self.classesh + [self.corner_string], lambda p:
(1, p + 2))):
for p, label in enumerate(headers):
i, j = ix(p)
item = self._item(i, j)
item.setData(label, Qt.DisplayRole)
if self.bold_headers:
item.setFont(bold_font)
if not (i == 1 and self.circles):
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
if p < len(headers) - 1:
item.setData("br"[j == 1], BorderRole)
item.setData(QColor(192, 192, 192), BorderColorRole)
else:
item.setData("", BorderRole)
self._set_item(i, j, item)
def _resize(self):
"""
Resize table to fit new contents and style.
"""
if self.circles:
self.resizeRowToContents(1)
self.horizontalHeader().setDefaultSectionSize(self.rowHeight(2))
self.resizeColumnToContents(1)
self.tablemodel.setRowCount(len(self.classesv) + 2)
self.tablemodel.setColumnCount(len(self.classesh) + 2)
else:
if len(' '.join(self.classesh + [self.corner_string])) < 120:
self.horizontalHeader().setSectionResizeMode(
QHeaderView.ResizeToContents)
else:
self.horizontalHeader().setDefaultSectionSize(60)
self.tablemodel.setRowCount(len(self.classesv) + 3)
self.tablemodel.setColumnCount(len(self.classesh) + 3)
def initialize(self, circles=False, bold_headers=True):
"""
Initializes table structure. Class headers must be set beforehand.
Parameters
----------
circles : :obj:`bool`, optional
Turns on circle display. All table values should be between 0 and 1 (inclusive). Defaults to False.
bold_headers : :obj:`bool`, optional
Whether the headers are bold or not. Defaults to True.
"""
assert self.classesv is not None and self.classesh is not None
self.circles = circles
self.bold_headers = bold_headers
self._style_cells()
self._initialize_headers()
self._resize()
def get_selection(self):
"""
Get indexes of selected cells.
Returns
-------
:obj:`set` of :obj:`tuple` of :obj:`int`
Set of pairs of indexes.
"""
return {(ind.row() - 2, ind.column() - 2)
for ind in self.selectedIndexes()}
def set_selection(self, indexes):
"""
Set indexes of selected cells.
Parameters
----------
indexes : :obj:`set` of :obj:`tuple` of :obj:`int`
Set of pairs of indexes.
"""
selection = QItemSelection()
index = self.model().index
for row, col in indexes:
sel = index(row + 2, col + 2)
selection.select(sel, sel)
self.selectionModel().select(selection,
QItemSelectionModel.ClearAndSelect)
def _set_sums(self, colsum, rowsum):
"""
Set content of cells on bottom and right edge.
Parameters
----------
colsum : numpy.array
Content of cells on bottom edge.
rowsum : numpy.array
Content of cells on right edge.
"""
bold_font = self.tablemodel.invisibleRootItem().font()
bold_font.setBold(True)
def _sum_item(value, border=""):
item = QStandardItem()
item.setData(value, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
item.setFont(bold_font)
item.setData(border, BorderRole)
item.setData(QColor(192, 192, 192), BorderColorRole)
return item
for i in range(len(self.classesh)):
self._set_item(
len(self.classesv) + 2, i + 2, _sum_item(int(colsum[i]), "t"))
for i in range(len(self.classesv)):
self._set_item(i + 2,
len(self.classesh) + 2,
_sum_item(int(rowsum[i]), "l"))
self._set_item(
len(self.classesv) + 2,
len(self.classesh) + 2, _sum_item(int(rowsum.sum())))
def _set_values(self, matrix, colors, formatstr, tooltip):
"""
Set content of cells which aren't headers and don't represent aggregate values.
Parameters
----------
matrix : numpy.array
2D array to be set as data.
colors : :obj:`numpy.array`
2D array with color values.
formatstr : :obj:`str`, optional
Format string for cell data.
tooltip : :obj:`(int, int) -> str`
Function which takes vertical index and horizontal index as arguments and returns
desired tooltip as a string.
"""
def _isinvalid(x):
return isnan(x) or isinf(x)
for i in range(len(self.classesv)):
for j in range(len(self.classesh)):
val = matrix[i, j]
col_val = float('nan') if colors is None else colors[i, j]
item = QStandardItem()
if self.circles:
item.setData(val, CircleAreaRole)
else:
item.setData(
"NA" if _isinvalid(val) else formatstr.format(val),
Qt.DisplayRole)
bkcolor = QColor.fromHsl(
[0, 240][i == j], 160,
255 if _isinvalid(col_val) else int(255 -
30 * col_val))
item.setData(QBrush(bkcolor), Qt.BackgroundRole)
item.setData("trbl", BorderRole)
if tooltip is not None:
item.setToolTip(tooltip(i, j))
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
self._set_item(i + 2, j + 2, item)
def update_table(self,
matrix,
colsum=None,
rowsum=None,
colors=None,
formatstr="{}",
tooltip=None):
"""
Sets ``matrix`` as data of the table.
Parameters
----------
matrix : numpy.array
2D array to be set as data.
colsum : :obj:`numpy.array`, optional
1D optional array with aggregate values of columns, defaults to sum.
rowsum : :obj:`numpy.array`, optional
1D optional array with aggregate values of rows, defaults to sum.
colors : :obj:`numpy.array`, optional
2D array with color values, defaults to no color.
formatstr : :obj:`str`, optional
Format string for cell data, defaults to ``"{}"``.
tooltip : :obj:`(int, int) -> str`, optional
Function which takes vertical index and horizontal index as arguments and returns
desired tooltip as a string. Defaults to no tooltips.
"""
selected_indexes = self.get_selection()
self._set_values(matrix, colors, formatstr, tooltip)
if not self.circles:
if colsum is None:
colsum = matrix.sum(axis=0)
if rowsum is None:
rowsum = matrix.sum(axis=1)
self._set_sums(colsum, rowsum)
self.set_selection(selected_indexes)
def clear(self):
"""
Clears the table.
"""
self.tablemodel.clear()
class OWConfusionMatrix(widget.OWWidget):
"""Confusion matrix widget"""
name = "Confusion Matrix"
description = "Display a confusion matrix constructed from " \
"the results of classifier evaluations."
icon = "icons/ConfusionMatrix.svg"
priority = 1001
class Inputs:
evaluation_results = Input("Evaluation Results", Orange.evaluation.Results)
class Outputs:
selected_data = Output("Selected Data", Orange.data.Table, default=True)
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Orange.data.Table)
quantities = ["Number of instances",
"Proportion of predicted",
"Proportion of actual"]
settings_version = 1
settingsHandler = settings.ClassValuesContextHandler()
selected_learner = settings.Setting([0], schema_only=True)
selection = settings.ContextSetting(set())
selected_quantity = settings.Setting(0)
append_predictions = settings.Setting(True)
append_probabilities = settings.Setting(False)
autocommit = settings.Setting(True)
UserAdviceMessages = [
widget.Message(
"Clicking on cells or in headers outputs the corresponding "
"data instances",
"click_cell")]
class Error(widget.OWWidget.Error):
no_regression = Msg("Confusion Matrix cannot show regression results.")
invalid_values = Msg("Evaluation Results input contains invalid values")
def __init__(self):
super().__init__()
self.data = None
self.results = None
self.learners = []
self.headers = []
self.learners_box = gui.listBox(
self.controlArea, self, "selected_learner", "learners", box=True,
callback=self._learner_changed
)
self.outputbox = gui.vBox(self.controlArea, "Output")
box = gui.hBox(self.outputbox)
gui.checkBox(box, self, "append_predictions",
"Predictions", callback=self._invalidate)
gui.checkBox(box, self, "append_probabilities",
"Probabilities",
callback=self._invalidate)
gui.auto_commit(self.outputbox, self, "autocommit",
"Send Selected", "Send Automatically", box=False)
self.mainArea.layout().setContentsMargins(0, 0, 0, 0)
box = gui.vBox(self.mainArea, box=True)
sbox = gui.hBox(box)
gui.rubber(sbox)
gui.comboBox(sbox, self, "selected_quantity",
items=self.quantities, label="Show: ",
orientation=Qt.Horizontal, callback=self._update)
self.tablemodel = QStandardItemModel(self)
view = self.tableview = QTableView(
editTriggers=QTableView.NoEditTriggers)
view.setModel(self.tablemodel)
view.horizontalHeader().hide()
view.verticalHeader().hide()
view.horizontalHeader().setMinimumSectionSize(60)
view.selectionModel().selectionChanged.connect(self._invalidate)
view.setShowGrid(False)
view.setItemDelegate(BorderedItemDelegate(Qt.white))
view.setSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding)
view.clicked.connect(self.cell_clicked)
box.layout().addWidget(view)
selbox = gui.hBox(box)
gui.button(selbox, self, "Select Correct",
callback=self.select_correct, autoDefault=False)
gui.button(selbox, self, "Select Misclassified",
callback=self.select_wrong, autoDefault=False)
gui.button(selbox, self, "Clear Selection",
callback=self.select_none, autoDefault=False)
def sizeHint(self):
"""Initial size"""
return QSize(750, 340)
def _item(self, i, j):
return self.tablemodel.item(i, j) or QStandardItem()
def _set_item(self, i, j, item):
self.tablemodel.setItem(i, j, item)
def _init_table(self, nclasses):
item = self._item(0, 2)
item.setData("Predicted", Qt.DisplayRole)
item.setTextAlignment(Qt.AlignCenter)
item.setFlags(Qt.NoItemFlags)
self._set_item(0, 2, item)
item = self._item(2, 0)
item.setData("Actual", Qt.DisplayRole)
item.setTextAlignment(Qt.AlignHCenter | Qt.AlignBottom)
item.setFlags(Qt.NoItemFlags)
self.tableview.setItemDelegateForColumn(0, gui.VerticalItemDelegate())
self._set_item(2, 0, item)
self.tableview.setSpan(0, 2, 1, nclasses)
self.tableview.setSpan(2, 0, nclasses, 1)
font = self.tablemodel.invisibleRootItem().font()
bold_font = QFont(font)
bold_font.setBold(True)
for i in (0, 1):
for j in (0, 1):
item = self._item(i, j)
item.setFlags(Qt.NoItemFlags)
self._set_item(i, j, item)
for p, label in enumerate(self.headers):
for i, j in ((1, p + 2), (p + 2, 1)):
item = self._item(i, j)
item.setData(label, Qt.DisplayRole)
item.setFont(bold_font)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
if p < len(self.headers) - 1:
item.setData("br"[j == 1], BorderRole)
item.setData(QColor(192, 192, 192), BorderColorRole)
self._set_item(i, j, item)
hor_header = self.tableview.horizontalHeader()
if len(' '.join(self.headers)) < 120:
hor_header.setSectionResizeMode(QHeaderView.ResizeToContents)
else:
hor_header.setDefaultSectionSize(60)
self.tablemodel.setRowCount(nclasses + 3)
self.tablemodel.setColumnCount(nclasses + 3)
@Inputs.evaluation_results
def set_results(self, results):
"""Set the input results."""
prev_sel_learner = self.selected_learner.copy()
self.clear()
self.warning()
self.closeContext()
data = None
if results is not None and results.data is not None:
data = results.data[results.row_indices]
if data is not None and not data.domain.has_discrete_class:
self.Error.no_regression()
data = results = None
else:
self.Error.no_regression.clear()
nan_values = False
if results is not None:
assert isinstance(results, Orange.evaluation.Results)
if np.any(np.isnan(results.actual)) or \
np.any(np.isnan(results.predicted)):
# Error out here (could filter them out with a warning
# instead).
nan_values = True
results = data = None
if nan_values:
self.Error.invalid_values()
else:
self.Error.invalid_values.clear()
self.results = results
self.data = data
if data is not None:
class_values = data.domain.class_var.values
elif results is not None:
raise NotImplementedError
if results is None:
self.report_button.setDisabled(True)
else:
self.report_button.setDisabled(False)
nmodels = results.predicted.shape[0]
self.headers = class_values + \
[unicodedata.lookup("N-ARY SUMMATION")]
# NOTE: The 'learner_names' is set in 'Test Learners' widget.
if hasattr(results, "learner_names"):
self.learners = results.learner_names
else:
self.learners = ["Learner #{}".format(i + 1)
for i in range(nmodels)]
self._init_table(len(class_values))
self.openContext(data.domain.class_var)
if not prev_sel_learner or prev_sel_learner[0] >= len(self.learners):
if self.learners:
self.selected_learner[:] = [0]
else:
self.selected_learner[:] = prev_sel_learner
self._update()
self._set_selection()
self.unconditional_commit()
def clear(self):
"""Reset the widget, clear controls"""
self.results = None
self.data = None
self.tablemodel.clear()
self.headers = []
# Clear learners last. This action will invoke `_learner_changed`
self.learners = []
def select_correct(self):
"""Select the diagonal elements of the matrix"""
selection = QItemSelection()
n = self.tablemodel.rowCount()
for i in range(2, n):
index = self.tablemodel.index(i, i)
selection.select(index, index)
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def select_wrong(self):
"""Select the off-diagonal elements of the matrix"""
selection = QItemSelection()
n = self.tablemodel.rowCount()
for i in range(2, n):
for j in range(i + 1, n):
index = self.tablemodel.index(i, j)
selection.select(index, index)
index = self.tablemodel.index(j, i)
selection.select(index, index)
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def select_none(self):
"""Reset selection"""
self.tableview.selectionModel().clear()
def cell_clicked(self, model_index):
"""Handle cell click event"""
i, j = model_index.row(), model_index.column()
if not i or not j:
return
n = self.tablemodel.rowCount()
index = self.tablemodel.index
selection = None
if i == j == 1 or i == j == n - 1:
selection = QItemSelection(index(2, 2), index(n - 1, n - 1))
elif i in (1, n - 1):
selection = QItemSelection(index(2, j), index(n - 1, j))
elif j in (1, n - 1):
selection = QItemSelection(index(i, 2), index(i, n - 1))
if selection is not None:
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def _prepare_data(self):
indices = self.tableview.selectedIndexes()
indices = {(ind.row() - 2, ind.column() - 2) for ind in indices}
actual = self.results.actual
learner_name = self.learners[self.selected_learner[0]]
predicted = self.results.predicted[self.selected_learner[0]]
selected = [i for i, t in enumerate(zip(actual, predicted))
if t in indices]
extra = []
class_var = self.data.domain.class_var
metas = self.data.domain.metas
if self.append_predictions:
extra.append(predicted.reshape(-1, 1))
var = Orange.data.DiscreteVariable(
"{}({})".format(class_var.name, learner_name),
class_var.values
)
metas = metas + (var,)
if self.append_probabilities and \
self.results.probabilities is not None:
probs = self.results.probabilities[self.selected_learner[0]]
extra.append(np.array(probs, dtype=object))
pvars = [Orange.data.ContinuousVariable("p({})".format(value))
for value in class_var.values]
metas = metas + tuple(pvars)
domain = Orange.data.Domain(self.data.domain.attributes,
self.data.domain.class_vars,
metas)
data = self.data.transform(domain)
if len(extra):
data.metas[:, len(self.data.domain.metas):] = \
np.hstack(tuple(extra))
data.name = learner_name
if selected:
annotated_data = create_annotated_table(data, selected)
data = data[selected]
else:
annotated_data = create_annotated_table(data, [])
data = None
return data, annotated_data
def commit(self):
"""Output data instances corresponding to selected cells"""
if self.results is not None and self.data is not None \
and self.selected_learner:
data, annotated_data = self._prepare_data()
else:
data = None
annotated_data = None
self.Outputs.selected_data.send(data)
self.Outputs.annotated_data.send(annotated_data)
def _invalidate(self):
indices = self.tableview.selectedIndexes()
self.selection = {(ind.row() - 2, ind.column() - 2) for ind in indices}
self.commit()
def _set_selection(self):
selection = QItemSelection()
index = self.tableview.model().index
for row, col in self.selection:
sel = index(row + 2, col + 2)
selection.select(sel, sel)
self.tableview.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
def _learner_changed(self):
self._update()
self._set_selection()
self.commit()
def _update(self):
def _isinvalid(x):
return isnan(x) or isinf(x)
# Update the displayed confusion matrix
if self.results is not None and self.selected_learner:
cmatrix = confusion_matrix(self.results, self.selected_learner[0])
colsum = cmatrix.sum(axis=0)
rowsum = cmatrix.sum(axis=1)
n = len(cmatrix)
diag = np.diag_indices(n)
colors = cmatrix.astype(np.double)
colors[diag] = 0
if self.selected_quantity == 0:
normalized = cmatrix.astype(np.int)
formatstr = "{}"
div = np.array([colors.max()])
else:
if self.selected_quantity == 1:
normalized = 100 * cmatrix / colsum
div = colors.max(axis=0)
else:
normalized = 100 * cmatrix / rowsum[:, np.newaxis]
div = colors.max(axis=1)[:, np.newaxis]
formatstr = "{:2.1f} %"
div[div == 0] = 1
colors /= div
colors[diag] = normalized[diag] / normalized[diag].max()
for i in range(n):
for j in range(n):
val = normalized[i, j]
col_val = colors[i, j]
item = self._item(i + 2, j + 2)
item.setData(
"NA" if _isinvalid(val) else formatstr.format(val),
Qt.DisplayRole)
bkcolor = QColor.fromHsl(
[0, 240][i == j], 160,
255 if _isinvalid(col_val) else int(255 - 30 * col_val))
item.setData(QBrush(bkcolor), Qt.BackgroundRole)
item.setData("trbl", BorderRole)
item.setToolTip("actual: {}\npredicted: {}".format(
self.headers[i], self.headers[j]))
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable)
self._set_item(i + 2, j + 2, item)
bold_font = self.tablemodel.invisibleRootItem().font()
bold_font.setBold(True)
def _sum_item(value, border=""):
item = QStandardItem()
item.setData(value, Qt.DisplayRole)
item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter)
item.setFlags(Qt.ItemIsEnabled)
item.setFont(bold_font)
item.setData(border, BorderRole)
item.setData(QColor(192, 192, 192), BorderColorRole)
return item
for i in range(n):
self._set_item(n + 2, i + 2, _sum_item(int(colsum[i]), "t"))
self._set_item(i + 2, n + 2, _sum_item(int(rowsum[i]), "l"))
self._set_item(n + 2, n + 2, _sum_item(int(rowsum.sum())))
def send_report(self):
"""Send report"""
if self.results is not None and self.selected_learner:
self.report_table(
"Confusion matrix for {} (showing {})".
format(self.learners[self.selected_learner[0]],
self.quantities[self.selected_quantity].lower()),
self.tableview)
@classmethod
def migrate_settings(cls, settings, version):
if not version:
# For some period of time the 'selected_learner' property was
# changed from List[int] -> int
# (commit 4e49bb3fd0e11262f3ebf4b1116a91a4b49cc982) and then back
# again (commit 8a492d79a2e17154a0881e24a05843406c8892c0)
if "selected_learner" in settings and \
isinstance(settings["selected_learner"], int):
settings["selected_learner"] = [settings["selected_learner"]]