class OWCalibrationPlot(widget.OWWidget):
name = "Calibration Plot"
description = "Calibration plot based on evaluation of classifiers."
icon = "icons/CalibrationPlot.svg"
priority = 1030
keywords = []
class Inputs:
evaluation_results = Input("Evaluation Results", Results)
class Outputs:
calibrated_model = Output("Calibrated Model", Model)
class Error(widget.OWWidget.Error):
non_discrete_target = Msg("Calibration plot requires a categorical "
"target variable.")
empty_input = widget.Msg("Empty result on input. Nothing to display.")
nan_classes = \
widget.Msg("Remove test data instances with unknown classes.")
all_target_class = widget.Msg(
"All data instances belong to target class.")
no_target_class = widget.Msg(
"No data instances belong to target class.")
class Warning(widget.OWWidget.Warning):
omitted_folds = widget.Msg(
"Test folds where all data belongs to (non)-target are not shown.")
omitted_nan_prob_points = widget.Msg(
"Instance for which the model couldn't compute probabilities are"
"skipped.")
no_valid_data = widget.Msg("No valid data for model(s) {}")
class Information(widget.OWWidget.Information):
no_output = Msg("Can't output a model: {}")
settingsHandler = EvaluationResultsContextHandler()
target_index = settings.ContextSetting(0)
selected_classifiers = settings.ContextSetting([])
score = settings.Setting(0)
output_calibration = settings.Setting(0)
fold_curves = settings.Setting(False)
display_rug = settings.Setting(True)
threshold = settings.Setting(0.5)
visual_settings = settings.Setting({}, schema_only=True)
auto_commit = settings.Setting(True)
graph_name = "plot"
def __init__(self):
super().__init__()
self.results = None
self.scores = None
self.classifier_names = []
self.colors = []
self.line = None
self._last_score_value = -1
box = gui.vBox(self.controlArea, box="Settings")
self.target_cb = gui.comboBox(
box, self, "target_index", label="Target:",
orientation=Qt.Horizontal, callback=self.target_index_changed,
contentsLength=8, searchable=True)
gui.checkBox(
box, self, "display_rug", "Show rug",
callback=self._on_display_rug_changed)
gui.checkBox(
box, self, "fold_curves", "Curves for individual folds",
callback=self._replot)
self.classifiers_list_box = gui.listBox(
self.controlArea, self, "selected_classifiers", "classifier_names",
box="Classifier", selectionMode=QListWidget.ExtendedSelection,
sizePolicy=(QSizePolicy.Preferred, QSizePolicy.Preferred),
sizeHint=QSize(150, 40),
callback=self._on_selection_changed)
box = gui.vBox(self.controlArea, "Metrics")
combo = gui.comboBox(
box, self, "score", items=(metric.name for metric in Metrics),
callback=self.score_changed)
self.explanation = gui.widgetLabel(
box, wordWrap=True, fixedWidth=combo.sizeHint().width())
self.explanation.setContentsMargins(8, 8, 0, 0)
font = self.explanation.font()
font.setPointSizeF(0.85 * font.pointSizeF())
self.explanation.setFont(font)
gui.radioButtons(
box, self, value="output_calibration",
btnLabels=("Sigmoid calibration", "Isotonic calibration"),
label="Output model calibration", callback=self.commit.deferred)
self.info_box = gui.widgetBox(self.controlArea, "Info")
self.info_label = gui.widgetLabel(self.info_box)
gui.auto_apply(self.buttonsArea, self, "auto_commit")
self.plotview = GraphicsView()
self.plot = PlotItem(enableMenu=False)
self.plot.parameter_setter = ParameterSetter(self.plot)
self.plot.setMouseEnabled(False, False)
self.plot.hideButtons()
for axis_name in ("bottom", "left"):
axis = self.plot.getAxis(axis_name)
# Remove the condition (that is, allow setting this for bottom
# axis) when pyqtgraph is fixed
# Issue: https://github.com/pyqtgraph/pyqtgraph/issues/930
# Pull request: https://github.com/pyqtgraph/pyqtgraph/pull/932
if axis_name != "bottom": # remove if when pyqtgraph is fixed
axis.setStyle(stopAxisAtTick=(True, True))
self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)
self.plotview.setCentralItem(self.plot)
self.mainArea.layout().addWidget(self.plotview)
self._set_explanation()
VisualSettingsDialog(self, self.plot.parameter_setter.initial_settings)
@Inputs.evaluation_results
def set_results(self, results):
self.closeContext()
self.clear()
self.Error.clear()
self.Information.clear()
self.results = None
if results is not None:
if not results.domain.has_discrete_class:
self.Error.non_discrete_target()
elif not results.actual.size:
self.Error.empty_input()
elif np.any(np.isnan(results.actual)):
self.Error.nan_classes()
else:
self.results = results
self._initialize(results)
class_var = self.results.domain.class_var
self.target_index = int(len(class_var.values) == 2)
self.openContext(class_var, self.classifier_names)
self._replot()
self.commit.now()
def clear(self):
self.plot.clear()
self.results = None
self.classifier_names = []
self.selected_classifiers = []
self.target_cb.clear()
self.colors = []
def target_index_changed(self):
if len(self.results.domain.class_var.values) == 2:
self.threshold = 1 - self.threshold
self._set_explanation()
self._replot()
self.commit.deferred()
def score_changed(self):
self._set_explanation()
self._replot()
if self._last_score_value != self.score:
self.commit.deferred()
self._last_score_value = self.score
def _set_explanation(self):
explanation = Metrics[self.score].explanation
if explanation:
self.explanation.setText(explanation)
self.explanation.show()
else:
self.explanation.hide()
if self.score == 0:
self.controls.output_calibration.show()
self.info_box.hide()
else:
self.controls.output_calibration.hide()
self.info_box.show()
axis = self.plot.getAxis("bottom")
axis.setLabel("Predicted probability" if self.score == 0
else "Threshold probability to classify as positive")
axis = self.plot.getAxis("left")
axis.setLabel(Metrics[self.score].name)
def _initialize(self, results):
n = len(results.predicted)
names = getattr(results, "learner_names", None)
if names is None:
names = ["#{}".format(i + 1) for i in range(n)]
self.classifier_names = names
self.colors = colorpalettes.get_default_curve_colors(n)
for i in range(n):
item = self.classifiers_list_box.item(i)
item.setIcon(colorpalettes.ColorIcon(self.colors[i]))
self.selected_classifiers = list(range(n))
self.target_cb.addItems(results.domain.class_var.values)
self.target_index = 0
def _rug(self, data, pen_args):
color = pen_args["pen"].color()
rh = 0.025
rug_x = np.c_[data.probs[:-1], data.probs[:-1]]
rug_x_true = rug_x[data.ytrue].ravel()
rug_x_false = rug_x[~data.ytrue].ravel()
rug_y_true = np.ones_like(rug_x_true)
rug_y_true[1::2] = 1 - rh
rug_y_false = np.zeros_like(rug_x_false)
rug_y_false[1::2] = rh
self.plot.plot(
rug_x_false, rug_y_false,
pen=color, connect="pairs", antialias=True)
self.plot.plot(
rug_x_true, rug_y_true,
pen=color, connect="pairs", antialias=True)
def plot_metrics(self, data, metrics, pen_args):
if metrics is None:
return self._prob_curve(data.ytrue, data.probs[:-1], pen_args)
ys = [metric(data) for metric in metrics]
for y in ys:
self.plot.plot(data.probs, y, **pen_args)
return data.probs, ys
def _prob_curve(self, ytrue, probs, pen_args):
xmin, xmax = probs.min(), probs.max()
x = np.linspace(xmin, xmax, 100)
if xmax != xmin:
f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin))
y = f(x)
else:
y = np.full(100, xmax)
self.plot.plot(x, y, symbol="+", symbolSize=4, **pen_args)
return x, (y, )
def _setup_plot(self):
target = self.target_index
results = self.results
metrics = Metrics[self.score].functions
plot_folds = self.fold_curves and results.folds is not None
self.scores = []
if not self._check_class_presence(results.actual == target):
return
self.Warning.omitted_folds.clear()
self.Warning.omitted_nan_prob_points.clear()
no_valid_models = []
shadow_width = 4 + 4 * plot_folds
for clsf in self.selected_classifiers:
data = Curves.from_results(results, target, clsf)
if data.tot == 0: # all probabilities are nan
no_valid_models.append(clsf)
continue
if data.tot != results.probabilities.shape[1]: # some are nan
self.Warning.omitted_nan_prob_points()
color = self.colors[clsf]
pen_args = dict(
pen=pg.mkPen(color, width=1), antiAlias=True,
shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))
self.scores.append(
(self.classifier_names[clsf],
self.plot_metrics(data, metrics, pen_args)))
if self.display_rug:
self._rug(data, pen_args)
if plot_folds:
pen_args = dict(
pen=pg.mkPen(color, width=1, style=Qt.DashLine),
antiAlias=True)
for fold in range(len(results.folds)):
fold_results = results.get_fold(fold)
fold_curve = Curves.from_results(fold_results, target, clsf)
# Can't check this before: p and n can be 0 because of
# nan probabilities
if fold_curve.p * fold_curve.n == 0:
self.Warning.omitted_folds()
self.plot_metrics(fold_curve, metrics, pen_args)
if no_valid_models:
self.Warning.no_valid_data(
", ".join(self.classifier_names[i] for i in no_valid_models))
if self.score == 0:
self.plot.plot([0, 1], [0, 1], antialias=True)
else:
self.line = pg.InfiniteLine(
pos=self.threshold, movable=True,
pen=pg.mkPen(color="k", style=Qt.DashLine, width=2),
hoverPen=pg.mkPen(color="k", style=Qt.DashLine, width=3),
bounds=(0, 1),
)
self.line.sigPositionChanged.connect(self.threshold_change)
self.line.sigPositionChangeFinished.connect(
self.threshold_change_done)
self.plot.addItem(self.line)
def _check_class_presence(self, ytrue):
self.Error.all_target_class.clear()
self.Error.no_target_class.clear()
if np.max(ytrue) == 0:
self.Error.no_target_class()
return False
if np.min(ytrue) == 1:
self.Error.all_target_class()
return False
return True
def _replot(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
self._update_info()
def _on_display_rug_changed(self):
self._replot()
def _on_selection_changed(self):
self._replot()
self.commit.deferred()
def threshold_change(self):
self.threshold = round(self.line.pos().x(), 2)
self.line.setPos(self.threshold)
self._update_info()
def get_info_text(self, short):
if short:
def elided(s):
return s[:17] + "..." if len(s) > 20 else s
text = f"""<table>
<tr>
<th align='right'>Threshold: p=</th>
<td colspan='4'>{self.threshold:.2f}<br/></td>
</tr>"""
else:
def elided(s):
return s
text = f"""<table>
<tr>
<th align='right'>Threshold:</th>
<td colspan='4'>p = {self.threshold:.2f}<br/>
</td>
<tr/>
</tr>"""
if self.scores is not None:
short_names = Metrics[self.score].short_names
if short_names:
text += f"""<tr>
<th></th>
{"<td></td>".join(f"<td align='right'>{n}</td>"
for n in short_names)}
</tr>"""
for name, (probs, curves) in self.scores:
ind = min(np.searchsorted(probs, self.threshold),
len(probs) - 1)
text += f"<tr><th align='right'>{elided(name)}:</th>"
text += "<td>/</td>".join(f'<td>{curve[ind]:.3f}</td>'
for curve in curves)
text += "</tr>"
text += "<table>"
return text
return None
def _update_info(self):
self.info_label.setText(self.get_info_text(short=True))
def threshold_change_done(self):
self.commit.deferred()
@gui.deferred
def commit(self):
self.Information.no_output.clear()
wrapped = None
results = self.results
if results is not None:
problems = [
msg for condition, msg in (
(results.folds is not None and len(results.folds) > 1,
"each training data sample produces a different model"),
(results.models is None,
"test results do not contain stored models - try testing "
"on separate data or on training data"),
(len(self.selected_classifiers) != 1,
"select a single model - the widget can output only one"),
(self.score != 0 and len(results.domain.class_var.values) != 2,
"cannot calibrate non-binary classes"))
if condition]
if len(problems) == 1:
self.Information.no_output(problems[0])
elif problems:
self.Information.no_output(
"".join(f"\n - {problem}" for problem in problems))
else:
clsf_idx = self.selected_classifiers[0]
model = results.models[0, clsf_idx]
if self.score == 0:
cal_learner = CalibratedLearner(
None, self.output_calibration)
wrapped = cal_learner.get_model(
model, results.actual, results.probabilities[clsf_idx])
else:
threshold = [1 - self.threshold,
self.threshold][self.target_index]
wrapped = ThresholdClassifier(model, threshold)
self.Outputs.calibrated_model.send(wrapped)
def send_report(self):
if self.results is None:
return
self.report_items((
("Target class", self.target_cb.currentText()),
("Output model calibration",
self.score == 0
and ("Sigmoid calibration",
"Isotonic calibration")[self.output_calibration])
))
caption = report.list_legend(self.classifiers_list_box,
self.selected_classifiers)
self.report_plot()
self.report_caption(caption)
self.report_caption(self.controls.score.currentText())
if self.score != 0:
self.report_raw(self.get_info_text(short=False))
def set_visual_settings(self, key, value):
self.plot.parameter_setter.set_parameter(key, value)
self.visual_settings[key] = value
class OWLiftCurve(widget.OWWidget):
name = "Lift Curve"
description = "Construct and display a lift curve " \
"from the evaluation of classifiers."
icon = "icons/LiftCurve.svg"
priority = 1020
keywords = []
class Inputs:
evaluation_results = Input("Evaluation Results",
Orange.evaluation.Results)
settingsHandler = EvaluationResultsContextHandler()
target_index = settings.ContextSetting(0)
selected_classifiers = settings.ContextSetting([])
display_convex_hull = settings.Setting(False)
display_cost_func = settings.Setting(True)
fp_cost = settings.Setting(500)
fn_cost = settings.Setting(500)
target_prior = settings.Setting(50.0)
graph_name = "plot"
def __init__(self):
super().__init__()
self.results = None
self.classifier_names = []
self.colors = []
self._curve_data = {}
box = gui.vBox(self.controlArea, "Plot")
tbox = gui.vBox(box, "Target Class")
tbox.setFlat(True)
self.target_cb = gui.comboBox(tbox,
self,
"target_index",
callback=self._on_target_changed,
contentsLength=8)
cbox = gui.vBox(box, "Classifiers")
cbox.setFlat(True)
self.classifiers_list_box = gui.listBox(
cbox,
self,
"selected_classifiers",
"classifier_names",
selectionMode=QListView.MultiSelection,
callback=self._on_classifiers_changed)
gui.checkBox(box,
self,
"display_convex_hull",
"Show lift convex hull",
callback=self._replot)
self.plotview = pg.GraphicsView(background="w")
self.plotview.setFrameStyle(QFrame.StyledPanel)
self.plot = pg.PlotItem(enableMenu=False)
self.plot.setMouseEnabled(False, False)
self.plot.hideButtons()
pen = QPen(self.palette().color(QPalette.Text))
tickfont = QFont(self.font())
tickfont.setPixelSize(max(int(tickfont.pixelSize() * 2 // 3), 11))
axis = self.plot.getAxis("bottom")
axis.setTickFont(tickfont)
axis.setPen(pen)
axis.setLabel("P Rate")
axis = self.plot.getAxis("left")
axis.setTickFont(tickfont)
axis.setPen(pen)
axis.setLabel("TP Rate")
self.plot.showGrid(True, True, alpha=0.1)
self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)
self.plotview.setCentralItem(self.plot)
self.mainArea.layout().addWidget(self.plotview)
@Inputs.evaluation_results
def set_results(self, results):
"""Set the input evaluation results."""
self.closeContext()
self.clear()
self.results = check_results_adequacy(results, self.Error)
if self.results is not None:
self._initialize(results)
self.openContext(self.results.domain.class_var,
self.classifier_names)
self._setup_plot()
def clear(self):
"""Clear the widget state."""
self.plot.clear()
self.results = None
self.target_cb.clear()
self.target_index = 0
self.classifier_names = []
self.colors = []
self._curve_data = {}
def _initialize(self, results):
N = len(results.predicted)
names = getattr(results, "learner_names", None)
if names is None:
names = ["#{}".format(i + 1) for i in range(N)]
scheme = colorbrewer.colorSchemes["qualitative"]["Dark2"]
if N > len(scheme):
scheme = colorpalette.DefaultRGBColors
self.colors = colorpalette.ColorPaletteGenerator(N, scheme)
self.classifier_names = names
self.selected_classifiers = list(range(N))
for i in range(N):
item = self.classifiers_list_box.item(i)
item.setIcon(colorpalette.ColorPixmap(self.colors[i]))
self.target_cb.addItems(results.data.domain.class_var.values)
def plot_curves(self, target, clf_idx):
if (target, clf_idx) not in self._curve_data:
curve = liftCurve_from_results(self.results, clf_idx, target)
color = self.colors[clf_idx]
pen = QPen(color, 1)
pen.setCosmetic(True)
shadow_pen = QPen(pen.color().lighter(160), 2.5)
shadow_pen.setCosmetic(True)
item = pg.PlotDataItem(curve.points[0],
curve.points[1],
pen=pen,
shadowPen=shadow_pen,
symbol="+",
symbolSize=3,
symbolPen=shadow_pen,
antialias=True)
hull_item = pg.PlotDataItem(curve.hull[0],
curve.hull[1],
pen=pen,
antialias=True)
self._curve_data[target, clf_idx] = \
PlotCurve(curve, item, hull_item)
return self._curve_data[target, clf_idx]
def _setup_plot(self):
target = self.target_index
selected = self.selected_classifiers
curves = [self.plot_curves(target, clf_idx) for clf_idx in selected]
for curve in curves:
self.plot.addItem(curve.curve_item)
if self.display_convex_hull:
hull = convex_hull([c.curve.hull for c in curves])
self.plot.plot(hull[0], hull[1], pen="y", antialias=True)
pen = QPen(QColor(100, 100, 100, 100), 1, Qt.DashLine)
pen.setCosmetic(True)
self.plot.plot([0, 1], [0, 1], pen=pen, antialias=True)
warning = ""
if not all(c.curve.is_valid for c in curves):
if any(c.curve.is_valid for c in curves):
warning = "Some lift curves are undefined"
else:
warning = "All lift curves are undefined"
self.warning(warning)
def _replot(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
def _on_target_changed(self):
self._replot()
def _on_classifiers_changed(self):
self._replot()
def send_report(self):
if self.results is None:
return
caption = report.list_legend(self.classifiers_list_box,
self.selected_classifiers)
self.report_items((("Target class", self.target_cb.currentText()), ))
self.report_plot()
self.report_caption(caption)
class OWROCAnalysis(widget.OWWidget):
name = "ROC Analysis"
description = "Display the Receiver Operating Characteristics curve " \
"based on the evaluation of classifiers."
icon = "icons/ROCAnalysis.svg"
priority = 1010
keywords = []
class Inputs:
evaluation_results = Input("Evaluation Results",
Orange.evaluation.Results)
settingsHandler = EvaluationResultsContextHandler()
target_index = settings.ContextSetting(0)
selected_classifiers = settings.ContextSetting([])
display_perf_line = settings.Setting(True)
display_def_threshold = settings.Setting(True)
fp_cost = settings.Setting(500)
fn_cost = settings.Setting(500)
target_prior = settings.Setting(50.0, schema_only=True)
#: ROC Averaging Types
Merge, Vertical, Threshold, NoAveraging = 0, 1, 2, 3
roc_averaging = settings.Setting(Merge)
display_convex_hull = settings.Setting(False)
display_convex_curve = settings.Setting(False)
graph_name = "plot"
def __init__(self):
super().__init__()
self.results = None
self.classifier_names = []
self.perf_line = None
self.colors = []
self._curve_data = {}
self._plot_curves = {}
self._rocch = None
self._perf_line = None
self._tooltip_cache = None
box = gui.vBox(self.controlArea, "Plot")
self.target_cb = gui.comboBox(box,
self,
"target_index",
label="Target",
orientation=Qt.Horizontal,
callback=self._on_target_changed,
contentsLength=8,
searchable=True)
gui.widgetLabel(box, "Classifiers")
line_height = 4 * QFontMetrics(self.font()).lineSpacing()
self.classifiers_list_box = gui.listBox(
box,
self,
"selected_classifiers",
"classifier_names",
selectionMode=QListView.MultiSelection,
callback=self._on_classifiers_changed,
sizeHint=QSize(0, line_height))
abox = gui.vBox(self.controlArea, "Curves")
gui.comboBox(abox,
self,
"roc_averaging",
items=[
"Merge Predictions from Folds", "Mean TP Rate",
"Mean TP and FP at Threshold",
"Show Individual Curves"
],
callback=self._replot)
gui.checkBox(abox,
self,
"display_convex_curve",
"Show convex ROC curves",
callback=self._replot)
gui.checkBox(abox,
self,
"display_convex_hull",
"Show ROC convex hull",
callback=self._replot)
box = gui.vBox(self.controlArea, "Analysis")
gui.checkBox(box,
self,
"display_def_threshold",
"Default threshold (0.5) point",
callback=self._on_display_def_threshold_changed)
gui.checkBox(box,
self,
"display_perf_line",
"Show performance line",
callback=self._on_display_perf_line_changed)
grid = QGridLayout()
gui.indentedBox(box, orientation=grid)
sp = gui.spin(box,
self,
"fp_cost",
1,
1000,
10,
alignment=Qt.AlignRight,
callback=self._on_display_perf_line_changed)
grid.addWidget(QLabel("FP Cost:"), 0, 0)
grid.addWidget(sp, 0, 1)
sp = gui.spin(box,
self,
"fn_cost",
1,
1000,
10,
alignment=Qt.AlignRight,
callback=self._on_display_perf_line_changed)
grid.addWidget(QLabel("FN Cost:"))
grid.addWidget(sp, 1, 1)
self.target_prior_sp = gui.spin(box,
self,
"target_prior",
1,
99,
alignment=Qt.AlignRight,
callback=self._on_target_prior_changed)
self.target_prior_sp.setSuffix(" %")
self.target_prior_sp.addAction(QAction("Auto", sp))
grid.addWidget(QLabel("Prior probability:"))
grid.addWidget(self.target_prior_sp, 2, 1)
self.plotview = pg.GraphicsView(background="w")
self.plotview.setFrameStyle(QFrame.StyledPanel)
self.plotview.scene().sigMouseMoved.connect(self._on_mouse_moved)
self.plot = pg.PlotItem(enableMenu=False)
self.plot.setMouseEnabled(False, False)
self.plot.hideButtons()
pen = QPen(self.palette().color(QPalette.Text))
tickfont = QFont(self.font())
tickfont.setPixelSize(max(int(tickfont.pixelSize() * 2 // 3), 11))
axis = self.plot.getAxis("bottom")
axis.setTickFont(tickfont)
axis.setPen(pen)
axis.setLabel("FP Rate (1-Specificity)")
axis = self.plot.getAxis("left")
axis.setTickFont(tickfont)
axis.setPen(pen)
axis.setLabel("TP Rate (Sensitivity)")
self.plot.showGrid(True, True, alpha=0.1)
self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0), padding=0.05)
self.plotview.setCentralItem(self.plot)
self.mainArea.layout().addWidget(self.plotview)
@Inputs.evaluation_results
def set_results(self, results):
"""Set the input evaluation results."""
self.closeContext()
self.clear()
self.results = check_results_adequacy(results, self.Error)
if self.results is not None:
self._initialize(self.results)
self.openContext(self.results.domain.class_var,
self.classifier_names)
self._setup_plot()
else:
self.warning()
def clear(self):
"""Clear the widget state."""
self.results = None
self.plot.clear()
self.classifier_names = []
self.selected_classifiers = []
self.target_cb.clear()
self.colors = []
self._curve_data = {}
self._plot_curves = {}
self._rocch = None
self._perf_line = None
self._tooltip_cache = None
def _initialize(self, results):
names = getattr(results, "learner_names", None)
if names is None:
names = [
"#{}".format(i + 1) for i in range(len(results.predicted))
]
self.colors = colorpalettes.get_default_curve_colors(len(names))
self.classifier_names = names
self.selected_classifiers = list(range(len(names)))
for i in range(len(names)):
listitem = self.classifiers_list_box.item(i)
listitem.setIcon(colorpalettes.ColorIcon(self.colors[i]))
class_var = results.data.domain.class_var
self.target_cb.addItems(class_var.values)
self.target_index = 0
self._set_target_prior()
def _set_target_prior(self):
"""
This function sets the initial target class probability prior value
based on the input data.
"""
if self.results.data:
# here we can use target_index directly since values in the
# dropdown are sorted in same order than values in the table
target_values_cnt = np.count_nonzero(
self.results.data.Y == self.target_index)
count_all = np.count_nonzero(~np.isnan(self.results.data.Y))
self.target_prior = np.round(target_values_cnt / count_all * 100)
# set the spin text to gray color when set automatically
self.target_prior_sp.setStyleSheet("color: gray;")
def curve_data(self, target, clf_idx):
"""Return `ROCData' for the given target and classifier."""
if (target, clf_idx) not in self._curve_data:
# pylint: disable=no-member
data = ROCData.from_results(self.results, clf_idx, target)
self._curve_data[target, clf_idx] = data
return self._curve_data[target, clf_idx]
def plot_curves(self, target, clf_idx):
"""Return a set of functions `plot_curves` generating plot curves."""
def generate_pens(basecolor):
pen = QPen(basecolor, 1)
pen.setCosmetic(True)
shadow_pen = QPen(pen.color().lighter(160), 2.5)
shadow_pen.setCosmetic(True)
return pen, shadow_pen
data = self.curve_data(target, clf_idx)
if (target, clf_idx) not in self._plot_curves:
pen, shadow_pen = generate_pens(self.colors[clf_idx])
name = self.classifier_names[clf_idx]
@once
def merged():
return plot_curve(data.merged,
pen=pen,
shadow_pen=shadow_pen,
name=name)
@once
def folds():
return [
plot_curve(fold, pen=pen, shadow_pen=shadow_pen)
for fold in data.folds
]
@once
def avg_vert():
return plot_avg_curve(data.avg_vertical,
pen=pen,
shadow_pen=shadow_pen,
name=name)
@once
def avg_thres():
return plot_avg_curve(data.avg_threshold,
pen=pen,
shadow_pen=shadow_pen,
name=name)
self._plot_curves[target,
clf_idx] = PlotCurves(merge=merged,
folds=folds,
avg_vertical=avg_vert,
avg_threshold=avg_thres)
return self._plot_curves[target, clf_idx]
def _setup_plot(self):
def merge_averaging():
for curve in curves:
graphics = curve.merge()
curve = graphics.curve
self.plot.addItem(graphics.curve_item)
if self.display_convex_curve:
self.plot.addItem(graphics.hull_item)
if self.display_def_threshold and curve.is_valid:
points = curve.points
ind = np.argmin(np.abs(points.thresholds - 0.5))
item = pg.TextItem(text="{:.3f}".format(
points.thresholds[ind]), )
item.setPos(points.fpr[ind], points.tpr[ind])
self.plot.addItem(item)
hull_curves = [curve.merged.hull for curve in selected]
if hull_curves:
self._rocch = convex_hull(hull_curves)
iso_pen = QPen(QColor(Qt.black), 1)
iso_pen.setCosmetic(True)
self._perf_line = InfiniteLine(pen=iso_pen, antialias=True)
self.plot.addItem(self._perf_line)
return hull_curves
def vertical_averaging():
for curve in curves:
graphics = curve.avg_vertical()
self.plot.addItem(graphics.curve_item)
self.plot.addItem(graphics.confint_item)
return [curve.avg_vertical.hull for curve in selected]
def threshold_averaging():
for curve in curves:
graphics = curve.avg_threshold()
self.plot.addItem(graphics.curve_item)
self.plot.addItem(graphics.confint_item)
return [curve.avg_threshold.hull for curve in selected]
def no_averaging():
for curve in curves:
graphics = curve.folds()
for fold in graphics:
self.plot.addItem(fold.curve_item)
if self.display_convex_curve:
self.plot.addItem(fold.hull_item)
return [fold.hull for curve in selected for fold in curve.folds]
averagings = {
OWROCAnalysis.Merge: merge_averaging,
OWROCAnalysis.Vertical: vertical_averaging,
OWROCAnalysis.Threshold: threshold_averaging,
OWROCAnalysis.NoAveraging: no_averaging
}
target = self.target_index
selected = self.selected_classifiers
curves = [self.plot_curves(target, i) for i in selected]
selected = [self.curve_data(target, i) for i in selected]
hull_curves = averagings[self.roc_averaging]()
if self.display_convex_hull and hull_curves:
hull = convex_hull(hull_curves)
hull_pen = QPen(QColor(200, 200, 200, 100), 2)
hull_pen.setCosmetic(True)
item = self.plot.plot(hull.fpr,
hull.tpr,
pen=hull_pen,
brush=QBrush(QColor(200, 200, 200, 50)),
fillLevel=0)
item.setZValue(-10000)
pen = QPen(QColor(100, 100, 100, 100), 1, Qt.DashLine)
pen.setCosmetic(True)
self.plot.plot([0, 1], [0, 1], pen=pen, antialias=True)
if self.roc_averaging == OWROCAnalysis.Merge:
self._update_perf_line()
warning = ""
if not all(c.is_valid for c in hull_curves):
if any(c.is_valid for c in hull_curves):
warning = "Some ROC curves are undefined"
else:
warning = "All ROC curves are undefined"
self.warning(warning)
def _on_mouse_moved(self, pos):
target = self.target_index
selected = self.selected_classifiers
curves = [(clf_idx, self.plot_curves(target, clf_idx))
for clf_idx in selected
] # type: List[Tuple[int, PlotCurves]]
valid_thresh, valid_clf = [], []
pt, ave_mode = None, self.roc_averaging
for clf_idx, crv in curves:
if self.roc_averaging == OWROCAnalysis.Merge:
curve = crv.merge()
elif self.roc_averaging == OWROCAnalysis.Vertical:
curve = crv.avg_vertical()
elif self.roc_averaging == OWROCAnalysis.Threshold:
curve = crv.avg_threshold()
else:
# currently not implemented for 'Show Individual Curves'
return
sp = curve.curve_item.childItems()[0] # type: pg.ScatterPlotItem
act_pos = sp.mapFromScene(pos)
pts = sp.pointsAt(act_pos)
if pts:
mouse_pt = pts[0].pos()
if self._tooltip_cache:
cache_pt, cache_thresh, cache_clf, cache_ave = self._tooltip_cache
curr_thresh, curr_clf = [], []
if np.linalg.norm(mouse_pt - cache_pt) < 10e-6 \
and cache_ave == self.roc_averaging:
mask = np.equal(cache_clf, clf_idx)
curr_thresh = np.compress(mask, cache_thresh).tolist()
curr_clf = np.compress(mask, cache_clf).tolist()
else:
QToolTip.showText(QCursor.pos(), "")
self._tooltip_cache = None
if curr_thresh:
valid_thresh.append(*curr_thresh)
valid_clf.append(*curr_clf)
pt = cache_pt
continue
curve_pts = curve.curve.points
roc_points = np.column_stack((curve_pts.fpr, curve_pts.tpr))
diff = np.subtract(roc_points, mouse_pt)
# Find closest point on curve and save the corresponding threshold
idx_closest = np.argmin(np.linalg.norm(diff, axis=1))
thresh = curve_pts.thresholds[idx_closest]
if not np.isnan(thresh):
valid_thresh.append(thresh)
valid_clf.append(clf_idx)
pt = [
curve_pts.fpr[idx_closest], curve_pts.tpr[idx_closest]
]
if valid_thresh:
clf_names = self.classifier_names
msg = "Thresholds:\n" + "\n".join([
"({:s}) {:.3f}".format(clf_names[i], thresh)
for i, thresh in zip(valid_clf, valid_thresh)
])
QToolTip.showText(QCursor.pos(), msg)
self._tooltip_cache = (pt, valid_thresh, valid_clf, ave_mode)
def _on_target_changed(self):
self.plot.clear()
self._set_target_prior()
self._setup_plot()
def _on_classifiers_changed(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
def _on_target_prior_changed(self):
self.target_prior_sp.setStyleSheet("color: black;")
self._on_display_perf_line_changed()
def _on_display_perf_line_changed(self):
if self.roc_averaging == OWROCAnalysis.Merge:
self._update_perf_line()
if self.perf_line is not None:
self.perf_line.setVisible(self.display_perf_line)
def _on_display_def_threshold_changed(self):
self._replot()
def _replot(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
def _update_perf_line(self):
if self._perf_line is None:
return
self._perf_line.setVisible(self.display_perf_line)
if self.display_perf_line:
m = roc_iso_performance_slope(self.fp_cost, self.fn_cost,
self.target_prior / 100.0)
hull = self._rocch
if hull.is_valid:
ind = roc_iso_performance_line(m, hull)
angle = np.arctan2(m, 1) # in radians
self._perf_line.setAngle(angle * 180 / np.pi)
self._perf_line.setPos((hull.fpr[ind[0]], hull.tpr[ind[0]]))
else:
self._perf_line.setVisible(False)
def onDeleteWidget(self):
self.clear()
def send_report(self):
if self.results is None:
return
items = OrderedDict()
items["Target class"] = self.target_cb.currentText()
if self.display_perf_line:
items["Costs"] = \
"FP = {}, FN = {}".format(self.fp_cost, self.fn_cost)
items["Target probability"] = "{} %".format(self.target_prior)
caption = report.list_legend(self.classifiers_list_box,
self.selected_classifiers)
self.report_items(items)
self.report_plot()
self.report_caption(caption)
class OWLiftCurve(widget.OWWidget):
name = "Lift Curve"
description = "Construct and display a lift curve " \
"from the evaluation of classifiers."
icon = "icons/LiftCurve.svg"
priority = 1020
keywords = ["lift", "cumulative gain"]
class Inputs:
evaluation_results = Input(
"Evaluation Results", Orange.evaluation.Results)
class Warning(widget.OWWidget.Warning):
undefined_curves = Msg(
"Some curves are undefined; check models and data")
class Error(widget.OWWidget.Error):
undefined_curves = Msg(
"No defined curves; check models and data")
settingsHandler = EvaluationResultsContextHandler()
target_index = settings.ContextSetting(0)
selected_classifiers = settings.ContextSetting([])
display_convex_hull = settings.Setting(True)
curve_type = settings.Setting(CurveTypes.LiftCurve)
graph_name = "plot"
YLabels = ("Lift", "TP Rate")
def __init__(self):
super().__init__()
self.results = None
self.classifier_names = []
self.colors = []
self._points_hull: Dict[Tuple[int, int], PointsAndHull] = {}
box = gui.vBox(self.controlArea, box="Curve")
self.target_cb = gui.comboBox(
box, self, "target_index",
label="Target: ", orientation=Qt.Horizontal,
callback=self._on_target_changed,
contentsLength=8, searchable=True
)
gui.radioButtons(
box, self, "curve_type", ("Lift Curve", "Cumulative Gains"),
callback=self._on_curve_type_changed
)
self.classifiers_list_box = gui.listBox(
self.controlArea, self, "selected_classifiers", "classifier_names",
box="Models",
selectionMode=QListView.MultiSelection,
callback=self._on_classifiers_changed
)
self.classifiers_list_box.setMaximumHeight(100)
gui.checkBox(self.controlArea, self, "display_convex_hull",
"Show convex hull", box="Settings", callback=self._replot)
gui.rubber(self.controlArea)
self.plotview = pg.GraphicsView(background="w")
self.plotview.setFrameStyle(QFrame.StyledPanel)
self.plot = pg.PlotItem(enableMenu=False)
self.plot.setMouseEnabled(False, False)
self.plot.hideButtons()
pen = QPen(self.palette().color(QPalette.Text))
tickfont = QFont(self.font())
tickfont.setPixelSize(max(int(tickfont.pixelSize() * 2 // 3), 11))
for pos, label in (("bottom", "P Rate"), ("left", "")):
axis = self.plot.getAxis(pos)
axis.setTickFont(tickfont)
axis.setPen(pen)
axis.setLabel(label)
self._set_left_label()
self.plot.showGrid(True, True, alpha=0.1)
self.plotview.setCentralItem(self.plot)
self.mainArea.layout().addWidget(self.plotview)
@Inputs.evaluation_results
def set_results(self, results):
self.closeContext()
self.clear()
self.results = check_results_adequacy(results, self.Error)
if self.results is not None:
self._initialize(results)
self.openContext(self.results.domain.class_var,
self.classifier_names)
self._setup_plot()
def clear(self):
self.plot.clear()
self.Warning.clear()
self.Error.clear()
self.results = None
self.target_cb.clear()
self.classifier_names = []
self.colors = []
self._points_hull = {}
def _initialize(self, results):
n_models = len(results.predicted)
self.classifier_names = getattr(results, "learner_names", None) \
or [f"#{i}" for i in range(n_models)]
self.selected_classifiers = list(range(n_models))
self.colors = colorpalettes.get_default_curve_colors(n_models)
for i, color in enumerate(self.colors):
item = self.classifiers_list_box.item(i)
item.setIcon(colorpalettes.ColorIcon(color))
class_values = results.data.domain.class_var.values
self.target_cb.addItems(class_values)
if class_values:
self.target_index = 0
def _replot(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
_on_target_changed = _replot
_on_classifiers_changed = _replot
def _on_curve_type_changed(self):
self._set_left_label()
self._replot()
def _set_left_label(self):
self.plot.getAxis("left").setLabel(self.YLabels[self.curve_type])
def _setup_plot(self):
self._plot_default_line()
is_valid = [
self._plot_curve(self.target_index, clf_idx)
for clf_idx in self.selected_classifiers
]
self.plot.autoRange()
self._set_undefined_curves_err_warn(is_valid)
def _plot_curve(self, target, clf_idx):
key = (target, clf_idx)
if key not in self._points_hull:
self._points_hull[key] = \
points_from_results(self.results, target, clf_idx)
points, hull = self._points_hull[key]
if not points.is_valid:
return False
color = self.colors[clf_idx]
pen = QPen(color, 1)
pen.setCosmetic(True)
wide_pen = QPen(color, 3)
wide_pen.setCosmetic(True)
def _plot(points, pen):
contacted, respondents, _ = points
if self.curve_type == CurveTypes.LiftCurve:
respondents = respondents / contacted
self.plot.plot(contacted, respondents, pen=pen, antialias=True)
_plot(points, wide_pen if not self.display_convex_hull else pen)
if self.display_convex_hull:
_plot(hull, wide_pen)
return True
def _plot_default_line(self):
pen = QPen(QColor(20, 20, 20), 1, Qt.DashLine)
pen.setCosmetic(True)
y0 = 1 if self.curve_type == CurveTypes.LiftCurve else 0
self.plot.plot([0, 1], [y0, 1], pen=pen, antialias=True)
def _set_undefined_curves_err_warn(self, is_valid):
self.Error.undefined_curves.clear()
self.Warning.undefined_curves.clear()
if not all(is_valid):
if any(is_valid):
self.Warning.undefined_curves()
else:
self.Error.undefined_curves()
def send_report(self):
if self.results is None:
return
caption = report.list_legend(self.classifiers_list_box,
self.selected_classifiers)
self.report_items((("Target class", self.target_cb.currentText()),))
self.report_plot()
self.report_caption(caption)