class OWTestLearners(widget.OWWidget): name = "Test Learners" description = "" icon = "icons/TestLearners1.svg" priority = 100 inputs = [("Learner", Orange.classification.Learner, "set_learner", widget.Multiple), ("Data", Orange.data.Table, "set_train_data", widget.Default), ("Test Data", Orange.data.Table, "set_test_data")] outputs = [("Evaluation Results", testing.Results)] #: Resampling/testing types KFold, LeaveOneOut, Bootstrap, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4 #: Selected resampling type resampling = settings.Setting(0) #: Number of folds for K-fold cross validation k_folds = settings.Setting(10) #: Number of repeats for bootstrap sampling n_repeat = settings.Setting(10) #: Bootstrap sampling p sample_p = settings.Setting(75) def __init__(self, parent=None): super().__init__(parent) self.train_data = None self.test_data = None #: An Ordered dictionary with current inputs and their testing #: results. self.learners = OrderedDict() sbox = gui.widgetBox(self.controlArea, "Sampling") rbox = gui.radioButtons( sbox, self, "resampling", callback=self._param_changed ) gui.appendRadioButton(rbox, "Cross validation") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:", callback=self.kfold_changed) gui.appendRadioButton(rbox, "Leave one out") gui.appendRadioButton(rbox, "Random sampling") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test", callback=self.bootstrap_changed) gui.widgetLabel(ibox, "Relative training set size:") gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=100, ticks=20, vertical=False, labelFormat="%d %%", callback=self.bootstrap_changed) gui.appendRadioButton(rbox, "Test on train data") gui.appendRadioButton(rbox, "Test on test data") rbox.layout().addSpacing(5) gui.button(rbox, self, "Apply", callback=self.apply) gui.rubber(self.controlArea) self.view = QTreeView( rootIsDecorated=False, uniformRowHeights=True, wordWrap=True, editTriggers=QTreeView.NoEditTriggers ) header = self.view.header() header.setResizeMode(QHeaderView.ResizeToContents) header.setDefaultAlignment(Qt.AlignCenter) header.setStretchLastSection(False) self.result_model = QStandardItemModel() self.view.setModel(self.result_model) self.view.setItemDelegate(ItemDelegate()) self._update_header() box = gui.widgetBox(self.mainArea, "Evaluation Results") box.layout().addWidget(self.view) def set_learner(self, learner, key): if key in self.learners and learner is None: del self.learners[key] else: self.learners[key] = Input(learner, None, ()) self._update_stats_model() def set_train_data(self, data): self.error(0) if data is not None: if data.domain.class_var is None: self.error(0, "Train data input requires a class variable") data = None self.train_data = data self._update_header() self._invalidate() def set_test_data(self, data): self.error(1) if data is not None: if data.domain.class_var is None: self.error(1, "Test data input requires a class variable") data = None self.test_data = data if self.resampling == OWTestLearners.TestOnTest: self._invalidate() def handleNewSignals(self): self.update_results() self.commit() def kfold_changed(self): self.resampling = OWTestLearners.KFold self._param_changed() def bootstrap_changed(self): self.resampling = OWTestLearners.Bootstrap self._param_changed() def _param_changed(self): self._invalidate() def update_results(self): self.warning([1, 2]) self.error(2) if self.train_data is None: return if self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: self.warning(2, "Missing separate test data input") return elif self.test_data.domain.class_var != \ self.train_data.domain.class_var: self.error(2, ("Inconsistent class variable between test " + "and train data sets")) return # items in need of an update items = [(key, input) for key, input in self.learners.items() if input.results is None] learners = [input.learner for _, input in items] self.setStatusMessage("Running") if self.test_data is not None and \ self.resampling != OWTestLearners.TestOnTest: self.warning(1, "Test data is present but unused. " "Select 'Test on test data' to use it.") # TODO: Test each learner individually if self.resampling == OWTestLearners.KFold: results = testing.CrossValidation( self.train_data, learners, k=self.k_folds, store_data=True ) elif self.resampling == OWTestLearners.LeaveOneOut: results = testing.LeaveOneOut( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.Bootstrap: p = self.sample_p / 100.0 results = testing.Bootstrap( self.train_data, learners, n_resamples=self.n_repeat, p=p, store_data=True ) elif self.resampling == OWTestLearners.TestOnTrain: results = testing.TestOnTrainingData( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: return results = testing.TestOnTestData( self.train_data, self.test_data, learners, store_data=True ) else: assert False results = list(split_by_model(results)) class_var = self.train_data.domain.class_var if is_discrete(class_var): test_stats = classification_stats else: test_stats = regression_stats self._update_header() stats = [test_stats(res) for res in results] for (key, input), res, stat in zip(items, results, stats): self.learners[key] = input._replace(results=res, stats=stat) self.setStatusMessage("") self._update_stats_model() def _update_header(self): headers = ["Method"] if self.train_data is not None: if is_discrete(self.train_data.domain.class_var): headers.extend(classification_stats.headers) else: headers.extend(regression_stats.headers) for i in reversed(range(len(headers), self.result_model.columnCount())): self.result_model.takeColumn(i) self.result_model.setHorizontalHeaderLabels(headers) def _update_stats_model(self): model = self.view.model() for r in reversed(range(model.rowCount())): model.takeRow(r) for input in self.learners.values(): name = learner_name(input.learner) row = [] head = QStandardItem() head.setData(name, Qt.DisplayRole) row.append(head) for stat in input.stats: item = QStandardItem() item.setData(" {:.3f} ".format(stat[0]), Qt.DisplayRole) row.append(item) model.appendRow(row) def _invalidate(self, which=None): if which is None: which = self.learners.keys() all_keys = list(self.learners.keys()) model = self.view.model() for key in which: self.learners[key] = \ self.learners[key]._replace(results=None, stats=None) if key in self.learners: row = all_keys.index(key) for c in range(1, model.columnCount()): item = model.item(row, c) if item is not None: item.setData(None, Qt.DisplayRole) def apply(self): self.update_results() self.commit() def commit(self): results = [val.results for val in self.learners.values() if val.results is not None] if results: combined = results_merge(results) combined.learner_names = [learner_name(val.learner) for val in self.learners.values()] else: combined = None self.send("Evaluation Results", combined)
class OWTestLearners(widget.OWWidget): name = "Test & Score" description = "Cross-validation accuracy estimation." icon = "icons/TestLearners1.svg" priority = 100 inputs = [("Learner", Learner, "set_learner", widget.Multiple), ("Data", Table, "set_train_data", widget.Default), ("Test Data", Table, "set_test_data"), ("Preprocessor", Preprocess, "set_preprocessor")] outputs = [("Predictions", Orange.data.Table), ("Evaluation Results", Orange.evaluation.Results)] settingsHandler = settings.ClassValuesContextHandler() #: Resampling/testing types KFold, LeaveOneOut, ShuffleSplit, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4 #: Selected resampling type resampling = settings.Setting(0) #: Number of folds for K-fold cross validation k_folds = settings.Setting(10) #: Number of repeats for ShuffleSplit sampling n_repeat = settings.Setting(10) #: ShuffleSplit sampling p sample_p = settings.Setting(75) TARGET_AVERAGE = "(Average over classes)" class_selection = settings.ContextSetting(TARGET_AVERAGE) def __init__(self): super().__init__() self.data = None self.test_data = None self.preprocessor = None self.train_data_missing_vals = False self.test_data_missing_vals = False #: An Ordered dictionary with current inputs and their testing #: results. self.learners = OrderedDict() sbox = gui.widgetBox(self.controlArea, "Sampling") rbox = gui.radioButtons( sbox, self, "resampling", callback=self._param_changed ) gui.appendRadioButton(rbox, "Cross validation") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:", callback=self.kfold_changed) gui.appendRadioButton(rbox, "Leave one out") gui.appendRadioButton(rbox, "Random sampling") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test", callback=self.shuffle_split_changed) gui.widgetLabel(ibox, "Relative training set size:") gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=99, ticks=20, vertical=False, labelFormat="%d %%", callback=self.shuffle_split_changed) gui.appendRadioButton(rbox, "Test on train data") gui.appendRadioButton(rbox, "Test on test data") rbox.layout().addSpacing(5) self.apply_button = gui.button( rbox, self, "Apply", callback=self.apply, default=True) self.cbox = gui.widgetBox(self.controlArea, "Target class") self.class_selection_combo = gui.comboBox( self.cbox, self, "class_selection", items=[], sendSelectedValue=True, valueType=str, callback=self._on_target_class_changed, contentsLength=8) gui.rubber(self.controlArea) self.view = QTreeView( rootIsDecorated=False, uniformRowHeights=True, wordWrap=True, editTriggers=QTreeView.NoEditTriggers ) header = self.view.header() header.setResizeMode(QHeaderView.ResizeToContents) header.setDefaultAlignment(Qt.AlignCenter) header.setStretchLastSection(False) self.result_model = QStandardItemModel(self) self.result_model.setHorizontalHeaderLabels(["Method"]) self.view.setModel(self.result_model) self.view.setItemDelegate(ItemDelegate()) box = gui.widgetBox(self.mainArea, "Evaluation Results") box.layout().addWidget(self.view) def sizeHint(self): return QSize(780, 1) def set_learner(self, learner, key): """ Set the input `learner` for `key`. """ if key in self.learners and learner is None: # Removed del self.learners[key] else: self.learners[key] = Input(learner, None, None) self._invalidate([key]) def set_train_data(self, data): """ Set the input training dataset. """ self.error(0) self.information(0) if data and not data.domain.class_var: self.error(0, "Train data input requires a class variable") data = None if isinstance(data, SqlTable): if data.approx_len() < AUTO_DL_LIMIT: data = Table(data) else: self.information(0, "Train data has been sampled") data_sample = data.sample_time(1, no_cache=True) data_sample.download_data(AUTO_DL_LIMIT, partial=True) data = Table(data_sample) self.warning(4) self.train_data_missing_vals = data is not None and \ np.isnan(data.Y).any() if self.train_data_missing_vals or self.test_data_missing_vals: self.warning(4, self._get_missing_data_warning( self.train_data_missing_vals, self.test_data_missing_vals )) if data: data = RemoveNaNClasses(data) self.data = data self.closeContext() if data is not None: self._update_class_selection() self.openContext(data.domain.class_var) self._invalidate() def set_test_data(self, data): """ Set the input separate testing dataset. """ self.error(1) self.information(1) if data and not data.domain.class_var: self.error(1, "Test data input requires a class variable") data = None if isinstance(data, SqlTable): if data.approx_len() < AUTO_DL_LIMIT: data = Table(data) else: self.information(1, "Test data has been sampled") data_sample = data.sample_time(1, no_cache=True) data_sample.download_data(AUTO_DL_LIMIT, partial=True) data = Table(data_sample) self.warning(4) self.test_data_missing_vals = data is not None and \ np.isnan(data.Y).any() if self.train_data_missing_vals or self.test_data_missing_vals: self.warning(4, self._get_missing_data_warning( self.train_data_missing_vals, self.test_data_missing_vals )) if data: data = RemoveNaNClasses(data) self.test_data = data if self.resampling == OWTestLearners.TestOnTest: self._invalidate() def _get_missing_data_warning(self, train_missing, test_missing): return "Instances with unknown target values were removed from{}data"\ .format(train_missing * test_missing * " " or train_missing * " train " or test_missing * " test ") def set_preprocessor(self, preproc): """ Set the input preprocessor to apply on the training data. """ self.preprocessor = preproc self._invalidate() def handleNewSignals(self): """Reimplemented from OWWidget.handleNewSignals.""" self._update_class_selection() self.apply() def kfold_changed(self): self.resampling = OWTestLearners.KFold self._param_changed() def shuffle_split_changed(self): self.resampling = OWTestLearners.ShuffleSplit self._param_changed() def _param_changed(self): self._invalidate() def _update_results(self): """ Run/evaluate the learners. """ self.warning([1, 2]) self.error(2) if self.data is None: return class_var = self.data.domain.class_var if self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: self.warning(2, "Missing separate test data input") return elif self.test_data.domain.class_var != class_var: self.error(2, ("Inconsistent class variable between test " + "and train data sets")) return # items in need of an update items = [(key, slot) for key, slot in self.learners.items() if slot.results is None] learners = [slot.learner for _, slot in items] if len(items) == 0: return if self.test_data is not None and \ self.resampling != OWTestLearners.TestOnTest: self.warning(1, "Test data is present but unused. " "Select 'Test on test data' to use it.") rstate = 42 def update_progress(finished): self.progressBarSet(100 * finished) common_args = dict( store_data=True, preprocessor=self.preprocessor, callback=update_progress) self.setStatusMessage("Running") self.progressBarInit() try: if self.resampling == OWTestLearners.KFold: warnings = [] results = Orange.evaluation.CrossValidation( self.data, learners, k=self.k_folds, random_state=rstate, warnings=warnings, **common_args) if warnings: self.warning(2, warnings[0]) elif self.resampling == OWTestLearners.LeaveOneOut: results = Orange.evaluation.LeaveOneOut( self.data, learners, **common_args) elif self.resampling == OWTestLearners.ShuffleSplit: train_size = self.sample_p / 100 results = Orange.evaluation.ShuffleSplit( self.data, learners, n_resamples=self.n_repeat, train_size=train_size, test_size=None, random_state=rstate, **common_args) elif self.resampling == OWTestLearners.TestOnTrain: results = Orange.evaluation.TestOnTrainingData( self.data, learners, **common_args) elif self.resampling == OWTestLearners.TestOnTest: results = Orange.evaluation.TestOnTestData( self.data, self.test_data, learners, **common_args) else: assert False except RuntimeError as e: self.error(2, str(e)) self.setStatusMessage("") self.progressBarFinished() return learner_key = {slot.learner: key for key, slot in self.learners.items()} for learner, result in zip(learners, split_by_model(results)): stats = None if class_var.is_discrete: scorers = classification_stats.scores elif class_var.is_continuous: scorers = regression_stats.scores else: scorers = None if scorers: ex = result.failed[0] if ex: stats = [Try.Fail(ex)] * len(scorers) result = Try.Fail(ex) else: stats = [Try(lambda: score(result)) for score in scorers] result = Try.Success(result) key = learner_key[learner] self.learners[key] = \ self.learners[key]._replace(results=result, stats=stats) self.setStatusMessage("") self.progressBarFinished() def _update_header(self): # Set the correct horizontal header labels on the results_model. headers = ["Method"] if self.data is not None: if self.data.domain.has_discrete_class: headers.extend(classification_stats.headers) else: headers.extend(regression_stats.headers) # remove possible extra columns from the model. for i in reversed(range(len(headers), self.result_model.columnCount())): self.result_model.takeColumn(i) self.result_model.setHorizontalHeaderLabels(headers) def _update_stats_model(self): # Update the results_model with up to date scores. # Note: The target class specific scores (if requested) are # computed as needed in this method. model = self.view.model() # clear the table model, but preserving the header labels for r in reversed(range(model.rowCount())): model.takeRow(r) target_index = None if self.data is not None: class_var = self.data.domain.class_var if class_var.is_discrete and \ self.class_selection != self.TARGET_AVERAGE: target_index = class_var.values.index(self.class_selection) else: class_var = None errors = [] has_missing_scores = False for key, slot in self.learners.items(): name = learner_name(slot.learner) head = QStandardItem(name) head.setData(key, Qt.UserRole) if isinstance(slot.results, Try.Fail): head.setToolTip(str(slot.results.exception)) head.setText("{} (error)".format(name)) head.setForeground(QtGui.QBrush(Qt.red)) errors.append("{name} failed with error:\n" "{exc.__class__.__name__}: {exc!s}" .format(name=name, exc=slot.results.exception)) row = [head] if class_var is not None and class_var.is_discrete and \ target_index is not None: if slot.results is not None and slot.results.success: ovr_results = results_one_vs_rest( slot.results.value, target_index) stats = [Try(lambda: score(ovr_results)) for score in classification_stats.scores] else: stats = None else: stats = slot.stats if stats is not None: for stat in stats: item = QStandardItem() if stat.success: item.setText("{:.3f}".format(stat.value[0])) else: item.setToolTip(str(stat.exception)) has_missing_scores = True row.append(item) model.appendRow(row) if errors: self.error(3, "\n".join(errors)) else: self.error(3) if has_missing_scores: self.warning(3, "Some scores could not be computed") else: self.warning(3) def _update_class_selection(self): self.class_selection_combo.setCurrentIndex(-1) self.class_selection_combo.clear() if not self.data: return if self.data.domain.has_discrete_class: self.cbox.setVisible(True) class_var = self.data.domain.class_var items = [self.TARGET_AVERAGE] + class_var.values self.class_selection_combo.addItems(items) class_index = 0 if self.class_selection in class_var.values: class_index = class_var.values.index(self.class_selection) + 1 self.class_selection_combo.setCurrentIndex(class_index) self.class_selection = items[class_index] else: self.cbox.setVisible(False) def _on_target_class_changed(self): self._update_stats_model() def _invalidate(self, which=None): # Invalidate learner results for `which` input keys # (if None then all learner results are invalidated) if which is None: which = self.learners.keys() model = self.view.model() statmodelkeys = [model.item(row, 0).data(Qt.UserRole) for row in range(model.rowCount())] for key in which: self.learners[key] = \ self.learners[key]._replace(results=None, stats=None) if key in statmodelkeys: row = statmodelkeys.index(key) for c in range(1, model.columnCount()): item = model.item(row, c) if item is not None: item.setData(None, Qt.DisplayRole) item.setData(None, Qt.ToolTipRole) self.apply_button.setEnabled(True) def apply(self): self.apply_button.setEnabled(False) self._update_header() # Update the view to display the model names self._update_stats_model() self._update_results() self._update_stats_model() self.commit() def commit(self): valid = [slot for slot in self.learners.values() if slot.results is not None and slot.results.success] if valid: # Evaluation results combined = results_merge([slot.results.value for slot in valid]) combined.learner_names = [learner_name(slot.learner) for slot in valid] # Predictions & Probabilities predictions = combined.get_augmented_data(combined.learner_names) else: combined = None predictions = None self.send("Evaluation Results", combined) self.send("Predictions", predictions)
class OWTestLearners(widget.OWWidget): name = "Test & Score" description = "Cross-validation accuracy estimation." icon = "icons/TestLearners1.svg" priority = 100 inputs = [("Learner", Learner, "set_learner", widget.Multiple), ("Data", Table, "set_train_data", widget.Default), ("Test Data", Table, "set_test_data"), ("Preprocessor", Preprocess, "set_preprocessor")] outputs = [("Predictions", Orange.data.Table), ("Evaluation Results", Orange.evaluation.Results)] settingsHandler = settings.ClassValuesContextHandler() #: Resampling/testing types KFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4 #: Selected resampling type resampling = settings.Setting(0) #: Number of folds for K-fold cross validation k_folds = settings.Setting(10) #: Stratified sampling for K-fold cv_stratified = settings.Setting(True) #: Number of repeats for ShuffleSplit sampling n_repeat = settings.Setting(10) #: ShuffleSplit sampling p sample_p = settings.Setting(75) #: Stratified sampling for Random Sampling shuffle_stratified = settings.Setting(True) TARGET_AVERAGE = "(Average over classes)" class_selection = settings.ContextSetting(TARGET_AVERAGE) def __init__(self): super().__init__() self.data = None self.test_data = None self.preprocessor = None self.train_data_missing_vals = False self.test_data_missing_vals = False #: An Ordered dictionary with current inputs and their testing #: results. self.learners = OrderedDict() sbox = gui.widgetBox(self.controlArea, "Sampling") rbox = gui.radioButtons(sbox, self, "resampling", callback=self._param_changed) gui.appendRadioButton(rbox, "Cross validation") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:", callback=self.kfold_changed) gui.checkBox(ibox, self, "cv_stratified", "Stratified", callback=self.kfold_changed) gui.appendRadioButton(rbox, "Random sampling") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test", callback=self.shuffle_split_changed) gui.widgetLabel(ibox, "Relative training set size:") gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=99, ticks=20, vertical=False, labelFormat="%d %%", callback=self.shuffle_split_changed) gui.checkBox(ibox, self, "shuffle_stratified", "Stratified", callback=self.shuffle_split_changed) gui.appendRadioButton(rbox, "Leave one out") gui.appendRadioButton(rbox, "Test on train data") gui.appendRadioButton(rbox, "Test on test data") rbox.layout().addSpacing(5) self.apply_button = gui.button(rbox, self, "Apply", callback=self.apply, default=True) self.cbox = gui.widgetBox(self.controlArea, "Target class") self.class_selection_combo = gui.comboBox( self.cbox, self, "class_selection", items=[], sendSelectedValue=True, valueType=str, callback=self._on_target_class_changed, contentsLength=8) gui.rubber(self.controlArea) self.view = QTreeView(rootIsDecorated=False, uniformRowHeights=True, wordWrap=True, editTriggers=QTreeView.NoEditTriggers) header = self.view.header() header.setResizeMode(QHeaderView.ResizeToContents) header.setDefaultAlignment(Qt.AlignCenter) header.setStretchLastSection(False) self.result_model = QStandardItemModel(self) self.result_model.setHorizontalHeaderLabels(["Method"]) self.view.setModel(self.result_model) self.view.setItemDelegate(ItemDelegate()) box = gui.widgetBox(self.mainArea, "Evaluation Results") box.layout().addWidget(self.view) def sizeHint(self): return QSize(780, 1) def set_learner(self, learner, key): """ Set the input `learner` for `key`. """ if key in self.learners and learner is None: # Removed del self.learners[key] else: self.learners[key] = Input(learner, None, None) self._invalidate([key]) def set_train_data(self, data): """ Set the input training dataset. """ self.error(0) self.information(0) if data and not data.domain.class_var: self.error(0, "Train data input requires a class variable") data = None if isinstance(data, SqlTable): if data.approx_len() < AUTO_DL_LIMIT: data = Table(data) else: self.information(0, "Train data has been sampled") data_sample = data.sample_time(1, no_cache=True) data_sample.download_data(AUTO_DL_LIMIT, partial=True) data = Table(data_sample) self.warning(4) self.train_data_missing_vals = data is not None and \ np.isnan(data.Y).any() if self.train_data_missing_vals or self.test_data_missing_vals: self.warning( 4, self._get_missing_data_warning(self.train_data_missing_vals, self.test_data_missing_vals)) if data: data = RemoveNaNClasses(data) self.data = data self.closeContext() if data is not None: self._update_class_selection() self.openContext(data.domain.class_var) self._invalidate() def set_test_data(self, data): """ Set the input separate testing dataset. """ self.error(1) self.information(1) if data and not data.domain.class_var: self.error(1, "Test data input requires a class variable") data = None if isinstance(data, SqlTable): if data.approx_len() < AUTO_DL_LIMIT: data = Table(data) else: self.information(1, "Test data has been sampled") data_sample = data.sample_time(1, no_cache=True) data_sample.download_data(AUTO_DL_LIMIT, partial=True) data = Table(data_sample) self.warning(4) self.test_data_missing_vals = data is not None and \ np.isnan(data.Y).any() if self.train_data_missing_vals or self.test_data_missing_vals: self.warning( 4, self._get_missing_data_warning(self.train_data_missing_vals, self.test_data_missing_vals)) if data: data = RemoveNaNClasses(data) self.test_data = data if self.resampling == OWTestLearners.TestOnTest: self._invalidate() def _get_missing_data_warning(self, train_missing, test_missing): return "Instances with unknown target values were removed from{}data"\ .format(train_missing * test_missing * " " or train_missing * " train " or test_missing * " test ") def set_preprocessor(self, preproc): """ Set the input preprocessor to apply on the training data. """ self.preprocessor = preproc self._invalidate() def handleNewSignals(self): """Reimplemented from OWWidget.handleNewSignals.""" self._update_class_selection() self.apply() def kfold_changed(self): self.resampling = OWTestLearners.KFold self._param_changed() def shuffle_split_changed(self): self.resampling = OWTestLearners.ShuffleSplit self._param_changed() def _param_changed(self): self._invalidate() def _update_results(self): """ Run/evaluate the learners. """ self.warning([1, 2]) self.error([2, 4]) if self.data is None: return class_var = self.data.domain.class_var if self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: self.warning(2, "Missing separate test data input") return elif self.test_data.domain.class_var != class_var: self.error(2, ("Inconsistent class variable between test " + "and train data sets")) return # items in need of an update items = [(key, slot) for key, slot in self.learners.items() if slot.results is None] learners = [slot.learner for _, slot in items] if len(items) == 0: return if self.test_data is not None and \ self.resampling != OWTestLearners.TestOnTest: self.warning( 1, "Test data is present but unused. " "Select 'Test on test data' to use it.") rstate = 42 def update_progress(finished): self.progressBarSet(100 * finished) common_args = dict(store_data=True, preprocessor=self.preprocessor, callback=update_progress) self.setStatusMessage("Running") with self.progressBar(): try: if self.resampling == OWTestLearners.KFold: if len(self.data) < self.k_folds: self.error(4, "Number of folds exceeds the data size") return warnings = [] results = Orange.evaluation.CrossValidation( self.data, learners, k=self.k_folds, random_state=rstate, warnings=warnings, **common_args) if warnings: self.warning(2, warnings[0]) elif self.resampling == OWTestLearners.LeaveOneOut: results = Orange.evaluation.LeaveOneOut( self.data, learners, **common_args) elif self.resampling == OWTestLearners.ShuffleSplit: train_size = self.sample_p / 100 results = Orange.evaluation.ShuffleSplit( self.data, learners, n_resamples=self.n_repeat, train_size=train_size, test_size=None, stratified=self.shuffle_stratified, random_state=rstate, **common_args) elif self.resampling == OWTestLearners.TestOnTrain: results = Orange.evaluation.TestOnTrainingData( self.data, learners, **common_args) elif self.resampling == OWTestLearners.TestOnTest: results = Orange.evaluation.TestOnTestData( self.data, self.test_data, learners, **common_args) else: assert False except RuntimeError as e: self.error(2, str(e)) self.setStatusMessage("") return learner_key = { slot.learner: key for key, slot in self.learners.items() } for learner, result in zip(learners, split_by_model(results)): stats = None if class_var.is_discrete: scorers = classification_stats.scores elif class_var.is_continuous: scorers = regression_stats.scores else: scorers = None if scorers: ex = result.failed[0] if ex: stats = [Try.Fail(ex)] * len(scorers) result = Try.Fail(ex) else: stats = [Try(lambda: score(result)) for score in scorers] result = Try.Success(result) key = learner_key[learner] self.learners[key] = \ self.learners[key]._replace(results=result, stats=stats) self.setStatusMessage("") def _update_header(self): # Set the correct horizontal header labels on the results_model. headers = ["Method"] if self.data is not None: if self.data.domain.has_discrete_class: headers.extend(classification_stats.headers) else: headers.extend(regression_stats.headers) # remove possible extra columns from the model. for i in reversed(range(len(headers), self.result_model.columnCount())): self.result_model.takeColumn(i) self.result_model.setHorizontalHeaderLabels(headers) def _update_stats_model(self): # Update the results_model with up to date scores. # Note: The target class specific scores (if requested) are # computed as needed in this method. model = self.view.model() # clear the table model, but preserving the header labels for r in reversed(range(model.rowCount())): model.takeRow(r) target_index = None if self.data is not None: class_var = self.data.domain.class_var if self.data.domain.has_discrete_class and \ self.class_selection != self.TARGET_AVERAGE: target_index = class_var.values.index(self.class_selection) else: class_var = None errors = [] has_missing_scores = False for key, slot in self.learners.items(): name = learner_name(slot.learner) head = QStandardItem(name) head.setData(key, Qt.UserRole) if isinstance(slot.results, Try.Fail): head.setToolTip(str(slot.results.exception)) head.setText("{} (error)".format(name)) head.setForeground(QtGui.QBrush(Qt.red)) errors.append("{name} failed with error:\n" "{exc.__class__.__name__}: {exc!s}".format( name=name, exc=slot.results.exception)) row = [head] if class_var is not None and class_var.is_discrete and \ target_index is not None: if slot.results is not None and slot.results.success: ovr_results = results_one_vs_rest(slot.results.value, target_index) stats = [ Try(lambda: score(ovr_results)) for score in classification_stats.scores ] else: stats = None else: stats = slot.stats if stats is not None: for stat in stats: item = QStandardItem() if stat.success: item.setText("{:.3f}".format(stat.value[0])) else: item.setToolTip(str(stat.exception)) has_missing_scores = True row.append(item) model.appendRow(row) if errors: self.error(3, "\n".join(errors)) else: self.error(3) if has_missing_scores: self.warning(3, "Some scores could not be computed") else: self.warning(3) def _update_class_selection(self): self.class_selection_combo.setCurrentIndex(-1) self.class_selection_combo.clear() if not self.data: return if self.data.domain.has_discrete_class: self.cbox.setVisible(True) class_var = self.data.domain.class_var items = [self.TARGET_AVERAGE] + class_var.values self.class_selection_combo.addItems(items) class_index = 0 if self.class_selection in class_var.values: class_index = class_var.values.index(self.class_selection) + 1 self.class_selection_combo.setCurrentIndex(class_index) self.class_selection = items[class_index] else: self.cbox.setVisible(False) def _on_target_class_changed(self): self._update_stats_model() def _invalidate(self, which=None): # Invalidate learner results for `which` input keys # (if None then all learner results are invalidated) if which is None: which = self.learners.keys() model = self.view.model() statmodelkeys = [ model.item(row, 0).data(Qt.UserRole) for row in range(model.rowCount()) ] for key in which: self.learners[key] = \ self.learners[key]._replace(results=None, stats=None) if key in statmodelkeys: row = statmodelkeys.index(key) for c in range(1, model.columnCount()): item = model.item(row, c) if item is not None: item.setData(None, Qt.DisplayRole) item.setData(None, Qt.ToolTipRole) self.apply_button.setEnabled(True) def apply(self): self.apply_button.setEnabled(False) self._update_header() # Update the view to display the model names self._update_stats_model() self._update_results() self._update_stats_model() self.commit() def commit(self): valid = [ slot for slot in self.learners.values() if slot.results is not None and slot.results.success ] if valid: # Evaluation results combined = results_merge([slot.results.value for slot in valid]) combined.learner_names = [ learner_name(slot.learner) for slot in valid ] # Predictions & Probabilities predictions = combined.get_augmented_data(combined.learner_names) else: combined = None predictions = None self.send("Evaluation Results", combined) self.send("Predictions", predictions) def send_report(self): if not self.data or not self.learners: return if self.resampling == self.KFold: stratified = 'Stratified ' if self.cv_stratified else '' items = [ ("Sampling type", "{}{}-fold Cross validation".format(stratified, self.k_folds)) ] elif self.resampling == self.LeaveOneOut: items = [("Sampling type", "Leave one out")] elif self.resampling == self.ShuffleSplit: stratified = 'Stratified ' if self.shuffle_stratified else '' items = [ ("Sampling type", "{}Shuffle split, {} random samples with {}% data ".format( stratified, self.n_repeat, self.sample_p)) ] elif self.resampling == self.TestOnTrain: items = [("Sampling type", "No sampling, test on training data")] elif self.resampling == self.TestOnTest: items = [("Sampling type", "No sampling, test on testing data")] else: items = [] if self.data.domain.has_discrete_class: items += [("Target class", self.class_selection.strip("()"))] if items: self.report_items("Settings", items) self.report_table("Scores", self.view)
class OWTestLearners(widget.OWWidget): name = "Test Learners" description = "Cross-validation accuracy estimation." icon = "icons/TestLearners1.svg" priority = 100 inputs = [("Learner", Orange.classification.Learner, "set_learner", widget.Multiple), ("Data", Orange.data.Table, "set_train_data", widget.Default), ("Test Data", Orange.data.Table, "set_test_data")] outputs = [("Evaluation Results", Orange.evaluation.Results)] settingsHandler = settings.ClassValuesContextHandler() #: Resampling/testing types KFold, LeaveOneOut, Bootstrap, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4 #: Selected resampling type resampling = settings.Setting(0) #: Number of folds for K-fold cross validation k_folds = settings.Setting(10) #: Number of repeats for bootstrap sampling n_repeat = settings.Setting(10) #: Bootstrap sampling p sample_p = settings.Setting(75) class_selection = settings.ContextSetting("(None)") def __init__(self, parent=None): super().__init__(parent) self.train_data = None self.test_data = None #: An Ordered dictionary with current inputs and their testing #: results. self.learners = OrderedDict() sbox = gui.widgetBox(self.controlArea, "Sampling") rbox = gui.radioButtons( sbox, self, "resampling", callback=self._param_changed ) gui.appendRadioButton(rbox, "Cross validation") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:", callback=self.kfold_changed) gui.appendRadioButton(rbox, "Leave one out") gui.appendRadioButton(rbox, "Random sampling") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test", callback=self.bootstrap_changed) gui.widgetLabel(ibox, "Relative training set size:") gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=100, ticks=20, vertical=False, labelFormat="%d %%", callback=self.bootstrap_changed) gui.appendRadioButton(rbox, "Test on train data") gui.appendRadioButton(rbox, "Test on test data") rbox.layout().addSpacing(5) gui.button(rbox, self, "Apply", callback=self.apply) self.cbox = gui.widgetBox(self.controlArea, "Target class") self.class_selection_combo = gui.comboBox(self.cbox, self, "class_selection", items=[], callback=self._select_class, sendSelectedValue=True, valueType=str) gui.rubber(self.controlArea) self.view = QTreeView( rootIsDecorated=False, uniformRowHeights=True, wordWrap=True, editTriggers=QTreeView.NoEditTriggers ) header = self.view.header() header.setResizeMode(QHeaderView.ResizeToContents) header.setDefaultAlignment(Qt.AlignCenter) header.setStretchLastSection(False) self.result_model = QStandardItemModel() self.view.setModel(self.result_model) self.view.setItemDelegate(ItemDelegate()) self._update_header() box = gui.widgetBox(self.mainArea, "Evaluation Results") box.layout().addWidget(self.view) def set_learner(self, learner, key): if key in self.learners and learner is None: del self.learners[key] else: self.learners[key] = Input(learner, None, ()) self._update_stats_model() def set_train_data(self, data): self.error(0) if data and not data.domain.class_var: self.error(0, "Train data input requires a class variable") data = None self.train_data = data self.closeContext() if data is not None: self.openContext(data.domain.class_var) self._update_header() self._update_class_selection() self._invalidate() def set_test_data(self, data): self.error(1) if data and not data.domain.class_var: self.error(1, "Test data input requires a class variable") data = None self.test_data = data if self.resampling == OWTestLearners.TestOnTest: self._invalidate() def handleNewSignals(self): self.update_results() self.commit() def kfold_changed(self): self.resampling = OWTestLearners.KFold self._param_changed() def bootstrap_changed(self): self.resampling = OWTestLearners.Bootstrap self._param_changed() def _param_changed(self): self._invalidate() def update_results(self): self.warning([1, 2]) self.error(2) if self.train_data is None: return if self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: self.warning(2, "Missing separate test data input") return elif self.test_data.domain.class_var != \ self.train_data.domain.class_var: self.error(2, ("Inconsistent class variable between test " + "and train data sets")) return # items in need of an update items = [(key, input) for key, input in self.learners.items() if input.results is None] learners = [input.learner for _, input in items] self.setStatusMessage("Running") if self.test_data is not None and \ self.resampling != OWTestLearners.TestOnTest: self.warning(1, "Test data is present but unused. " "Select 'Test on test data' to use it.") # TODO: Test each learner individually try: if self.resampling == OWTestLearners.KFold: results = Orange.evaluation.CrossValidation( self.train_data, learners, k=self.k_folds, store_data=True ) elif self.resampling == OWTestLearners.LeaveOneOut: results = Orange.evaluation.LeaveOneOut( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.Bootstrap: p = self.sample_p / 100.0 results = Orange.evaluation.Bootstrap( self.train_data, learners, n_resamples=self.n_repeat, p=p, store_data=True ) elif self.resampling == OWTestLearners.TestOnTrain: results = Orange.evaluation.TestOnTrainingData( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: return results = Orange.evaluation.TestOnTestData( self.train_data, self.test_data, learners, store_data=True ) else: assert False except Exception as e: self.error(2, str(e)) return self.results = results results = list(split_by_model(results)) class_var = self.train_data.domain.class_var if class_var.is_discrete: stats = [classification_stats(self.one_vs_rest(res)) for res in results] else: stats = [regression_stats(res) for res in results] self._update_header() for (key, input), res, stat in zip(items, results, stats): self.learners[key] = input._replace(results=res, stats=stat) self.setStatusMessage("") self._update_stats_model() def _update_header(self): headers = ["Method"] if self.train_data is not None: if self.train_data.domain.class_var.is_discrete: headers.extend(classification_stats.headers) else: headers.extend(regression_stats.headers) for i in reversed(range(len(headers), self.result_model.columnCount())): self.result_model.takeColumn(i) self.result_model.setHorizontalHeaderLabels(headers) def _update_stats_model(self): model = self.view.model() for r in reversed(range(model.rowCount())): model.takeRow(r) for input in self.learners.values(): name = learner_name(input.learner) row = [] head = QStandardItem() head.setData(name, Qt.DisplayRole) row.append(head) for stat in input.stats: item = QStandardItem() item.setData(" {:.3f} ".format(stat[0]), Qt.DisplayRole) row.append(item) model.appendRow(row) def _update_class_selection(self): self.class_selection_combo.clear() if not self.train_data: return if self.train_data.domain.class_var.is_discrete: self.cbox.setVisible(True) values = self.train_data.domain.class_var.values self.class_selection_combo.addItem("(None)") self.class_selection_combo.addItems(values) class_index = 0 if self.class_selection in self.train_data.domain.class_var.values: class_index = self.train_data.domain.class_var.values.index(self.class_selection)+1 else: self.class_selection = '(None)' self.class_selection_combo.setCurrentIndex(class_index) self.previous_class_selection = "(None)" else: self.cbox.setVisible(False) def one_vs_rest(self, res): if self.class_selection != '(None)' and self.class_selection != 0: class_ = self.train_data.domain.class_var.values.index(self.class_selection) actual = res.actual == class_ predicted = res.predicted == class_ return Results( nmethods=1, domain=self.train_data.domain, actual=actual, predicted=predicted) else: return res def _select_class(self): if self.previous_class_selection == self.class_selection: return results = list(split_by_model(self.results)) items = [(key, input) for key, input in self.learners.items()] learners = [input.learner for _, input in items] class_var = self.train_data.domain.class_var if class_var.is_discrete: stats = [classification_stats(self.one_vs_rest(res)) for res in results] else: stats = [regression_stats(res) for res in results] for (key, input), res, stat in zip(items, results, stats): self.learners[key] = input._replace(results=res, stats=stat) self.setStatusMessage("") self._update_stats_model() self.previous_class_selection = self.class_selection def _invalidate(self, which=None): if which is None: which = self.learners.keys() all_keys = list(self.learners.keys()) model = self.view.model() for key in which: self.learners[key] = \ self.learners[key]._replace(results=None, stats=None) if key in self.learners: row = all_keys.index(key) for c in range(1, model.columnCount()): item = model.item(row, c) if item is not None: item.setData(None, Qt.DisplayRole) def apply(self): self.update_results() self.commit() def commit(self): results = [val.results for val in self.learners.values() if val.results is not None] if results: combined = results_merge(results) combined.learner_names = [learner_name(val.learner) for val in self.learners.values()] else: combined = None self.send("Evaluation Results", combined)
class OWTestLearners(OWWidget): name = "Test & Score" description = "Cross-validation accuracy estimation." icon = "icons/TestLearners1.svg" priority = 100 inputs = [ ("Learner", Learner, "set_learner", widget.Multiple), ("Data", Table, "set_train_data", widget.Default), ("Test Data", Table, "set_test_data"), ("Preprocessor", Preprocess, "set_preprocessor"), ] outputs = [("Predictions", Table), ("Evaluation Results", Results)] settingsHandler = settings.ClassValuesContextHandler() #: Resampling/testing types KFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4 #: Numbers of folds NFolds = [2, 3, 5, 10, 20] #: Number of repetitions NRepeats = [2, 3, 5, 10, 20, 50, 100] #: Sample sizes SampleSizes = [5, 10, 20, 25, 30, 33, 40, 50, 60, 66, 70, 75, 80, 90, 95] #: Selected resampling type resampling = settings.Setting(0) #: Number of folds for K-fold cross validation n_folds = settings.Setting(3) #: Stratified sampling for K-fold cv_stratified = settings.Setting(True) #: Number of repeats for ShuffleSplit sampling n_repeats = settings.Setting(3) #: ShuffleSplit sample size sample_size = settings.Setting(9) #: Stratified sampling for Random Sampling shuffle_stratified = settings.Setting(True) TARGET_AVERAGE = "(Average over classes)" class_selection = settings.ContextSetting(TARGET_AVERAGE) class Error(OWWidget.Error): class_required = Msg("Train data input requires a class variable") class_required_test = Msg("Test data input requires a class variable") too_many_folds = Msg("Number of folds exceeds the data size") class_inconsistent = Msg("Test and train data sets " "have different classes") class Warning(OWWidget.Warning): missing_data = Msg("Instances with unknown target values were removed from{}data") test_data_missing = Msg("Missing separate test data input") scores_not_computed = Msg("Some scores could not be computed") test_data_unused = Msg("Test data is present but unused. " "Select 'Test on test data' to use it.") class Information(OWWidget.Information): data_sampled = Msg("Train data has been sampled") test_data_sampled = Msg("Test data has been sampled") def __init__(self): super().__init__() self.data = None self.test_data = None self.preprocessor = None self.train_data_missing_vals = False self.test_data_missing_vals = False #: An Ordered dictionary with current inputs and their testing results. self.learners = OrderedDict() sbox = gui.vBox(self.controlArea, "Sampling") rbox = gui.radioButtons(sbox, self, "resampling", callback=self._param_changed) gui.appendRadioButton(rbox, "Cross validation") ibox = gui.indentedBox(rbox) gui.comboBox( ibox, self, "n_folds", label="Number of folds: ", items=[str(x) for x in self.NFolds], maximumContentsLength=3, orientation=Qt.Horizontal, callback=self.kfold_changed, ) gui.checkBox(ibox, self, "cv_stratified", "Stratified", callback=self.kfold_changed) gui.appendRadioButton(rbox, "Random sampling") ibox = gui.indentedBox(rbox) gui.comboBox( ibox, self, "n_repeats", label="Repeat train/test: ", items=[str(x) for x in self.NRepeats], maximumContentsLength=3, orientation=Qt.Horizontal, callback=self.shuffle_split_changed, ) gui.comboBox( ibox, self, "sample_size", label="Training set size: ", items=["{} %".format(x) for x in self.SampleSizes], maximumContentsLength=5, orientation=Qt.Horizontal, callback=self.shuffle_split_changed, ) gui.checkBox(ibox, self, "shuffle_stratified", "Stratified", callback=self.shuffle_split_changed) gui.appendRadioButton(rbox, "Leave one out") gui.appendRadioButton(rbox, "Test on train data") gui.appendRadioButton(rbox, "Test on test data") self.cbox = gui.vBox(self.controlArea, "Target Class") self.class_selection_combo = gui.comboBox( self.cbox, self, "class_selection", items=[], sendSelectedValue=True, valueType=str, callback=self._on_target_class_changed, contentsLength=8, ) gui.rubber(self.controlArea) self.view = gui.TableView(wordWrap=True) header = self.view.horizontalHeader() header.setResizeMode(QHeaderView.ResizeToContents) header.setDefaultAlignment(Qt.AlignCenter) header.setStretchLastSection(False) self.result_model = QStandardItemModel(self) self.result_model.setHorizontalHeaderLabels(["Method"]) self.view.setModel(self.result_model) self.view.setItemDelegate(ItemDelegate()) box = gui.vBox(self.mainArea, "Evaluation Results") box.layout().addWidget(self.view) def sizeHint(self): return QSize(780, 1) def set_learner(self, learner, key): """ Set the input `learner` for `key`. """ if key in self.learners and learner is None: # Removed del self.learners[key] else: self.learners[key] = Input(learner, None, None) self._invalidate([key]) def set_train_data(self, data): """ Set the input training dataset. """ self.Information.data_sampled.clear() if data and not data.domain.class_var: self.Error.class_required() data = None else: self.Error.class_required.clear() if isinstance(data, SqlTable): if data.approx_len() < AUTO_DL_LIMIT: data = Table(data) else: self.Information.data_sampled() data_sample = data.sample_time(1, no_cache=True) data_sample.download_data(AUTO_DL_LIMIT, partial=True) data = Table(data_sample) self.train_data_missing_vals = data is not None and np.isnan(data.Y).any() if self.train_data_missing_vals or self.test_data_missing_vals: self.Warning.missing_data(self._which_missing_data()) if data: data = RemoveNaNClasses(data) else: self.Warning.missing_data.clear() self.data = data self.closeContext() if data is not None: self._update_class_selection() self.openContext(data.domain.class_var) self._invalidate() def set_test_data(self, data): """ Set the input separate testing dataset. """ self.Information.test_data_sampled.clear() if data and not data.domain.class_var: self.Error.class_required() data = None else: self.Error.class_required_test.clear() if isinstance(data, SqlTable): if data.approx_len() < AUTO_DL_LIMIT: data = Table(data) else: self.Information.test_data_sampled() data_sample = data.sample_time(1, no_cache=True) data_sample.download_data(AUTO_DL_LIMIT, partial=True) data = Table(data_sample) self.test_data_missing_vals = data is not None and np.isnan(data.Y).any() if self.train_data_missing_vals or self.test_data_missing_vals: self.Warning.missing_data(self._which_missing_data()) if data: data = RemoveNaNClasses(data) else: self.Warning.missing_data.clear() self.test_data = data if self.resampling == OWTestLearners.TestOnTest: self._invalidate() def _which_missing_data(self): return {(True, True): " ", (True, False): " train ", (False, True): " test "}[ # both, don't specify (self.train_data_missing_vals, self.test_data_missing_vals) ] def set_preprocessor(self, preproc): """ Set the input preprocessor to apply on the training data. """ self.preprocessor = preproc self._invalidate() def handleNewSignals(self): """Reimplemented from OWWidget.handleNewSignals.""" self._update_class_selection() self.commit() def kfold_changed(self): self.resampling = OWTestLearners.KFold self._param_changed() def shuffle_split_changed(self): self.resampling = OWTestLearners.ShuffleSplit self._param_changed() def _param_changed(self): self._invalidate() def _update_results(self): """ Run/evaluate the learners. """ self.Warning.test_data_unused.clear() self.Warning.test_data_missing.clear() self.warning() self.Error.class_inconsistent.clear() self.Error.too_many_folds.clear() self.error() if self.data is None: return class_var = self.data.domain.class_var if self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: self.Warning.test_data_missing() return elif self.test_data.domain.class_var != class_var: self.Error.class_inconsistent() return # items in need of an update items = [(key, slot) for key, slot in self.learners.items() if slot.results is None] learners = [slot.learner for _, slot in items] if len(items) == 0: return if self.test_data is not None and self.resampling != OWTestLearners.TestOnTest: self.Warning.test_data_unused() rstate = 42 def update_progress(finished): self.progressBarSet(100 * finished) common_args = dict(store_data=True, preprocessor=self.preprocessor, callback=update_progress, n_jobs=-1) self.setStatusMessage("Running") with self.progressBar(): try: folds = self.NFolds[self.n_folds] if self.resampling == OWTestLearners.KFold: if len(self.data) < folds: self.Error.too_many_folds() return warnings = [] results = Orange.evaluation.CrossValidation( self.data, learners, k=folds, random_state=rstate, warnings=warnings, **common_args ) if warnings: self.warning(warnings[0]) elif self.resampling == OWTestLearners.LeaveOneOut: results = Orange.evaluation.LeaveOneOut(self.data, learners, **common_args) elif self.resampling == OWTestLearners.ShuffleSplit: train_size = self.SampleSizes[self.sample_size] / 100 results = Orange.evaluation.ShuffleSplit( self.data, learners, n_resamples=self.NRepeats[self.n_repeats], train_size=train_size, test_size=None, stratified=self.shuffle_stratified, random_state=rstate, **common_args ) elif self.resampling == OWTestLearners.TestOnTrain: results = Orange.evaluation.TestOnTrainingData(self.data, learners, **common_args) elif self.resampling == OWTestLearners.TestOnTest: results = Orange.evaluation.TestOnTestData(self.data, self.test_data, learners, **common_args) else: assert False except (RuntimeError, ValueError) as e: self.error(str(e)) self.setStatusMessage("") return else: self.error() learner_key = {slot.learner: key for key, slot in self.learners.items()} for learner, result in zip(learners, results.split_by_model()): stats = None if class_var.is_discrete: scorers = classification_stats.scores elif class_var.is_continuous: scorers = regression_stats.scores else: scorers = None if scorers: ex = result.failed[0] if ex: stats = [Try.Fail(ex)] * len(scorers) result = Try.Fail(ex) else: stats = [Try(lambda: score(result)) for score in scorers] result = Try.Success(result) key = learner_key[learner] self.learners[key] = self.learners[key]._replace(results=result, stats=stats) self.setStatusMessage("") def _update_header(self): # Set the correct horizontal header labels on the results_model. headers = ["Method"] if self.data is not None: if self.data.domain.has_discrete_class: headers.extend(classification_stats.headers) else: headers.extend(regression_stats.headers) # remove possible extra columns from the model. for i in reversed(range(len(headers), self.result_model.columnCount())): self.result_model.takeColumn(i) self.result_model.setHorizontalHeaderLabels(headers) def _update_stats_model(self): # Update the results_model with up to date scores. # Note: The target class specific scores (if requested) are # computed as needed in this method. model = self.view.model() # clear the table model, but preserving the header labels for r in reversed(range(model.rowCount())): model.takeRow(r) target_index = None if self.data is not None: class_var = self.data.domain.class_var if self.data.domain.has_discrete_class and self.class_selection != self.TARGET_AVERAGE: target_index = class_var.values.index(self.class_selection) else: class_var = None errors = [] has_missing_scores = False for key, slot in self.learners.items(): name = learner_name(slot.learner) head = QStandardItem(name) head.setData(key, Qt.UserRole) if isinstance(slot.results, Try.Fail): head.setToolTip(str(slot.results.exception)) head.setText("{} (error)".format(name)) head.setForeground(QtGui.QBrush(Qt.red)) errors.append( "{name} failed with error:\n" "{exc.__class__.__name__}: {exc!s}".format(name=name, exc=slot.results.exception) ) row = [head] if class_var is not None and class_var.is_discrete and target_index is not None: if slot.results is not None and slot.results.success: ovr_results = results_one_vs_rest(slot.results.value, target_index) stats = [Try(lambda: score(ovr_results)) for score in classification_stats.scores] else: stats = None else: stats = slot.stats if stats is not None: for stat in stats: item = QStandardItem() if stat.success: item.setText("{:.3f}".format(stat.value[0])) else: item.setToolTip(str(stat.exception)) has_missing_scores = True row.append(item) model.appendRow(row) self.error("\n".join(errors), shown=bool(errors)) self.Warning.scores_not_computed(shown=has_missing_scores) def _update_class_selection(self): self.class_selection_combo.setCurrentIndex(-1) self.class_selection_combo.clear() if not self.data: return if self.data.domain.has_discrete_class: self.cbox.setVisible(True) class_var = self.data.domain.class_var items = [self.TARGET_AVERAGE] + class_var.values self.class_selection_combo.addItems(items) class_index = 0 if self.class_selection in class_var.values: class_index = class_var.values.index(self.class_selection) + 1 self.class_selection_combo.setCurrentIndex(class_index) self.class_selection = items[class_index] else: self.cbox.setVisible(False) def _on_target_class_changed(self): self._update_stats_model() def _invalidate(self, which=None): # Invalidate learner results for `which` input keys # (if None then all learner results are invalidated) if which is None: which = self.learners.keys() model = self.view.model() statmodelkeys = [model.item(row, 0).data(Qt.UserRole) for row in range(model.rowCount())] for key in which: self.learners[key] = self.learners[key]._replace(results=None, stats=None) if key in statmodelkeys: row = statmodelkeys.index(key) for c in range(1, model.columnCount()): item = model.item(row, c) if item is not None: item.setData(None, Qt.DisplayRole) item.setData(None, Qt.ToolTipRole) self.commit() def commit(self): """Recompute and output the results""" self._update_header() # Update the view to display the model names self._update_stats_model() self._update_results() self._update_stats_model() valid = [slot for slot in self.learners.values() if slot.results is not None and slot.results.success] if valid: # Evaluation results combined = results_merge([slot.results.value for slot in valid]) combined.learner_names = [learner_name(slot.learner) for slot in valid] # Predictions & Probabilities predictions = combined.get_augmented_data(combined.learner_names) else: combined = None predictions = None self.send("Evaluation Results", combined) self.send("Predictions", predictions) def send_report(self): """Report on the testing schema and results""" if not self.data or not self.learners: return if self.resampling == self.KFold: stratified = "Stratified " if self.cv_stratified else "" items = [("Sampling type", "{}{}-fold Cross validation".format(stratified, self.NFolds[self.n_folds]))] elif self.resampling == self.LeaveOneOut: items = [("Sampling type", "Leave one out")] elif self.resampling == self.ShuffleSplit: stratified = "Stratified " if self.shuffle_stratified else "" items = [ ( "Sampling type", "{}Shuffle split, {} random samples with {}% data ".format( stratified, self.NRepeats[self.n_repeats], self.SampleSizes[self.sample_size] ), ) ] elif self.resampling == self.TestOnTrain: items = [("Sampling type", "No sampling, test on training data")] elif self.resampling == self.TestOnTest: items = [("Sampling type", "No sampling, test on testing data")] else: items = [] if self.data.domain.has_discrete_class: items += [("Target class", self.class_selection.strip("()"))] if items: self.report_items("Settings", items) self.report_table("Scores", self.view)
class OWTestLearners(widget.OWWidget): name = "Test Learners" description = "" icon = "icons/TestLearners1.svg" priority = 100 inputs = [("Learner", Orange.classification.Fitter, "set_learner", widget.Multiple), ("Data", Orange.data.Table, "set_train_data", widget.Default), ("Test Data", Orange.data.Table, "set_test_data")] outputs = [("Evaluation Results", testing.Results)] #: Resampling/testing types KFold, LeaveOneOut, Bootstrap, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4 #: Selected resampling type resampling = settings.Setting(0) #: Number of folds for K-fold cross validation k_folds = settings.Setting(10) #: Number of repeats for bootstrap sampling n_repeat = settings.Setting(10) #: Bootstrap sampling p sample_p = settings.Setting(75) def __init__(self, parent=None): super().__init__(parent) self.train_data = None self.test_data = None #: An Ordered dictionary with current inputs and their testing #: results. self.learners = OrderedDict() sbox = gui.widgetBox(self.controlArea, "Sampling") rbox = gui.radioButtons( sbox, self, "resampling", callback=self._param_changed ) gui.appendRadioButton(rbox, "Cross validation") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:", callback=self._param_changed) gui.appendRadioButton(rbox, "Leave one out") gui.appendRadioButton(rbox, "Random sampling") ibox = gui.indentedBox(rbox) gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test", callback=self._param_changed) gui.widgetLabel(ibox, "Relative training set size:") gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=100, ticks=20, vertical=False, callback=self._param_changed) gui.appendRadioButton(rbox, "Test on train data") gui.appendRadioButton(rbox, "Test on test data") rbox.layout().addSpacing(5) gui.button(rbox, self, "Apply", callback=self.apply) gui.rubber(self.controlArea) self.view = QTreeView( rootIsDecorated=False, uniformRowHeights=True, wordWrap=True, editTriggers=QTreeView.NoEditTriggers ) self.result_model = QStandardItemModel() self.view.setModel(self.result_model) self._update_header() box = gui.widgetBox(self.mainArea, "Evaluation Results") box.layout().addWidget(self.view) def set_learner(self, learner, key): if key in self.learners and learner is None: del self.learners[key] else: self.learners[key] = Input(learner, None, ()) self._update_stats_model() def set_train_data(self, data): self.train_data = data self._update_header() self._invalidate() def set_test_data(self, data): self.test_data = data if self.resampling == OWTestLearners.TestOnTest: self._invalidate() def handleNewSignals(self): self.update_results() self.commit() def _param_changed(self): self._invalidate() def update_results(self): self.warning(1, "") if self.train_data is None: return # items in need of an update items = [(key, input) for key, input in self.learners.items() if input.results is None] learners = [input.learner for _, input in items] self.setStatusMessage("Running") if self.test_data is not None and \ self.resampling != OWTestLearners.TestOnTest: self.warning(1, "Select 'Test on test data' to use the test data") # TODO: Test each learner individually if self.resampling == OWTestLearners.KFold: results = testing.CrossValidation( self.train_data, learners, k=self.k_folds, store_data=True ) elif self.resampling == OWTestLearners.LeaveOneOut: results = testing.LeaveOneOut( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.Bootstrap: p = self.sample_p / 100.0 results = testing.Bootstrap( self.train_data, learners, n_resamples=self.n_repeat, p=p, store_data=True ) elif self.resampling == OWTestLearners.TestOnTrain: results = testing.TestOnTrainingData( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: return results = testing.TestOnTestData( self.train_data, self.test_data, learners, store_data=True ) else: assert False results = list(split_by_model(results)) class_var = self.train_data.domain.class_var if is_discrete(class_var): test_stats = classification_stats else: test_stats = regression_stats self._update_header() stats = [test_stats(res) for res in results] for (key, input), res, stat in zip(items, results, stats): self.learners[key] = input._replace(results=res, stats=stat) self.setStatusMessage("") self._update_stats_model() def _update_header(self): headers = ["Method"] if self.train_data is not None: if is_discrete(self.train_data.domain.class_var): if len(self.train_data.domain.class_var.values) == 2: headers.extend(classification_stats.headers_binary) headers.extend(classification_stats.headers) else: headers.extend(regression_stats.headers) for i in reversed(range(len(headers), self.result_model.columnCount())): self.result_model.takeColumn(i) self.result_model.setHorizontalHeaderLabels(headers) def _update_stats_model(self): model = self.view.model() for r in reversed(range(model.rowCount())): model.takeRow(r) for input in self.learners.values(): name = learner_name(input.learner) row = [] head = QStandardItem() head.setData(name, Qt.DisplayRole) row.append(head) for stat in input.stats: item = QStandardItem() item.setData(float(stat[0]), Qt.DisplayRole) row.append(item) model.appendRow(row) self.view.resizeColumnToContents(0) def _invalidate(self, which=None): if which is None: which = self.learners.keys() all_keys = list(self.learners.keys()) model = self.view.model() for key in which: self.learners[key] = \ self.learners[key]._replace(results=None, stats=None) if key in self.learners: row = all_keys.index(key) for c in range(1, model.columnCount()): item = model.item(row, c) if item is not None: item.setData(None, Qt.DisplayRole) def apply(self): self.update_results() self.commit() def commit(self): results = [val.results for val in self.learners.values() if val.results is not None] if results: combined = results_merge(results) combined.fitter_names = [learner_name(val.learner) for val in self.learners.values()] else: combined = None self.send("Evaluation Results", combined)