def test_on_data(self, classifiers, examples, store_classifiers=False, store_examples=False): """ Test classifiers on the given data :param classifiers: a list of classifiers :param examples: testing data :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results """ examples, weight = demangle_examples(examples) test_type = self.check_test_type(examples, classifiers) test_results = ExperimentResults(1, classifier_names = [getobjectname(l) for l in classifiers], domain=examples.domain, test_type = test_type, weights=weight) test_results.results = [test_results.create_tested_example(0, example) for i, example in enumerate(examples)] if store_examples: test_results.examples = examples if store_classifiers: test_results.classifiers = classifiers results = self._test_on_data(classifiers, examples) for example, classifier, result in results: test_results.results[example].set_result(classifier, *result) return test_results
def encode_PP(self, pps): for pp in pps: objname = getobjectname(pp[1]) if len(objname): pps += "_" + objname else: return "*" return pps
def encode_PP(self, pps): for pp in pps: objname = getobjectname(pp[1]) if len(objname): pps+="_"+objname else: return "*" return pps
def learning_curve_with_test_data(self, learners, learn_set, test_set, times=10, proportions=Orange.core.frange(0.1), stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible, preprocessors=(), random_generator=0, store_classifiers=False, store_examples=False): """ Compute a learning curve given two datasets. Models are learned on proportion of the first dataset and then tested on the second. :param learners: list of learning algorithms :param learn_set: training data :param test_set: testing data :param times: number of iterations :param straitification: use stratified sampling :param proportions: a list of proportions of training data to be used :param preprocessors: a list of preprocessors (obsolete) :param random_generator: random seed or generator (see above) :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: list of :obj:`ExperimentResults` """ learn_set, learn_weight = demangle_examples(learn_set) test_set, test_weight = demangle_examples(test_set) test_type = self.check_test_type(learn_set, learners) self.check_test_type(test_set, learners) indices = Orange.core.MakeRandomIndices2(stratified = stratification, randomGenerator = random_generator) all_results=[] for p in proportions: test_results = ExperimentResults(times, classifier_names = [getobjectname(l) for l in learners], domain=test_set.domain, test_type = test_type, weights=test_weight) offset = 0 for t in xrange(times): test_results.results.extend(test_results.create_tested_example(t, example) for i, example in enumerate(test_set)) learn_examples = learn_set.selectref(indices(learn_set, p), 0) classifiers, results = self._learn_and_test_on_test_data\ (learners, learn_examples, learn_weight, test_set, preprocessors=preprocessors) for example, classifier, result in results: test_results.results[offset+example].set_result(classifier, *result) offset += len(test_set) if store_classifiers: test_results.classifiers.append(classifiers) if store_examples: test_results.examples = learn_examples all_results.append(test_results) return all_results
def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(), callback=None, store_classifiers=False, store_examples=False): """ Train learning algorithms and test them on the same data. :param learners: list of learning algorithms :param examples: data instances used for training and testing :param preprocessors: a list of preprocessors (obsolete) :param callback: a function that is called after each learning :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ examples, weight = demangle_examples(examples) test_type = self.check_test_type(examples, learners) # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to # clone it. if not filter(lambda x: x[0] != "B", preprocessors): learn_set, test_set = self._preprocess_data( examples, Orange.data.Table(examples.domain), preprocessors) test_set = learn_set else: learn_set, test_set = self._preprocess_data( examples, Orange.data.Table(examples), preprocessors) classifiers = self._train_with_callback(learners, learn_set, weight, callback) test_results = ExperimentResults( 1, classifier_names=[getobjectname(l) for l in learners], test_type=test_type, domain=examples.domain, weights=weight) test_results.results = [ test_results.create_tested_example(0, example) for i, example in enumerate(examples) ] if store_classifiers: test_results.classifiers = classifiers if store_examples: test_results.examples = test_set results = self._test_on_data(classifiers, test_set) for example, classifier, result in results: test_results.results[example].set_result(classifier, *result) return test_results
def test_with_indices(self, learners, examples, indices, preprocessors=(), callback=None, store_classifiers=False, store_examples=False): """ Perform a cross-validation-like test. Examples for each fold are selected based on given indices. :param learners: list of learning algorithms :param examples: data instances used for training and testing :param indices: a list of integer indices that sort examples into folds; each index corresponds to an example from ``examples`` :param preprocessors: a list of preprocessors (obsolete) :param callback: a function that is called after each fold :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ examples, weight = demangle_examples(examples) if not examples: raise ValueError("Test data set with no examples") test_type = self.check_test_type(examples, learners) niterations = max(indices) + 1 test_result = ExperimentResults( niterations, classifier_names=[getobjectname(l) for l in learners], domain=examples.domain, weights=weight, test_type=test_type) test_result.results = [ test_result.create_tested_example(indices[i], example) for i, example in enumerate(examples) ] if store_examples: test_result.examples = examples for fold in xrange(niterations): results, classifiers = self.one_fold_with_indices( learners, examples, fold, indices, preprocessors, weight) for example, learner, result in results: test_result.results[example].set_result(learner, *result) if store_classifiers: test_result.classifiers.append(classifiers) if callback: callback() return test_result
def proportion_test(self, learners, examples, learning_proportion=.7, times=10, stratification=Orange.core.MakeRandomIndices.StratifiedIfPossible, preprocessors=(), random_generator=0, callback=None, store_classifiers=False, store_examples=False): """ Iteratively split the data into training and testing set, and train and test the learnign algorithms. :param learners: list of learning algorithms :param examples: data instances used for training and testing :param learning_proportion: proportion of data used for training :param times: number of iterations :param stratification: use stratified sampling :param preprocessors: a list of preprocessors (obsolete) :param random_generator: random seed or generator (see above) :param callback: a function that is called after each fold :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ pick = Orange.core.MakeRandomIndices2(stratified = stratification, p0 = learning_proportion, randomGenerator = random_generator) examples, weight = demangle_examples(examples) test_type = self.check_test_type(examples, learners) test_results = ExperimentResults(times, classifier_names = [getobjectname(l) for l in learners], domain=examples.domain, test_type = test_type, weights=weight) if store_examples: test_results.examples = [] test_results.classifiers = [] offset=0 for time in xrange(times): indices = pick(examples) learn_set = examples.selectref(indices, 0) test_set = examples.selectref(indices, 1) classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, weight, test_set, preprocessors) if store_classifiers: test_results.classifiers.append(classifiers) if store_examples: test_results.examples.append(learn_set) test_results.results.extend(test_results.create_tested_example(time, example) for i, example in enumerate(test_set)) for example, classifier, result in results: test_results.results[offset+example].set_result(classifier, *result) offset += len(test_set) if callback: callback() return test_results
def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(), callback=None, store_classifiers=False, store_examples=False): """ Train learning algorithms on one data sets and test them on another. :param learners: list of learning algorithms :param learn_set: training instances :param test_set: testing instances :param preprocessors: a list of preprocessors (obsolete) :param callback: a function that is called after each learning :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ learn_set, learn_weight = demangle_examples(learn_set) test_set, test_weight = demangle_examples(test_set) test_type = self.check_test_type(learn_set, learners) self.check_test_type(test_set, learners) test_results = ExperimentResults( 1, classifier_names=[getobjectname(l) for l in learners], domain=test_set.domain, test_type=test_type, weights=test_weight) test_results.results = [ test_results.create_tested_example(0, example) for i, example in enumerate(test_set) ] classifiers, results = self._learn_and_test_on_test_data( learners, learn_set, learn_weight, test_set, preprocessors, callback) if store_classifiers: test_results.classifiers = classifiers if store_examples: test_results.examples = test_set for example, classifier, result in results: test_results.results[example].set_result(classifier, *result) return test_results
def test_with_indices(self, learners, examples, indices, preprocessors=(), callback=None, store_classifiers=False, store_examples=False): """ Perform a cross-validation-like test. Examples for each fold are selected based on given indices. :param learners: list of learning algorithms :param examples: data instances used for training and testing :param indices: a list of integer indices that sort examples into folds; each index corresponds to an example from ``examples`` :param preprocessors: a list of preprocessors (obsolete) :param callback: a function that is called after each fold :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ examples, weight = demangle_examples(examples) if not examples: raise ValueError("Test data set with no examples") test_type = self.check_test_type(examples, learners) niterations = max(indices)+1 test_result = ExperimentResults(niterations, classifier_names = [getobjectname(l) for l in learners], domain=examples.domain, weights=weight, test_type=test_type) test_result.results = [test_result.create_tested_example(indices[i], example) for i, example in enumerate(examples)] if store_examples: test_result.examples = examples for fold in xrange(niterations): results, classifiers = self.one_fold_with_indices(learners, examples, fold, indices, preprocessors, weight) for example, learner, result in results: test_result.results[example].set_result(learner, *result) if store_classifiers: test_result.classifiers.append(classifiers) if callback: callback() return test_result
def learn_and_test_on_learn_data(self, learners, examples, preprocessors=(), callback=None, store_classifiers=False, store_examples=False): """ Train learning algorithms and test them on the same data. :param learners: list of learning algorithms :param examples: data instances used for training and testing :param preprocessors: a list of preprocessors (obsolete) :param callback: a function that is called after each learning :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ examples, weight = demangle_examples(examples) test_type = self.check_test_type(examples, learners) # If preprocessors are not used, we use the same dataset for learning and testing. Otherwise we need to # clone it. if not filter(lambda x:x[0]!="B", preprocessors): learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples.domain), preprocessors) test_set = learn_set else: learn_set, test_set = self._preprocess_data(examples, Orange.data.Table(examples), preprocessors) classifiers = self._train_with_callback(learners, learn_set, weight, callback) test_results = ExperimentResults(1, classifier_names = [getobjectname(l) for l in learners], test_type = test_type, domain=examples.domain, weights=weight) test_results.results = [test_results.create_tested_example(0, example) for i, example in enumerate(examples)] if store_classifiers: test_results.classifiers = classifiers if store_examples: test_results.examples = test_set results = self._test_on_data(classifiers, test_set) for example, classifier, result in results: test_results.results[example].set_result(classifier, *result) return test_results
def learn_and_test_on_test_data(self, learners, learn_set, test_set, preprocessors=(), callback=None, store_classifiers=False, store_examples=False): """ Train learning algorithms on one data sets and test them on another. :param learners: list of learning algorithms :param learn_set: training instances :param test_set: testing instances :param preprocessors: a list of preprocessors (obsolete) :param callback: a function that is called after each learning :param store_classifiers: if ``True``, classifiers are stored in results :param store_examples: if ``True``, examples are stored in results :return: :obj:`ExperimentResults` """ learn_set, learn_weight = demangle_examples(learn_set) test_set, test_weight = demangle_examples(test_set) test_type = self.check_test_type(learn_set, learners) self.check_test_type(test_set, learners) test_results = ExperimentResults(1, classifier_names = [getobjectname(l) for l in learners], domain=test_set.domain, test_type = test_type, weights=test_weight) test_results.results = [test_results.create_tested_example(0, example) for i, example in enumerate(test_set)] classifiers, results = self._learn_and_test_on_test_data(learners, learn_set, learn_weight, test_set, preprocessors, callback) if store_classifiers: test_results.classifiers = classifiers if store_examples: test_results.examples = test_set for example, classifier, result in results: test_results.results[example].set_result(classifier, *result) return test_results