def _test_tasks_orange(self, models, measures): """Test the given tasks' models on their testing data sets. Compute the given scoring measures of the testing results. Return a two-dimensional dictionary with the first key corresponding to the task's id and the second key corresponding to the measure's name. The value corresponds to the score for the given task and scoring measure. Note: If a particular scoring measure couldn't be computed for a task, its value is set to None. Note: This function works with Orange classifiers and Orange data tables. Arguments: models -- dictionary mapping from tasks' ids to their models measures -- list of strings representing measure's names (currently, only CA and AUC are supported) """ scores = dict() comp_errors = {measure: 0 for measure in measures} for tid, task_test_data in self._test_data_orange.iteritems(): scores[tid] = dict() test_res = Orange.evaluation.testing.test_on_data([models[tid]], task_test_data) for measure in measures: if measure == "AUC": try: score = Orange.evaluation.scoring.AUC(test_res)[0] except ValueError as e: if (e.args[0] == "Cannot compute AUC on a single-class problem" ): # AUC cannot be computed because all instances # belong to the same class score = None comp_errors[measure] += 1 else: raise e elif measure == "CA": score = Orange.evaluation.scoring.CA(test_res)[0] else: raise ValueError("Unknown scoring measure: {}".\ format(measure)) scores[tid][measure] = score # report the number of errors when computing the scoring measures n = len(self._tasks) for m_name, m_errors in comp_errors.iteritems(): if m_errors > 0: logger.info("Scoring measure {} could not be computed for {}" " out of {} tasks ({:.1f}%)".format( m_name, m_errors, n, 100. * m_errors / n)) return scores
def _test_tasks_orange(self, models, measures): """Test the given tasks' models on their testing data sets. Compute the given scoring measures of the testing results. Return a two-dimensional dictionary with the first key corresponding to the task's id and the second key corresponding to the measure's name. The value corresponds to the score for the given task and scoring measure. Note: If a particular scoring measure couldn't be computed for a task, its value is set to None. Note: This function works with Orange classifiers and Orange data tables. Arguments: models -- dictionary mapping from tasks' ids to their models measures -- list of strings representing measure's names (currently, only CA and AUC are supported) """ scores = dict() comp_errors = {measure : 0 for measure in measures} for tid, task_test_data in self._test_data_orange.iteritems(): scores[tid] = dict() test_res = Orange.evaluation.testing.test_on_data([models[tid]], task_test_data) for measure in measures: if measure == "AUC": try: score = Orange.evaluation.scoring.AUC(test_res)[0] except ValueError as e: if (e.args[0] == "Cannot compute AUC on a single-class problem"): # AUC cannot be computed because all instances # belong to the same class score = None comp_errors[measure] += 1 else: raise e elif measure == "CA": score = Orange.evaluation.scoring.CA(test_res)[0] else: raise ValueError("Unknown scoring measure: {}".\ format(measure)) scores[tid][measure] = score # report the number of errors when computing the scoring measures n = len(self._tasks) for m_name, m_errors in comp_errors.iteritems(): if m_errors > 0: logger.info("Scoring measure {} could not be computed for {}" " out of {} tasks ({:.1f}%)".format(m_name, m_errors, n, 100.*m_errors/n)) return scores
def test_tasks(self, learners, base_learners, measures, results_path, save_orange_data=False): """Repeat the following experiment self._repeats times: Prepare tasks' data with the _prepare_tasks_data() function. Test the performance of the given learning algorithms with the given base learning algorithms and compute the testing results using the given scoring measures. Process the obtained repetition scores with the _process_repetition_scores() function. Note: This function only test some specific combinations of base_learners and learners as used by the binarization experiment. Arguments: learners -- ordered dictionary with items of the form (name, learner), where name is a string representing the learner's name and learner is a MTL method (e.g. ERM, NoMerging, ...) base learners -- ordered dictionary with items of the form (name, learner), where name is a string representing the base learner's name and learner is a scikit-learn estimator object measures -- list of strings representing measure's names (currently, only CA and AUC are supported) results_path -- string representing the path where to save any extra information about the running of this test (currently, only used for pickling the results when there is an error in calling the learner) save_orange_data -- boolean indicating whether to save the Orange data tables created with the call to self._prepare_tasks_data() function """ rpt_scores = OrderedDict() dend_info = {bl : OrderedDict() for bl in base_learners.iterkeys()} for i in range(self._repeats): self._repetition_number = i self._prepare_tasks_data(**self._tasks_data_params) if save_orange_data: self._save_orange_data(i, results_path) rpt_scores[i] = {bl : dict() for bl in base_learners.iterkeys()} for bl in base_learners: for l in learners: start = time.clock() try: if isinstance(learners[l], bin_exp.TreeMarkedAndMergedLearner): R = learners[l](self._tasks.keys(), self._merged_learn_data_orange, base_learners[bl]) elif isinstance(base_learners[bl], Orange.core.Learner): wrapped_bl = OrangeClassifierWrapper( orange_learner=base_learners[bl]) R = learners[l](self._tasks, wrapped_bl) else: raise ValueError("An unexpected combination of " "base_learner and leaner detected: {} and " "{}".format(type(base_learners[bl]), type(learners[l]))) except Exception as e: logger.exception("There was an error during repetition:" " {} with base learner: {} and learner: {}.".\ format(i, bl, l)) if i > 0: logger.info("Saving the results of previous " "repetitions.") # remove the scores of the last repetition del rpt_scores[i] # process the remaining repetition scores self._process_repetition_scores(rpt_scores, dend_info) # pickle them to a file pickle_path_fmt = os.path.join(results_path, "bl-{}.pkl") self.pickle_test_results(pickle_path_fmt) # re-raise the original exception import sys exc_info = sys.exc_info() raise exc_info[1], None, exc_info[2] rpt_scores[i][bl][l] = self._test_tasks(R["task_models"], measures) end = time.clock() logger.debug("Finished repetition: {}, base learner: {}, " "learner: {} in {:.2f}s".format(i, bl, l, end-start)) # store dendrogram info if the results contain it if "dend_info" in R: dend_info[bl][i] = R["dend_info"] # pickle and visualize the decision tree if the learner is a # (sub)class of TreeMarkedAndMergedLearner if isinstance(learners[l], bin_exp.TreeMarkedAndMergedLearner): tree = R["task_models"].values()[0] pickle_path = os.path.join(results_path, "{}-{}-" "repeat{}.pkl".format(bl, l, i)) svg_path = os.path.join(results_path, "{}-{}-repeat{}" ".svg".format(bl, l, i)) tikz_path = os.path.join(results_path, "{}-{}-repeat{}" "-tikz.tex".format(bl, l, i)) pickle_obj(tree, pickle_path) save_treegraph_image(tree, svg_path) draw_and_save_tikz_tree_document(tree, tikz_path) self._process_repetition_scores(rpt_scores, dend_info)
def test_tasks(self, learners, base_learners, measures, results_path, save_orange_data=False): """Repeat the following experiment self._repeats times: Prepare tasks' data with the _prepare_tasks_data() function. Test the performance of the given learning algorithms with the given base learning algorithms and compute the testing results using the given scoring measures. Process the obtained repetition scores with the _process_repetition_scores() function. Note: This function only test some specific combinations of base_learners and learners as used by the binarization experiment. Arguments: learners -- ordered dictionary with items of the form (name, learner), where name is a string representing the learner's name and learner is a MTL method (e.g. ERM, NoMerging, ...) base learners -- ordered dictionary with items of the form (name, learner), where name is a string representing the base learner's name and learner is a scikit-learn estimator object measures -- list of strings representing measure's names (currently, only CA and AUC are supported) results_path -- string representing the path where to save any extra information about the running of this test (currently, only used for pickling the results when there is an error in calling the learner) save_orange_data -- boolean indicating whether to save the Orange data tables created with the call to self._prepare_tasks_data() function """ rpt_scores = OrderedDict() dend_info = {bl: OrderedDict() for bl in base_learners.iterkeys()} for i in range(self._repeats): self._repetition_number = i self._prepare_tasks_data(**self._tasks_data_params) if save_orange_data: self._save_orange_data(i, results_path) rpt_scores[i] = {bl: dict() for bl in base_learners.iterkeys()} for bl in base_learners: for l in learners: start = time.clock() try: if isinstance(learners[l], bin_exp.TreeMarkedAndMergedLearner): R = learners[l](self._tasks.keys(), self._merged_learn_data_orange, base_learners[bl]) elif isinstance(base_learners[bl], Orange.core.Learner): wrapped_bl = OrangeClassifierWrapper( orange_learner=base_learners[bl]) R = learners[l](self._tasks, wrapped_bl) else: raise ValueError( "An unexpected combination of " "base_learner and leaner detected: {} and " "{}".format(type(base_learners[bl]), type(learners[l]))) except Exception as e: logger.exception("There was an error during repetition:" " {} with base learner: {} and learner: {}.".\ format(i, bl, l)) if i > 0: logger.info("Saving the results of previous " "repetitions.") # remove the scores of the last repetition del rpt_scores[i] # process the remaining repetition scores self._process_repetition_scores( rpt_scores, dend_info) # pickle them to a file pickle_path_fmt = os.path.join( results_path, "bl-{}.pkl") self.pickle_test_results(pickle_path_fmt) # re-raise the original exception import sys exc_info = sys.exc_info() raise exc_info[1], None, exc_info[2] rpt_scores[i][bl][l] = self._test_tasks( R["task_models"], measures) end = time.clock() logger.debug("Finished repetition: {}, base learner: {}, " "learner: {} in {:.2f}s".format( i, bl, l, end - start)) # store dendrogram info if the results contain it if "dend_info" in R: dend_info[bl][i] = R["dend_info"] # pickle and visualize the decision tree if the learner is a # (sub)class of TreeMarkedAndMergedLearner if isinstance(learners[l], bin_exp.TreeMarkedAndMergedLearner): tree = R["task_models"].values()[0] pickle_path = os.path.join( results_path, "{}-{}-" "repeat{}.pkl".format(bl, l, i)) svg_path = os.path.join( results_path, "{}-{}-repeat{}" ".svg".format(bl, l, i)) tikz_path = os.path.join( results_path, "{}-{}-repeat{}" "-tikz.tex".format(bl, l, i)) pickle_obj(tree, pickle_path) save_treegraph_image(tree, svg_path) draw_and_save_tikz_tree_document(tree, tikz_path) self._process_repetition_scores(rpt_scores, dend_info)
def __call__(self, tasks, base_learner): """Run the merging algorithm for the given tasks. Perform the intelligent merging of tasks' data according to the ERM learning method. After the merging is complete, build a model for each remaining (merged) task and assign this model to each original task of this (merged) task. Return a dictionary of data structures computed within this call to ERM. It has the following keys: task_models -- dictionary mapping from each original task id to its model dend_info -- list of tuples (one for each merged task) as returned by the convert_merg_history_to_scipy_linkage function Arguments: tasks -- dictionary mapping from tasks' ids to their Task objects base_learner -- scikit-learn estimator """ self._base_learner = base_learner # create an ordered dictionary of MergedTask objects from the given # dictionary of tasks self._tasks = OrderedDict() for _, task in sorted(tasks.iteritems()): merg_task = MergedTask(task) self._tasks[merg_task.id] = merg_task # populate the dictionary of task pairs that are candidates for merging C = dict() pairs = list(combinations(self._tasks, 2)) n_pairs = len(pairs) msg = "Computing candidate pairs for merging ({} pairs)".format(n_pairs) logger.debug(msg) print msg for i, (tid_i, tid_j) in enumerate(pairs): if self._prefilter(tid_i, tid_j): avg_pred_errs, p_values_ij = \ self._estimate_errors_significances(tid_i, tid_j) er_ij = error_reduction(avg_pred_errs["data1"]["data1"], avg_pred_errs["data2"]["data2"], avg_pred_errs["dataM"]["dataM"], self._tasks[tid_i].get_data_size(), self._tasks[tid_j].get_data_size()) min_ij = min(avg_pred_errs["data1"]["dataM"], avg_pred_errs["data2"]["dataM"]) if er_ij >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_ij: cp = CandidatePair(tid_i, tid_j, p_values_ij) C[cp.key] = cp update_progress(1.* (i + 1) / n_pairs) print # iteratively merge the most similar pair of tasks, until such pairs # exist n_cand = len(C) msg = "Processing {} candidate pairs for merging".format(n_cand) logger.debug(msg) print msg while len(C) > 0: # find the task pair with the minimal maximal p-value maxes = [(cp_key, cp.get_max_p_value()) for cp_key, cp in C.iteritems()] (min_tid_i, min_tid_j), _ = min(maxes, key=lambda x: x[1]) # merge the pair of tasks and update self._tasks task_M = MergedTask(self._tasks[min_tid_i], self._tasks[min_tid_j]) tid_M = task_M.id del self._tasks[min_tid_i] del self._tasks[min_tid_j] self._tasks[tid_M] = task_M # remove task pairs that don't exist anymore from C for (tid_i, tid_j) in C.keys(): if ((tid_i == min_tid_i) or (tid_i == min_tid_j) or (tid_j == min_tid_i) or (tid_j == min_tid_j)): del C[(tid_i, tid_j)] # find new task pairs that are candidates for merging for tid_i in self._tasks: if tid_i != tid_M and self._prefilter(tid_i, tid_M): avg_pred_errs, p_values_iM = \ self._estimate_errors_significances(tid_i, tid_M) er_iM = error_reduction(avg_pred_errs["data1"]["data1"], avg_pred_errs["data2"]["data2"], avg_pred_errs["dataM"]["dataM"], self._tasks[tid_i].get_data_size(), self._tasks[tid_M].get_data_size()) min_iM = min(avg_pred_errs["data1"]["dataM"], avg_pred_errs["data2"]["dataM"]) if er_iM >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_iM: cp = CandidatePair(tid_i, tid_M, p_values_iM) C[cp.key] = cp update_progress(1.* len(C) / n_cand, invert=True) print # build a model for each remaining (merged) task and store the info # for drawing a dendrogram showing the merging history task_models = dict() dend_info = [] for merg_task in self._tasks.itervalues(): # NOTE: When the number of unique class values is less than 2, we # cannot fit an ordinary model (e.g. logistic regression). Instead, # we have to use a dummy classifier which is subsequently augmented # to handle all the other class values. # NOTE: The scikit-learn estimator must be cloned so that each # (merged) task gets its own classifier X, y = merg_task.get_learn_data() if len(np.unique(y)) < 2: logger.info("Learning data for merged task {} has less than 2 " "class values. Using DummyClassifier.".\ format(merg_task)) model = DummyClassifier() model.fit(X, y) change_dummy_classes(model, np.array([0, 1])) else: model = clone(self._base_learner) model.fit(X, y) # assign this model to each original task of this (merged) task original_ids = merg_task.get_original_ids() for tid in original_ids: task_models[tid] = model # store the dendrogram info (if the task is truly a merged task) if len(original_ids) > 1: dend_info.append(convert_merg_history_to_scipy_linkage( merg_task.merg_history)) # create and fill the return dictionary R = dict() R["task_models"] = task_models R["dend_info"] = dend_info return R
def __call__(self, tasks, base_learner): """Run the merging algorithm for the given tasks. Perform the intelligent merging of tasks' data according to the ERM learning method. After the merging is complete, build a model for each remaining (merged) task and assign this model to each original task of this (merged) task. Return a dictionary of data structures computed within this call to ERM. It has the following keys: task_models -- dictionary mapping from each original task id to its model dend_info -- list of tuples (one for each merged task) as returned by the convert_merg_history_to_scipy_linkage function Arguments: tasks -- dictionary mapping from tasks' ids to their Task objects base_learner -- scikit-learn estimator """ self._base_learner = base_learner # create an ordered dictionary of MergedTask objects from the given # dictionary of tasks self._tasks = OrderedDict() for _, task in sorted(tasks.iteritems()): merg_task = MergedTask(task) self._tasks[merg_task.id] = merg_task # populate the dictionary of task pairs that are candidates for merging C = dict() pairs = list(combinations(self._tasks, 2)) n_pairs = len(pairs) msg = "Computing candidate pairs for merging ({} pairs)".format( n_pairs) logger.debug(msg) print msg for i, (tid_i, tid_j) in enumerate(pairs): if self._prefilter(tid_i, tid_j): avg_pred_errs, p_values_ij = \ self._estimate_errors_significances(tid_i, tid_j) er_ij = error_reduction(avg_pred_errs["data1"]["data1"], avg_pred_errs["data2"]["data2"], avg_pred_errs["dataM"]["dataM"], self._tasks[tid_i].get_data_size(), self._tasks[tid_j].get_data_size()) min_ij = min(avg_pred_errs["data1"]["dataM"], avg_pred_errs["data2"]["dataM"]) if er_ij >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_ij: cp = CandidatePair(tid_i, tid_j, p_values_ij) C[cp.key] = cp update_progress(1. * (i + 1) / n_pairs) print # iteratively merge the most similar pair of tasks, until such pairs # exist n_cand = len(C) msg = "Processing {} candidate pairs for merging".format(n_cand) logger.debug(msg) print msg while len(C) > 0: # find the task pair with the minimal maximal p-value maxes = [(cp_key, cp.get_max_p_value()) for cp_key, cp in C.iteritems()] (min_tid_i, min_tid_j), _ = min(maxes, key=lambda x: x[1]) # merge the pair of tasks and update self._tasks task_M = MergedTask(self._tasks[min_tid_i], self._tasks[min_tid_j]) tid_M = task_M.id del self._tasks[min_tid_i] del self._tasks[min_tid_j] self._tasks[tid_M] = task_M # remove task pairs that don't exist anymore from C for (tid_i, tid_j) in C.keys(): if ((tid_i == min_tid_i) or (tid_i == min_tid_j) or (tid_j == min_tid_i) or (tid_j == min_tid_j)): del C[(tid_i, tid_j)] # find new task pairs that are candidates for merging for tid_i in self._tasks: if tid_i != tid_M and self._prefilter(tid_i, tid_M): avg_pred_errs, p_values_iM = \ self._estimate_errors_significances(tid_i, tid_M) er_iM = error_reduction(avg_pred_errs["data1"]["data1"], avg_pred_errs["data2"]["data2"], avg_pred_errs["dataM"]["dataM"], self._tasks[tid_i].get_data_size(), self._tasks[tid_M].get_data_size()) min_iM = min(avg_pred_errs["data1"]["dataM"], avg_pred_errs["data2"]["dataM"]) if er_iM >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_iM: cp = CandidatePair(tid_i, tid_M, p_values_iM) C[cp.key] = cp update_progress(1. * len(C) / n_cand, invert=True) print # build a model for each remaining (merged) task and store the info # for drawing a dendrogram showing the merging history task_models = dict() dend_info = [] for merg_task in self._tasks.itervalues(): # NOTE: When the number of unique class values is less than 2, we # cannot fit an ordinary model (e.g. logistic regression). Instead, # we have to use a dummy classifier which is subsequently augmented # to handle all the other class values. # NOTE: The scikit-learn estimator must be cloned so that each # (merged) task gets its own classifier X, y = merg_task.get_learn_data() if len(np.unique(y)) < 2: logger.info("Learning data for merged task {} has less than 2 " "class values. Using DummyClassifier.".\ format(merg_task)) model = DummyClassifier() model.fit(X, y) change_dummy_classes(model, np.array([0, 1])) else: model = clone(self._base_learner) model.fit(X, y) # assign this model to each original task of this (merged) task original_ids = merg_task.get_original_ids() for tid in original_ids: task_models[tid] = model # store the dendrogram info (if the task is truly a merged task) if len(original_ids) > 1: dend_info.append( convert_merg_history_to_scipy_linkage( merg_task.merg_history)) # create and fill the return dictionary R = dict() R["task_models"] = task_models R["dend_info"] = dend_info return R