def test_tasks(self, learners, base_learners, measures, results_path,
                save_orange_data=False):
     """Repeat the following experiment self._repeats times:
     Prepare tasks' data with the _prepare_tasks_data() function.
     Test the performance of the given learning algorithms with the given
     base learning algorithms and compute the testing results using the
     given scoring measures.
     Process the obtained repetition scores with the
     _process_repetition_scores() function.
     Note: This function only test some specific combinations of
     base_learners and learners as used by the binarization experiment.
     
     Arguments:
     learners -- ordered dictionary with items of the form (name, learner),
         where name is a string representing the learner's name and
         learner is a MTL method (e.g. ERM, NoMerging, ...) 
     base learners -- ordered dictionary with items of the form (name,
         learner), where name is a string representing the base learner's
         name and learner is a scikit-learn estimator object
     measures -- list of strings representing measure's names (currently,
         only CA and AUC are supported)
     results_path -- string representing the path where to save any extra
         information about the running of this test (currently, only used
         for pickling the results when there is an error in calling the
         learner)
     save_orange_data -- boolean indicating whether to save the Orange data
         tables created with the call to self._prepare_tasks_data() function
     
     """
     rpt_scores = OrderedDict()
     dend_info = {bl : OrderedDict() for bl in base_learners.iterkeys()}
     for i in range(self._repeats):
         self._repetition_number = i
         self._prepare_tasks_data(**self._tasks_data_params)
         if save_orange_data:
             self._save_orange_data(i, results_path)
         rpt_scores[i] = {bl : dict() for bl in base_learners.iterkeys()}
         for bl in base_learners:
             for l in learners:
                 start = time.clock()
                 try: 
                     if isinstance(learners[l],
                                   bin_exp.TreeMarkedAndMergedLearner):
                         R = learners[l](self._tasks.keys(),
                                         self._merged_learn_data_orange,
                                         base_learners[bl])
                     elif isinstance(base_learners[bl], Orange.core.Learner):
                         wrapped_bl = OrangeClassifierWrapper(
                                         orange_learner=base_learners[bl])
                         R = learners[l](self._tasks, wrapped_bl)
                     else:
                         raise ValueError("An unexpected combination of "
                                 "base_learner and leaner detected: {} and "
                                 "{}".format(type(base_learners[bl]),
                                             type(learners[l])))
                 except Exception as e:
                     logger.exception("There was an error during repetition:"
                         " {} with base learner: {} and learner: {}.".\
                         format(i, bl, l))
                     if i > 0:
                         logger.info("Saving the results of previous "
                                     "repetitions.")
                         # remove the scores of the last repetition
                         del rpt_scores[i]
                         # process the remaining repetition scores
                         self._process_repetition_scores(rpt_scores,
                                                         dend_info)
                         # pickle them to a file
                         pickle_path_fmt = os.path.join(results_path,
                                                        "bl-{}.pkl")
                         self.pickle_test_results(pickle_path_fmt)
                     # re-raise the original exception
                     import sys
                     exc_info = sys.exc_info()
                     raise exc_info[1], None, exc_info[2]
                 rpt_scores[i][bl][l] = self._test_tasks(R["task_models"],
                                                         measures)
                 end = time.clock()
                 logger.debug("Finished repetition: {}, base learner: {}, "
                     "learner: {} in {:.2f}s".format(i, bl, l, end-start))
                 # store dendrogram info if the results contain it 
                 if "dend_info" in R:
                     dend_info[bl][i] = R["dend_info"]
                 # pickle and visualize the decision tree if the learner is a
                 # (sub)class of TreeMarkedAndMergedLearner
                 if isinstance(learners[l],
                               bin_exp.TreeMarkedAndMergedLearner):
                     tree = R["task_models"].values()[0]
                     pickle_path = os.path.join(results_path, "{}-{}-"
                                     "repeat{}.pkl".format(bl, l, i))
                     svg_path = os.path.join(results_path, "{}-{}-repeat{}"
                                             ".svg".format(bl, l, i))
                     tikz_path = os.path.join(results_path, "{}-{}-repeat{}"
                                              "-tikz.tex".format(bl, l, i))
                     pickle_obj(tree, pickle_path)
                     save_treegraph_image(tree, svg_path)
                     draw_and_save_tikz_tree_document(tree, tikz_path)
     self._process_repetition_scores(rpt_scores, dend_info)
 def test_tasks(self,
                learners,
                base_learners,
                measures,
                results_path,
                save_orange_data=False):
     """Repeat the following experiment self._repeats times:
     Prepare tasks' data with the _prepare_tasks_data() function.
     Test the performance of the given learning algorithms with the given
     base learning algorithms and compute the testing results using the
     given scoring measures.
     Process the obtained repetition scores with the
     _process_repetition_scores() function.
     Note: This function only test some specific combinations of
     base_learners and learners as used by the binarization experiment.
     
     Arguments:
     learners -- ordered dictionary with items of the form (name, learner),
         where name is a string representing the learner's name and
         learner is a MTL method (e.g. ERM, NoMerging, ...) 
     base learners -- ordered dictionary with items of the form (name,
         learner), where name is a string representing the base learner's
         name and learner is a scikit-learn estimator object
     measures -- list of strings representing measure's names (currently,
         only CA and AUC are supported)
     results_path -- string representing the path where to save any extra
         information about the running of this test (currently, only used
         for pickling the results when there is an error in calling the
         learner)
     save_orange_data -- boolean indicating whether to save the Orange data
         tables created with the call to self._prepare_tasks_data() function
     
     """
     rpt_scores = OrderedDict()
     dend_info = {bl: OrderedDict() for bl in base_learners.iterkeys()}
     for i in range(self._repeats):
         self._repetition_number = i
         self._prepare_tasks_data(**self._tasks_data_params)
         if save_orange_data:
             self._save_orange_data(i, results_path)
         rpt_scores[i] = {bl: dict() for bl in base_learners.iterkeys()}
         for bl in base_learners:
             for l in learners:
                 start = time.clock()
                 try:
                     if isinstance(learners[l],
                                   bin_exp.TreeMarkedAndMergedLearner):
                         R = learners[l](self._tasks.keys(),
                                         self._merged_learn_data_orange,
                                         base_learners[bl])
                     elif isinstance(base_learners[bl],
                                     Orange.core.Learner):
                         wrapped_bl = OrangeClassifierWrapper(
                             orange_learner=base_learners[bl])
                         R = learners[l](self._tasks, wrapped_bl)
                     else:
                         raise ValueError(
                             "An unexpected combination of "
                             "base_learner and leaner detected: {} and "
                             "{}".format(type(base_learners[bl]),
                                         type(learners[l])))
                 except Exception as e:
                     logger.exception("There was an error during repetition:"
                         " {} with base learner: {} and learner: {}.".\
                         format(i, bl, l))
                     if i > 0:
                         logger.info("Saving the results of previous "
                                     "repetitions.")
                         # remove the scores of the last repetition
                         del rpt_scores[i]
                         # process the remaining repetition scores
                         self._process_repetition_scores(
                             rpt_scores, dend_info)
                         # pickle them to a file
                         pickle_path_fmt = os.path.join(
                             results_path, "bl-{}.pkl")
                         self.pickle_test_results(pickle_path_fmt)
                     # re-raise the original exception
                     import sys
                     exc_info = sys.exc_info()
                     raise exc_info[1], None, exc_info[2]
                 rpt_scores[i][bl][l] = self._test_tasks(
                     R["task_models"], measures)
                 end = time.clock()
                 logger.debug("Finished repetition: {}, base learner: {}, "
                              "learner: {} in {:.2f}s".format(
                                  i, bl, l, end - start))
                 # store dendrogram info if the results contain it
                 if "dend_info" in R:
                     dend_info[bl][i] = R["dend_info"]
                 # pickle and visualize the decision tree if the learner is a
                 # (sub)class of TreeMarkedAndMergedLearner
                 if isinstance(learners[l],
                               bin_exp.TreeMarkedAndMergedLearner):
                     tree = R["task_models"].values()[0]
                     pickle_path = os.path.join(
                         results_path, "{}-{}-"
                         "repeat{}.pkl".format(bl, l, i))
                     svg_path = os.path.join(
                         results_path, "{}-{}-repeat{}"
                         ".svg".format(bl, l, i))
                     tikz_path = os.path.join(
                         results_path, "{}-{}-repeat{}"
                         "-tikz.tex".format(bl, l, i))
                     pickle_obj(tree, pickle_path)
                     save_treegraph_image(tree, svg_path)
                     draw_and_save_tikz_tree_document(tree, tikz_path)
     self._process_repetition_scores(rpt_scores, dend_info)
                           "nls10-seed63-complete_test/orange_merged_learn-"
                           "repetition{}.tab".format(i)))
     tree = tree_learner(data)
     pickle_path = os.path.join(results_path, "test-pickle.pkl")
     pickle_obj(tree, pickle_path)
     unpickled_tree = unpickle_obj(pickle_path)
     print ("Repetition {} original vs. pickled/unpickled tree equality:".
            format(i)),
     print np.all(tree[e] == unpickled_tree[e] for e in data)
 os.remove(pickle_path)
 
 data = Table(os.path.join(results_path, "bool_func-a8d4n100g2tg5nse0.0rs15"
                           "nls10-seed63-complete_test/orange_merged_learn-"
                           "repetition0.tab"))
 tree = tree_learner(data)
 tex_file = os.path.join(results_path, "test-tree.tex")
 pdf_file = os.path.join(results_path, "test-tree.pdf")
 draw_and_save_tikz_tree_document(tree, tex_file)
 import subprocess
 subprocess.call(["-c", "pdflatex -interaction=batchmode {0} && "
                  "rm {1}.{{aux,log}} && pdfcrop --margins 10 {1}.pdf {2}".
                  format(tex_file, tex_file[:-4], pdf_file)],
                  shell=True, cwd=results_path)
 
 # initialize Qt application
 import sys
 from OWWidget import QApplication
 a = QApplication(sys.argv)
 from PyMTL.orange_visualizations import save_treegraph_image
 save_treegraph_image(tree, os.path.join(results_path, "test-tree.svg"))
 
        unpickled_tree = unpickle_obj(pickle_path)
        print("Repetition {} original vs. pickled/unpickled tree equality:".
              format(i)),
        print np.all(tree[e] == unpickled_tree[e] for e in data)
    os.remove(pickle_path)

    data = Table(
        os.path.join(
            results_path, "bool_func-a8d4n100g2tg5nse0.0rs15"
            "nls10-seed63-complete_test/orange_merged_learn-"
            "repetition0.tab"))
    tree = tree_learner(data)
    tex_file = os.path.join(results_path, "test-tree.tex")
    pdf_file = os.path.join(results_path, "test-tree.pdf")
    draw_and_save_tikz_tree_document(tree, tex_file)
    import subprocess
    subprocess.call([
        "-c", "pdflatex -interaction=batchmode {0} && "
        "rm {1}.{{aux,log}} && pdfcrop --margins 10 {1}.pdf {2}".format(
            tex_file, tex_file[:-4], pdf_file)
    ],
                    shell=True,
                    cwd=results_path)

    # initialize Qt application
    import sys
    from OWWidget import QApplication
    a = QApplication(sys.argv)
    from PyMTL.orange_visualizations import save_treegraph_image
    save_treegraph_image(tree, os.path.join(results_path, "test-tree.svg"))