def _test_tasks_orange(self, models, measures):
"""Test the given tasks' models on their testing data sets. Compute
the given scoring measures of the testing results.
Return a two-dimensional dictionary with the first key corresponding to
the task's id and the second key corresponding to the measure's name.
The value corresponds to the score for the given task and scoring
measure.
Note: If a particular scoring measure couldn't be computed for a task,
its value is set to None.
Note: This function works with Orange classifiers and Orange data
tables.
Arguments:
models -- dictionary mapping from tasks' ids to their models
measures -- list of strings representing measure's names (currently,
only CA and AUC are supported)
"""
scores = dict()
comp_errors = {measure: 0 for measure in measures}
for tid, task_test_data in self._test_data_orange.iteritems():
scores[tid] = dict()
test_res = Orange.evaluation.testing.test_on_data([models[tid]],
task_test_data)
for measure in measures:
if measure == "AUC":
try:
score = Orange.evaluation.scoring.AUC(test_res)[0]
except ValueError as e:
if (e.args[0] ==
"Cannot compute AUC on a single-class problem"
):
# AUC cannot be computed because all instances
# belong to the same class
score = None
comp_errors[measure] += 1
else:
raise e
elif measure == "CA":
score = Orange.evaluation.scoring.CA(test_res)[0]
else:
raise ValueError("Unknown scoring measure: {}".\
format(measure))
scores[tid][measure] = score
# report the number of errors when computing the scoring measures
n = len(self._tasks)
for m_name, m_errors in comp_errors.iteritems():
if m_errors > 0:
logger.info("Scoring measure {} could not be computed for {}"
" out of {} tasks ({:.1f}%)".format(
m_name, m_errors, n, 100. * m_errors / n))
return scores
def _test_tasks_orange(self, models, measures):
"""Test the given tasks' models on their testing data sets. Compute
the given scoring measures of the testing results.
Return a two-dimensional dictionary with the first key corresponding to
the task's id and the second key corresponding to the measure's name.
The value corresponds to the score for the given task and scoring
measure.
Note: If a particular scoring measure couldn't be computed for a task,
its value is set to None.
Note: This function works with Orange classifiers and Orange data
tables.
Arguments:
models -- dictionary mapping from tasks' ids to their models
measures -- list of strings representing measure's names (currently,
only CA and AUC are supported)
"""
scores = dict()
comp_errors = {measure : 0 for measure in measures}
for tid, task_test_data in self._test_data_orange.iteritems():
scores[tid] = dict()
test_res = Orange.evaluation.testing.test_on_data([models[tid]],
task_test_data)
for measure in measures:
if measure == "AUC":
try:
score = Orange.evaluation.scoring.AUC(test_res)[0]
except ValueError as e:
if (e.args[0] ==
"Cannot compute AUC on a single-class problem"):
# AUC cannot be computed because all instances
# belong to the same class
score = None
comp_errors[measure] += 1
else:
raise e
elif measure == "CA":
score = Orange.evaluation.scoring.CA(test_res)[0]
else:
raise ValueError("Unknown scoring measure: {}".\
format(measure))
scores[tid][measure] = score
# report the number of errors when computing the scoring measures
n = len(self._tasks)
for m_name, m_errors in comp_errors.iteritems():
if m_errors > 0:
logger.info("Scoring measure {} could not be computed for {}"
" out of {} tasks ({:.1f}%)".format(m_name, m_errors, n,
100.*m_errors/n))
return scores
def test_tasks(self, learners, base_learners, measures, results_path,
save_orange_data=False):
"""Repeat the following experiment self._repeats times:
Prepare tasks' data with the _prepare_tasks_data() function.
Test the performance of the given learning algorithms with the given
base learning algorithms and compute the testing results using the
given scoring measures.
Process the obtained repetition scores with the
_process_repetition_scores() function.
Note: This function only test some specific combinations of
base_learners and learners as used by the binarization experiment.
Arguments:
learners -- ordered dictionary with items of the form (name, learner),
where name is a string representing the learner's name and
learner is a MTL method (e.g. ERM, NoMerging, ...)
base learners -- ordered dictionary with items of the form (name,
learner), where name is a string representing the base learner's
name and learner is a scikit-learn estimator object
measures -- list of strings representing measure's names (currently,
only CA and AUC are supported)
results_path -- string representing the path where to save any extra
information about the running of this test (currently, only used
for pickling the results when there is an error in calling the
learner)
save_orange_data -- boolean indicating whether to save the Orange data
tables created with the call to self._prepare_tasks_data() function
"""
rpt_scores = OrderedDict()
dend_info = {bl : OrderedDict() for bl in base_learners.iterkeys()}
for i in range(self._repeats):
self._repetition_number = i
self._prepare_tasks_data(**self._tasks_data_params)
if save_orange_data:
self._save_orange_data(i, results_path)
rpt_scores[i] = {bl : dict() for bl in base_learners.iterkeys()}
for bl in base_learners:
for l in learners:
start = time.clock()
try:
if isinstance(learners[l],
bin_exp.TreeMarkedAndMergedLearner):
R = learners[l](self._tasks.keys(),
self._merged_learn_data_orange,
base_learners[bl])
elif isinstance(base_learners[bl], Orange.core.Learner):
wrapped_bl = OrangeClassifierWrapper(
orange_learner=base_learners[bl])
R = learners[l](self._tasks, wrapped_bl)
else:
raise ValueError("An unexpected combination of "
"base_learner and leaner detected: {} and "
"{}".format(type(base_learners[bl]),
type(learners[l])))
except Exception as e:
logger.exception("There was an error during repetition:"
" {} with base learner: {} and learner: {}.".\
format(i, bl, l))
if i > 0:
logger.info("Saving the results of previous "
"repetitions.")
# remove the scores of the last repetition
del rpt_scores[i]
# process the remaining repetition scores
self._process_repetition_scores(rpt_scores,
dend_info)
# pickle them to a file
pickle_path_fmt = os.path.join(results_path,
"bl-{}.pkl")
self.pickle_test_results(pickle_path_fmt)
# re-raise the original exception
import sys
exc_info = sys.exc_info()
raise exc_info[1], None, exc_info[2]
rpt_scores[i][bl][l] = self._test_tasks(R["task_models"],
measures)
end = time.clock()
logger.debug("Finished repetition: {}, base learner: {}, "
"learner: {} in {:.2f}s".format(i, bl, l, end-start))
# store dendrogram info if the results contain it
if "dend_info" in R:
dend_info[bl][i] = R["dend_info"]
# pickle and visualize the decision tree if the learner is a
# (sub)class of TreeMarkedAndMergedLearner
if isinstance(learners[l],
bin_exp.TreeMarkedAndMergedLearner):
tree = R["task_models"].values()[0]
pickle_path = os.path.join(results_path, "{}-{}-"
"repeat{}.pkl".format(bl, l, i))
svg_path = os.path.join(results_path, "{}-{}-repeat{}"
".svg".format(bl, l, i))
tikz_path = os.path.join(results_path, "{}-{}-repeat{}"
"-tikz.tex".format(bl, l, i))
pickle_obj(tree, pickle_path)
save_treegraph_image(tree, svg_path)
draw_and_save_tikz_tree_document(tree, tikz_path)
self._process_repetition_scores(rpt_scores, dend_info)
def test_tasks(self,
learners,
base_learners,
measures,
results_path,
save_orange_data=False):
"""Repeat the following experiment self._repeats times:
Prepare tasks' data with the _prepare_tasks_data() function.
Test the performance of the given learning algorithms with the given
base learning algorithms and compute the testing results using the
given scoring measures.
Process the obtained repetition scores with the
_process_repetition_scores() function.
Note: This function only test some specific combinations of
base_learners and learners as used by the binarization experiment.
Arguments:
learners -- ordered dictionary with items of the form (name, learner),
where name is a string representing the learner's name and
learner is a MTL method (e.g. ERM, NoMerging, ...)
base learners -- ordered dictionary with items of the form (name,
learner), where name is a string representing the base learner's
name and learner is a scikit-learn estimator object
measures -- list of strings representing measure's names (currently,
only CA and AUC are supported)
results_path -- string representing the path where to save any extra
information about the running of this test (currently, only used
for pickling the results when there is an error in calling the
learner)
save_orange_data -- boolean indicating whether to save the Orange data
tables created with the call to self._prepare_tasks_data() function
"""
rpt_scores = OrderedDict()
dend_info = {bl: OrderedDict() for bl in base_learners.iterkeys()}
for i in range(self._repeats):
self._repetition_number = i
self._prepare_tasks_data(**self._tasks_data_params)
if save_orange_data:
self._save_orange_data(i, results_path)
rpt_scores[i] = {bl: dict() for bl in base_learners.iterkeys()}
for bl in base_learners:
for l in learners:
start = time.clock()
try:
if isinstance(learners[l],
bin_exp.TreeMarkedAndMergedLearner):
R = learners[l](self._tasks.keys(),
self._merged_learn_data_orange,
base_learners[bl])
elif isinstance(base_learners[bl],
Orange.core.Learner):
wrapped_bl = OrangeClassifierWrapper(
orange_learner=base_learners[bl])
R = learners[l](self._tasks, wrapped_bl)
else:
raise ValueError(
"An unexpected combination of "
"base_learner and leaner detected: {} and "
"{}".format(type(base_learners[bl]),
type(learners[l])))
except Exception as e:
logger.exception("There was an error during repetition:"
" {} with base learner: {} and learner: {}.".\
format(i, bl, l))
if i > 0:
logger.info("Saving the results of previous "
"repetitions.")
# remove the scores of the last repetition
del rpt_scores[i]
# process the remaining repetition scores
self._process_repetition_scores(
rpt_scores, dend_info)
# pickle them to a file
pickle_path_fmt = os.path.join(
results_path, "bl-{}.pkl")
self.pickle_test_results(pickle_path_fmt)
# re-raise the original exception
import sys
exc_info = sys.exc_info()
raise exc_info[1], None, exc_info[2]
rpt_scores[i][bl][l] = self._test_tasks(
R["task_models"], measures)
end = time.clock()
logger.debug("Finished repetition: {}, base learner: {}, "
"learner: {} in {:.2f}s".format(
i, bl, l, end - start))
# store dendrogram info if the results contain it
if "dend_info" in R:
dend_info[bl][i] = R["dend_info"]
# pickle and visualize the decision tree if the learner is a
# (sub)class of TreeMarkedAndMergedLearner
if isinstance(learners[l],
bin_exp.TreeMarkedAndMergedLearner):
tree = R["task_models"].values()[0]
pickle_path = os.path.join(
results_path, "{}-{}-"
"repeat{}.pkl".format(bl, l, i))
svg_path = os.path.join(
results_path, "{}-{}-repeat{}"
".svg".format(bl, l, i))
tikz_path = os.path.join(
results_path, "{}-{}-repeat{}"
"-tikz.tex".format(bl, l, i))
pickle_obj(tree, pickle_path)
save_treegraph_image(tree, svg_path)
draw_and_save_tikz_tree_document(tree, tikz_path)
self._process_repetition_scores(rpt_scores, dend_info)
def __call__(self, tasks, base_learner):
"""Run the merging algorithm for the given tasks. Perform the
intelligent merging of tasks' data according to the ERM learning method.
After the merging is complete, build a model for each remaining (merged)
task and assign this model to each original task of this (merged) task.
Return a dictionary of data structures computed within this call to ERM.
It has the following keys:
task_models -- dictionary mapping from each original task id to its
model
dend_info -- list of tuples (one for each merged task) as returned
by the convert_merg_history_to_scipy_linkage function
Arguments:
tasks -- dictionary mapping from tasks' ids to their Task objects
base_learner -- scikit-learn estimator
"""
self._base_learner = base_learner
# create an ordered dictionary of MergedTask objects from the given
# dictionary of tasks
self._tasks = OrderedDict()
for _, task in sorted(tasks.iteritems()):
merg_task = MergedTask(task)
self._tasks[merg_task.id] = merg_task
# populate the dictionary of task pairs that are candidates for merging
C = dict()
pairs = list(combinations(self._tasks, 2))
n_pairs = len(pairs)
msg = "Computing candidate pairs for merging ({} pairs)".format(n_pairs)
logger.debug(msg)
print msg
for i, (tid_i, tid_j) in enumerate(pairs):
if self._prefilter(tid_i, tid_j):
avg_pred_errs, p_values_ij = \
self._estimate_errors_significances(tid_i, tid_j)
er_ij = error_reduction(avg_pred_errs["data1"]["data1"],
avg_pred_errs["data2"]["data2"],
avg_pred_errs["dataM"]["dataM"],
self._tasks[tid_i].get_data_size(),
self._tasks[tid_j].get_data_size())
min_ij = min(avg_pred_errs["data1"]["dataM"],
avg_pred_errs["data2"]["dataM"])
if er_ij >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_ij:
cp = CandidatePair(tid_i, tid_j, p_values_ij)
C[cp.key] = cp
update_progress(1.* (i + 1) / n_pairs)
print
# iteratively merge the most similar pair of tasks, until such pairs
# exist
n_cand = len(C)
msg = "Processing {} candidate pairs for merging".format(n_cand)
logger.debug(msg)
print msg
while len(C) > 0:
# find the task pair with the minimal maximal p-value
maxes = [(cp_key, cp.get_max_p_value()) for cp_key, cp in
C.iteritems()]
(min_tid_i, min_tid_j), _ = min(maxes, key=lambda x: x[1])
# merge the pair of tasks and update self._tasks
task_M = MergedTask(self._tasks[min_tid_i], self._tasks[min_tid_j])
tid_M = task_M.id
del self._tasks[min_tid_i]
del self._tasks[min_tid_j]
self._tasks[tid_M] = task_M
# remove task pairs that don't exist anymore from C
for (tid_i, tid_j) in C.keys():
if ((tid_i == min_tid_i) or (tid_i == min_tid_j) or
(tid_j == min_tid_i) or (tid_j == min_tid_j)):
del C[(tid_i, tid_j)]
# find new task pairs that are candidates for merging
for tid_i in self._tasks:
if tid_i != tid_M and self._prefilter(tid_i, tid_M):
avg_pred_errs, p_values_iM = \
self._estimate_errors_significances(tid_i, tid_M)
er_iM = error_reduction(avg_pred_errs["data1"]["data1"],
avg_pred_errs["data2"]["data2"],
avg_pred_errs["dataM"]["dataM"],
self._tasks[tid_i].get_data_size(),
self._tasks[tid_M].get_data_size())
min_iM = min(avg_pred_errs["data1"]["dataM"],
avg_pred_errs["data2"]["dataM"])
if er_iM >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_iM:
cp = CandidatePair(tid_i, tid_M, p_values_iM)
C[cp.key] = cp
update_progress(1.* len(C) / n_cand, invert=True)
print
# build a model for each remaining (merged) task and store the info
# for drawing a dendrogram showing the merging history
task_models = dict()
dend_info = []
for merg_task in self._tasks.itervalues():
# NOTE: When the number of unique class values is less than 2, we
# cannot fit an ordinary model (e.g. logistic regression). Instead,
# we have to use a dummy classifier which is subsequently augmented
# to handle all the other class values.
# NOTE: The scikit-learn estimator must be cloned so that each
# (merged) task gets its own classifier
X, y = merg_task.get_learn_data()
if len(np.unique(y)) < 2:
logger.info("Learning data for merged task {} has less than 2 "
"class values. Using DummyClassifier.".\
format(merg_task))
model = DummyClassifier()
model.fit(X, y)
change_dummy_classes(model, np.array([0, 1]))
else:
model = clone(self._base_learner)
model.fit(X, y)
# assign this model to each original task of this (merged) task
original_ids = merg_task.get_original_ids()
for tid in original_ids:
task_models[tid] = model
# store the dendrogram info (if the task is truly a merged task)
if len(original_ids) > 1:
dend_info.append(convert_merg_history_to_scipy_linkage(
merg_task.merg_history))
# create and fill the return dictionary
R = dict()
R["task_models"] = task_models
R["dend_info"] = dend_info
return R
def __call__(self, tasks, base_learner):
"""Run the merging algorithm for the given tasks. Perform the
intelligent merging of tasks' data according to the ERM learning method.
After the merging is complete, build a model for each remaining (merged)
task and assign this model to each original task of this (merged) task.
Return a dictionary of data structures computed within this call to ERM.
It has the following keys:
task_models -- dictionary mapping from each original task id to its
model
dend_info -- list of tuples (one for each merged task) as returned
by the convert_merg_history_to_scipy_linkage function
Arguments:
tasks -- dictionary mapping from tasks' ids to their Task objects
base_learner -- scikit-learn estimator
"""
self._base_learner = base_learner
# create an ordered dictionary of MergedTask objects from the given
# dictionary of tasks
self._tasks = OrderedDict()
for _, task in sorted(tasks.iteritems()):
merg_task = MergedTask(task)
self._tasks[merg_task.id] = merg_task
# populate the dictionary of task pairs that are candidates for merging
C = dict()
pairs = list(combinations(self._tasks, 2))
n_pairs = len(pairs)
msg = "Computing candidate pairs for merging ({} pairs)".format(
n_pairs)
logger.debug(msg)
print msg
for i, (tid_i, tid_j) in enumerate(pairs):
if self._prefilter(tid_i, tid_j):
avg_pred_errs, p_values_ij = \
self._estimate_errors_significances(tid_i, tid_j)
er_ij = error_reduction(avg_pred_errs["data1"]["data1"],
avg_pred_errs["data2"]["data2"],
avg_pred_errs["dataM"]["dataM"],
self._tasks[tid_i].get_data_size(),
self._tasks[tid_j].get_data_size())
min_ij = min(avg_pred_errs["data1"]["dataM"],
avg_pred_errs["data2"]["dataM"])
if er_ij >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_ij:
cp = CandidatePair(tid_i, tid_j, p_values_ij)
C[cp.key] = cp
update_progress(1. * (i + 1) / n_pairs)
print
# iteratively merge the most similar pair of tasks, until such pairs
# exist
n_cand = len(C)
msg = "Processing {} candidate pairs for merging".format(n_cand)
logger.debug(msg)
print msg
while len(C) > 0:
# find the task pair with the minimal maximal p-value
maxes = [(cp_key, cp.get_max_p_value())
for cp_key, cp in C.iteritems()]
(min_tid_i, min_tid_j), _ = min(maxes, key=lambda x: x[1])
# merge the pair of tasks and update self._tasks
task_M = MergedTask(self._tasks[min_tid_i], self._tasks[min_tid_j])
tid_M = task_M.id
del self._tasks[min_tid_i]
del self._tasks[min_tid_j]
self._tasks[tid_M] = task_M
# remove task pairs that don't exist anymore from C
for (tid_i, tid_j) in C.keys():
if ((tid_i == min_tid_i) or (tid_i == min_tid_j)
or (tid_j == min_tid_i) or (tid_j == min_tid_j)):
del C[(tid_i, tid_j)]
# find new task pairs that are candidates for merging
for tid_i in self._tasks:
if tid_i != tid_M and self._prefilter(tid_i, tid_M):
avg_pred_errs, p_values_iM = \
self._estimate_errors_significances(tid_i, tid_M)
er_iM = error_reduction(avg_pred_errs["data1"]["data1"],
avg_pred_errs["data2"]["data2"],
avg_pred_errs["dataM"]["dataM"],
self._tasks[tid_i].get_data_size(),
self._tasks[tid_M].get_data_size())
min_iM = min(avg_pred_errs["data1"]["dataM"],
avg_pred_errs["data2"]["dataM"])
if er_iM >= 0 and avg_pred_errs["dataM"]["dataM"] <= min_iM:
cp = CandidatePair(tid_i, tid_M, p_values_iM)
C[cp.key] = cp
update_progress(1. * len(C) / n_cand, invert=True)
print
# build a model for each remaining (merged) task and store the info
# for drawing a dendrogram showing the merging history
task_models = dict()
dend_info = []
for merg_task in self._tasks.itervalues():
# NOTE: When the number of unique class values is less than 2, we
# cannot fit an ordinary model (e.g. logistic regression). Instead,
# we have to use a dummy classifier which is subsequently augmented
# to handle all the other class values.
# NOTE: The scikit-learn estimator must be cloned so that each
# (merged) task gets its own classifier
X, y = merg_task.get_learn_data()
if len(np.unique(y)) < 2:
logger.info("Learning data for merged task {} has less than 2 "
"class values. Using DummyClassifier.".\
format(merg_task))
model = DummyClassifier()
model.fit(X, y)
change_dummy_classes(model, np.array([0, 1]))
else:
model = clone(self._base_learner)
model.fit(X, y)
# assign this model to each original task of this (merged) task
original_ids = merg_task.get_original_ids()
for tid in original_ids:
task_models[tid] = model
# store the dendrogram info (if the task is truly a merged task)
if len(original_ids) > 1:
dend_info.append(
convert_merg_history_to_scipy_linkage(
merg_task.merg_history))
# create and fill the return dictionary
R = dict()
R["task_models"] = task_models
R["dend_info"] = dend_info
return R
