def format_method_comparison_heatmaps(real_data_results, sim_data_results,
heatmap_methods):
shared_studies = None
shared_categories = {}
for depth_desc, depth_res in real_data_results.items():
for metric, metric_res in depth_res.items():
for method, method_res in metric_res.items():
matched_method = False
for m in heatmap_methods:
if method == m.DirectoryName:
matched_method = True
break
if matched_method:
studies = sorted(method_res.keys())
if shared_studies is None:
shared_studies = studies
elif studies != shared_studies:
raise ValueError("Not all methods to include in "
"the heatmap have results for "
"the same studies.")
for study, study_res in sorted(method_res.items()):
categories = [cat for cat, cat_res in \
sorted(study_res.items()) if not
is_empty(cat_res)]
if study not in shared_categories:
shared_categories[study] = set(categories)
else:
shared_categories[study] &= set(categories)
# Gather real data effect sizes for each method (in the same order for each
# method!).
method_data = defaultdict(list)
for depth_desc, depth_res in real_data_results.items():
for metric, metric_res in depth_res.items():
for method, method_res in metric_res.items():
matched_method = False
for m in heatmap_methods:
if method == m.DirectoryName:
matched_method = True
break
if not matched_method:
continue
for study, study_res in sorted(method_res.items()):
for category, category_res in sorted(study_res.items()):
if category in shared_categories[study]:
method_data[method].append(
category_res['original'].effect_size)
method_data[method].append(
category_res['shuffled'].effect_size)
# Gather simulated data effect sizes.
for method, method_res in sim_data_results.items():
matched_method = False
for m in heatmap_methods:
if method == m.DirectoryName:
matched_method = True
break
if not matched_method:
continue
for study, study_res in sorted(method_res.items()):
for depth, depth_res in sorted(study_res.items()):
for category, category_res in sorted(depth_res.items()):
for trial_num, trial_num_res in sorted(category_res.items()):
for samp_size, samp_size_res in sorted(trial_num_res.items()):
for dissim, dissim_res in sorted(samp_size_res.items()):
for metric, metric_res in sorted(dissim_res.items()):
if metric_res.isEmpty():
raise ValueError("Encountered empty "
"simulated data results.")
else:
method_data[method].append(
metric_res.effect_size)
# Make sure our data looks sane. We should have the same number of
# observations (i.e. effect sizes) for each method.
data_length = None
for method, data in method_data.items():
if data_length is None:
data_length = len(data)
elif len(data) != data_length:
raise ValueError("The number of observations (i.e. effect sizes) "
"is not the same between all methods, so we "
"can't compare them.")
# Compute the correlation coefficient between each pair of methods and put
# the output in an array. This array can then be used to generate a
# text-based table or heatmap.
results = {}
for correlation_name, correlation_fn in \
('pearson', pearson), ('spearman', spearman):
num_methods = len(heatmap_methods)
heatmap_data = ones((num_methods, num_methods))
# I know this is inefficient, but it really doesn't matter for what
# we're doing here.
for method1_idx, method1 in enumerate(heatmap_methods):
for method2_idx, method2 in enumerate(heatmap_methods):
corr_coeff = correlation_fn(method_data[method1.DirectoryName],
method_data[method2.DirectoryName])
heatmap_data[method1_idx][method2_idx] = corr_coeff
# Mask out the upper triangle. Taken from
# http://stackoverflow.com/a/2332520
mask = invert(tri(heatmap_data.shape[0], k=0, dtype=bool))
heatmap_data = numpy.ma.array(heatmap_data, mask=mask)
results[correlation_name] = heatmap_data
return results
def test_is_empty(self):
"""Test checking if category results are empty or not."""
self.assertTrue(is_empty(self.cat_res1))
self.assertTrue(is_empty(self.cat_res2))
self.assertFalse(is_empty(self.cat_res3))
self.assertTrue(is_empty({}))
