def test_mpl_to_bezier_path(self):
"""test_mpl_to_bezier_path"""
star_path = Path.unit_regular_star(10)
r = RendererCocoa(None, None)
bpath = r.mpl_to_bezier_path(star_path)
yield self.assert_paths_equal, star_path, bpath
wedge_path = Path.wedge(10, 25)
bpath = r.mpl_to_bezier_path(wedge_path)
yield self.assert_paths_equal, wedge_path, bpath
def test_mpl_to_bezier_path(self):
"""test_mpl_to_bezier_path"""
star_path = Path.unit_regular_star(10)
r = RendererCocoa(None, None)
bpath = r.mpl_to_bezier_path(star_path)
yield self.assert_paths_equal, star_path, bpath
wedge_path = Path.wedge(10, 25)
bpath = r.mpl_to_bezier_path(wedge_path)
yield self.assert_paths_equal, wedge_path, bpath
def __init__(self, center, r, theta1, theta2, **kwargs):
"""
Draw a wedge centered at x,y tuple center with radius r that
sweeps theta1 to theta2 (angles)
Valid kwargs are:
%(Patch)s
"""
Patch.__init__(self, **kwargs)
self.center = center
self.r = r
self.theta1 = theta1
self.theta2 = theta2
self._patch_transform = transforms.IdentityTransform()
self._path = Path.wedge(self.theta1, self.theta2)
def __init__(self, center, r, theta1, theta2, **kwargs):
"""
Draw a wedge centered at x,y tuple center with radius r that
sweeps theta1 to theta2 (angles)
Valid kwargs are:
%(Patch)s
"""
Patch.__init__(self, **kwargs)
self.center = center
self.r = r
self.theta1 = theta1
self.theta2 = theta2
self._patch_transform = transforms.IdentityTransform()
self._path = Path.wedge(self.theta1, self.theta2)
def __init__(self, center, r, theta1, theta2, **kwargs):
"""
Draw a wedge centered at *x*, *y* center with radius *r* that
sweeps *theta1* to *theta2* (in degrees).
Valid kwargs are:
%(Patch)s
"""
Patch.__init__(self, **kwargs)
self.center = center
self.r = r
self.theta1 = theta1
self.theta2 = theta2
self._patch_transform = transforms.IdentityTransform()
self._path = Path.wedge(self.theta1, self.theta2)
def __init__(self, center, r, theta1, theta2, **kwargs):
"""
Draw a wedge centered at *x*, *y* center with radius *r* that
sweeps *theta1* to *theta2* (in degrees).
Valid kwargs are:
%(Patch)s
"""
Patch.__init__(self, **kwargs)
self.center = center
self.r = r
self.theta1 = theta1
self.theta2 = theta2
self._patch_transform = transforms.IdentityTransform()
self._path = Path.wedge(self.theta1, self.theta2)
but will then be redundant and should be removed.
This is fixed in MPL v1.3, raising a RuntimeError. A check is performed to
allow for backward compatibility with MPL v1.2.x.
'''
if wedge_path.vertices.flags.writeable:
wedge_path.vertices[0] = 0
wedge_path.vertices[-2:] = 0
return wedge_path
CLOUD_COVER = {
0: [_ring_path()],
1: [_ring_path(), _vertical_bar_path()],
2: [_ring_path(), _wedge_fix(Path.wedge(0, 90))],
3: [_ring_path(),
_wedge_fix(Path.wedge(0, 90)),
_vertical_bar_path()],
4: [_ring_path(), Path.unit_circle_righthalf()],
5: [_ring_path(),
Path.unit_circle_righthalf(),
_left_bar_path()],
6: [_ring_path(), _wedge_fix(Path.wedge(-180, 90))],
7: [_slot_path()],
8: [Path.unit_circle()],
9: [_ring_path(), _slash_path(),
_backslash_path()],
}
"""
A dictionary mapping WMO cloud cover codes to their corresponding symbol.
def _wedge_fix(wedge_path):
# Fixes the problem with Path.wedge where it doesn't initialise the first,
# and last two vertices.
# This fix should not have any side-effects once Path.wedge has been fixed,
# but will then be redundant and should be removed.
wedge_path.vertices[0] = 0
wedge_path.vertices[-2:] = 0
return wedge_path
CLOUD_COVER = {
0: [_ring_path()],
1: [_ring_path(), _vertical_bar_path()],
2: [_ring_path(), _wedge_fix(Path.wedge(0, 90))],
3: [_ring_path(), _wedge_fix(Path.wedge(0, 90)), _vertical_bar_path()],
4: [_ring_path(), Path.unit_circle_righthalf()],
5: [_ring_path(), Path.unit_circle_righthalf(), _left_bar_path()],
6: [_ring_path(), _wedge_fix(Path.wedge(-180, 90))],
7: [_slot_path()],
8: [Path.unit_circle()],
9: [_ring_path(), _slash_path(), _backslash_path()],
}
"""
A dictionary mapping WMO cloud cover codes to their corresponding symbol.
See http://www.wmo.int/pages/prog/www/DPFS/documents/485_Vol_I_en_colour.pdf
Part II, Appendix II.4, Graphical Representation of Data, Analyses and Forecasts
"""
def plot_model_comparison(result,
sort=False,
colors=None,
alpha=0.01,
test_pair_comparisons=True,
multiple_pair_testing='fdr',
test_above_0=True,
test_below_noise_ceil=True,
error_bars='sem'):
""" Plots the results of RSA inference on a set of models as a bar graph
with one bar for each model indicating its predictive performance. The
function also shows the noise ceiling whose upper edge is an upper bound
on the performance the true model could achieve (given noise and inter-
subject variability) and whose lower edge is an estimate of a lower bound
on the performance of the true model. In addition, all pairwise inferential
model comparisons are shown in the upper part of the figure.
The only mandatory input is a "result" object containing model evaluations
for bootstrap samples and crossvalidation folds. These are used here to
construct confidence intervals and perform the significance tests.
Args (All strings case insensitive):
result (pyrsa.inference.result.Result):
model evaluation result
sort (Boolean or string):
False (default): plot bars in the order passed
'descend[ing]': plot bars in descending order of model performance
'ascend[ing]': plot bars in ascending order of model performance
colors (list of lists, numpy array, matplotlib colormap):
None (default): default blue for all bars
single color: list or numpy array of 3 or 4 values (RGB, RGBA)
specifying the color for all bars
multiple colors: list of lists or numpy array (number of colors by
3 or 4 channels -- RGB, RGBA). If the number of colors
matches the number of models, each color is used for the
bar corresponding to one model (in the order of the models
as passed). If the number of colors does not match the
number of models, the list is linearly interpolated to
assign a color to each model (in the order of the models as
passed). For example, two colors will become a gradation,
unless there are exactly two model. Instead of a list of
lists or numpy array, a matplotlib colormap object may also
be passed (e.g. colors = cm.coolwarm).
alpha (float):
significance threshold (p threshold or FDR q threshold)
test_pair_comparisons (Boolean or string):
False or None: do not plot pairwise model comparison results
True (default): plot pairwise model comparison results using
default settings
'arrows': plot results in arrows style, indicating pairs of sets
between which all differences are significant
'nili': plot results as Nili bars (Nili et al. 2014), indicating
each significant difference by a horizontal line (or each
nonsignificant difference if the string contains a '2', e.g.
'nili2')
'golan': plot results as Golan wings (Golan et al. 2020), with one
wing (graphical element) indicating all dominance relationships
for one model.
'cliques': plot results as cliques of insignificant differences
multiple_pair_testing (Boolean or string):
False or 'none': do not adjust for multiple testing for the
pairwise model comparisons
'FDR' or 'fdr' (default): control the false-discorvery rate at
q = alpha
'FWER',' fwer', or 'Bonferroni': control the familywise error rate
using the Bonferroni method
test_above_0 (Boolean or string):
False or None: do not plot results of statistical comparison of
each model performance against 0
True (default): plot results of statistical comparison of each
model performance against 0 using default settings ('dewdrops')
'dewdrops': place circular "dewdrops" at the baseline to indicate
models whose performance is significantly greater than 0
'icicles': place triangular "icicles" at the baseline to indicate
models whose performance is significantly greater than 0
Tests are one-sided, use the global alpha threshold and are
automatically Bonferroni-corrected for the number of models tested.
test_below_noise_ceil (Boolean or string):
False or None: do not plot results of statistical comparison of
each model performance against the lower-bound estimate of the
noise ceiling
True (default): plot results of statistical comparison of each
model performance against the lower-bound estimate of the noise
ceiling using default settings ('dewdrops')
'dewdrops': use circular "dewdrops" at the lower bound of the
noise ceiling to indicate models whose performance is
significantly below the lower-bound estimate of the noise
ceiling
'icicles': use triangular "icicles" at the lower bound of the noise
ceiling to indicate models whose performance is significantly
below the lower-bound estimate of the noise ceiling
Tests are one-sided, use the global alpha threshold and are
automatically Bonferroni-corrected for the number of models tested.
error_bars (Boolean or string):
False or None: do not plot error bars
True (default) or 'SEM': plot the standard error of the mean
'CI': plot 95%-confidence intervals (exluding 2.5% on each side)
'CI[x]': plot x%-confidence intervals (exluding 2.5% on each side)
Confidence intervals are based on the bootstrap procedure,
reflecting variability of the estimate across subjects and/or
experimental conditions.
Returns:
---
"""
# Prepare and sort data
evaluations = result.evaluations
models = result.models
noise_ceiling = result.noise_ceiling
method = result.method
while len(evaluations.shape) > 2:
evaluations = np.nanmean(evaluations, axis=-1)
if noise_ceiling.ndim > 1:
noise_ceiling = noise_ceiling[:, ~np.isnan(evaluations[:, 0])]
evaluations = evaluations[~np.isnan(evaluations[:, 0])]
perf = np.mean(evaluations, axis=0)
n_bootstraps, n_models = evaluations.shape
if sort is True:
sort = 'descending' # descending by default if sort is True
elif sort is False:
sort = 'unsorted'
if sort != 'unsorted': # 'descending' or 'ascending'
idx = np.argsort(perf)
if 'descend' in sort.lower():
idx = np.flip(idx)
perf = perf[idx]
evaluations = evaluations[:, idx]
models = [models[i] for i in idx]
if not ('descend' in sort.lower() or 'ascend' in sort.lower()):
raise Exception('plot_model_comparison: Argument ' +
'sort is incorrectly defined as ' + sort + '.')
# Prepare axes for bars and pairwise comparisons
fs, fs2 = 18, 14 # axis label font sizes
l, b, w, h = 0.15, 0.15, 0.8, 0.8
fig = plt.figure(figsize=(12.5, 10))
if test_pair_comparisons is True:
test_pair_comparisons = 'arrows'
if test_pair_comparisons:
if test_pair_comparisons.lower() in ['arrows', 'cliques']:
h_pair_tests = 0.25
elif 'golan' in test_pair_comparisons.lower():
h_pair_tests = 0.3
elif 'nili' in test_pair_comparisons.lower():
h_pair_tests = 0.4
else:
raise Exception(
'plot_model_comparison: Argument ' +
'test_pair_comparisons is incorrectly defined as ' +
test_pair_comparisons + '.')
ax = plt.axes((l, b, w, h * (1 - h_pair_tests)))
axbar = plt.axes(
(l, b + h * (1 - h_pair_tests), w, h * h_pair_tests * 0.7))
else:
ax = plt.axes((l, b, w, h))
# Define the model colors
if colors is None: # no color passed...
colors = [0, 0.4, 0.9, 1] # use default blue
elif isinstance(colors, cm.colors.LinearSegmentedColormap):
cmap = cm.get_cmap(colors)
colors = cmap(np.linspace(0, 1, 100))[np.newaxis, :, :3].squeeze()
colors = np.array([np.array(col) for col in colors])
if len(colors.shape) == 1: # one color passed...
n_col, n_chan = 1, colors.shape[0]
colors.shape = (n_col, n_chan)
else: # multiple colors passed...
n_col, n_chan = colors.shape
if n_col == n_models: # one color passed for each model...
cols2 = colors
else: # number of colors passed does not match number of models...
# interpolate colors to define a color for each model
cols2 = np.empty((n_models, n_chan))
for c in range(n_chan):
cols2[:,
c] = np.interp(np.array(range(n_models)),
np.array(range(n_col)) / n_col * n_models,
colors[:, c])
if sort != 'unsorted':
colors = cols2[idx, :]
else:
colors = cols2
if colors.shape[1] == 3:
colors = np.concatenate((colors, np.ones((colors.shape[0], 1))),
axis=1)
# Plot bars and error bars
ax.bar(np.arange(evaluations.shape[1]), perf, color=colors)
if error_bars is True:
error_bars = 'sem'
if error_bars.lower() == 'sem':
errorbar_low = np.std(evaluations, axis=0)
errorbar_high = np.std(evaluations, axis=0)
elif error_bars[0:2].lower() == 'ci':
if len(error_bars) == 2:
CI_percent = 95
else:
CI_percent = int(error_bars[2:])
prop_cut = (1 - CI_percent / 100) / 2
framed_evals = np.concatenate(
(np.tile(np.array((-np.inf, np.inf)).reshape(2, 1),
(1, n_models)), evaluations),
axis=0)
errorbar_low = -(np.quantile(framed_evals, prop_cut, axis=0) - perf)
errorbar_high = (np.quantile(framed_evals, 1 - prop_cut, axis=0) -
perf)
limits = np.concatenate((errorbar_low, errorbar_high))
if np.isnan(limits).any() or (abs(limits) == np.inf).any():
raise Exception(
'plot_model_comparison: Too few bootstrap samples for the ' +
'requested confidence interval: ' + error_bars + '.')
elif error_bars:
raise Exception('plot_model_comparison: Argument ' +
'error_bars is incorrectly defined as ' + error_bars +
'.')
if error_bars:
ax.errorbar(np.arange(evaluations.shape[1]),
perf,
yerr=[errorbar_low, errorbar_high],
fmt='none',
ecolor='k',
capsize=0,
linewidth=3)
# Test whether model performance exceeds 0 (one sided)
if test_above_0 is True:
test_above_0 = 'dewdrops'
if test_above_0:
p = ((evaluations < 0).sum(axis=0) + 1) / n_bootstraps
model_significant = p < alpha / n_models
half_sym_size = 9
if test_above_0.lower() == 'dewdrops':
halfmoonup = Path.wedge(0, 180)
ax.plot(model_significant.nonzero()[0],
np.tile(0, model_significant.sum()),
'w',
marker=halfmoonup,
markersize=half_sym_size,
linewidth=0)
elif test_above_0.lower() == 'icicles':
ax.plot(model_significant.nonzero()[0],
np.tile(0, model_significant.sum()),
'w',
marker=10,
markersize=half_sym_size,
linewidth=0)
else:
raise Exception('plot_model_comparison: Argument test_above_0' +
' is incorrectly defined as ' + test_above_0 + '.')
# Plot noise ceiling
noise_ceil_col = [0.5, 0.5, 0.5, 0.2]
if noise_ceiling is not None:
noise_lower = np.nanmean(noise_ceiling[0])
noise_upper = np.nanmean(noise_ceiling[1])
noiserect = patches.Rectangle((-0.5, noise_lower),
len(perf),
noise_upper - noise_lower,
linewidth=0,
facecolor=noise_ceil_col,
zorder=1e6)
ax.add_patch(noiserect)
# Test whether model performance is below the noise ceiling's lower bound
# (one sided)
if test_below_noise_ceil is True:
test_below_noise_ceil = 'dewdrops'
if test_below_noise_ceil:
if len(noise_ceiling.shape) > 1:
noise_lower_bs = noise_ceiling[0]
noise_lower_bs.shape = (noise_lower_bs.shape[0], 1)
else:
noise_lower_bs = noise_ceiling[0].reshape(1, 1)
diffs = noise_lower_bs - evaluations # positive if below lower bound
p = ((diffs < 0).sum(axis=0) + 1) / n_bootstraps
model_below_lower_bound = p < alpha / n_models
if test_below_noise_ceil.lower() == 'dewdrops':
halfmoondown = Path.wedge(180, 360)
ax.plot(model_below_lower_bound.nonzero()[0],
np.tile(noise_lower + 0.0000,
model_below_lower_bound.sum()),
color='none',
marker=halfmoondown,
markersize=half_sym_size,
markerfacecolor=noise_ceil_col,
markeredgecolor='none',
linewidth=0)
elif test_below_noise_ceil.lower() == 'icicles':
ax.plot(model_below_lower_bound.nonzero()[0],
np.tile(noise_lower + 0.0007,
model_below_lower_bound.sum()),
color='none',
marker=11,
markersize=half_sym_size,
markerfacecolor=noise_ceil_col,
markeredgecolor='none',
linewidth=0)
else:
raise Exception(
'plot_model_comparison: Argument ' +
'test_below_noise_ceil is incorrectly defined as ' +
test_below_noise_ceil + '.')
# Pairwise model comparisons
if test_pair_comparisons:
model_comp_descr = 'Model comparisons: two-tailed, '
p_values = pair_tests(evaluations)
n_tests = int((n_models**2 - n_models) / 2)
if multiple_pair_testing is None:
multiple_pair_testing = 'uncorrected'
if multiple_pair_testing.lower() == 'bonferroni' or \
multiple_pair_testing.lower() == 'fwer':
significant = p_values < (alpha / n_tests)
model_comp_descr = (model_comp_descr +
'p < {:<.5g}'.format(alpha) +
', Bonferroni-corrected for ' + str(n_tests) +
' model-pair comparisons')
elif multiple_pair_testing.lower() == 'fdr':
ps = batch_to_vectors(np.array([p_values]))[0][0]
ps = np.sort(ps)
criterion = alpha * (np.arange(ps.shape[0]) + 1) / ps.shape[0]
k_ok = ps < criterion
if np.any(k_ok):
k_max = np.max(np.where(ps < criterion)[0])
crit = criterion[k_max]
else:
crit = 0
significant = p_values < crit
model_comp_descr = (model_comp_descr +
'FDR q < {:<.5g}'.format(alpha) + ' (' +
str(n_tests) + ' model-pair comparisons)')
else:
if 'uncorrected' not in multiple_pair_testing.lower():
raise Exception(
'plot_model_comparison: Argument ' +
'multiple_pair_testing is incorrectly defined as ' +
multiple_pair_testing + '.')
significant = p_values < alpha
model_comp_descr = (model_comp_descr +
'p < {:<.5g}'.format(alpha) +
', uncorrected (' + str(n_tests) +
' model-pair comparisons)')
if result.cv_method in [
'bootstrap_rdm', 'bootstrap_pattern', 'bootstrap_crossval'
]:
model_comp_descr = model_comp_descr + \
'\nInference by bootstrap resampling ' + \
'({:<,.0f}'.format(n_bootstraps) + ' bootstrap samples) of '
if result.cv_method == 'bootstrap_rdm':
model_comp_descr = model_comp_descr + 'subjects. '
elif result.cv_method == 'bootstrap_pattern':
model_comp_descr = model_comp_descr + 'experimental conditions. '
elif result.cv_method in ['bootstrap', 'bootstrap_crossval']:
model_comp_descr = model_comp_descr + \
'subjects and experimental conditions. '
model_comp_descr = model_comp_descr + 'Error bars indicate the'
if error_bars[0:2].lower() == 'ci':
model_comp_descr = (model_comp_descr + ' ' + str(CI_percent) +
'% confidence interval.')
elif error_bars.lower() == 'sem':
model_comp_descr = (model_comp_descr +
' standard error of the mean.')
if test_above_0 or test_below_noise_ceil:
model_comp_descr = (
model_comp_descr +
'\nOne-sided comparisons of each model performance ')
if test_above_0:
model_comp_descr = model_comp_descr + 'against 0 '
if test_above_0 and test_below_noise_ceil:
model_comp_descr = model_comp_descr + 'and '
if test_below_noise_ceil:
model_comp_descr = (
model_comp_descr +
'against the lower-bound estimate of the noise ceiling ')
if test_above_0 or test_below_noise_ceil:
model_comp_descr = (model_comp_descr +
'are Bonferroni-corrected for ' +
str(n_models) + ' models.')
fig.suptitle(model_comp_descr, fontsize=fs2 / 2)
axbar.set_xlim(ax.get_xlim())
digits = [d for d in list(test_pair_comparisons) if d.isdigit()]
if len(digits) > 0:
v = int(digits[0])
else:
v = None
if 'nili' in test_pair_comparisons.lower():
if v:
plot_nili_bars(axbar, significant, version=v)
else:
plot_nili_bars(axbar, significant)
elif 'golan' in test_pair_comparisons.lower():
if v:
plot_golan_wings(axbar,
significant,
perf,
sort,
colors,
version=v)
else:
plot_golan_wings(axbar, significant, perf, sort, colors)
elif 'arrows' in test_pair_comparisons.lower():
plot_arrows(axbar, significant)
elif 'cliques' in test_pair_comparisons.lower():
plot_cliques(axbar, significant)
# Floating axes
ytoptick = np.floor(min(1, noise_upper) * 10) / 10
ax.set_yticks(np.arange(0, ytoptick + 1e-6, step=0.1))
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.set_xticks(np.arange(n_models))
ax.spines['left'].set_bounds(0, ytoptick)
ax.spines['bottom'].set_bounds(0, n_models - 1)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
plt.rc('ytick', labelsize=fs2)
# Axis labels
ylabel_fig_x, ysublabel_fig_x = 0.07, 0.095
trans = transforms.blended_transform_factory(fig.transFigure,
ax.get_yaxis_transform())
ax.text(ylabel_fig_x,
ytoptick / 2,
'RDM prediction accuracy',
horizontalalignment='center',
verticalalignment='center',
rotation='vertical',
fontsize=fs,
fontweight='bold',
transform=trans)
if method.lower() == 'cosine':
ax.set_ylabel('[across-subject mean of cosine similarity]',
fontsize=fs2)
if method.lower() in ['cosine_cov', 'whitened cosine']:
ax.set_ylabel('[across-subject mean of whitened-RDM cosine]',
fontsize=fs2)
elif method.lower() == 'spearman':
ax.set_ylabel('[across-subject mean of Spearman r rank correlation]',
fontsize=fs2)
elif method.lower() in ['corr', 'pearson']:
ax.text(ysublabel_fig_x,
ytoptick / 2,
'[across-subject mean of Pearson r correlation]',
horizontalalignment='center',
verticalalignment='center',
rotation='vertical',
fontsize=fs2,
fontweight='normal',
transform=trans)
# ax.set_ylabel('[across-subject mean of Pearson r correlation]',
# fontsize=fs2)
elif method.lower() in ['whitened pearson', 'corr_cov']:
ax.set_ylabel('[across-subject mean of whitened-RDM Pearson r ' +
'correlation]',
fontsize=fs2)
elif method.lower() in ['kendall', 'tau-b']:
ax.set_ylabel('[across-subject mean of Kendall tau-b rank ' +
'correlation]',
fontsize=fs2)
elif method.lower() == 'tau-a':
ax.set_ylabel('[across-subject mean of ' +
'Kendall tau-a rank correlation]',
fontsize=fs2)
if models is not None:
ax.set_xticklabels([m.name for m in models], fontsize=fs2, rotation=45)
def _wedge_fix(wedge_path):
# Fixes the problem with Path.wedge where it doesn't initialise the first,
# and last two vertices.
# This fix should not have any side-effects once Path.wedge has been fixed,
# but will then be redundant and should be removed.
wedge_path.vertices[0] = 0
wedge_path.vertices[-2:] = 0
return wedge_path
CLOUD_COVER = {
0: [_ring_path()],
1: [_ring_path(), _vertical_bar_path()],
2: [_ring_path(), _wedge_fix(Path.wedge(0, 90))],
3: [_ring_path(), _wedge_fix(Path.wedge(0, 90)), _vertical_bar_path()],
4: [_ring_path(), Path.unit_circle_righthalf()],
5: [_ring_path(), Path.unit_circle_righthalf(), _left_bar_path()],
6: [_ring_path(), _wedge_fix(Path.wedge(-180, 90))],
7: [_slot_path()],
8: [Path.unit_circle()],
9: [_ring_path(), _slash_path(), _backslash_path()],
}
"""
A dictionary mapping WMO cloud cover codes to their corresponding symbol.
See http://www.wmo.int/pages/prog/www/DPFS/documents/485_Vol_I_en_colour.pdf
Part II, Appendix II.4, Graphical Representation of Data, Analyses and Forecasts
"""
