def analyze(results, out_path, name="analysis", plot=False):
"""Analyze results.
Given a results dataframe, generates a folder with
results and a dataframe of the exact data used to generate those results,
aswell as introspection to return information on the computer
parameters
----------
out_path: location to store analysis folder
results: Dataframe generated from Results object
path: string/None
plot: whether to plot results
Either path or results is necessary
"""
# input checks #
if not isinstance(out_path, str):
raise ValueError("Given out_path argument is not string")
elif not os.path.isdir(out_path):
raise IOError("Given directory does not exist")
else:
analysis_path = os.path.join(out_path, name)
unique_ids = [plt._simplify_names(x) for x in results.pipeline.unique()]
simplify = True
print(unique_ids)
print(set(unique_ids))
if len(unique_ids) != len(set(unique_ids)):
log.warning(
"Pipeline names are too similar, turning off name shortening")
simplify = False
os.makedirs(analysis_path, exist_ok=True)
# TODO: no good cross-platform way of recording CPU info?
with open(os.path.join(analysis_path, "info.txt"), "a") as f:
dt = datetime.now()
f.write("Date: {:%Y-%m-%d}\n Time: {:%H:%M}\n".format(dt, dt))
f.write("System: {}\n".format(platform.system()))
f.write("CPU: {}\n".format(platform.processor()))
results.to_csv(os.path.join(analysis_path, "data.csv"))
stats = compute_dataset_statistics(results)
stats.to_csv(os.path.join(analysis_path, "stats.csv"))
P, T = find_significant_differences(stats)
if plot:
fig, color_dict = plt.score_plot(results)
fig.savefig(os.path.join(analysis_path, "scores.pdf"))
fig = plt.summary_plot(P, T, simplify=simplify)
fig.savefig(os.path.join(analysis_path, "ordering.pdf"))
def analyze(results, out_path, name='analysis', plot=False):
'''Analyze results.
Given a results dataframe, generates a folder with
results and a dataframe of the exact data used to generate those results,
aswell as introspection to return information on the computer
parameters
----------
out_path: location to store analysis folder
results: Dataframe generated from Results object
path: string/None
plot: whether to plot results
Either path or results is necessary
'''
# input checks #
if type(out_path) is not str:
raise ValueError('Given out_path argument is not string')
elif not os.path.isdir(out_path):
raise IOError('Given directory does not exist')
else:
analysis_path = os.path.join(out_path, name)
os.makedirs(analysis_path, exist_ok=True)
# TODO: no good cross-platform way of recording CPU info?
with open(os.path.join(analysis_path, 'info.txt'), 'a') as f:
dt = datetime.now()
f.write('Date: {:%Y-%m-%d}\n Time: {:%H:%M}\n'.format(dt, dt))
f.write('System: {}\n'.format(platform.system()))
f.write('CPU: {}\n'.format(platform.processor()))
results.to_csv(os.path.join(analysis_path, 'data.csv'))
stats = compute_dataset_statistics(results)
stats.to_csv(os.path.join(analysis_path, 'stats.csv'))
P, T = find_significant_differences(stats)
if plot:
fig, color_dict = plt.score_plot(results)
fig.savefig(os.path.join(analysis_path, 'scores.pdf'))
fig = plt.summary_plot(P, T)
fig.savefig(os.path.join(analysis_path, 'ordering.pdf'))
fig = moabb_plt.paired_plot(results, "CSP+LDA", "RG+LDA") plt.show() ############################################################################### # Statistical testing and further plots # ---------------------------------------- # # If the statistical significance of results is of interest, the method # compute_dataset_statistics allows one to show a meta-analysis style plot as # well. For an overview of how all algorithms perform in comparison with each # other, the method find_significant_differences and the summary_plot are # possible. stats = compute_dataset_statistics(results) P, T = find_significant_differences(stats) ############################################################################### # The meta-analysis style plot shows the standardized mean difference within # each tested dataset for the two algorithms in question, in addition to a # meta-effect and significances both per-dataset and overall. fig = moabb_plt.meta_analysis_plot(stats, "CSP+LDA", "RG+LDA") plt.show() ############################################################################### # The summary plot shows the effect and significance related to the hypothesis # that the algorithm on the y-axis significantly out-performed the algorithm on # the x-axis over all datasets moabb_plt.summary_plot(P, T) plt.show()
