def test_greater_is_better_distinction(self):
"""
Test for method greater_is_better_distinction.
Should return Boolean or raise NotImplementedError.
"""
for implemented_metric in self.all_implemented_metrics:
self.assertIn(Scorer.greater_is_better_distinction(implemented_metric), [True, False])
for not_implemented_metric in self.some_not_implemented_metrics:
with self.assertRaises(NameError):
Scorer.greater_is_better_distinction(not_implemented_metric)
def get_minimum_config_evaluations(self):
config_evaluations = self.get_config_evaluations()
minimum_config_evaluations = dict()
for metric, evaluations in config_evaluations.items():
minimum_config_evaluations[metric] = list()
greater_is_better = Scorer.greater_is_better_distinction(metric)
for fold in evaluations:
fold_evaluations = list()
if greater_is_better:
for i, config in enumerate(fold):
if i == 0:
last_config = config
else:
if config > last_config:
last_config = config
fold_evaluations.append(last_config)
else:
last_config = np.inf
for i, config in enumerate(fold):
if i == 0:
last_config = config
else:
if config < last_config:
last_config = config
fold_evaluations.append(last_config)
minimum_config_evaluations[metric].append(fold_evaluations)
return minimum_config_evaluations
def metric(self, value):
"""
Setter for attribute metric.
:param value: metric value
:return:
"""
try:
self._metric = value
self._greater_is_better = Scorer.greater_is_better_distinction(
self._metric)
except NameError:
self._metric = "unknown"
logger.warning(
"Your metric is not supported. Performance constraints are constantly False."
)
def plot_optimizer_history(self,
metric,
title: str = 'Optimizer History',
type: str = 'plot',
reduce_scatter_by: Union[int, str] = 'auto',
file: str = None):
"""
:param metric: specify metric that has been stored within the PHOTON results tree
:param type: 'plot' or 'scatter'
:param reduce_scatter_by: integer or string ('auto'), reduce the number of points plotted by scatter
:param file: specify a filename if you want to save the plot
:return:
"""
if metric not in self.results.hyperpipe_info.metrics:
raise ValueError(
'Metric "{}" not stored in results tree'.format(metric))
config_evaluations = self.get_config_evaluations()
minimum_config_evaluations = self.get_minimum_config_evaluations()
# handle different lengths
min_corresponding = len(min(config_evaluations[metric], key=len))
config_evaluations_corres = [
configs[:min_corresponding]
for configs in config_evaluations[metric]
]
minimum_config_evaluations_corres = [
configs[:min_corresponding]
for configs in minimum_config_evaluations[metric]
]
mean = np.nanmean(np.asarray(config_evaluations_corres), axis=0)
mean_min = np.nanmean(np.asarray(minimum_config_evaluations_corres),
axis=0)
greater_is_better = Scorer.greater_is_better_distinction(metric)
if greater_is_better:
caption = 'Maximum'
else:
caption = 'Minimum'
plt.figure()
if type == 'plot':
plt.plot(np.arange(0, len(mean)),
mean,
'-',
color='gray',
label='Mean Performance')
elif type == 'scatter':
# now do smoothing
if isinstance(reduce_scatter_by, str):
if reduce_scatter_by != 'auto':
logger.warning(
'{} is not a valid smoothing_kernel specifier. Falling back to "auto".'
.format(reduce_scatter_by))
# if auto, then calculate size of reduce_scatter_by so that 75 points on x remain
# smallest reduce_scatter_by should be 1
reduce_scatter_by = max(
[np.floor(min_corresponding / 75).astype(int), 1])
if reduce_scatter_by > 1:
plt.plot([], [],
' ',
label="scatter reduced by factor {}".format(
reduce_scatter_by))
for i, fold in enumerate(config_evaluations[metric]):
# add a few None so that list can be divided by smoothing_kernel
remaining = len(fold) % reduce_scatter_by
if remaining:
fold.extend([np.nan] * (reduce_scatter_by - remaining))
# calculate mean over every n named_steps so that plot is less cluttered
reduced_fold = np.nanmean(np.asarray(fold).reshape(
-1, reduce_scatter_by),
axis=1)
reduced_xfit = np.arange(reduce_scatter_by / 2,
len(fold),
step=reduce_scatter_by)
if i == len(config_evaluations[metric]) - 1:
plt.scatter(reduced_xfit,
np.asarray(reduced_fold),
color='gray',
alpha=0.5,
label='Performance',
marker='.')
else:
plt.scatter(reduced_xfit,
np.asarray(reduced_fold),
color='gray',
alpha=0.5,
marker='.')
else:
raise ValueError('Please specify either "plot" or "scatter".')
plt.plot(np.arange(0, len(mean_min)),
mean_min,
'-',
color='black',
label='Mean {} Performance'.format(caption))
for i, fold in enumerate(minimum_config_evaluations[metric]):
xfit = np.arange(0, len(fold))
plt.plot(xfit, fold, '-', color='black', alpha=0.5)
plt.ylabel(metric.replace('_', ' '))
plt.xlabel('No of Evaluations')
plt.legend()
plt.title(title)
if file:
plt.savefig(file)
else:
if self.output_settings:
file = os.path.join(self.output_settings.results_folder,
"optimizer_history.png")
plt.savefig(file)
plt.close()
def test_greater_is_better_distinction_ZERO():
with pytest.raises(PhotonaiError):
assert Scorer.greater_is_better_distinction("SC") == True
def test_greater_is_better_distinction_POS():
assert Scorer.greater_is_better_distinction("FM") == True
def test_greater_is_better_distinction_NEG():
assert Scorer.greater_is_better_distinction("hamming_loss") == False
def plotly_optimizer_history(name, config_evaluations,
minimum_config_evaluations, metric):
# handle different lengths
min_corresponding = len(min(config_evaluations[metric], key=len))
config_evaluations_corres = [
configs[:min_corresponding] for configs in config_evaluations[metric]
]
minimum_config_evaluations_corres = [
configs[:min_corresponding]
for configs in minimum_config_evaluations[metric]
]
mean = np.nanmean(np.asarray(config_evaluations_corres), axis=0)
mean_min = np.nanmean(np.asarray(minimum_config_evaluations_corres),
axis=0)
greater_is_better = Scorer.greater_is_better_distinction(metric)
if greater_is_better:
caption = "Maximum"
else:
caption = "Minimum"
# now do smoothing
reduce_scatter_by = max([np.floor(min_corresponding / 75).astype(int), 1])
traces = list()
for i, fold in enumerate(config_evaluations[metric]):
trace = PlotlyTrace(
"Fold_{}".format(i + 1),
trace_type="scatter",
trace_size=6,
trace_color="rgba(42, 54, 62, 0.5)",
)
# add a few None so that list can be divided by smoothing_kernel
remaining = len(fold) % reduce_scatter_by
if remaining:
fold.extend([np.nan] * (reduce_scatter_by - remaining))
# calculate mean over every n elements so that plot is less cluttered
reduced_fold = np.nanmean(np.asarray(fold).reshape(
-1, reduce_scatter_by),
axis=1)
reduced_xfit = np.arange(max(1, reduce_scatter_by / 2),
len(fold) + 1,
step=reduce_scatter_by)
trace.x = reduced_xfit
trace.y = np.asarray(reduced_fold)
traces.append(trace)
trace = PlotlyTrace(
"Mean_{}_Performance".format(caption),
trace_type="scatter",
mode="lines",
trace_size=8,
trace_color="rgb(214, 123, 25)",
)
trace.x = np.arange(1, len(mean_min) + 1)
trace.y = mean_min
traces.append(trace)
for i, fold in enumerate(minimum_config_evaluations[metric]):
trace = PlotlyTrace(
"Fold_{}_{}_Performance".format(i + 1, caption),
trace_type="scatter",
mode="lines",
trace_size=8,
trace_color="rgba(214, 123, 25, 0.5)",
)
xfit = np.arange(1, len(fold) + 1)
trace.x = xfit
trace.y = fold
traces.append(trace)
plot = PlotlyPlot(
plot_name=name,
title="Optimizer History",
traces=traces,
xlabel="No of Evaluations",
ylabel=metric.replace("_", " "),
show_legend=False,
)
return plot.to_plot()