def test_add_raw_value_stat(self):
sb = StatBook('test_sb')
func = lambda x: x + 1
sb.add_raw_value_stat('test_ms', func)
self.assertIn('test_ms', sb.multi_stats)
self.assertIn(MultiStat.RAW_VALUE_STAT_NAME, sb.multi_stats['test_ms'].stats)
self.assertEqual(sb.multi_stats['test_ms'].stats.get(MultiStat.RAW_VALUE_STAT_NAME).function, func)
def test_multi_stats(self):
sb = StatBook('test_sb')
ms1 = MultiStat(name='test_stat1')
ms2 = MultiStat(name='test_stat2')
sb.add_multi_stat(ms1)
sb.add_multi_stat(ms2)
result = sb.multi_stats_list()
self.assertListEqual(result, [ms1, ms2])
def test_running_std(self):
sb = StatBook('test_sb')
ms = MultiStat(name='test_stat')
ms.running_std = MagicMock()
ms.running_std.return_value = [3., 4., 5.]
sb.add_multi_stat(ms)
result = sb.running_std(ms.name)
self.assertListEqual(result, [3., 4., 5.])
ms.running_std.assert_called_once()
def test_maximum(self):
sb = StatBook('test_sb')
ms = MultiStat(name='test_stat')
ms.maximum = MagicMock()
ms.maximum.return_value = [3., 4., 5.]
sb.add_multi_stat(ms)
result = sb.maximum(ms.name)
self.assertListEqual(result, [3., 4., 5.])
ms.maximum.assert_called_once()
def update_stat_book(stat_book: StatBook, gp_models: List[GPModel], x_train,
base_kernel_names: List[str], n_dims: int) -> None:
"""Update model population statistics.
:param stat_book:
:param gp_models:
:return:
"""
stat_book.update_stat_book(data=gp_models,
x=x_train,
base_kernels=base_kernel_names,
n_dims=n_dims)
def plot_cov_dist_summary(cov_dists_name: str, stat_book: StatBook,
x_label: str) -> None:
"""Plot a summary of the homogeneity of models over each generation.
:return:
"""
mean_cov_dists = stat_book.mean(cov_dists_name)
std_cov_dists = stat_book.std(cov_dists_name)
ax = plot_distribution(mean_cov_dists,
std_cov_dists,
metric_name='covariance distance',
x_label=x_label)
fig = ax.figure
fig.suptitle(f'{stat_book.label}', y=1)
fig.subplots_adjust(top=0.88)
plt.show()
def plot_kernel_diversity_summary(diversity_scores_name: str,
stat_book: StatBook, x_label: str) -> None:
"""Plot a summary of the diversity of models over each generation.
:return:
"""
mean_diversity_scores = stat_book.running_mean(diversity_scores_name)
std_diversity_scores = stat_book.running_std(diversity_scores_name)
ax = plot_distribution(mean_diversity_scores,
std_diversity_scores,
metric_name='diversity',
value_name='population',
x_label=x_label)
fig = ax.figure
fig.suptitle(f'{stat_book.label}', y=1)
fig.subplots_adjust(top=0.88)
plt.show()
def test_add_stat_book(self):
sbc = StatBookCollection()
sbc.create_shared_stat_books(self.sb_names, self.ms_names)
sb = StatBook('test_sb')
sbc.add_stat_book(sb)
self.assertIn(sb.name, sbc.stat_book_names())
self.assertIn(sb, sbc.stat_book_list())
def plot_best_scores(score_name: str, evaluations_name: str,
stat_book: StatBook) -> None:
"""Plot the best models scores
:return:
"""
if stat_book.name == evaluations_name:
best_scores = stat_book.running_max(score_name)
x_label = 'evaluations'
else:
best_scores = stat_book.maximum(score_name)
x_label = 'generation'
ax = plot_best_so_far(best_scores, x_label=x_label)
fig = ax.figure
fig.suptitle(f'{stat_book.label}', y=1)
fig.subplots_adjust(top=0.88)
plt.show()
def plot_score_summary(score_name: str, evaluations_name: str,
stat_book: StatBook) -> None:
"""Plot a summary of model scores
:return:
"""
if stat_book.name == evaluations_name:
best_scores = stat_book.running_max(score_name)
mean_scores = stat_book.running_mean(score_name)
std_scores = stat_book.running_std(score_name)
x_label = 'evaluations'
else:
best_scores = stat_book.maximum(score_name)
mean_scores = stat_book.mean(score_name)
std_scores = stat_book.std(score_name)
x_label = 'generation'
ax = plot_distribution(mean_scores,
std_scores,
best_scores,
x_label=x_label)
fig = ax.figure
fig.suptitle(f'{stat_book.label}', y=1)
fig.subplots_adjust(top=0.88)
plt.show()
def test_to_dict(self):
sb = StatBook('test_sb')
ms = MultiStat(name='test_ms')
sb.add_multi_stat(ms)
output_dict = sb.to_dict()
self.assertEqual(sb.name, output_dict["name"])
self.assertEqual([ms.to_dict() for ms in sb.multi_stats_list()], output_dict["multi_stats"])
def plot_n_operands_summary(n_operands_name: str, best_stat_name: str,
stat_book: StatBook, x_label: str) -> None:
"""Plot a summary of the number of operands
:return:
"""
if best_stat_name in stat_book.multi_stats[n_operands_name].stats:
best_n_operands = stat_book.multi_stats[n_operands_name].stats[
best_stat_name].data
else:
best_n_operands = None
median_n_operands = stat_book.median(n_operands_name)
std_n_operands = stat_book.std(n_operands_name)
ax = plot_distribution(median_n_operands,
std_n_operands,
best_n_operands,
value_name='median',
metric_name='# Operands',
x_label=x_label)
fig = ax.figure
fig.suptitle(f'{stat_book.label}', y=1)
fig.subplots_adjust(top=0.88)
plt.show()
def test_update_stat_book(self):
sb = StatBook('test_sb')
sb.add_raw_value_stat('test_stat1')
sb.add_raw_value_stat('test_stat2')
sb.update_stat_book(data=[1, 2, 3])
result = sb.multi_stats['test_stat1'].get_raw_values()
self.assertListEqual(result, [[1, 2, 3]])
result = sb.multi_stats['test_stat2'].get_raw_values()
self.assertListEqual(result, [[1, 2, 3]])
def test_clear_all_values(self):
sb = StatBook('test_sb')
ms1 = MultiStat(name='test_stat1')
ms1.clear_all_values = MagicMock()
ms2 = MultiStat(name='test_stat2')
ms2.clear_all_values = MagicMock()
sb.add_multi_stat(ms1)
sb.add_multi_stat(ms2)
sb.clear_all_values()
ms1.clear_all_values.assert_called_once()
ms2.clear_all_values.assert_called_once()
def plot_base_kernel_freqs(base_kern_freq_names: list, stat_book: StatBook,
x_label: str) -> None:
"""Plot base kernel frequency across generations.
:param base_kern_freq_names:
:param stat_book:
:param x_label:
:return:
"""
freqs = [(stat_book.sum(key), key) for key in base_kern_freq_names]
plt.title('Base Kernel Frequency')
plt.xlabel(x_label)
plt.ylabel('Frequency')
ax = plt.gca()
ax.xaxis.set_major_locator(
MaxNLocator(integer=True)) # x-axis will have integer ticks
for freq, key in freqs:
plt.plot(freq, label=key, marker='o', markerfacecolor='black')
plt.legend()
plt.gcf().suptitle(f'{stat_book.label}', y=1)
plt.gcf().subplots_adjust(top=0.88)
plt.show()
def test_add_multistat(self):
sb = StatBook('test_sb')
ms = MultiStat(name='test_stat')
sb.add_multi_stat(ms)
self.assertIn(ms.name, sb.multi_stats)
self.assertEqual(sb.multi_stats.get(ms.name), ms)