def _plot_selection_probabilities(self):
r"""
Plot pie charts of the selection probabilities of prospective parents at each generation.
Models are signified by their F score.
"""
generations = sorted(
self.genetic_algorithm.gene_pool.generation.unique())
self.log_print(
["[_plot_selection_probabilities] generations:", generations])
lf = LatexFigure(auto_gridspec=len(generations))
for g in generations:
ax = lf.new_axis()
this_gen_genes = self.genetic_algorithm.gene_pool[
self.genetic_algorithm.gene_pool.generation == g]
f_scores = this_gen_genes.f_score
colours = [self.f_score_cmap(f) for f in f_scores]
probabilities = this_gen_genes.probability
ax.pie(
probabilities,
colors=colours,
radius=2,
)
save_to_file = os.path.join(self.save_directory,
"selection_probabilities")
lf.save(save_to_file, figure_format=self.figure_format)
def plot_heuristic_attributes(self, save_to_file, **kwargs):
"""
Summarise the heuristic used for the model training through several plots.
volume of distribution at each experiment
time designed by heuristic for each experiment
effecitve sample size at each experiment, used to determine when to resample
:param save_to_file: path to which the summary figure is stored
:type save_to_file: path
"""
plots_to_include = ["volume", "times_used", "effective_sample_size"]
plt.clf()
nrows = len(plots_to_include)
lf = LatexFigure(gridspec_layout=(nrows, 1))
if "volume" in plots_to_include:
ax = lf.new_axis()
self._plot_volumes(ax=ax)
ax.legend()
if "times_used" in plots_to_include:
ax = lf.new_axis()
self._plot_suggested_times(ax=ax)
ax.legend()
if "effective_sample_size" in plots_to_include:
ax = lf.new_axis()
self._plot_effective_sample_size(ax=ax)
ax.legend()
# Save figure
self.log_print(["LatexFigure has size:", lf.size])
lf.save(save_to_file, file_format=self._figure_format)
def __plot_gene_pool_progression(self, ):
r"""
Succinct representation of the progression of gene pool with respect to F score.
"""
lf = LatexFigure()
ax = lf.new_axis()
gene_pool = self.genetic_algorithm.gene_pool
gene_pool.sort_values('f_score', inplace=True, ascending=False)
self.gene_pool_progression(
gene_pool=gene_pool,
ax=ax,
f_score_cmap=self.f_score_cmap,
)
lf.save(save_to_file=os.path.join(self.save_directory,
'gene_pool_progression'),
file_format=self.figure_format)
def _plot_model_ratings(self, ):
r"""
Plot ratings of models on all generations, as determined by the RatingSystem
"""
plt.clf()
ratings = self.ratings_class.all_ratings
generations = [int(g) for g in ratings.generation.unique()]
num_generations = len(generations)
lf = LatexFigure(use_gridspec=True,
gridspec_layout=(num_generations, 1))
# TODO : unique linestyle and colour combo for each model ID and tracks across subplots
ratings["Model ID"] = ratings["model_id"]
for gen in generations:
ax = lf.new_axis()
this_gen_ratings = ratings[ratings.generation == gen]
colours = {
m: self.f_score_cmap(self.model_f_scores[m])
for m in this_gen_ratings["model_id"]
}
sns.lineplot(
x="idx",
y="rating",
hue=r"Model ID",
hue_order=sorted(this_gen_ratings.model_id.unique()),
data=this_gen_ratings,
ax=ax,
legend="full",
palette=colours,
)
ax.set_title("Generation {}".format(gen), pad=-15)
ax.set_xlabel("")
ax.set_ylabel("Elo rating")
ax.legend(bbox_to_anchor=(1, 1))
save_to_file = os.path.join(self.save_directory,
"ratings".format(self.qmla_id))
lf.save(save_to_file, file_format=self.figure_format)
def plot_heuristic_attributes(self, save_to_file, **kwargs):
"""Plot results related to the experiment design heuristic.
Plots
times generated by the heuristic;
metric distances: distance between the two sampled particles under various measures;
volume of the parameter distribution at each experiment.
:param save_to_file: path to which to save the resultant figure.
:type save_to_file: path
"""
plots_to_include = [
"volume",
"metric_distances",
"times_used", #'derivatives',
]
plt.clf()
nrows = len(plots_to_include)
lf = LatexFigure(gridspec_layout=(nrows, 1))
# Volume
if "volume" in plots_to_include:
ax = lf.new_axis()
self._plot_volumes(ax=ax)
self.add_time_factor_change_points_to_ax(ax=ax)
ax.legend()
if "times_used" in plots_to_include:
ax = lf.new_axis()
self._plot_suggested_times(ax=ax)
self.add_time_factor_change_points_to_ax(ax=ax)
ax.legend()
if "derivatives" in plots_to_include:
# Volume Derivatives
ax = lf.new_axis()
## first derivatives
derivs = self.derivatives[1]
epochs = sorted(derivs.keys())
first_derivatives = [
derivs[e] if e in derivs else None for e in epochs
]
ax.plot(
epochs,
first_derivatives,
label=r"$\frac{dV}{dE}$",
color="darkblue",
marker="x",
)
## second derivatives
derivs = self.derivatives[2]
epochs = sorted(derivs.keys())
second_derivatives = [
derivs[e] if e in derivs else None for e in epochs
]
ax.plot(
epochs,
second_derivatives,
label=r"$\frac{d^2V}{dE^2}$",
color="maroon",
marker="+",
)
ax.axhline(0, ls="--", alpha=0.3)
ax.legend(loc="upper right")
ax.set_yscale("symlog")
ax.set_ylabel("Derivatives", fontsize=self._label_fontsize)
ax.set_xlabel("Epoch", fontsize=self._label_fontsize)
if len(self.time_factor_changes["decreasing"]) > 0:
ax.axvline(
self.time_factor_changes["decreasing"][0],
ls="--",
color="red",
label="decrease k",
alpha=0.5,
)
for e in self.time_factor_changes["decreasing"][1:]:
ax.axvline(e, ls="--", color="red", alpha=0.5)
if len(self.time_factor_changes["increasing"]) > 0:
ax.axvline(
self.time_factor_changes["increasing"][0],
ls="--",
color="green",
label="increase k",
alpha=0.5,
)
for e in self.time_factor_changes["increasing"][1:]:
ax.axvline(e, ls="--", color="green", alpha=0.5)
ax.legend(loc="lower right")
if "metric_distances" in plots_to_include:
# Times by distance metrics
ax = lf.new_axis()
linestyles = itertools.cycle(["--", ":", "-."])
for method in self.designed_times:
times_of_method = self.designed_times[method]
epochs = sorted(times_of_method.keys())
time_by_epoch = [times_of_method[e] for e in epochs]
lb = method
if self.distance_metric_to_use == method:
ls = "-"
lb += " (used)"
alpha = 1
else:
ls = next(linestyles)
alpha = 0.75
ax.plot(epochs, time_by_epoch, label=lb, ls=ls, alpha=alpha)
ax.legend(title="Distance metric")
ax.set_title("Raw time chosen by distance metrics",
fontsize=self._label_fontsize)
ax.set_ylabel("Time", fontsize=self._label_fontsize)
ax.set_xlabel("Epoch", fontsize=self._label_fontsize)
ax.semilogy()
# Save figure
self.log_print(["LatexFigure has size:", lf.size])
lf.save(save_to_file, file_format=self._figure_format)
def plot_heuristic_attributes(self, save_to_file, **kwargs):
"""Plot results related to the experiment design heuristic.
Plots
times generated by the heuristic;
metric distances: distance between the two sampled particles under various measures;
volume of the parameter distribution at each experiment.
:param save_to_file: path to which to save the resultant figure.
:type save_to_file: path
"""
plots_to_include = [
'volume',
'metric_distances',
'times_used', #'derivatives',
]
plt.clf()
nrows = len(plots_to_include)
lf = LatexFigure(gridspec_layout=(nrows, 1))
# Volume
if 'volume' in plots_to_include:
ax = lf.new_axis()
self._plot_volumes(ax=ax)
self.add_time_factor_change_points_to_ax(ax=ax)
ax.legend()
if 'times_used' in plots_to_include:
ax = lf.new_axis()
self._plot_suggested_times(ax=ax)
self.add_time_factor_change_points_to_ax(ax=ax)
ax.legend()
if 'derivatives' in plots_to_include:
# Volume Derivatives
ax = lf.new_axis()
## first derivatives
derivs = self.derivatives[1]
epochs = sorted(derivs.keys())
first_derivatives = [
derivs[e] if e in derivs else None for e in epochs
]
ax.plot(epochs,
first_derivatives,
label=r"$\frac{dV}{dE}$",
color='darkblue',
marker='x')
## second derivatives
derivs = self.derivatives[2]
epochs = sorted(derivs.keys())
second_derivatives = [
derivs[e] if e in derivs else None for e in epochs
]
ax.plot(epochs,
second_derivatives,
label=r"$\frac{d^2V}{dE^2}$",
color='maroon',
marker='+')
ax.axhline(0, ls='--', alpha=0.3)
ax.legend(loc='upper right')
ax.set_yscale('symlog')
ax.set_ylabel('Derivatives', fontsize=self._label_fontsize)
ax.set_xlabel('Epoch', fontsize=self._label_fontsize)
if len(self.time_factor_changes['decreasing']) > 0:
ax.axvline(self.time_factor_changes['decreasing'][0],
ls='--',
color='red',
label='decrease k',
alpha=0.5)
for e in self.time_factor_changes['decreasing'][1:]:
ax.axvline(e, ls='--', color='red', alpha=0.5)
if len(self.time_factor_changes['increasing']) > 0:
ax.axvline(self.time_factor_changes['increasing'][0],
ls='--',
color='green',
label='increase k',
alpha=0.5)
for e in self.time_factor_changes['increasing'][1:]:
ax.axvline(e, ls='--', color='green', alpha=0.5)
ax.legend(loc='lower right')
if 'metric_distances' in plots_to_include:
# Times by distance metrics
ax = lf.new_axis()
linestyles = itertools.cycle(['--', ':', '-.'])
for method in self.designed_times:
times_of_method = self.designed_times[method]
epochs = sorted(times_of_method.keys())
time_by_epoch = [times_of_method[e] for e in epochs]
lb = method
if self.distance_metric_to_use == method:
ls = '-'
lb += ' (used)'
alpha = 1
else:
ls = next(linestyles)
alpha = 0.75
ax.plot(epochs, time_by_epoch, label=lb, ls=ls, alpha=alpha)
ax.legend(title='Distance metric')
ax.set_title('Raw time chosen by distance metrics',
fontsize=self._label_fontsize)
ax.set_ylabel('Time', fontsize=self._label_fontsize)
ax.set_xlabel('Epoch', fontsize=self._label_fontsize)
ax.semilogy()
# Save figure
self.log_print(["LatexFigure has size:", lf.size])
lf.save(save_to_file, file_format=self._figure_format)
def plot_dynamics_from_models(models,
exp_msmts,
bf_times,
bayes_factor,
save_directory,
figure_format="png"):
"""Plot the dynamics of the pair of models considered in a Bayes factor comparison.
:param models: list of 2 models which were compared during this calculation, [model_a, model_b].
:type models: :class:`~qmla.ModelInstanceForLearning`
:param exp_msmts: times and expectation values for the system.
:type exp_msmts: dict
:param bf_times: Times used for the BF calculation
:type bf_times: list
:param bayes_factor: Bayes factor between the two input models, to be read as BF(model_a, model_b)
:type bayes_factor: float
:param save_directory: path where the generated figure is to be saved
:type save_directory: path
"""
times = list(sorted(exp_msmts.keys()))
lf = LatexFigure(fraction=0.45, auto_label=False)
ax1 = lf.new_axis()
lines = []
for model in models:
l = model.plot_dynamics(ax=ax1, times=times)
lines.extend(l)
ax1.set_xlim((min(times), max(times)))
# Plot system measurements
l = ax1.scatter(times, [exp_msmts[t] for t in times],
label=r"$Q$",
color='red',
alpha=0.6,
s=5)
lines.append(l)
# Overlay times
try:
# in background, show how often that time was considered
ax2 = ax1.twinx()
num_times = int(len(times)) - 1
l = ax2.hist(
bf_times,
bins=num_times,
# TODO put on separate plot to see when higher times compared on
range=(min(times), max(times)),
histtype='stepfilled',
fill=False,
label=r"$t$",
alpha=0.25)
ax2.set_ylabel('Frequency')
max_freq = max(l[0])
ax2.set_ylim(0, 1.6 * max_freq)
ax2.set_yticks([0, int(max_freq / 2), max_freq])
lines.append(l[-1][0])
except BaseException:
raise
# pass
bf = np.log10(bayes_factor)
labels = [l.get_label() for l in lines]
ax1.set_ylim(0, 1.6)
ax1.set_yticks([0, 0.5, 1])
ax1.set_ylabel('Expectation Value')
ax1.set_xlabel("Time")
ax1.legend(lines, labels, ncol=2, loc="upper center")
plot_path = os.path.join(
save_directory, 'BF_{}_{}'.format(str(models[0].model_id),
str(models[1].model_id)))
# plt.savefig(plot_path)
lf.save(plot_path, file_format=figure_format)
def plot_qmla_branches(q, show_fscore_cmap=False, return_graphs=False):
trees = list(q.trees.values())
q.log_print(["Plotting QMLA branch graphs. Trees:", trees])
plt.rcParams.update(
get_latex_rc_params(
font_scale=1.5,
)
)
for tree in trees:
# tree = trees[0] # TODO loop over trees
q.log_print(["Working on tree ", tree.exploration_strategy])
graphs = {}
branches = [tree.branches[b] for b in sorted(tree.branches.keys())]
if len(branches) > 1:
branches = branches[:-1]
num_branches = len(branches)
ncols = int(np.ceil(np.sqrt(num_branches)))
if num_branches == 1:
ncols = 1
nrows = 1
elif num_branches == 2:
ncols = 1
nrows = 2
else:
nrows = int(np.ceil(num_branches / ncols))
# Generate plot
widths = [1] * ncols
widths.append(0.1)
if show_fscore_cmap:
widths.append(0.1)
total_ncols = len(widths)
size_scaler = min(2, 4 / num_branches)
label_fontsize = 15 * size_scaler
lf = LatexFigure(
auto_label=False,
gridspec_layout=(nrows, total_ncols),
gridspec_params={
"width_ratios": widths,
"wspace": 0.3,
"hspace": 0.2,
},
)
# colour maps
f_score_cmap = q.exploration_class.f_score_cmap
bf_cmap = q.exploration_class.bf_cmap
min_bf = q.bayes_factors_df.log10_bayes_factor.min()
max_bf = q.bayes_factors_df.log10_bayes_factor.max()
norm = plt.Normalize(vmin=min_bf, vmax=max_bf)
bf_cmapper = plt.cm.ScalarMappable(norm=norm, cmap=bf_cmap)
bf_cmapper.set_array([]) # still no idea what this is for??
# BF cmap
cbar_ax = lf.new_axis(
force_position=(0, total_ncols - 1 - int(show_fscore_cmap)), span=("all", 1)
)
lf.fig.colorbar(bf_cmapper, cax=cbar_ax, ticks=[min_bf, 0, max_bf])
cbar_ax.set_title(r"$\\log_{10}(BF)$", loc="center")
cbar_ax.yaxis.set_ticks_position("left")
# F score cmap
if show_fscore_cmap:
ax = lf.new_axis(force_position=[0, total_ncols - 1], span=("all", 1))
sm = plt.cm.ScalarMappable(
cmap=f_score_cmap, norm=plt.Normalize(vmin=0, vmax=1)
)
sm.set_array([])
lf.fig.colorbar(
sm,
cax=ax,
orientation="vertical",
label=r"$F_1$-score",
ticks=[0, 0.5, 1],
)
# ax.set_ylabel(
# r"F-score",
# )
# ax.set_yticks([0, 0.5, 1])
# Plot graphs
min_seen_bf = 0
max_seen_bf = 0
for branch in branches:
ax = lf.new_axis()
models = branch.models.keys()
pairs = branch.pairs_to_compare
graph = nx.Graph()
for m in models:
graph.add_node(
m,
model_id=int(m),
model_name=branch.models[m],
f_score=q.model_f_scores[m],
colour=f_score_cmap(q.model_f_scores[m]),
longest_path_to_any_target=0,
)
for edge in pairs:
e = (min(edge), max(edge)) # edges go A->B low -> high
bf = q.all_bayes_factors[e[0]][e[1]][-1]
lbf = np.log10(bf)
colour = bf_cmapper.to_rgba(lbf)
graph.add_edge(e[0], e[1], log_bf=lbf, colour=colour)
pos = nx.kamada_kawai_layout(graph) # could use spring_layout or other
labels = nx.get_node_attributes(graph, "model_id")
node_colours = [graph.nodes[m]["colour"] for m in models]
edge_colours = [graph.edges[e]["colour"] for e in graph.edges]
nx.draw_networkx_nodes(
graph,
pos,
alpha=0.7,
node_color=node_colours,
node_size=400 * size_scaler,
ax=ax,
)
nx.draw_networkx_labels(graph, pos, labels, font_size=label_fontsize, ax=ax)
nx.draw_networkx_edges(
graph,
pos,
edgelist=graph.edges,
edge_color=edge_colours,
ax=ax,
alpha=1,
width=4,
)
# summarise this graph in a text box
node_edges = {n: len(graph.edges(n)) for n in graph.nodes}
lowest_connectivity = min(node_edges.values())
highest_connectivity = max(node_edges.values())
for n in graph:
for target in graph:
if n != target:
try:
shortest_path = nx.shortest_path_length(
graph, source=n, target=target
)
except:
# can't connect to some other node
shortest_path = -1
this_node_current_longest_path = graph.nodes[n][
"longest_path_to_any_target"
]
if shortest_path > this_node_current_longest_path:
graph.nodes[n]["longest_path_to_any_target"] = shortest_path
distance_between_nodes = [
graph.nodes[n]["longest_path_to_any_target"] for n in graph.nodes
]
lowest_distance_between_nodes = min(distance_between_nodes)
highest_distance_between_nodes = max(distance_between_nodes)
short_summary = "{}".format(
branch.branch_id,
)
summary = "Branch {} \n{} models \n {} edges\n ${} \leq C \leq{}$\n $d \leq {}$".format(
branch.branch_id,
len(graph.nodes),
len(graph.edges),
lowest_connectivity,
highest_connectivity,
highest_distance_between_nodes,
)
props = dict(boxstyle="round", facecolor="wheat", alpha=0.5)
ax.text(
-0.05,
0.95,
short_summary, # summary,
transform=ax.transAxes,
# fontsize = label_fontsize,
bbox=props,
ha="center",
va="center",
)
ax.axis("off")
graphs[branch.branch_id] = graph
# Save figure
save_file = "graphs_of_branches_{}".format(tree.exploration_strategy)
q.log_print(["Storing ", save_file])
lf.save(
os.path.join(q.qmla_controls.plots_directory, save_file),
file_format=q.qmla_controls.figure_format,
)
def plot_rating_progress_single_model_static(
ratings_df, target_model_id, return_df=False, return_lf=None, save_to_file=None
):
# First isolate the ratings for this model
model_identifiers = ["a", "b"]
ratings_of_single_model = pd.DataFrame(
columns=[
"target",
"opponent",
"initial_rating_target",
"initial_rating_opponent",
"f_score_target",
"f_score_opponent",
"delta_r_target",
"final_rating_target",
"idx",
"bayes_factor",
"generation",
"weight",
]
)
for target in model_identifiers:
opponent = list(set(model_identifiers) - set(target))[0]
target_ratings = ratings_df[
ratings_df["model_{}".format(target)] == target_model_id
][
[
"model_{}".format(target),
"model_{}".format(opponent),
"r_{}_initial".format(target),
"r_{}_initial".format(opponent),
"f_score_{}".format(target),
"f_score_{}".format(opponent),
"delta_r_{}".format(target),
"r_{}_new".format(target),
"idx",
"bayes_factor",
"generation",
"weight",
"winner",
]
]
target_ratings.rename(
columns={
"model_{}".format(target): "target",
"model_{}".format(opponent): "opponent",
"r_{}_initial".format(target): "initial_rating_target",
"r_{}_initial".format(opponent): "initial_rating_opponent",
"f_score_{}".format(target): "f_score_target",
"f_score_{}".format(opponent): "f_score_opponent",
"r_{}_new".format(target): "final_rating_target",
"delta_r_{}".format(target): "delta_r_target",
},
inplace=True,
)
ratings_of_single_model = ratings_of_single_model.append(
target_ratings, ignore_index=True
)
ratings_of_single_model["won_comparison"] = (
ratings_of_single_model.winner == ratings_of_single_model.target
)
ratings_of_single_model.won_comparison.replace(
{True: "Won", False: "Lost"}, inplace=True
)
ratings_of_single_model.sort_values("idx", inplace=True)
ratings_of_single_model["new_idx"] = list(range(len(ratings_of_single_model)))
# Plot 3 perspectives on this data:
# 1. How rating changes
# 2. relative change per comparison
# 3. strength of evidence each comparison
plots_to_include = ["ratings_progress", "delta_r", "bf"]
include_f_ax = True
lf = LatexFigure(
fraction=0.9,
use_gridspec=True,
gridspec_layout=(len(plots_to_include), 1),
gridspec_params={"height_ratios": [0.35, 0.95, 0.35]},
)
if "bf" in plots_to_include:
# Bayes factor comparisons
ax2 = lf.new_axis()
sns.barplot(
data=ratings_of_single_model,
y="weight",
x="new_idx",
hue="won_comparison",
palette={"Won": "green", "Lost": "purple"},
dodge=False,
ax=ax2,
)
ax2.legend(title=None, loc="upper left")
ax2.set_ylabel(r"$\left| \log_{10}(B_{ij}) \right|$")
generations = ratings_of_single_model.generation.unique()
generation_change_indices = {
g: ratings_of_single_model[
ratings_of_single_model.generation == g
].new_idx.max()
+ 0.5
for g in generations
}
# Actual ratings
if "ratings_progress" in plots_to_include:
ax0 = lf.new_axis(ax_params={"sharex": ax2})
sns.scatterplot(
data=ratings_of_single_model,
x="new_idx",
y="initial_rating_target",
label=r"$\hat{H}_i$ \ before",
color="green",
marker="x",
s=50,
ax=ax0,
)
sns.lineplot(
data=ratings_of_single_model,
x="new_idx",
y="final_rating_target",
label=r"$\hat{H}_i$ \ after",
color="green",
ax=ax0,
)
sns.scatterplot(
x="new_idx",
y="initial_rating_opponent",
data=ratings_of_single_model,
label=r"$\hat{H}_j$",
color="purple",
marker="+",
s=50,
ax=ax0,
)
ax0.legend()
ax0.set_ylabel("Rating")
ax0.set_xticks([])
ax0.set_xlabel("")
ax0.legend(title="Ratings", loc="upper left", ncol=3)
if include_f_ax:
f_score_ax = ax0.twinx()
sns.scatterplot(
x="new_idx",
y="f_score_opponent",
data=ratings_of_single_model,
label=r"$\hat{H}_j$",
marker="d",
s=50,
color="purple",
ax=f_score_ax,
)
target_f_score = ratings_of_single_model.f_score_target.values[0]
f_score_ax.axhline(
target_f_score,
color="green",
ls="--",
label=r"$\hat{H}_i$",
)
f_score_ax.set_ylabel(r"$F_1$-score")
f_score_ax.legend(title=r"$F_1$-score", loc="lower left", ncol=2)
f_score_ax.set_ylim(-0.1, 1.1)
f_score_ax.set_yticks([0, 0.5, 1])
if "delta_r" in plots_to_include:
ax1 = lf.new_axis(ax_params={"sharex": ax2})
sns.barplot(
data=ratings_of_single_model,
y="delta_r_target",
x="new_idx",
hue="won_comparison",
palette={"Won": "green", "Lost": "purple"},
dodge=False,
ax=ax1,
)
ax1.set_ylabel(r"$\Delta R_i$")
ax1.set_xticks([])
ax1.set_xlabel("")
ax1.get_legend().remove()
for ax in lf.gridspec_axes.values():
ax.set_xlabel("")
ax.set_xticks([])
bottom_ax = lf.gridspec_axes[(lf.num_rows - 1, 0)]
if len(generations) > 1:
# vertical lines separating generations
for ax in lf.gridspec_axes.values():
for g in generation_change_indices.values():
ax.axvline(
g,
ls="--",
c="grey",
alpha=0.6,
)
# label x-axis with generations
xtick_locations = [generation_change_indices[g] for g in generations]
xtick_locations.insert(0, 0)
centred_xticks = [
np.mean([xtick_locations[i], xtick_locations[i + 1]])
for i in range(len(xtick_locations) - 1)
]
bottom_ax.set_xticklabels(generations)
bottom_ax.set_xticks(centred_xticks)
bottom_ax.set_xlabel("Generation")
else:
bottom_ax.set_xticks(
range(len(ratings_of_single_model["new_idx"])),
)
bottom_ax.set_xticklabels(
range(1, 1 + len(ratings_of_single_model["new_idx"]))
)
bottom_ax.set_xlabel("Comparison")
if save_to_file is not None:
lf.save(save_to_file)
if return_df:
return ratings_of_single_model
def plot_models_ratings_against_generation(
self,
f_scores,
f_score_cmap,
save_directory,
show_fscore_cmap=False,
figure_format="png",
):
self.figure_format = figure_format
all_model_ratings_by_generation = pd.DataFrame()
models = self.all_ratings.model_id.unique()
for model in models:
model_ratings = self.all_ratings[self.all_ratings.model_id == model]
generations = model_ratings.generation.unique()
for g in generations:
mod_ratings_this_generation = model_ratings[
model_ratings.generation == g
]
start_idx = mod_ratings_this_generation.idx.min()
final_idx = mod_ratings_this_generation.idx.max()
start_rating = self.all_ratings[
(self.all_ratings.model_id == model)
& (self.all_ratings.generation == g)
& (self.all_ratings.idx == start_idx)
].rating.item()
final_rating = self.all_ratings[
(self.all_ratings.model_id == model)
& (self.all_ratings.generation == g)
& (self.all_ratings.idx == final_idx)
].rating.item()
new_data = [
pd.Series(
{"model_id": model, "generation": g, "rating": start_rating}
),
pd.Series(
{
"model_id": model,
"generation": g + 0.8,
"rating": final_rating,
}
),
]
for d in new_data:
all_model_ratings_by_generation = (
all_model_ratings_by_generation.append(d, ignore_index=True)
)
# First prepare a dictionary to map model id to a colour corresponding to F-score
f_granularity = 0.05
available_f_scores = np.linspace(0, 1, 1 + (1 / f_granularity))
model_coloured_by_f = {
# m : colour_by_f[ qmla.utilities.round_nearest(f_scores[m], f_granularity) ]
m: f_score_cmap(f_scores[m])
for m in all_model_ratings_by_generation.model_id.unique()
}
# Plot
widths = [1]
if show_fscore_cmap:
widths.append(0.1)
legend_axis = (0, 1)
else:
legend_axis = None
lf = LatexFigure(
# via https://github.com/flynnbr11/lfig-py
use_gridspec=True,
gridspec_layout=(1, len(widths)),
gridspec_params={
"width_ratios": widths,
},
legend_axis=legend_axis,
)
ax = lf.new_axis()
sns.lineplot(
x="generation",
y="rating",
hue="model_id",
data=all_model_ratings_by_generation,
palette=model_coloured_by_f,
legend=False,
ax=ax,
)
for g in self.all_ratings.generation.unique():
ax.axvline(g, ls="--", c="black")
ax.axhline(self.initial_rating, ls=":", color="black")
label_fontsize = 25
ax.set_xlabel(
"Generation",
# fontsize = label_fontsize
)
ax.set_ylabel(r"$R$")
ax.set_xticks(list(self.all_ratings.generation.unique()))
if show_fscore_cmap:
ax = lf.legend_ax
sm = plt.cm.ScalarMappable(
cmap=f_score_cmap, norm=plt.Normalize(vmin=0, vmax=1)
)
sm.set_array(available_f_scores)
plt.colorbar(sm, cax=ax, orientation="vertical")
ax.set_ylabel(
"F-score",
# fontsize=label_fontsize
)
lf.save(
os.path.join(save_directory, "elo_ratings_of_all_models"),
file_format=self.figure_format,
)