def sample_plot(self, sample_space, n_iter, chart_scale=15):
"""
Documentation:
---
Definition:
Visualizes a single hyperopt theoretical distribution. Useful for helping to determine a
distribution to use when setting up hyperopt distribution objects for actual parameter
tuning.
---
Parameters:
sample_space : dictionary
Dictionary of 'param name: hyperopt distribution object' key/value pairs. The name can
be arbitrarily chosen, and the value is a defined hyperopt distribution.
n_iter : int
Number of iterations to draw from theoretical distribution in order to visualize the
theoretical distribution. Higher number leads to more robust distribution but can take
considerably longer to create.
chart_scale : float, default=15
Controls proportions of visualizations. larger values scale visual up in size, smaller values
scale visual down in size.
"""
# iterate through each parameter
for param in sample_space.keys():
# sample from theoretical distribution for n_iters
theoretical_dist = []
for _ in range(n_iter):
theoretical_dist.append(sample(sample_space)[param])
theoretical_dist = np.array(theoretical_dist)
# create prettierplot object
p = PrettierPlot(chart_scale=chart_scale)
# add canvas to prettierplot object
ax = p.make_canvas(
title="actual vs. theoretical plot\n* {}".format(param),
y_shift=0.8,
position=111,
)
# add kernel density plot to canvas
p.kde_plot(
theoretical_dist,
color=style.style_grey,
y_units="p",
x_units="fff" if np.nanmax(theoretical_dist) <= 5.0 else "ff",
ax=ax,
)
def model_param_plot(self, bayes_optim_summary, estimator_class, estimator_parameter_space, n_iter, chart_scale=15,
color_map="viridis", title_scale=1.2, show_single_str_params=False):
"""
Documentation:
---
Definition:
Visualize hyperparameter optimization over all iterations. Compares theoretical distribution to
the distribution of values that were actually chosen, and visualizes how parameter value
selections changes over time.
---
Parameters:
bayes_optim_summary : Pandas DataFrame
Pandas DataFrame containing results from bayesian optimization process.
estimator_class : str or sklearn api object
Name of estimator to visualize.
estimator_parameter_space : dictionary of dictionaries
Dictionary of nested dictionaries. Outer key is an estimator, and the corresponding value is
a dictionary. Each nested dictionary contains 'parameter: value distribution' key/value
pairs. The inner dictionary key specifies the parameter of the model to be tuned, and the
value is a distribution of values from which trial values are drawn.
n_iter : int
Number of iterations to draw from theoretical distribution in order to visualize the
theoretical distribution. Higher number leader to more robust distribution but can take
considerably longer to create.
chart_scale : float, default=15
Controls proportions of visualizations. larger values scale visual up in size, smaller values
scale visual down in size.
color_map : str specifying built-in matplotlib colormap, default="viridis"
Color map applied to plots.
title_scale : float, default=1.2
Controls the scaling up (higher value) and scaling down (lower value) of the size of
the main chart title, the x_axis title and the y_axis title.
show_single_str_params : boolean, default=False
Controls whether to display visuals for string attributes where there is only one unique value,
i.e. there was only one choice for the optimization procedure to choose from during each iteration.
"""
# unpack bayes_optim_summary parameters for an estimator_class
estimator_summary = self.unpack_bayes_optim_summary(
bayes_optim_summary=bayes_optim_summary, estimator_class=estimator_class
)
# override None with string representation
estimator_summary = estimator_summary.replace([None], "None")
# subset estimator_parameter_space to space for the specified estimator_class
estimator_space = estimator_parameter_space[estimator_class]
print("*" * 100)
print("* {}".format(estimator_class))
print("*" * 100)
# iterate through each parameter
for param in estimator_space.keys():
# sample from theoretical distribution for n_iters
theoretical_dist = []
for _ in range(n_iter):
theoretical_dist.append(sample(estimator_space)[param])
## override None with string representation
# theoretical distribution
theoretical_dist = ["none" if v is None else v for v in theoretical_dist]
theoretical_dist = np.array(theoretical_dist)
# actual distribution
actual_dist = estimator_summary[param].tolist()
actual_dist = ["none" if v is None else v for v in actual_dist]
actual_dist = np.array(actual_dist)
# limit estimator_summary to "iteration" and current "param" columns
actual_iter_df = estimator_summary[["iteration", param]]
# identify how many values in param column are zero or one
zeros_and_ones = (actual_iter_df[param].eq(True) | actual_iter_df[param].eq(False)).sum()
# param column only contains zeros and ones, store string representations of "TRUE" and "FALSE"
if zeros_and_ones == actual_iter_df.shape[0]:
actual_iter_df = actual_iter_df.replace({True: "TRUE", False: "FALSE"})
# if theoreitcal distribution has dtype -- np.bool_, store string representations of "TRUE" and "FALSE"
if isinstance(theoretical_dist[0], np.bool_):
theoretical_dist = np.array(["TRUE" if i == True else "FALSE" for i in theoretical_dist.tolist()])
estimator_summary = estimator_summary.replace([True], "TRUE")
estimator_summary = estimator_summary.replace([False], "FALSE")
# if theoretical distribution contains str data, then treat this as an object/category parameter
if any(isinstance(d, str) for d in theoretical_dist):
# generate color list for stripplot
stripplot_color_list = style.color_gen(name=color_map, num=len(actual_iter_df[param].unique()) + 1)
# generate color list for bar chart
bar_color_list = style.color_gen(name=color_map, num=3)
# identify unique values and associated count in theoretical distribution
unique_vals_theo, unique_counts_theo = np.unique(theoretical_dist, return_counts=True)
# if theoretical distribution only has one unique value and show_single_str_params is set to True
if len(unique_vals_theo) > 1 or show_single_str_params:
# identify unique values and associated count in actual distribution
unique_vals_actual, unique_counts_actual = np.unique(actual_dist, return_counts=True)
# store data in DataFrame
df = pd.DataFrame({"param": unique_vals_actual, "Theorical": unique_counts_theo, "Actual": unique_counts_actual})
# create prettierplot object
p = PrettierPlot(chart_scale=chart_scale, plot_orientation = "wide_narrow")
# add canvas to prettierplot object
ax = p.make_canvas(
title="Selection vs. theoretical distribution\n* {0} - {1}".format(estimator_class, param),
y_shift=0.8,
position=121,
title_scale=title_scale,
)
# add faceted bar chart to canvas
p.facet_cat(
df=df,
feature="param",
color_map=bar_color_list[:-1],
bbox=(1.0, 1.15),
alpha=1.0,
legend_labels=df.columns[1:].values,
x_units=None,
ax=ax,
)
# add canvas to prettierplot object
ax = p.make_canvas(
title="Selection by iteration\n* {0} - {1}".format(estimator_class, param),
y_shift=0.5,
position=122,
title_scale=title_scale,
)
# add stripply to canvas
sns.stripplot(
x="iteration",
y=param,
data=estimator_summary,
jitter=0.3,
alpha=1.0,
size=0.7 * chart_scale,
palette=sns.color_palette(stripplot_color_list[:-1]),
ax=ax,
).set(xlabel=None, ylabel=None)
# set tick label font size
ax.tick_params(axis="both", colors=style.style_grey, labelsize=1.2 * chart_scale)
plt.show()
# otherwise treat it as a numeric parameter
else:
# cast "iteration" as an int and the param values as float
convert_dict = {"iteration": int, param: float}
actual_iter_df = actual_iter_df.astype(convert_dict)
# create color map
color_list = style.color_gen(name=color_map, num=3)
# create prettierplot object
p = PrettierPlot(chart_scale=chart_scale, plot_orientation = "wide_narrow")
# add canvas to prettierplot object
ax = p.make_canvas(
title="Selection vs. theoretical distribution\n* {0} - {1}".format(estimator_class, param),
y_shift=0.8,
position=121,
title_scale=title_scale,
)
# dynamically set x-unit precision based on max value
if -1.0 <= np.nanmax(theoretical_dist) <= 1.0:
x_units = "fff"
elif 1.0 < np.nanmax(theoretical_dist) <= 5.0:
x_units = "ff"
elif np.nanmax(theoretical_dist) > 5.0:
x_units = "f"
# add kernsel density plot for theoretical distribution to canvas
p.kde_plot(
theoretical_dist,
color=color_list[0],
y_units="ffff",
x_units=x_units,
line_width=0.4,
bw=0.4,
ax=ax,
)
# add kernsel density plot for actual distribution to canvas
p.kde_plot(
actual_dist,
color=color_list[1],
y_units="ffff",
x_units=x_units,
line_width=0.4,
bw=0.4,
ax=ax,
)
## create custom legend
# create labels
label_color = {}
legend_labels = ["Theoretical", "Actual"]
for ix, i in enumerate(legend_labels):
label_color[i] = color_list[ix]
# create legend Patches
Patches = [Patch(color=v, label=k, alpha=1.0) for k, v in label_color.items()]
# draw legend
leg = plt.legend(
handles=Patches,
fontsize=1.1 * chart_scale,
loc="upper right",
markerscale=0.6 * chart_scale,
ncol=1,
bbox_to_anchor=(.95, 1.1),
)
# label font color
for text in leg.get_texts():
plt.setp(text, color="grey")
# dynamically set y-unit precision based on max value
if -1.0 <= np.nanmax(actual_iter_df[param]) <= 1.0:
y_units = "fff"
elif 1.0 < np.nanmax(actual_iter_df[param]) <= 5.0:
y_units = "ff"
elif np.nanmax(actual_iter_df[param]) > 5.0:
y_units = "f"
# add canvas to prettierplot object
ax = p.make_canvas(
title="Selection by iteration\n* {0} - {1}".format(estimator_class, param),
y_shift=0.8,
position=122,
title_scale=title_scale,
)
# add regression plot to canvas
p.reg_plot(
x="iteration",
y=param,
data=actual_iter_df,
y_units=y_units,
x_units="f",
line_color=color_list[0],
line_width=0.4,
dot_color=color_list[1],
dot_size=10.0,
alpha=0.6,
ax=ax
)
plt.show()