def test_apply_theme(self, mock_OFD, test_plot):
# the subsequent call inside ODF will fail since the model was never
# added to a document. Ignoring that since we just want to make sure
# ODF is called with the expected theme arg.
try:
bes.json_item(test_plot, theme="foo")
except ValueError:
pass
mock_OFD.assert_called_once_with([test_plot], apply_theme="foo")
def test_apply_theme(self, mock_OFD, test_plot):
# the subsequent call inside ODF will fail since the model was never
# added to a document. Ignoring that since we just want to make sure
# ODF is called with the expected theme arg.
try:
bes.json_item(test_plot, theme="foo")
except ValueError:
pass
mock_OFD.assert_called_once_with([test_plot], apply_theme="foo")
def test_version(self, monkeypatch: pytest.MonkeyPatch,
test_plot: figure) -> None:
from bokeh import __version__
out = bes.json_item(test_plot, target=ID("foo"))
assert set(out.keys()) == JSON_ITEMS_KEYS
assert out['doc']['version'] == __version__
out = bes.json_item(test_plot)
assert set(out.keys()) == JSON_ITEMS_KEYS
assert out['doc']['version'] == __version__
def test_without_target_id(self, test_plot: figure) -> None:
out = bes.json_item(test_plot)
assert set(out.keys()) == JSON_ITEMS_KEYS
assert out['target_id'] == None
def test_root_id(self, test_plot):
out = bes.json_item(test_plot, target="foo")
assert out['doc']['roots']['root_ids'][0] == out['root_id']
def test_doc_title(self, test_plot):
out = bes.json_item(test_plot, target="foo")
assert out['doc']['title'] == ""
def test_doc_json(self, test_plot):
out = bes.json_item(test_plot, target="foo")
expected = list(standalone_docs_json([test_plot]).values())[0]
assert out['doc'] == expected
def test_with_target_id(self, test_plot: Figure) -> None:
out = bes.json_item(test_plot, target=ID("foo"))
assert set(out.keys()) == JSON_ITEMS_KEYS
assert out['target_id'] == "foo"
def test_root_id(self, test_plot: figure) -> None:
out = bes.json_item(test_plot, target=ID("foo"))
assert set(out.keys()) == JSON_ITEMS_KEYS
assert out['doc']['roots'][0]["id"] == out['root_id']
def test_root_id(self, test_plot):
out = bes.json_item(test_plot, target="foo")
assert out['doc']['roots']['root_ids'][0] == out['root_id']
def test_doc_title(self, test_plot):
out = bes.json_item(test_plot, target="foo")
assert out['doc']['title'] == ""
def test_doc_json(self, test_plot):
out = bes.json_item(test_plot, target="foo")
expected = list(standalone_docs_json([test_plot]).values())[0]
assert out['doc'] == expected
def test_without_target_id(self, test_plot):
out = bes.json_item(test_plot)
assert out['target_id'] == None
def test_with_target_id(self, test_plot):
out = bes.json_item(test_plot, target="foo")
assert out['target_id'] == "foo"
def test_doc_json(self, test_plot: figure) -> None:
out = bes.json_item(test_plot, target=ID("foo"))
assert set(out.keys()) == JSON_ITEMS_KEYS
expected = list(standalone_docs_json([test_plot]).values())[0]
assert out['doc'] == expected
def test_doc_title(self, test_plot: figure) -> None:
out = bes.json_item(test_plot, target=ID("foo"))
assert set(out.keys()) == JSON_ITEMS_KEYS
assert out['doc']['title'] == ""
def test_with_target_id(self, test_plot):
out = bes.json_item(test_plot, target="foo")
assert out['target_id'] == "foo"
def test_without_target_id(self, test_plot):
out = bes.json_item(test_plot)
assert out['target_id'] == None
def plot_runs(
run_config_base_names,
smooth=100,
validation_scores=None,
higher_is_betters=None,
plot_losses=None,
return_json=False,
):
print("PLOTTING RUNS")
runs = get_runs_info(run_config_base_names, validation_scores, plot_losses)
print("GOT RUNS INFO")
colors = itertools.cycle(palette[20])
loss_tooltips = [
("task", "@task"),
("architecture", "@architecture"),
("trainer", "@trainer"),
("datasplit", "@datasplit"),
("iteration", "@iteration"),
("loss", "@loss"),
]
loss_figure = bokeh.plotting.figure(
tools="pan, wheel_zoom, reset, save, hover",
x_axis_label="iterations",
tooltips=loss_tooltips,
plot_width=2048,
)
loss_figure.background_fill_color = "#efefef"
validation_figures = {}
validation_datasets = set(
itertools.chain(*[list(run.validation_scores.datasets) for run in runs])
)
if validation_scores:
validation_score_names = set()
validation_postprocessor_parameter_names = set()
for r in runs:
if r.validation_scores.validated_until() > 0:
validation_score_names = validation_score_names.union(
r.validation_scores.criteria
)
validation_postprocessor_parameter_names = (
validation_postprocessor_parameter_names.union(
set(r.validation_scores.parameter_names)
)
)
validation_score_names = validation_score_names
validation_postprocessor_parameter_names = (
validation_postprocessor_parameter_names
)
validation_tooltips = (
[
("run", "@run"),
("task", "@task"),
("architecture", "@architecture"),
("trainer", "@trainer"),
("datasplit", "@datasplit"),
]
+ [(name, "@" + name) for name in validation_score_names]
+ [(name, "@" + name) for name in validation_postprocessor_parameter_names]
)
for dataset in validation_datasets:
validation_figure = bokeh.plotting.figure(
tools="pan, wheel_zoom, reset, save, hover",
x_axis_label="iterations",
tooltips=validation_tooltips,
plot_width=2048,
)
validation_figure.background_fill_color = "#efefef"
validation_figures[dataset.name] = validation_figure
print("VALIDATION SCORES TOOLTIP MADE")
summary_tooltips = [
("run", "@run"),
("task", "@task"),
("architecture", "@architecture"),
("trainer", "@trainer"),
("datasplit", "@datasplit"),
("best iteration", "@iteration"),
("best voi_split", "@voi_split"),
("best voi_merge", "@voi_merge"),
("best voi_sum", "@voi_sum"),
("num parameters", "@num_parameters"),
]
summary_figure = bokeh.plotting.figure(
tools="pan, wheel_zoom, reset, save, hover",
x_axis_label="model size",
y_axis_label="best validation",
tooltips=summary_tooltips,
plot_width=2048,
)
summary_figure.background_fill_color = "#efefef"
include_validation_figure = False
include_loss_figure = False
for run, color in zip(runs, colors):
name = run.name
if run.plot_loss:
iterations = [stat.iteration for stat in run.training_stats.iteration_stats]
losses = [stat.loss for stat in run.training_stats.iteration_stats]
print(F"Run {run.name} has {len(losses)} iterations")
if run.plot_loss:
include_loss_figure = True
smooth = int(np.maximum(len(iterations) / 2500, 1))
print(f"smoothing: {smooth}")
x, _ = smooth_values(iterations, smooth, stride=smooth)
y, s = smooth_values(losses, smooth, stride=smooth)
print(x, y)
print(f"plotting {(len(x), len(y))} points")
source = bokeh.plotting.ColumnDataSource(
{
"iteration": x,
"loss": y,
"task": [run.task] * len(x),
"architecture": [run.architecture] * len(x),
"trainer": [run.trainer] * len(x),
"datasplit": [run.datasplit] * len(x),
"run": [name] * len(x),
}
)
loss_figure.line(
"iteration",
"loss",
legend_label=name,
source=source,
color=color,
alpha=0.7,
)
loss_figure.patch(
np.concatenate([x, x[::-1]]),
np.concatenate([y + 3 * s, (y - 3 * s)[::-1]]),
legend_label=name,
color=color,
alpha=0.3,
)
print("LOSS PLOTTED")
if run.validation_score_name and run.validation_scores.validated_until() > 0:
validation_score_data = run.validation_scores.to_xarray().sel(
criteria=run.validation_score_name
)
for dataset in run.validation_scores.datasets:
dataset_data = validation_score_data.sel(datasets=dataset)
include_validation_figure = True
x = [
score.iteration for score in run.validation_scores.scores
]
source_dict = {
"iteration": x,
"task": [run.task] * len(x),
"architecture": [run.architecture] * len(x),
"trainer": [run.trainer] * len(x),
"datasplit": [run.datasplit] * len(x),
"run": [run.name] * len(x),
}
# TODO: get_best: higher_is_better is not true for all scores
best_parameters, best_scores = run.validation_scores.get_best(
dataset_data, dim="parameters"
)
source_dict.update(
{
name: np.array(
[
getattr(best_parameter, name)
for best_parameter in best_parameters.values
]
)
for name in run.validation_scores.parameter_names
}
)
source_dict.update(
{run.validation_score_name: np.array(best_scores.values)}
)
source = bokeh.plotting.ColumnDataSource(source_dict)
validation_figures[dataset.name].line(
"iteration",
run.validation_score_name,
legend_label=name + " " + run.validation_score_name,
source=source,
color=color,
alpha=0.7,
)
print("VALIDATION PLOTTED")
# Styling
# training
figures = []
if include_loss_figure:
loss_figure.title.text_font_size = "25pt"
loss_figure.title.text = "Training"
loss_figure.title.align = "center"
loss_figure.legend.label_text_font_size = "16pt"
loss_figure.xaxis.axis_label = "Iterations"
loss_figure.xaxis.axis_label_text_font_size = "20pt"
loss_figure.xaxis.major_label_text_font_size = "16pt"
loss_figure.xaxis.axis_label_text_font = "times"
loss_figure.xaxis.axis_label_text_color = "black"
loss_figure.yaxis.axis_label = "Loss"
loss_figure.yaxis.axis_label_text_font_size = "20pt"
loss_figure.yaxis.major_label_text_font_size = "16pt"
loss_figure.yaxis.axis_label_text_font = "times"
loss_figure.yaxis.axis_label_text_color = "black"
loss_figure.sizing_mode = "scale_width"
figures.append(loss_figure)
if include_validation_figure:
for dataset, validation_figure in validation_figures.items():
# validation
validation_figure.title.text_font_size = "25pt"
validation_figure.title.text = f"{dataset} Validation"
validation_figure.title.align = "center"
validation_figure.legend.label_text_font_size = "16pt"
validation_figure.xaxis.axis_label = "Iterations"
validation_figure.xaxis.axis_label_text_font_size = "20pt"
validation_figure.xaxis.major_label_text_font_size = "16pt"
validation_figure.xaxis.axis_label_text_font = "times"
validation_figure.xaxis.axis_label_text_color = "black"
validation_figure.yaxis.axis_label = "Validation Score"
validation_figure.yaxis.axis_label_text_font_size = "20pt"
validation_figure.yaxis.major_label_text_font_size = "16pt"
validation_figure.yaxis.axis_label_text_font = "times"
validation_figure.yaxis.axis_label_text_color = "black"
validation_figure.sizing_mode = "scale_width"
figures.append(validation_figure)
plot = bokeh.layouts.column(*figures)
plot.sizing_mode = "scale_width"
print("PLOTTING DONE")
if return_json:
print("Returning JSON")
return json.dumps(json_item(plot, "myplot"))
else:
bokeh.plotting.output_file("performance_plots.html")
bokeh.plotting.save(plot)
