def log_results(self, method: Method, results: IncrementalResults) -> None:
"""
TODO: Create the tabs we need to show up in wandb:
1. Final
- Average "Current/Online" performance (scalar)
- Average "Final" performance (scalar)
- Runtime
2. Test
- Task i (evolution over time (x axis is the task id, if possible))
"""
logger.info(results.summary())
if wandb.run:
wandb.summary["method"] = method.get_name()
wandb.summary["setting"] = self.get_name()
dataset = getattr(self, "dataset", "")
if dataset and isinstance(dataset, str):
wandb.summary["dataset"] = dataset
wandb.log(results.to_log_dict())
# BUG: Sometimes logging a matplotlib figure causes a crash:
# File "/home/fabrice/miniconda3/envs/sequoia/lib/python3.8/site-packages/plotly/matplotlylib/mplexporter/utils.py", line 246, in get_grid_style
# if axis._gridOnMajor and len(gridlines) > 0:
# AttributeError: 'XAxis' object has no attribute '_gridOnMajor'
wandb.log(results.make_plots())
wandb.run.finish()
def setup_wandb(self, method: Method) -> Run:
"""Call wandb.init, log the experiment configuration to the config dict.
This assumes that `self.wandb` is not None. This happens when one of the wandb
arguments is passed.
Parameters
----------
method : Method
Method to be applied.
"""
assert isinstance(self.wandb, WandbConfig)
method_name: str = method.get_name()
setting_name: str = self.get_name()
if not self.wandb.run_name:
# Set the default name for this run.
run_name = f"{method_name}-{setting_name}"
dataset = getattr(self, "dataset", None)
if isinstance(dataset, str):
run_name += f"-{dataset}"
if getattr(self, "nb_tasks", 0) > 1:
run_name += f"_{self.nb_tasks}t"
self.wandb.run_name = run_name
run: Run = self.wandb.wandb_init()
run.config["setting"] = setting_name
run.config["method"] = method_name
if hasattr(self, "to_dict"):
for k, value in self.to_dict().items():
if not k.startswith("_"):
run.config[f"setting/{k}"] = value
run.summary["setting"] = self.get_name()
run.summary["method"] = method.get_name()
assert wandb.run is run
return run