def _start_tracker(self):
tracker().reset_writers()
if self.is_evaluate:
return
if self.distributed_rank != 0:
return
if 'screen' in self.writers:
from labml.internal.tracker.writers import screen
tracker().add_writer(screen.ScreenWriter())
if 'sqlite' in self.writers:
from labml.internal.tracker.writers import sqlite
tracker().add_writer(
sqlite.Writer(self.run.sqlite_path, self.run.artifacts_folder))
if 'tensorboard' in self.writers:
from labml.internal.tracker.writers import tensorboard
tracker().add_writer(
tensorboard.Writer(self.run.tensorboard_log_path))
if 'wandb' in self.writers:
from labml.internal.tracker.writers import wandb
self.wandb = wandb.Writer()
tracker().add_writer(self.wandb)
else:
self.wandb = None
if 'comet' in self.writers:
from labml.internal.tracker.writers import comet
self.comet = comet.Writer()
tracker().add_writer(self.comet)
else:
self.comet = None
if 'file' in self.writers:
from labml.internal.tracker.writers import file
tracker().add_writer(file.Writer(self.run.log_file))
if 'web_api' in self.writers:
web_api_conf = lab_singleton().web_api
if web_api_conf is not None:
from labml.internal.tracker.writers import web_api
from labml.internal.api import ApiCaller
from labml.internal.api.experiment import ApiExperiment
api_caller = ApiCaller(web_api_conf.url,
{'run_uuid': self.run.uuid},
timeout_seconds=120)
self.web_api = ApiExperiment(
api_caller,
frequency=web_api_conf.frequency,
open_browser=web_api_conf.open_browser)
tracker().add_writer(
web_api.Writer(api_caller,
frequency=web_api_conf.frequency))
else:
self.web_api = None
def __init__(self):
self.__store = None
self.__writers: List[Writer] = []
self.__start_global_step: Optional[int] = None
self.__global_step: Optional[int] = None
self.__last_global_step: Optional[int] = None
self.__set_looping_indicators = None
self.__loop_counter = 0
self.indicators = {}
self.dot_indicators = {}
self.__indicators_file = None
self.namespaces = []
self.is_indicators_updated = True
self.reset_store()
from labml.internal.tracker.writers import screen
self.add_writer(screen.ScreenWriter())