def start_train(self, session: tf.Session, is_init: bool = True):
"""
## Start experiment
Load a checkpoint or reset based on `global_step`.
"""
global_step = 0
if not is_init:
# load checkpoint if we are starting from middle
with logger.section("Loading checkpoint") as m:
is_successful = self.__checkpoint_saver.load(session)
logger.set_successful(is_successful)
if is_successful:
global_step = self.__checkpoint_saver.max_step
self._start(global_step)
if global_step == 0:
# initialize variables and clear summaries if we are starting from scratch
with logger.section("Clearing summaries"):
self.clear_summaries()
with logger.section("Clearing checkpoints"):
self.clear_checkpoints()
with logger.section("Initializing variables"):
tf_util.init_variables(session)
self.create_writer(session)
def start(self,
*,
run: Optional[int] = None,
checkpoint: Optional[int] = None):
if run is not None:
with logger.section("Loading checkpoint"):
global_step = self._load_checkpoint(run, checkpoint)
if global_step is None:
logger.set_successful(False)
global_step = 0
else:
global_step = 0
self.run.start_step = global_step
logger.internal().set_start_global_step(global_step)
self.__print_info_and_check_repo()
self.configs_processor.print()
self.run.save_info()
if self.configs_processor is not None:
self.configs_processor.save(self.run.configs_path)
logger.internal().save_indicators(self.run.indicators_path)
with open(str(self.run.diff_path), "w") as f:
f.write(self.run.diff)
from lab.experiment.tensorflow import Experiment
# Create the sample experiment
EXPERIMENT = Experiment(name="sample",
python_file=__file__,
comment="Sample lab experiment",
check_repo_dirty=False)
# Sections are use to keep track of
# what's going on from the console output.
# It is also useful to organize the code into sections,
# when separating them into functions is difficult
with logger.section("Create model"):
# Indicate that this section failed. You don't have to set
# this if it is successful.
logger.set_successful(False)
# Sleep for a minute.
time.sleep(1)
# Print sample info
logger.info(one=1, two=2, string="string")
# ### Set logger indicators
# Reward is queued; this is useful when you want to track the moving
# average of something.
logger.add_indicator("reward", queue_limit=10)
# By default everything is a set of values and will create a TensorBoard histogram
# We specify that `fps` is a scalar.
def unsuccessful_section():
with logger.section("Unsuccessful section"):
time.sleep(1)
logger.set_successful(False)