def get_random_bool_streamer(true_rate, batch_size=1024):
return RandomStreamer(
lambda size: Random.random_bool(true_rate, size, "data"),
batch_size)
def run(self, train_stream, dev_streams):
rconf = self.rconf
last_report_uidx, last_dev_uidx = 0, 0
if rconf.validate_first:
self._validate(dev_streams)
# =====
# for lrate warmup and annealing
if rconf.lrate_warmup < 0:
# calculate epochs
steps_per_epoch = 0
for _ in train_stream:
steps_per_epoch += 1
n_epoch = -rconf.lrate_warmup
n_steps = n_epoch * steps_per_epoch
utils.zlog(f"Calculating warmup steps for {n_epoch} epochs: {steps_per_epoch} steps per epoch.")
elif rconf.lrate_warmup > 0:
n_steps = rconf.lrate_warmup
else:
n_steps = 0
max_lrate = self.lrate.value
# final_lrate = lrate * anneal_factor * (step)^lrate_anneal_alpha
# final_lrate(n_steps) = max_lrate
lrate_anneal_alpha = rconf.lrate_anneal_alpha
if n_steps > 0:
anneal_factor = 1. / (n_steps**lrate_anneal_alpha)
else:
anneal_factor = 1.
self.lrate_warmup_steps = n_steps
utils.zlog(f"For lrate-warmup, will go with the first {n_steps} steps up to {max_lrate}, "
f"then anneal with lrate*{anneal_factor}*step^{lrate_anneal_alpha}")
# =====
while not self._finished():
# todo(note): epoch start from 1!!
self._tp.eidx += 1
with Timer(tag="Train-Iter", info="Iter %s" % self._tp.eidx, print_date=True) as et:
self._adjust_scheduled_values() # adjust at the start of each epoch
act_lrate = 0.
# for batches
for insts in train_stream:
# skip this batch
if Random.random_bool(rconf.skip_batch):
continue
# train on batch, return a dictionary
# possibly split batch to save memory
self._fb_batch(insts)
self._tp.uidx += 1
# get the effective lrate
act_lrate = self.lrate.value
if self._tp.uidx < self.lrate_warmup_steps:
act_lrate *= (self._tp.uidx / self.lrate_warmup_steps)
else:
act_lrate *= anneal_factor * (self._tp.uidx**lrate_anneal_alpha)
#
self._run_update(act_lrate, 1.)
# report on training process
if rconf.flag_verbose and (self._tp.uidx-last_report_uidx)>=rconf.report_freq:
utils.zlog(f"Current act_lrate is {act_lrate}.")
self._run_train_report()
last_report_uidx = self._tp.uidx
# time for validating
if (self._tp.uidx-last_dev_uidx)>=rconf.valid_freq:
self._validate(dev_streams)
last_dev_uidx = self._tp.uidx
last_report_uidx = self._tp.uidx
# todo(+N): do we need to adjust sv at a finer grained?
self._adjust_scheduled_values() # adjust after uidx validation
if self._finished():
break
# validate at the end of epoch?
utils.zlog(f"End of epoch: Current act_lrate is {act_lrate}.")
if rconf.validate_epoch:
self._validate(dev_streams)
last_dev_uidx = self._tp.uidx
last_report_uidx = self._tp.uidx
utils.zlog("")
utils.zlog("zzzzzfinal: After training, the best point is: %s." % (str(self._tp.info_save_best())))