def experiment(variant):
cuda = True
from gym.envs.mujoco import HalfCheetahEnv
from mujoco_torch.core.bridge import MjCudaRender
R = 84
env = HalfCheetahEnv()
c = Convnet(6, output_activation=torch.tanh, input_channels=3)
if cuda:
c.cuda()
gt.stamp("start")
for i in range(100):
img = env.sim.render(R, R, device_id=1)
gt.stamp("warmstart")
for i in gt.timed_for(range(1000)):
env.step(np.random.rand(6))
gt.stamp('step')
img = env.sim.render(R, R, device_id=1)
gt.stamp('render')
x = np_to_var(img)
if cuda:
x = x.cuda()
torch.cuda.synchronize()
gt.stamp('transfer')
# cv2.imshow("img", img)
# cv2.waitKey(1)
gt.stamp("end")
print(img)
print(gt.report(include_itrs=False))
def main():
gt.reset()
tries = 0
# starting with just 9x9.
sudoku = [] # 1D list of all sudoku's elements.
while len(sudoku) < 81:
tries += 1
num_list = list(range(1, 10))
num = random.choice(num_list)
while inRow(sudoku, num) or inCol(sudoku, num) or inSquare(
sudoku, num):
try:
num_list.remove(num)
num = random.choice(num_list)
except IndexError:
sudoku = []
sudoku.append(num)
PrintSudoku(sudoku)
# substring in report with total time
total_time = float(gt.report()[91:97].split()[0])
total_time *= 1000 # convert seconds ot milliseconds
print("tries: " + str(tries))
print("total time: " + str(total_time) + " ms")
# sudoku attempts per millisecond
print("Attempts per ms: " + str(tries / total_time))
def experiment(variant):
root = 0
E = 20
R = 84
U = 6
cuda = True
envs = []
for e in range(E):
env = HalfCheetahEnv()
envs.append(env)
c = Convnet(6, output_activation=torch.tanh, input_channels=3)
if cuda:
c.cuda()
# viewer = mujoco_py.MjRenderContextOffscreen(env.sim, device_id=1)
# env.sim.add_render_context(viewer)
def step(i, stamp=True):
imgs = []
if i % 100 == 0:
for e in envs:
e.reset()
for e in envs:
img = e.sim.render(R, R, device_id=0).transpose()
imgs.append(img)
gt.stamp('render') if stamp else 0
imgs = np.array(imgs)
torch_img = np_to_var(imgs)
if cuda:
torch_img = torch_img.cuda()
torch.cuda.synchronize()
gt.stamp('transfer') if stamp else 0
u = get_numpy(c.forward(torch_img).cpu())
torch.cuda.synchronize()
gt.stamp('forward') if stamp else 0
for i, e in enumerate(envs):
e.step(u[i, :])
gt.stamp('step') if stamp else 0
for i in range(10):
step(i, False)
gt.stamp('start')
for i in gt.timed_for(range(100)):
step(i)
gt.stamp('end')
print(gt.report(include_itrs=False, format_options=dict(itr_num_width=10)))
def experiment(variant):
from gym.envs.mujoco import HalfCheetahEnv
from mujoco_torch.core.bridge import MjCudaRender
renderer = MjCudaRender(84, 84)
env = HalfCheetahEnv()
gt.stamp("start")
for i in range(100):
tensor, img = renderer.get_cuda_tensor(env.sim, False)
gt.stamp("warmstart")
for i in range(1000):
env.step(np.random.rand(6))
tensor, img = renderer.get_cuda_tensor(env.sim, True)
x = np_to_var(img).cuda()
torch.cuda.synchronize()
# cv2.imshow("img", img)
# cv2.waitKey(1)
gt.stamp("end")
print(img)
print(gt.report())
def experiment(variant):
from gym.envs.mujoco import HalfCheetahEnv
from mujoco_torch.core.bridge import MjCudaRender
renderer = MjCudaRender(84, 84)
env = HalfCheetahEnv()
gt.stamp("start")
for i in range(100):
tensor, img = renderer.get_cuda_tensor(env.sim, False)
gt.stamp("warmstart")
for i in gt.timed_for(range(1000)):
env.step(np.random.rand(6))
gt.stamp('step')
tensor, img = renderer.get_cuda_tensor(env.sim, False)
gt.stamp('render')
# cv2.imshow("img", img)
# cv2.waitKey(1)
gt.stamp("end")
print(img)
print(gt.report(include_itrs=False))
import gtimer as gt
import time
time.sleep(0.1)
gt.stamp('first')
time.sleep(0.3)
gt.stamp('second')
print gt.report()
def train_epoch(epoch):
print('\nTRAINING : Epoch ' + str(epoch))
model.train()
losses = []
logs = []
last_time = time.time()
metrics = Metrics(tok2i, i2tok, field=TRG)
for i, batch in enumerate(trainloader):
# -- Actual Training
gt.reset()
gt.stamp("load_data")
oracle = Oracle(batch.trg[0].detach(), model.n_classes, tok2i, i2tok, **oracle_flags)
gt.stamp("create_oracle")
max_steps = 2*batch.trg[0].detach().ne(tok2i[constants.PAD_WORD]).sum(1).max()+1
scores, samples, p_oracle = model.forward(xs=batch.src, oracle=oracle, max_steps=max_steps, num_samples=len(batch),
return_p_oracle=True)
gt.stamp("forward")
loss = loss_fn(scores, samples, p_oracle, end_idx=tok2i['<end>'], **loss_flags)
gt.stamp("loss")
optimizer.zero_grad()
loss.backward()
clip_grad_norm_(model.parameters(), args.max_norm)
optimizer.step()
gt.stamp("backward")
losses.append(loss.item())
# -- Report metrics every `print_every` batches.
if i % args.print_every == 0:
# Only compute training metrics once here for efficiency.
metrics.update(scores, samples, (batch.trg[0], None), kind='train')
gt.stamp("metrics.update")
# Training report computed over the last `print_every` batches.
ms = metrics.report('train')
ms['train/loss'] = round(np.mean(losses), 2)
logs.append('{0} ; loss {1} ; sentence/s {2} ; {3} train {4} '.format(
i+1,
round(np.mean(losses), 2),
int(len(losses) * args.batch_size / (time.time() - last_time)),
args.eval_metric,
ms['train/%s' % args.eval_metric],
))
args.logstep += 1
last_time = time.time()
losses = []
metrics.reset()
# -- Validation report with a single batch.
metrics.reset()
model.eval()
batch = next(iter(validloader))
scores, samples = predict_batch(batch)
model.train()
metrics.update(scores, samples, (batch.trg[0], None))
vms = metrics.report('valid_batch')
logs[-1] = logs[-1] + metrics.log(vms, 'valid_batch', ['bleu', 'avg_span', 'f1', 'em', 'depth_score'])
metrics.reset()
print_samples(samples, (batch.trg[0], None), n=len(batch))
gt.stamp("validation_batch")
log_tensorboard(ms, step=args.logstep)
log_tensorboard(vms, step=args.logstep)
print(logs[-1])
print(gt.report(include_itrs=False, format_options={'itr_name_width': 30}))
# -- Checkpointing
if i % args.save_every == 0:
print('saving checkpoint at epoch {0} batch {1}'.format(epoch, i))
print(os.path.join(args.log_directory, args.expr_name + '.checkpoint'))
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'optimizer_param': args.optimizer,
'loss': loss.item()
}, os.path.join(args.log_directory, args.expr_name + '.checkpoint'))
model_config['longest_label'] = model.longest_label
with open(os.path.join(args.log_directory, 'model_config.json'), 'w') as f:
json.dump(model_config, f)
print('end : epoch {0} '.format(epoch))
log_tensorboard({'lr': optimizer.param_groups[0]['lr']}, step=args.logstep)
import gtimer as gt
import time
time.sleep(0.1)
gt.stamp('first')
loop = gt.timed_loop('named_loop')
x = 0
while x < 3:
next(loop)
time.sleep(0.1)
x += 1
gt.stamp('loop')
loop.exit()
time.sleep(0.1)
gt.stamp('second')
print gt.report(include_itrs=False)
def _train(self, env, policy):
self._init_training(env, policy)
with self._sess.as_default():
observation = env.reset()
policy.reset()
itr = 0
path_length = 0
path_return = 0
gt.rename_root('online algo')
gt.reset()
gt.set_def_unique(False)
for epoch in gt.timed_for(range(self._n_epochs), save_itrs=True):
logger.push_prefix('Epoch #%d | ' % epoch)
for t in range(self._epoch_length):
# Sample next action and state.
action, _ = policy.get_action(observation)
gt.stamp('train: get actions')
action.squeeze()
if self._render:
env.render()
next_ob, raw_reward, terminal, info = env.step(action)
reward = raw_reward * self._scale_reward
path_length += 1
path_return += reward
gt.stamp('train: simulation')
# Add experience to replay pool.
self._pool.add_sample(observation, action, reward,
terminal, False)
should_reset = (terminal
or path_length >= self._max_path_length)
if should_reset:
# noinspection PyTypeChecker
self._pool.add_sample(next_ob, np.zeros_like(action),
np.zeros_like(reward),
np.zeros_like(terminal), True)
observation = env.reset()
policy.reset()
path_length = 0
path_return = 0
else:
observation = next_ob
gt.stamp('train: fill replay pool')
# Train.
if self._pool.size >= self._min_pool_size:
self._do_training(itr)
itr += 1
gt.stamp('train: updates')
# Evaluate.
self._evaluate(epoch)
gt.stamp("test")
# Log.
params = self.get_epoch_snapshot(epoch)
logger.save_itr_params(epoch, params)
times_itrs = gt.get_times().stamps.itrs
train_time = np.sum([
times_itrs[key][-1] for key in times_itrs.keys()
if 'train: ' in key
])
eval_time = times_itrs["test"][-1]
total_time = gt.get_times().total
logger.record_tabular("time: train", train_time)
logger.record_tabular("time: eval", eval_time)
logger.record_tabular("time: total", total_time)
logger.record_tabular("scale_reward", self._scale_reward)
logger.record_tabular("epochs", epoch)
logger.record_tabular("steps: all", itr)
logger.dump_tabular(with_prefix=False)
logger.pop_prefix()
gt.stamp("logging")
print(
gt.report(
include_itrs=False,
format_options={'itr_name_width': 30},
))
env.terminate()
def experiment(variant):
cuda = True
ingpu = False
R = 84
E = 100
N = 100
if ingpu:
from mujoco_torch.core.bridge import MjCudaRender
renderer = MjCudaRender(84, 84, E)
envs = []
for e in range(E):
env = HalfCheetahEnv()
envs.append(env)
c = Convnet(6, output_activation=torch.tanh, input_channels=3)
if cuda:
c.cuda()
def step(stamp=True):
for e in range(E):
env = envs[e]
env.step(np.random.rand(6))
gt.stamp('step') if stamp else 0
if ingpu:
sims = [env.sim for env in envs]
env = envs[e]
tensor, img = renderer.get_batch_cuda_tensor(sims, False)
tensor = Variable(tensor).float()
gt.stamp('render') if stamp else 0
else:
imgs = []
for e in range(E):
env = envs[e]
img = env.sim.render(R, R, device_id=1)
imgs.append(img)
gt.stamp('render') if stamp else 0
imgs = np.array(imgs)
tensor = np_to_var(imgs)
if cuda:
tensor = tensor.cuda()
torch.cuda.synchronize()
gt.stamp('transfer') if stamp else 0
u = get_numpy(c.forward(tensor).cpu())
torch.cuda.synchronize()
gt.stamp('forward') if stamp else 0
# cv2.imshow("img", img)
# cv2.waitKey(1)
gt.stamp("start")
for i in range(10):
step(False)
gt.stamp("warmstart")
for i in gt.timed_for(range(N)):
step()
gt.stamp("end")
print(gt.report(include_itrs=False))
def run_train(self, loaders, info, optimizer=None):
'''
training loop
'''
# prepare dictionaries
loaders_train = dict(filter(lambda x: 'train' in x[0], loaders.items()))
assert len(set([len(loader) for loader in loaders_train.values()])) == 1
loaders_valid = dict(filter(lambda x: 'train' not in x[0], loaders.items()))
vocabs_in = {'{};{}'.format(
loader.dataset.name, loader.dataset.ann_type): loader.dataset.vocab_in
for loader in loaders.values()}
epoch_length = len(next(iter(loaders_train.values())))
# initialize summary writer for tensorboardX
self.summary_writer = SummaryWriter(log_dir=self.args.dout)
# dump config
with open(os.path.join(self.args.dout, 'config.json'), 'wt') as f:
json.dump(vars(self.args), f, indent=2)
# optimizer
optimizer, schedulers = model_util.create_optimizer_and_schedulers(
info['progress'], self.args, self.parameters(), optimizer)
# make sure that all train loaders have the same length
assert len(set([len(loader) for loader in loaders_train.values()])) == 1
model_util.save_log(
self.args.dout, progress=info['progress'], total=self.args.epochs,
stage='train', best_loss=info['best_loss'], iters=info['iters'])
# display dout
print("Saving to: %s" % self.args.dout)
for epoch in range(info['progress'], self.args.epochs):
print('Epoch {}/{}'.format(epoch, self.args.epochs))
self.train()
train_iterators = {
key: iter(loader) for key, loader in loaders_train.items()}
metrics = {key: collections.defaultdict(list) for key in loaders_train}
gt.reset()
for _ in tqdm(range(epoch_length), desc='train'):
# sample batches
batches = data_util.sample_batches(
train_iterators, self.args.device, self.pad, self.args)
gt.stamp('data fetching', unique=False)
# do the forward passes
model_outs, losses_train = {}, {}
for batch_name, (traj_data, input_dict, gt_dict) in batches.items():
model_outs[batch_name] = self.model.forward(
vocabs_in[batch_name.split(':')[-1]],
action=gt_dict['action'], **input_dict)
info['iters']['train'] += (
len(traj_data) if ':' not in batch_name else 0)
gt.stamp('forward pass', unique=False)
# compute losses
losses_train = self.model.compute_loss(
model_outs,
{key: gt_dict for key, (_, _, gt_dict) in batches.items()})
# do the gradient step
optimizer.zero_grad()
sum_loss = sum(
[sum(loss.values()) for name, loss in losses_train.items()])
sum_loss.backward()
optimizer.step()
gt.stamp('optimizer', unique=False)
# compute metrics
for dataset_name in losses_train.keys():
self.model.compute_metrics(
model_outs[dataset_name], batches[dataset_name][2],
metrics['train:' + dataset_name])
for key, value in losses_train[dataset_name].items():
metrics['train:' + dataset_name]['loss/' + key].append(
value.item())
metrics['train:' + dataset_name]['loss/total'].append(
sum_loss.detach().cpu().item())
gt.stamp('metrics', unique=False)
if self.args.profile:
print(gt.report(include_itrs=False, include_stats=False))
# compute metrics for train
print('Computing train and validation metrics...')
metrics = {data: {k: sum(v) / len(v) for k, v in metr.items()}
for data, metr in metrics.items()}
# compute metrics for valid_seen
for loader_id, loader in loaders_valid.items():
with torch.no_grad():
metrics[loader_id] = self.run_validation(
loader, vocabs_in[loader_id.split(':')[-1]],
loader_id, info['iters'])
stats = {'epoch': epoch, 'general': {
'learning_rate': optimizer.param_groups[0]['lr']}, **metrics}
# save the checkpoint
print('Saving models...')
model_util.save_model(
self, 'model_{:02d}.pth'.format(epoch), stats, optimizer=optimizer)
model_util.save_model(self, 'latest.pth', stats, symlink=True)
# write averaged stats
for loader_id in stats.keys():
if isinstance(stats[loader_id], dict):
for stat_key, stat_value in stats[loader_id].items():
# for comparison with old epxs, maybe remove later
summary_key = '{}/{}'.format(
loader_id.replace(':', '/').replace(
'lmdb/', '').replace(';lang', '').replace(';', '_'),
stat_key.replace(':', '/').replace('lmdb/', ''))
self.summary_writer.add_scalar(
summary_key, stat_value, info['iters']['train'])
# dump the training info
model_util.save_log(
self.args.dout, progress=epoch+1, total=self.args.epochs,
stage='train', best_loss=info['best_loss'], iters=info['iters'])
model_util.adjust_lr(optimizer, self.args, epoch, schedulers)
print('{} epochs are completed, all the models were saved to: {}'.format(
self.args.epochs, self.args.dout))
def train_epoch(epoch):
print('\nTRAINING : Epoch ' + str(epoch))
model.train()
losses = []
logs = []
sample_avgs = []
update_avgs = []
last_time = time.time()
metrics = Metrics(tok2i, i2tok, field=TRG)
trajectory_sampler = buffer.TrajectorySampler(trainloader)
n_updates = 0
oracle_samples_only = args.rollin_beta == 1.0
while n_updates < updates_per_epoch:
gt.reset()
if oracle_samples_only:
start = time.time()
trajectory = trajectory_sampler.get_oracle_trajectory(model, Oracle, oracle_flags=oracle_flags)
sample_time = (time.time() - start)
start = time.time()
loss = trajectory_sampler.get_loss(model, trajectory, loss_flags)
update_time = (time.time() - start)
else:
start = time.time()
loss = trajectory_sampler.get_mixed_trajectory_loss(model, Oracle,
oracle_flags=oracle_flags,
beta=args.rollin_beta,
loss_flags=loss_flags)
sample_time = 0
update_time = (time.time() - start)
optimizer.zero_grad()
loss.backward()
clip_grad_norm_(model.parameters(), args.max_norm)
losses.append(loss.item())
optimizer.step()
n_updates += 1
sample_avgs.append(sample_time)
update_avgs.append(update_time)
gt.stamp("buffer updates")
if n_updates % 20 == 0:
print("%d|%d\t%.3f\tSample: %.3fs\tUpdate: %.3fs" % (epoch, n_updates, round(np.mean(losses), 3), np.mean(sample_avgs), np.mean(update_avgs)))
log_tensorboard({'sample_avgs': np.mean(sample_avgs),
'update_avgs': np.mean(update_avgs)}, step=args.logstep)
sample_avgs = []
update_avgs = []
# -- Report metrics every `print_every` batches.
if n_updates % args.print_every == 0:
gt.stamp("report")
# Training report computed over the last `print_every` batches.
ms = metrics.report('train')
ms['train/loss'] = round(np.mean(losses), 2)
logs.append('{0} ; loss {1} ; sentence/s {2} ; {3} train {4} '.format(
epoch,
round(np.mean(losses), 2),
int(len(losses) * args.batch_size / (time.time() - last_time)),
args.eval_metric,
ms.get('train/%s' % args.eval_metric, 0.0),
))
args.logstep += 1
last_time = time.time()
losses = []
metrics.reset()
# -- Validation report with a single batch.
metrics.reset()
model.eval()
batch = next(iter(validloader))
scores, samples = predict_batch(batch)
model.train()
metrics.update(scores, samples, (batch.trg[0], None))
vms = metrics.report('valid_batch')
logs[-1] = logs[-1] + metrics.log(vms, 'valid_batch', ['bleu', 'avg_span', 'f1', 'em', 'depth_score'])
metrics.reset()
print_samples(samples, (batch.trg[0], None), n=len(batch))
gt.stamp("validation_batch")
log_tensorboard(ms, step=args.logstep)
log_tensorboard(vms, step=args.logstep)
print(logs[-1])
print(gt.report(include_itrs=False, format_options={'itr_name_width': 30}))
# -- Checkpointing
if n_updates % args.save_every == 0:
print('saving checkpoint at epoch {0} batch {1}'.format(epoch, i))
print(os.path.join(args.log_directory, args.expr_name + '.checkpoint'))
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'optimizer_param': args.optimizer,
'loss': loss.item()
}, os.path.join(args.log_directory, args.expr_name + '.checkpoint'))
model_config.longest_label = model.longest_label
with open(os.path.join(args.log_directory, 'model_config.pkl'), 'wb') as f:
pickle.dump(model_config, f)
print('end : epoch {0} '.format(epoch))
log_tensorboard({'lr': optimizer.param_groups[0]['lr']}, step=args.logstep)
def experiment(variant):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
n_proc = comm.Get_size()
root = 0
gpus = GPUtil.getGPUs()
n_gpu = len(gpus)
torch.distributed.init_process_group(backend='mpi', world_size=n_proc)
E = 20
R = 84
U = 6
cuda = True
envs = []
for e in range(rank, E, n_proc):
env = HalfCheetahEnv()
envs.append(env)
sendcounts = np.array(comm.gather(len(envs), root))
i_sendcounts = None
u_sendcounts = None
if rank == root:
i_sendcounts = sendcounts * 3 * R * R
u_sendcounts = sendcounts * U
c = Convnet(6, output_activation=torch.tanh, input_channels=3)
c = torch.nn.parallel.DistributedDataParallel(c)
if cuda:
c.cuda()
# viewer = mujoco_py.MjRenderContextOffscreen(env.sim, device_id=1)
# env.sim.add_render_context(viewer)
def step(i, stamp=True):
imgs = []
if i % 100 == 0:
for e in envs:
e.reset()
for e in envs:
img = e.sim.render(R, R, device_id=rank % n_gpu).transpose()
imgs.append(img)
comm.Barrier()
if rank == 0:
gt.stamp('render') if stamp else 0
imgs = np.array(imgs)
r_imgs = None
if rank == 0:
r_imgs = np.empty((E, 3, R, R), dtype='uint8')
comm.Gatherv(sendbuf=imgs, recvbuf=(r_imgs, i_sendcounts), root=root)
if rank == 0:
gt.stamp('comm1') if stamp else 0
u = None
if rank == 0:
torch_img = np_to_var(r_imgs)
if cuda:
torch_img = torch_img.cuda()
torch.cuda.synchronize()
gt.stamp('transfer') if stamp else 0
u = get_numpy(c.forward(torch_img).cpu())
torch.cuda.synchronize()
gt.stamp('forward') if stamp else 0
r_u = np.empty((len(envs), U), dtype='float32')
comm.Scatterv(sendbuf=(u, u_sendcounts), recvbuf=r_u, root=root)
if rank == 0:
gt.stamp('comm2') if stamp else 0
for i, e in enumerate(envs):
e.step(r_u[i, :])
comm.Barrier()
if rank == 0:
gt.stamp('step') if stamp else 0
for i in range(10):
step(i, False)
if rank == 0:
gt.stamp('start')
for i in gt.timed_for(range(100)):
step(i)
if rank == 0:
gt.stamp('end')
print(
gt.report(include_itrs=False,
format_options=dict(itr_num_width=10)))
envs[e].reset()
for e in range(E):
img = envs[e].sim.render(R, R, device_id=1).transpose()
imgs.append(img)
gt.stamp('render') if stamp else 0
imgs = np.array(imgs)
torch_img = np_to_var(imgs)
if cuda:
torch_img = torch_img.cuda()
torch.cuda.synchronize()
gt.stamp('transfer') if stamp else 0
u = get_numpy(c.forward(torch_img).cpu())
torch.cuda.synchronize()
gt.stamp('forward') if stamp else 0
for e in range(E):
env.step(u[e, :])
gt.stamp('step') if stamp else 0
for i in range(10):
step(False)
gt.stamp('start')
for i in gt.timed_for(range(100)):
step()
gt.stamp('end')
print(gt.report(include_itrs=False, format_options=dict(itr_num_width=10)))
def train_epoch(epoch):
print('\nTRAINING : Epoch ' + str(epoch))
model.train()
losses = []
logs = []
last_time = time.time()
metrics = Metrics(tok2i, i2tok)
for i, data in enumerate(trainloader, 0):
# -- Actual Training
gt.reset()
xs, annots = data
xs = xs.to(args.device)
gt.stamp("load_data")
oracle = Oracle(xs, model.n_classes, tok2i, i2tok, **oracle_flags)
gt.stamp("create_oracle")
max_steps = 2*xs.ne(tok2i['<p>']).sum(1).max()+1
scores, samples, p_oracle = model.forward(num_samples=args.batch_size, oracle=oracle, max_steps=max_steps, return_p_oracle=True)
gt.stamp("forward")
loss = loss_fn(scores, samples, p_oracle, tok2i['<end>'], **loss_flags)
gt.stamp("loss")
optimizer.zero_grad()
loss.backward()
clip_grad_norm_(model.parameters(), args.max_norm)
optimizer.step()
gt.stamp("backward")
losses.append(loss.item())
# -- Report metrics every `print_every` batches.
if i % args.print_every == 0:
# Training report; loss averaged over the last `print_every` batches.
metrics.update(scores, samples, data)
gt.stamp("metrics.update")
ms = metrics.report('train')
ms['train/loss'] = round(np.mean(losses), 2)
logs.append('{0} ; loss {1} ; sentence/s {2} ; f1 train {3} '.format(
i+1,
round(np.mean(losses), 2),
int(len(losses) * args.batch_size / (time.time() - last_time)),
0,
))
args.logstep += 1
last_time = time.time()
losses = []
metrics.reset()
scores, samples = predict_batch(data)
print_samples(samples, data)
gt.stamp("validation_batch")
log_tensorboard(ms, step=args.logstep)
print(logs[-1])
print(gt.report(include_itrs=False, format_options={'itr_name_width': 30}))
# -- Checkpointing
if i % args.save_every == 0:
print('saving checkpoint at epoch {0} batch {1}'.format(epoch, i))
torch.save(model.state_dict(), os.path.join(args.log_directory, args.expr_name + '.checkpoint'))
model_config['longest_label'] = model.longest_label
with open(os.path.join(args.log_directory, 'model_config.json'), 'w') as f:
json.dump(model_config, f)
print('end : epoch {0} '.format(epoch))
log_tensorboard({'lr': optimizer.param_groups[0]['lr']}, step=args.logstep)
import gtimer as gt
import test1_b
import time
time.sleep(0.1)
gt.stamp('first')
time.sleep(0.2)
gt.stamp('second')
loop = gt.timed_loop('loop')
for i in [1, 2, 3]:
next(loop)
test1_b.func()
test1_b.func()
time.sleep(0.1)
gt.stamp('loop1')
loop.exit()
time.sleep(0.1)
gt.stamp('third')
x = gt.save_pkl('test.pkl')
y = gt.load_pkl('test.pkl')
print(gt.report(y))
