def on_end_epoch(hook_state, state):
if val_loader is not None:
if 'best_loss' not in hook_state:
hook_state['best_loss'] = np.inf
if 'wait' not in hook_state:
hook_state['wait'] = 0
if val_loader is not None:
model_utils.evaluate(state['model'],
val_loader,
meters['val'],
desc="Epoch {:d} valid".format(
state['epoch']))
#state['model'].propose_new()
#model_utils.evaluate(state['model'],
# val_loader,
# meters['val_new'],
# desc="Epoch {:d} valid (new struct)".format(state['epoch']))
meter_vals = log_utils.extract_meter_values(meters)
#if meter_vals['val']['loss'] < meter_vals['val_new']['loss']:
# state['model'].reject_new()
print("Epoch {:02d}: {:s}".format(
state['epoch'], log_utils.render_meter_values(meter_vals)))
meter_vals['epoch'] = state['epoch']
with open(trace_file, 'a') as f:
json.dump(meter_vals, f)
f.write('\n')
if val_loader is not None:
if meter_vals['val']['loss'] < hook_state['best_loss']:
hook_state['best_loss'] = meter_vals['val']['loss']
print(
"==> best model (loss = {:0.6f}), saving model...".format(
hook_state['best_loss']))
state['model'].cpu()
torch.save(
state['model'].state_dict(),
os.path.join(opt['log.exp_dir'],
'best_model_state_dict.pt'))
if opt['data.cuda']:
state['model'].cuda()
hook_state['wait'] = 0
else:
hook_state['wait'] += 1
if hook_state['wait'] > opt['train.patience']:
print("==> patience {:d} exceeded".format(
opt['train.patience']))
state['stop'] = True
else:
state['model'].cpu()
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.pt'))
if opt['data.cuda']:
state['model'].cuda()
def on_end_epoch(hook_state, state):
if val_loader is not None:
if 'best_loss' not in hook_state:
hook_state['best_loss'] = np.inf
if 'wait' not in hook_state:
hook_state['wait'] = 0
if val_loader is not None:
state['loader'].mode = 'val' # added
model_utils.evaluate(
state['model'],
state['loader'], #val_loader,
meters['val'],
desc="Epoch {:d} valid".format(state['epoch']))
meter_vals = log_utils.extract_meter_values(meters)
print("Epoch {:02d}: {:s}".format(
state['epoch'], log_utils.render_meter_values(meter_vals)))
meter_vals['epoch'] = state['epoch']
# this prevents cuda bugs
meter_vals = {
k: ({kk: float(vv)
for kk, vv in v.items()} if isinstance(v, dict) else v)
for k, v in meter_vals.items()
}
with open(trace_file, 'a') as f:
json.dump(meter_vals, f)
f.write('\n')
if val_loader is not None:
if meter_vals['val']['loss'] < hook_state['best_loss']:
hook_state['best_loss'] = meter_vals['val']['loss']
print(
"==> best model (loss = {:0.6f}), saving model...".format(
hook_state['best_loss']))
state['model'].cpu()
# used with inception
#torch.save(state['model'].encoder.added_layers, os.path.join(opt['log.exp_dir'], 'best_model.t7'))
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.t7'))
if opt['data.cuda']:
state['model'].cuda()
hook_state['wait'] = 0
else:
hook_state['wait'] += 1
if hook_state['wait'] > opt['train.patience']:
print("==> patience {:d} exceeded".format(
opt['train.patience']))
state['stop'] = True
else:
state['model'].cpu()
# used with inception
#torch.save(state['model'].encoder.added_layers, os.path.join(opt['log.exp_dir'], 'best_model.t7'))
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.t7'))
if opt['data.cuda']:
state['model'].cuda()
def on_update(state):
for field, meter in meters['train'].items():
meter.add(state['output'][field])
if state['batch'] % 5 == 0:
meter_vals = log_utils.extract_meter_values(meters)
print("Epoch {:02d}: {:s}".format(
state['epoch'], log_utils.render_meter_values(meter_vals)))
def on_end_epoch(hook_state, state):
if val_loader is not None:
if 'best_loss' not in hook_state:
hook_state['best_loss'] = np.inf
if 'wait' not in hook_state:
hook_state['wait'] = 0
if val_loader is not None:
model_utils.evaluate(state['model'],
val_loader,
meters['val'],
desc="Epoch {:d} valid".format(state['epoch']))
meter_vals = log_utils.extract_meter_values(meters)
print("Epoch {:02d}: {:s}".format(state['epoch'], log_utils.render_meter_values(meter_vals)))
meter_vals['epoch'] = state['epoch']
with open(trace_file, 'a') as f:
json.dump(meter_vals, f)
f.write('\n')
if state['epoch'] == 1 or state['epoch'] % 10 == 0:
state['model'].cpu()
torch.save(state['model'], os.path.join(opt['log.exp_dir'], f'epoch_{state["epoch"]}.pt'))
torch.save(state['optimizer'].state_dict(),
os.path.join(opt['log.exp_dir'], f'epoch_{state["epoch"]}_optim.pt'))
# save everything including model state into log dir
# torch.save(state, os.path.join(opt['log.exp_dir'], f'epoch_{state["epoch"]}.pt.tar'))
if opt['data.cuda']:
state['model'].cuda()
if val_loader is not None:
if meter_vals['val']['loss'] < hook_state['best_loss']:
hook_state['best_loss'] = meter_vals['val']['loss']
print("==> best model (loss = {:0.6f}), saving model...".format(hook_state['best_loss']))
state['model'].cpu()
torch.save(state['model'], os.path.join(opt['log.exp_dir'], 'best_model.pt'))
torch.save(state['optimizer'].state_dict(), os.path.join(opt['log.exp_dir'], 'best_model_optim.pt'))
# save everything including model state into log dir
# torch.save(state, os.path.join(opt['log.exp_dir'], 'best_model.pt.tar'))
if opt['data.cuda']:
state['model'].cuda()
hook_state['wait'] = 0
else:
hook_state['wait'] += 1
if hook_state['wait'] > opt['train.patience']:
print("==> patience {:d} exceeded".format(opt['train.patience']))
state['stop'] = True
else:
state['model'].cpu()
torch.save(state['model'], os.path.join(opt['log.exp_dir'], 'best_model.pt'))
if opt['data.cuda']:
state['model'].cuda()
def on_end_epoch(hook_state, state):
if val_loader is not None:
if 'best_loss' not in hook_state:
hook_state['best_loss'] = 0
if 'wait' not in hook_state:
hook_state['wait'] = 0
if val_loader is not None:
with torch.no_grad():
model_utils.evaluate(state['model'],
val_loader,
meters['val'],
opt['model.stage'],
desc="Epoch {:d} valid".format(
state['epoch']))
meter_vals = log_utils.extract_meter_values(meters)
print("Epoch {:02d}: {:s}".format(
state['epoch'], log_utils.render_meter_values(meter_vals)))
meter_vals['epoch'] = state['epoch']
with open(trace_file, 'a') as f:
json.dump(meter_vals, f)
f.write('\n')
if val_loader is not None:
if meter_vals['val']['acc'] > hook_state['best_loss']:
hook_state['best_loss'] = meter_vals['val']['acc']
print("==> best model (acc = {:0.6f}), saving model...".format(
hook_state['best_loss']))
state['model'].cpu()
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.pt'))
if opt['data.cuda']:
state['model'].cuda()
hook_state['wait'] = 0
else:
hook_state['wait'] += 1
if hook_state['wait'] > opt['train.patience']:
print("==> patience {:d} exceeded".format(
opt['train.patience']))
state['stop'] = True
else:
state['model'].cpu()
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.pt'))
if opt['data.cuda']:
state['model'].cuda()
def on_end_epoch(hook_state, state):
if val_loader is not None:
if 'best_loss' not in hook_state:
hook_state['best_loss'] = np.inf
if 'wait' not in hook_state:
hook_state['wait'] = 0
if val_loader is not None:
model_utils.evaluate(state['model'],
val_loader,
meters['val'],
desc="Epoch {:d} valid".format(
state['epoch']))
meter_vals = log_utils.extract_meter_values(meters)
print("Epoch {:02d}: {:s}".format(
state['epoch'], log_utils.render_meter_values(meter_vals)))
meter_vals['epoch'] = state['epoch']
with open(trace_file, 'a') as f:
json.dump(meter_vals, f)
f.write('\n')
tensorboard.add_scalar("Value MAP", meter_vals['val']['map'],
state['epoch'])
tensorboard.add_scalar("Value Loss", meter_vals['val']['loss'],
state['epoch'])
tensorboard.add_scalar("Value Accuracy", meter_vals['val']['acc'],
state['epoch'])
tensorboard.add_scalar("Train MAP", meter_vals['train']['map'],
state['epoch'])
tensorboard.add_scalar("Train Loss", meter_vals['train']['loss'],
state['epoch'])
tensorboard.add_scalar("Train Accuracy", meter_vals['train']['acc'],
state['epoch'])
tensorboard.add_scalar("Best Loss", hook_state['best_loss'],
state['epoch'])
if val_loader is not None:
if meter_vals['val']['loss'] < hook_state['best_loss']:
hook_state['best_loss'] = meter_vals['val']['loss']
print(
"==> best model (loss = {:0.6f}), saving model...".format(
hook_state['best_loss']))
state['model'].cpu()
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.pt'))
if opt['data.cuda']:
state['model'].cuda()
hook_state['wait'] = 0
else:
hook_state['wait'] += 1
if hook_state['wait'] > opt['train.patience']:
print("==> patience {:d} exceeded".format(
opt['train.patience']))
state['stop'] = True
else:
state['model'].cpu()
torch.save(state['model'],
os.path.join(opt['log.exp_dir'], 'best_model.pt'))
if opt['data.cuda']:
state['model'].cuda()
