net.debug = args.debug_mode
else:
raise NotImplementedError()
if not os.path.exists(args.save_path):
os.makedirs(args.save_path)
snapshot_pfx = 'snapshot.' + ".".join([key.upper()+str(config[key]) for key in config.keys() if key[0] != 'f'])
if args.out_file == '':
out_file = os.path.join(args.save_path, snapshot_pfx + '.tst-eval.json')
else:
print("will dump prediction to {}".format(args.out_file))
out_file = args.out_file
print("="*20)
print("Starting testing {} model".format(config['model']))
print("Snapshots {}.*\nDetails:".format(os.path.join(args.save_path, snapshot_pfx)))
for key in config:
print("{} : {}".format(key,config[key]))
print("="*20)
start_time = time.time()
print("startup time for {} model: {:5.3f} for {} parameters".format(config['model'],start_time - start, get_n_params(net)))
best_val , val_rate = evaluate(net, dev, CNN, config, '', verbose = True, no_lang = args.no_lang)
tst_score, tst_rate = evaluate(net, tst, CNN, config, '', verbose = True,
tst_json = tst_json,
out_file = out_file,
no_lang = args.no_lang)
print("model scores based on best validation accuracy\nval_acc:{:5.3f} test_acc: {:5.3f} test speed {:5.1f} inst/sec\n".format(best_val,tst_score,tst_rate))
criterion = nn.NLLLoss()
elif config['loss'] == 'smargin':
criterion = nn.MultiLabelSoftMarginLoss()
elif config['loss'] == 'lamm':
criterion = lamm
elif config['loss'] == 'mbr':
criterion = mbr
else:
raise NotImplementedError()
start_time = time.time()
if args.verbose:
print(
"startup time for {} model: {:5.3f} for {} instances for {} parameters"
.format(config['model'], start_time - start, len(indexes),
get_n_params(net)))
best_val = 0
timeout = False
for ITER in range(args.epochs):
net.train()
random.shuffle(indexes)
closs = 0.0
cinst = 0
correct = 0.0
trn_start = time.time()
if args.verbose and not args.debug_mode:
pbar = tqdm(indexes, desc='trn_loss')
else:
pbar = indexes
optimizer.zero_grad()
if config['loss'] == 'nll':
criterion = nn.NLLLoss()
elif config['loss'] == 'smargin':
criterion = nn.MultiLabelSoftMarginLoss()
elif config['loss'] == 'lamm':
criterion = lamm
elif config['loss'] == 'mbr':
criterion = mbr
else:
raise NotImplementedError()
start_time = time.time()
if args.verbose:
print("startup time for {} model: {:5.3f} for {} instances for {} parameters".format(config['model'],start_time - start, len(indexes), get_n_params(net)))
best_val = 0
timeout = False
for ITER in range(args.epochs):
net.train()
random.shuffle(indexes)
closs = 0.0
cinst = 0
correct = 0.0
trn_start = time.time()
if args.verbose and not args.debug_mode:
pbar = tqdm(indexes, desc='trn_loss')
else:
pbar = indexes
if args.out_file == '':
out_file = os.path.join(args.save_path, snapshot_pfx + '.tst-eval.json')
else:
print("will dump prediction to {}".format(args.out_file))
out_file = args.out_file
print("=" * 20)
print("Starting testing {} model".format(config['model']))
print("Snapshots {}.*\nDetails:".format(
os.path.join(args.save_path, snapshot_pfx)))
for key in config:
print("{} : {}".format(key, config[key]))
print("=" * 20)
start_time = time.time()
print("startup time for {} model: {:5.3f} for {} parameters".format(
config['model'], start_time - start, get_n_params(net)))
best_val, val_rate = evaluate(net, dev, CNN, config, '', verbose=True)
tst_score, tst_rate = evaluate(net,
tst,
CNN,
config,
'',
verbose=True,
tst_json=tst_json,
out_file=out_file)
print(
"model scores based on best validation accuracy\nval_acc:{:5.3f} test_acc: {:5.3f} test speed {:5.1f} inst/sec\n"
.format(best_val, tst_score, tst_rate))