def __init__(self, folderpath, create_if_notexists=False):
self.folderpath = folderpath
self.key_to_filename = {}
self.key_to_foldername = {}
self.key_to_memo = {}
tb_fs.create_folder(folderpath,
abort_if_exists=False,
create_parent_folders=create_if_notexists)
# initialize the memo based on the state of the memo folder:
for p in tb_fs.list_files(folderpath):
name_with_ext = tb_fs.path_last_element(p)
# for the files.
if name_with_ext.startswith(
'file_config-') and name_with_ext.endswith('.json'):
name = name_with_ext[len('file_config-'):-len('.json')]
config = tb_io.read_jsonfile(p)
key = self._key_from_config(config)
self.key_to_filename[key] = name
# for the sub-memos.
elif name_with_ext.startswith(
'memo_config-') and name_with_ext.endswith('.json'):
name = name_with_ext[len('memo_config-'):-len('.json')]
config = tb_io.read_jsonfile(p)
key = self._key_from_config(config)
self.key_to_foldername[key] = name
def _fn(e_folderpath):
cfg = tb_io.read_jsonfile(
tb_fs.join_paths([e_folderpath, 'config.json']))
res = None
if not use_checkpoints:
res_fpath = tb_fs.join_paths([e_folderpath, 'results.json'])
else:
res_fpath = tb_fs.join_paths([e_folderpath, 'checkpoint.json'])
if tb_fs.file_exists(res_fpath):
res = tb_io.read_jsonfile(res_fpath)
return (cfg, res)
def get_configs(self):
cfgs = []
for filename in self.key_to_filename.itervalues():
d = tb_io.read_jsonfile(
self._get_filepath('config', filename, 'json'))
cfgs.append(d)
return cfgs
def get_memo_configs(self):
lst = []
for foldername in itervalues(self.key_to_foldername):
filepath = self._get_memo_paths(foldername)[0]
cfg = tb_io.read_jsonfile(filepath)
lst.append(cfg)
return lst
def get_file_configs(self):
lst = []
for filename in itervalues(self.key_to_filename):
filepath = self._get_filepath("file_config", filename, "json")
cfg = tb_io.read_jsonfile(filepath)
lst.append(cfg)
return lst
def load(self, name, x):
cfg = self.name_to_cfg[name]
filepath = self._get_filepath(name, cfg['use_json'])
if tb_fs.file_exists(filepath):
if cfg['use_json']:
out = tb_io.read_jsonfile(filepath)
else:
out = tb_io.read_picklefile(filepath)
x = cfg['load_fn'](x, out)
return x
def _fn(cfg_path):
ds = []
for name in json_filename_lst:
p = tb_fs.join_paths([cfg_path, name])
if (not abort_if_notexists) and (not tb_fs.file_exists(p)):
d = None
# if abort if it does not exist, it is going to fail reading the file.
else:
d = tb_io.read_jsonfile(p)
ds.append(d)
return ds
def show_plots(xs,
key,
label_fn=None,
xlabel=None,
ylabel=None,
show_legend=True,
ref_line=None,
filepath=None):
for x in xs:
print(x)
y_lst = []
for i in range(num_repeats):
r = tb_io.read_jsonfile("out/cfg_r%d_%d/checkpoint.json" % (i, x))
c = tb_io.read_jsonfile_with_overlays("configs/cfg_r%d_%d.json" %
(i, x))
# assumes that for the repeats is always the same,
if label_fn is not None:
label = label_fn(c, r, x)
else:
label = str(x)
y_lst.append(r[key])
ys = np.stack(y_lst)
# plt.plot(np.arange(1, len(r[key]) + 1), np.mean(ys, axis=0), label=label)
plt.errorbar(np.arange(1,
len(r[key]) + 1),
np.mean(ys, axis=0),
yerr=np.std(ys, axis=0),
label=label)
if ref_line is not None:
plt.axhline(ref_line, color='k', linestyle='--')
# labelling the plot
if xlabel is not None:
plt.xlabel(xlabel)
if ylabel is not None:
plt.ylabel(ylabel)
if show_legend:
plt.legend()
if filepath != None:
plt.savefig(filepath, bbox_inches='tight')
if show_plot:
plt.show()
plt.close()
def delete_conditionally(self, delete_cond_fn):
del_lst = []
for filename in self.key_to_filename.itervalues():
config_filepath = self._get_filepath('config', filename, 'json')
config = tb_io.read_jsonfile(config_filepath)
if delete_cond_fn(config):
value_filepath = self._get_filepath('value', filename, 'pkl')
tb_fs.delete_file(config_filepath)
tb_fs.delete_file(value_filepath)
del_lst.append(config)
# remove the configs from the dictionary.
for config in del_lst:
key = self._key_from_config(config)
self.key_to_filename.pop(key)
def __init__(self, folderpath, create_if_notexists=False):
self.folderpath = folderpath
self.key_to_filename = {}
tb_fs.create_folder(folderpath,
abort_if_exists=False,
create_parent_folders=create_if_notexists)
# initialize the memo based on the state of the folder.
for fpath in tb_fs.list_files(folderpath):
fname_with_ext = tb_fs.path_last_element(fpath)
if fname_with_ext.startswith(
'config-') and fname_with_ext.endswith('.json'):
fname = fname_with_ext[len('config-'):-len('.json')]
config = tb_io.read_jsonfile(fpath)
key = self._key_from_config(config)
self.key_to_filename[key] = fname
def _read(self, filepath):
return tb_io.read_jsonfile(filepath)
def get_config():
cmd = CommandLineArgs()
cmd.add('config_filepath', 'str')
out = cmd.parse()
cfg = tb_io.read_jsonfile(out['config_filepath'])
return cfg
servername=servername,
username=username,
only_transfer_newer_files=True)
name_to_cmd['run_on_server'] = lambda: tb_re.run_on_server(
bash_command='sbatch run.sh',
servername=servername,
username=username,
folderpath=remote_folderpath,
wait_for_output=True,
prompt_for_password=False)
name_to_cmd['sync_and_run_on_server'] = lambda: (
write_server_run_script(), name_to_cmd['sync_to_server']
(), name_to_cmd['sync_from_server'](), name_to_cmd['run_on_server']())
if __name__ == '__main__':
d = tb_io.read_jsonfile('./interact_config.json')
parser = argparse.ArgumentParser()
parser.add_argument('cmd', type=str, choices=name_to_cmd.keys())
parser.add_argument('server', type=str, choices=d['servers'].keys())
parser.add_argument('job', type=str, choices=d['jobs'].keys())
args = parser.parse_args()
servertype = args.server
jobtype = args.job
local_folderpath = d['local_folderpath']
servername = d['servers'][args.server]['servername']
username = d['servers'][args.server]['username']
remote_folderpath = d['servers'][args.server]['remote_folderpath']
main_relfilepath = d['jobs'][args.job]['main_relfilepath']
def train_model_with_config():
import research_toolbox.tb_logging as tb_lg
if cfg["optimizer_type"] == "sgd":
trainer = dy.SimpleSGDTrainer(m, cfg["step_size_start"])
elif cfg["optimizer_type"] == "adam":
trainer = dy.AdamTrainer(m, cfg["step_size_start"])
elif cfg["optimizer_type"] == "sgd_mom":
trainer = dy.MomentumSGDTrainer(m, cfg["step_size_start"])
else:
raise ValueError
trainer.set_sparse_updates(0)
# restarting from a checkpoint if it exists.
# optimizer state is not kept.
ckpt_filepath = cfg["out_folder"] + "/checkpoint.json"
if tb_fs.file_exists(ckpt_filepath):
log_d = tb_io.read_jsonfile(ckpt_filepath)
current_epoch = len(log_d["dev_acc"])
best_dev_acc = np.max(log_d["dev_acc"])
m.populate(cfg["out_folder"] + '/model.ckpt')
else:
current_epoch = 0
best_dev_acc = 0.0
log_d = {
'dev_acc': [],
'avg_loss': [],
'train_tks/sec': [],
'eval_tks/sec': [],
'secs_per_epoch': [],
"lr": []
}
if cfg["debug"] or cfg["compute_train_acc"]:
log_d["train_acc"] = []
if cfg["loss_type"] == "log_neighbors":
loss_fn = loss_log_neighbors
elif cfg["loss_type"] == "log_beam":
loss_fn = loss_log_beam
elif cfg["loss_type"] == "cost_sensitive_margin_last":
loss_fn = loss_cost_sensitive_margin_last
elif cfg["loss_type"] == "margin_last":
loss_fn = loss_margin_last
elif cfg["loss_type"] == "perceptron_first":
loss_fn = loss_perceptron_first
elif cfg["loss_type"] == "perceptron_last":
loss_fn = loss_perceptron_last
elif cfg["loss_type"] == "upper_bound":
loss_fn = loss_upper_bound
else:
raise ValueError
cfg_accuracy = lambda data: beam_accuracy(data, cfg["beam_size"])
cfg_train_graph = lambda e: train_beam_graph(e, cfg["beam_size"], cfg[
"traj_type"], loss_fn)
for epoch in range(current_epoch, cfg["num_epochs"]):
if cfg["step_size_schedule_type"] == 'fixed':
lr = cfg["step_size_start"]
elif cfg["step_size_schedule_type"] == 'cosine':
lr = cosine_get_lr(cfg["step_size_start"], cfg["step_size_end"],
cfg["num_epochs"], epoch)
else:
raise ValueError
log_d['lr'].append(lr)
trainer.learning_rate = lr
acc_loss = 0.0
random.shuffle(train_data)
epoch_timer = tb_lg.TimeTracker()
train_timer = tb_lg.TimeTracker()
for i, e in enumerate(train_data):
if i % cfg["print_every_num_examples"] == 0 and i > 0:
print "Epoch %d - Example %d/%d" % (epoch, i, len(train_data))
loss = cfg_train_graph(e)
acc_loss += loss.value()
loss.backward()
trainer.update()
log_d["avg_loss"].append(acc_loss / len(train_data))
log_d["train_tks/sec"].append(num_train_tokens /
train_timer.time_since_start())
eval_timer = tb_lg.TimeTracker()
# log_d['train_acc'].append(accuracy(train_data))
log_d['dev_acc'].append(cfg_accuracy(dev_data))
# log_d['test_acc'].append(accuracy(test_data))
log_d['eval_tks/sec'].append(( #len(train_data) +
num_dev_tokens
# + num_test_tokens
) / eval_timer.time_since_start())
log_d["secs_per_epoch"].append(epoch_timer.time_since_start())
if cfg["debug"] or cfg["compute_train_acc"]:
train_acc = cfg_accuracy(train_data)
print "train_acc: ", train_acc
log_d["train_acc"].append(train_acc)
pprint({k: vs[-1] for k, vs in log_d.iteritems()})
if best_dev_acc < log_d["dev_acc"][-1]:
best_dev_acc = log_d["dev_acc"][-1]
m.save(cfg["out_folder"] + '/best_model.ckpt')
tb_io.write_jsonfile(log_d, cfg["out_folder"] + "/checkpoint.json")
m.save(cfg["out_folder"] + '/model.ckpt')
results_filepath = cfg["out_folder"] + "/results.json"
if not tb_fs.file_exists(results_filepath):
m.populate(cfg["out_folder"] + '/best_model.ckpt')
log_d['test_acc'] = cfg_accuracy(test_data)
tb_io.write_jsonfile(log_d, cfg["out_folder"] + "/results.json")
def read_results(n):
return tb_io.read_jsonfile("out/cfg%s/checkpoint.json" % n)
for k1 in ["vaswani", "lm"]:
print
for k2 in ["reset", "continue", "stop"]:
print " & ".join([
"$%0.2f_{%0.2f}$" % (round(100.0 * np.mean(k2r[(k1, k2, k3)]), 2),
round(100.0 * np.std(k2r[(k1, k2, k3)]), 2))
for k3 in [1, 2, 4, 8]
])
# %%
k2r = {}
for i in range(num_repeats):
for x in range(2000, 2040):
r = tb_io.read_jsonfile("out/cfg_r%d_%d/checkpoint.json" % (i, x))
c = tb_io.read_jsonfile_with_overlays("configs/cfg_r%d_%d.json" %
(i, x))
k = (c["model_type"], c["traj_type"], c['beam_size'])
if k not in k2r:
k2r[k] = []
k2r[k].append(np.max(r["dev_acc"]))
# generate the table.
for k1 in ["vaswani", "lm"]:
print
for k2 in ["oracle", "reset", "reset_multiple", "continue", "stop"]:
print " & ".join([
"$%0.2f_{%0.2f}$" % (round(100.0 * np.mean(k2r[(k1, k2, k3)]), 2),
round(100.0 * np.std(k2r[(k1, k2, k3)]), 2))
