def save_eval_file(opt, stats, eval_type="losses", split="dev", ext="pickle"):
if cfg.test_save:
name = "{}/{}.{}".format(
utils.make_name(opt,
prefix="garbage/{}/".format(eval_type),
is_dir=True,
eval_=True), split, ext)
else:
name = "{}/{}.{}".format(
utils.make_name(opt,
prefix="results/{}/".format(eval_type),
is_dir=True,
eval_=True), split, ext)
print("Saving {} {} to {}".format(split, eval_type, name))
if ext == "pickle":
with open(name, "wb") as f:
pickle.dump(stats, f)
elif ext == "txt":
with open(name, "w") as f:
f.write(stats)
elif ext == "json":
with open(name, "w") as f:
json.dump(stats, f)
else:
raise
def set_logger(self):
if cfg.toy:
self.logger = SummaryWriter(utils.make_name(
self.opt, prefix="garbage/logs/", eval_=True, do_epoch=False))
else:
self.logger = SummaryWriter(utils.make_name(
self.opt, prefix="logs/", eval_=True, do_epoch=False))
print("Logging Tensorboard Files at: {}".format(self.logger.logdir))
def save_step(model, vocab, optimizer, opt, length, lrs):
if cfg.test_save:
name = "{}.pickle".format(utils.make_name(
opt, prefix="garbage/models/", is_dir=False, eval_=True))
else:
name = "{}.pickle".format(utils.make_name(
opt, prefix="models/", is_dir=False, eval_=True))
save_checkpoint({
"epoch": length, "state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(), "opt": opt,
"vocab": vocab, "epoch_learning_rates": lrs},
name)
def save_eval_losses(opt, eval_losses, split="dev", ln="losses"):
if eval_losses:
name = "{}/{}.pickle".format(utils.make_name(
opt, prefix="results/{}/".format(ln), is_dir=True), split)
print "Saving evaluation losses to: {}".format(name)
with open(name, "w") as f:
pickle.dump(eval_losses, f)
def save_step(model, optimizer, opt, length):
name = "{}.pickle".format(utils.make_name(
opt, prefix="models/", is_dir=False))
save_checkpoint({
'epoch': length, 'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(), "opt": opt},
name)
print "Saving to: {}".format(name)
def do_class_eval_run(data_loader,
opt,
model,
gpu,
eval_losses,
l,
split="dev",
fn="class"):
# Run model on development set
print("Evaluating {}".format(split.capitalize()))
dev_loss, scores, counts, answers = evaluate.eval_function[fn](opt, model,
data_loader,
gpu, split)
# Print loss to stdout
print("{} Loss: \t {}".format(split.capitalize(),
dev_loss[opt.granularity]))
# Record loss
eval_losses[l] = dev_loss
# Save Recall, Precision, F1 scores computed
name = "{}/{}.scores".format(
utils.make_name(opt, prefix="results/scores/", is_dir=True,
eval_=True), split)
with open(name, "w") as f:
pickle.dump({"scores": scores, "counts": counts}, f)
# Save individual answers
name = "{}/{}.scores".format(
utils.make_name(opt,
prefix="results/answers/",
is_dir=True,
eval_=True), split)
with open(name, "w") as f:
pickle.dump(answers, f)
return eval_losses, scores, counts
def main(num):
# Generate configuration files depending on experiment being run
utils.generate_config_files("atomic", num)
# Loads the correct configuration file
config_file = "config/atomic/config_{}.json".format(num)
print(config_file)
# Read config file to option
config = cfg.read_config(cfg.load_config(config_file))
opt, meta = cfg.get_parameters(config)
# Set the random seeds
torch.manual_seed(opt.train.static.seed)
random.seed(opt.train.static.seed)
if config.gpu_mode:
torch.cuda.manual_seed_all(opt.train.static.seed)
# Where to find the data
splits = ["train", "dev", "test"]
opt.train.dynamic.epoch = 0
print("Loading Data")
categories = opt.data.categories
path = "data/atomic/processed/{}/{}.pickle".format(
opt.exp, utils.make_name_string(opt.data))
data_loader = data.make_data_loader(opt, categories)
loaded = data_loader.load_data(path)
print(data_loader.sequences["train"]["total"].size(0))
data_loader.opt = opt
data_loader.batch_size = opt.train.dynamic.bs
print("Done.")
# Initialize text_encoder
text_encoder = TextEncoder(config.encoder_path, config.bpe_path)
special = [data.start_token, data.end_token]
special += ["<{}>".format(cat) for cat in categories]
special += [data.blank_token]
text_encoder.encoder = data_loader.vocab_encoder
text_encoder.decoder = data_loader.vocab_decoder
opt.data.maxe1 = data_loader.max_event
opt.data.maxe2 = data_loader.max_effect
opt.data.maxr = data.atomic_data.num_delimiter_tokens["category"]
n_special = len(special)
n_ctx = opt.data.maxe1 + opt.data.maxe2
n_vocab = len(text_encoder.encoder) + n_ctx
print(data_loader.__dict__.keys())
opt.net.vSize = n_vocab
print("Building Model")
model = models.make_model(opt,
n_vocab,
n_ctx,
n_special,
load=(opt.net.init == "pt"))
print("Done.")
print("Files will be logged at: {}".format(
utils.make_name(opt, prefix="results/losses/", is_dir=True,
eval_=True)))
data_loader.reset_offsets("train")
# Get number of examples
data.set_max_sizes(data_loader)
if config.gpu_mode:
print("Pushing to GPU: {}".format(config.gpu_index))
cfg.device = config.gpu_index
cfg.do_gpu = True
torch.cuda.set_device(cfg.device)
if config.multigpu:
model = models.multi_gpu(model, config.gpu_indices).cuda()
else:
model.cuda(cfg.device)
print("Done.")
print("Training")
optimizer = OpenAIAdam(model.parameters(),
lr=opt.train.dynamic.lr,
schedule=opt.train.static.lrsched,
warmup=opt.train.static.lrwarm,
t_total=meta.iterations,
b1=opt.train.static.b1,
b2=opt.train.static.b2,
e=opt.train.static.e,
l2=opt.train.static.l2,
vector_l2=opt.train.static.vl2,
max_grad_norm=opt.train.static.clip)
scorers = ["bleu", "rouge", "cider"]
trainer = train.make_trainer(opt, meta, data_loader, model, optimizer)
trainer.set_evaluator(opt, model, data_loader)
trainer.run()
if len(opt.data.categories) == 1:
set_of_categories = [opt.data.categories]
else:
set_of_categories = [opt.data.categories] + [[i]
for i in opt.data.categories]
print(set_of_categories)
# Iterate over sets of categories to compute perplexities for
for eval_categories in set_of_categories:
print(eval_categories)
opt.eval.categories = eval_categories
results_name = "{}/{}.{}".format(
utils.make_name(opt,
prefix="results/{}/".format("losses"),
is_dir=True,
eval_=True), split, "pickle")
print("Will save {} losses to {}".format(split, results_name))
path = "data/atomic/processed/generation/{}.pickle".format(
utils.make_name_string(opt.data).replace(
"kr_{}".format(opt.data.get("kr", 1)), "kr_1"))
data_loader = data.make_data_loader(opt, opt.data.categories)
loaded = data_loader.load_data(path)
data_loader.batch_size = opt.train.dynamic.bs
print("Done.")
text_encoder = TextEncoder(config.encoder_path, config.bpe_path)
def main(num):
# Generate configuration files depending on experiment being run
utils.generate_config_files("conceptnet", num)
# Loads the correct configuration file
config_file = "config/conceptnet/config_{}.json".format(num)
print(config_file)
# Read config file to option
config = cfg.read_config(cfg.load_config(config_file))
opt, meta = cfg.get_parameters(config)
# config.gpu_mode = torch.cuda.is_available()
# Set the random seeds
torch.manual_seed(opt.train.static.seed)
random.seed(opt.train.static.seed)
if config.gpu_mode:
torch.cuda.manual_seed_all(opt.train.static.seed)
# Load the data
splits = ["train", "dev", "test"]
opt.train.dynamic.epoch = 0
print("Loading Data")
# Initialize path to pre-set data loader
path = "data/conceptnet/processed/{}/{}.pickle".format(
opt.exp, utils.make_name_string(opt.data))
# Make data loader
data_loader = data.make_data_loader(opt)
loaded = data_loader.load_data(path)
print(data_loader.sequences["train"]["total"].size(0))
data_loader.opt = opt
data_loader.batch_size = opt.train.dynamic.bs
print("Done.")
text_encoder = TextEncoder(config.encoder_path, config.bpe_path)
categories = data.conceptnet_data.conceptnet_relations
special = [data.start_token, data.end_token]
special += ["<{}>".format(cat) for cat in categories]
if loaded:
text_encoder.encoder = data_loader.vocab_encoder
text_encoder.decoder = data_loader.vocab_decoder
else:
for special_token in special:
text_encoder.decoder[len(encoder)] = special_token
text_encoder.encoder[special_token] = len(encoder)
data_loader.make_tensors(text_encoder, special)
# Set max size of different parts of relation
context_size_e1 = data_loader.max_e1
context_size_e2 = data_loader.max_e2
context_size_r = data_loader.max_r
opt.data.maxr = context_size_r
n_special = len(special)
n_ctx = context_size_e1 + context_size_r + context_size_e2
n_vocab = len(text_encoder.encoder) + n_ctx
print(data_loader.__dict__.keys())
opt.net.vSize = n_vocab
# Build Model
print("Building Model")
model = models.make_model(opt,
n_vocab,
n_ctx,
n_special,
load=(opt.net.init == "pt"))
print("Done.")
print("Files will be logged at: {}".format(
utils.make_name(opt, prefix="results/losses/", is_dir=True,
eval_=True)))
data_loader.reset_offsets("train", keys=["total"])
data.set_max_sizes(data_loader)
# Push to GPU
if config.gpu_mode:
print("Pushing to GPU: {}".format(config.gpu_index))
cfg.device = config.gpu_index
cfg.do_gpu = True
torch.cuda.set_device(cfg.device)
if config.multigpu:
model = models.multi_gpu(model, config.gpu_indices).cuda()
else:
model.cuda(cfg.device)
print("Done.")
print("Training")
optimizer = OpenAIAdam(model.parameters(),
lr=opt.train.dynamic.lr,
schedule=opt.train.static.lrsched,
warmup=opt.train.static.lrwarm,
t_total=meta.iterations,
b1=opt.train.static.b1,
b2=opt.train.static.b2,
e=opt.train.static.e,
l2=opt.train.static.l2,
vector_l2=opt.train.static.vl2,
max_grad_norm=opt.train.static.clip)
trainer = train.make_trainer(opt, meta, data_loader, model, optimizer)
print(data_loader.sequences["dev"]["total"].max())
trainer.set_generator(opt, model, data_loader)
trainer.set_evaluator(opt, model, data_loader)
trainer.run()
def main(num):
# Generate configuration files depending on experiment being run
#utils.generate_config_files("conceptnet", num)
# Loads the correct configuration file
config_file = "config/conceptnet/config_{}.json".format(num)
print(config_file)
# Read config file to option
config = cfg.read_config(cfg.load_config(config_file))
opt, meta = cfg.get_parameters(config)
# config.gpu_mode = torch.cuda.is_available()
# Set the random seeds
torch.manual_seed(opt.train.static.seed)
random.seed(opt.train.static.seed)
if config.gpu_mode:
torch.cuda.manual_seed_all(opt.train.static.seed)
# Load the data
splits = ["train", "dev", "test"]
opt.train.dynamic.epoch = 0
print("Loading Data")
# Initialize path to pre-set data loader
x = "data/conceptnet/processed/generation/rel_language-trainsize_100-devversion_12-maxe1_200-maxe2_200.pickle"
path = x.format(opt.exp, utils.make_name_string(opt.data))
print(path)
# Make data loader
data_loader = data.make_data_loader(opt)
loaded = data_loader.load_data(path)
#print(data_loader.sequences["train"]["total"].size(0))
data_loader.opt = opt
data_loader.batch_size = opt.train.dynamic.bs
print("Done.")
print(data_loader)
#text_encoder = TextEncoder(config.encoder_path, config.bpe_path)
text_encoder = GPT2Tokenizer.from_pretrained('gpt2')
special_tokens = {
"cls_token": "[CLS]",
"unk_token": "[UNK]"
} #, "mask": '["MASK"]',"separator": '["SEP"]', "start_of_sentence": '["SOS"]', "end_of_sentence": '["EOS"]'}
text_encoder = GPT2Tokenizer.from_pretrained("gpt2",
cls_token="[CLS]",
unk_token="[UNK]",
mask='["MASK"]',
separator='["SEP"]',
start_of_sentence='["SOS"]',
end_of_sentence='["EOS"]')
text_encoder.add_special_tokens(special_tokens)
#categories = data.conceptnet_data.conceptnet_relations
special = [data.start_token, data.end_token]
#special += ["<{}>".format(cat) for cat in categories]
if loaded:
text_encoder.encoder = data_loader.vocab_encoder
text_encoder.decoder = data_loader.vocab_decoder
else:
for special_token in special:
text_encoder.decoder[len(encoder)] = special_token
text_encoder.encoder[special_token] = len(encoder)
data_loader.make_tensors(text_encoder, special)
# Set max size of different parts of relation
context_size_i1 = data_loader.max_input1
context_size_i2 = data_loader.max_input2
context_size_i3 = data_loader.max_input3
context_size_i4 = data_loader.max_input4
context_size_o1 = data_loader.max_output1
context_size_o2 = data_loader.max_output2
context_size_o3 = data_loader.max_output3
context_size_o4 = data_loader.max_output4
#opt.data.maxr = context_size_r
n_special = len(special)
n_ctx = context_size_i1 + context_size_i2 + context_size_i3 + context_size_i4 + context_size_o1 + context_size_o2 + context_size_o3 + context_size_o4
n_vocab = len(text_encoder.encoder) + n_ctx
opt.net.vSize = n_vocab
# Build Model
print("Building Model")
print(opt.net.init == "pt")
model = models.make_model(opt, n_vocab, n_ctx, n_special)
model.resize_token_embeddings(len(text_encoder))
model_knowledge = model_knowledge_story.make_model(opt, n_vocab, n_ctx,
n_special)
model_knowledge.resize_token_embeddings(len(text_encoder))
print("Done.")
print("Files will be logged at: {}".format(
utils.make_name(opt, prefix="results/losses/", is_dir=True,
eval_=True)))
data_loader.reset_offsets("train", keys=["total"])
data.set_max_sizes(data_loader)
# Push to GPU
if config.gpu_mode:
print("Pushing to GPU: {}".format(config.gpu_index))
cfg.device = config.gpu_index
cfg.do_gpu = True
torch.cuda.set_device(cfg.device)
if config.multigpu:
#print("!!! I am here !!!")
model = models.multi_gpu(model, config.gpu_indices).cuda()
#model.to(f'cuda:{model.device_ids[0]}')
model_knowledge = model_knowledge_story.multi_gpu(
model_knowledge, config.gpu_indices).cuda()
#model_knowledge.to(f'cuda:{model.device_ids[1]}')
else:
model.cuda(cfg.device)
model_knowledge.cuda(cfg.device)
print("Done.")
print("Training")
optimizer_m = OpenAIAdam(model.parameters(),
lr=opt.train.dynamic.lr,
schedule=opt.train.static.lrsched,
warmup=opt.train.static.lrwarm,
t_total=meta.iterations,
b1=opt.train.static.b1,
b2=opt.train.static.b2,
e=opt.train.static.e,
l2=opt.train.static.l2,
vector_l2=opt.train.static.vl2,
max_grad_norm=opt.train.static.clip)
optimizer_k = Knowledge_Adam(model_knowledge.parameters(),
lr=opt.train.dynamic.lr,
schedule=opt.train.static.lrsched,
warmup=opt.train.static.lrwarm,
t_total=meta.iterations,
b1=opt.train.static.b1,
b2=opt.train.static.b2,
e=opt.train.static.e,
l2=opt.train.static.l2,
vector_l2=opt.train.static.vl2,
max_grad_norm=opt.train.static.clip)
trainer = train.make_trainer(opt, meta, data_loader, model,
model_knowledge, optimizer_m, optimizer_k)
trainer.set_generator(opt, model, model_knowledge, data_loader)
trainer.set_evaluator(opt, model, model_knowledge, data_loader)
trainer.run()
cw /= num
cw = 1 - torch.exp(-torch.sqrt(cw))
data_loader.class_weights[split] = cw
# Build Model
print("Building Model")
model = baselines.ClassModel(
opt.net, data_loader.vocabs["sentence"]._tokens,
ent_vocab=data_loader.vocabs["entity"]._tokens,
max_words=data_loader.max_words)
print("Done.")
print("Files will be logged at: {}".format(
utils.make_name(opt, prefix="results/losses/", is_dir=True)))
start = time.time()
if config.gpu_mode:
print("Pushing to GPU")
cfg.device = config.gpu_index
cfg.do_gpu = True
torch.cuda.set_device(config.gpu_index)
data_loader.cuda(cfg.device)
for split in data_loader.class_weights:
data_loader.class_weights[split] = \
data_loader.class_weights[split].cuda(cfg.device)
print "Data to gpu took {} s".format(time.time() - start)
model.cuda(cfg.device)
print "Model to gpu took {} s".format(time.time() - start)
def save_train_losses(opt, iter_loss, ln="losses"):
name = "{}/train.pickle".format(
utils.make_name(opt, prefix="results/{}/".format(ln), is_dir=True))
print "Saving training losses to: {}".format(name)
with open(name, "w") as f:
pickle.dump(iter_loss, f)
