def save_eval_file(opt, stats, eval_type="losses", split="dev", ext="pickle"): if cfg.test_save: name = "{}/{}.{}".format( utils.make_name(opt, prefix="garbage/{}/".format(eval_type), is_dir=True, eval_=True), split, ext) else: name = "{}/{}.{}".format( utils.make_name(opt, prefix="results/{}/".format(eval_type), is_dir=True, eval_=True), split, ext) print("Saving {} {} to {}".format(split, eval_type, name)) if ext == "pickle": with open(name, "wb") as f: pickle.dump(stats, f) elif ext == "txt": with open(name, "w") as f: f.write(stats) elif ext == "json": with open(name, "w") as f: json.dump(stats, f) else: raise
def set_logger(self): if cfg.toy: self.logger = SummaryWriter(utils.make_name( self.opt, prefix="garbage/logs/", eval_=True, do_epoch=False)) else: self.logger = SummaryWriter(utils.make_name( self.opt, prefix="logs/", eval_=True, do_epoch=False)) print("Logging Tensorboard Files at: {}".format(self.logger.logdir))
def save_step(model, vocab, optimizer, opt, length, lrs): if cfg.test_save: name = "{}.pickle".format(utils.make_name( opt, prefix="garbage/models/", is_dir=False, eval_=True)) else: name = "{}.pickle".format(utils.make_name( opt, prefix="models/", is_dir=False, eval_=True)) save_checkpoint({ "epoch": length, "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), "opt": opt, "vocab": vocab, "epoch_learning_rates": lrs}, name)
def save_eval_losses(opt, eval_losses, split="dev", ln="losses"): if eval_losses: name = "{}/{}.pickle".format(utils.make_name( opt, prefix="results/{}/".format(ln), is_dir=True), split) print "Saving evaluation losses to: {}".format(name) with open(name, "w") as f: pickle.dump(eval_losses, f)
def save_step(model, optimizer, opt, length): name = "{}.pickle".format(utils.make_name( opt, prefix="models/", is_dir=False)) save_checkpoint({ 'epoch': length, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), "opt": opt}, name) print "Saving to: {}".format(name)
def do_class_eval_run(data_loader, opt, model, gpu, eval_losses, l, split="dev", fn="class"): # Run model on development set print("Evaluating {}".format(split.capitalize())) dev_loss, scores, counts, answers = evaluate.eval_function[fn](opt, model, data_loader, gpu, split) # Print loss to stdout print("{} Loss: \t {}".format(split.capitalize(), dev_loss[opt.granularity])) # Record loss eval_losses[l] = dev_loss # Save Recall, Precision, F1 scores computed name = "{}/{}.scores".format( utils.make_name(opt, prefix="results/scores/", is_dir=True, eval_=True), split) with open(name, "w") as f: pickle.dump({"scores": scores, "counts": counts}, f) # Save individual answers name = "{}/{}.scores".format( utils.make_name(opt, prefix="results/answers/", is_dir=True, eval_=True), split) with open(name, "w") as f: pickle.dump(answers, f) return eval_losses, scores, counts
def main(num): # Generate configuration files depending on experiment being run utils.generate_config_files("atomic", num) # Loads the correct configuration file config_file = "config/atomic/config_{}.json".format(num) print(config_file) # Read config file to option config = cfg.read_config(cfg.load_config(config_file)) opt, meta = cfg.get_parameters(config) # Set the random seeds torch.manual_seed(opt.train.static.seed) random.seed(opt.train.static.seed) if config.gpu_mode: torch.cuda.manual_seed_all(opt.train.static.seed) # Where to find the data splits = ["train", "dev", "test"] opt.train.dynamic.epoch = 0 print("Loading Data") categories = opt.data.categories path = "data/atomic/processed/{}/{}.pickle".format( opt.exp, utils.make_name_string(opt.data)) data_loader = data.make_data_loader(opt, categories) loaded = data_loader.load_data(path) print(data_loader.sequences["train"]["total"].size(0)) data_loader.opt = opt data_loader.batch_size = opt.train.dynamic.bs print("Done.") # Initialize text_encoder text_encoder = TextEncoder(config.encoder_path, config.bpe_path) special = [data.start_token, data.end_token] special += ["<{}>".format(cat) for cat in categories] special += [data.blank_token] text_encoder.encoder = data_loader.vocab_encoder text_encoder.decoder = data_loader.vocab_decoder opt.data.maxe1 = data_loader.max_event opt.data.maxe2 = data_loader.max_effect opt.data.maxr = data.atomic_data.num_delimiter_tokens["category"] n_special = len(special) n_ctx = opt.data.maxe1 + opt.data.maxe2 n_vocab = len(text_encoder.encoder) + n_ctx print(data_loader.__dict__.keys()) opt.net.vSize = n_vocab print("Building Model") model = models.make_model(opt, n_vocab, n_ctx, n_special, load=(opt.net.init == "pt")) print("Done.") print("Files will be logged at: {}".format( utils.make_name(opt, prefix="results/losses/", is_dir=True, eval_=True))) data_loader.reset_offsets("train") # Get number of examples data.set_max_sizes(data_loader) if config.gpu_mode: print("Pushing to GPU: {}".format(config.gpu_index)) cfg.device = config.gpu_index cfg.do_gpu = True torch.cuda.set_device(cfg.device) if config.multigpu: model = models.multi_gpu(model, config.gpu_indices).cuda() else: model.cuda(cfg.device) print("Done.") print("Training") optimizer = OpenAIAdam(model.parameters(), lr=opt.train.dynamic.lr, schedule=opt.train.static.lrsched, warmup=opt.train.static.lrwarm, t_total=meta.iterations, b1=opt.train.static.b1, b2=opt.train.static.b2, e=opt.train.static.e, l2=opt.train.static.l2, vector_l2=opt.train.static.vl2, max_grad_norm=opt.train.static.clip) scorers = ["bleu", "rouge", "cider"] trainer = train.make_trainer(opt, meta, data_loader, model, optimizer) trainer.set_evaluator(opt, model, data_loader) trainer.run()
if len(opt.data.categories) == 1: set_of_categories = [opt.data.categories] else: set_of_categories = [opt.data.categories] + [[i] for i in opt.data.categories] print(set_of_categories) # Iterate over sets of categories to compute perplexities for for eval_categories in set_of_categories: print(eval_categories) opt.eval.categories = eval_categories results_name = "{}/{}.{}".format( utils.make_name(opt, prefix="results/{}/".format("losses"), is_dir=True, eval_=True), split, "pickle") print("Will save {} losses to {}".format(split, results_name)) path = "data/atomic/processed/generation/{}.pickle".format( utils.make_name_string(opt.data).replace( "kr_{}".format(opt.data.get("kr", 1)), "kr_1")) data_loader = data.make_data_loader(opt, opt.data.categories) loaded = data_loader.load_data(path) data_loader.batch_size = opt.train.dynamic.bs print("Done.") text_encoder = TextEncoder(config.encoder_path, config.bpe_path)
def main(num): # Generate configuration files depending on experiment being run utils.generate_config_files("conceptnet", num) # Loads the correct configuration file config_file = "config/conceptnet/config_{}.json".format(num) print(config_file) # Read config file to option config = cfg.read_config(cfg.load_config(config_file)) opt, meta = cfg.get_parameters(config) # config.gpu_mode = torch.cuda.is_available() # Set the random seeds torch.manual_seed(opt.train.static.seed) random.seed(opt.train.static.seed) if config.gpu_mode: torch.cuda.manual_seed_all(opt.train.static.seed) # Load the data splits = ["train", "dev", "test"] opt.train.dynamic.epoch = 0 print("Loading Data") # Initialize path to pre-set data loader path = "data/conceptnet/processed/{}/{}.pickle".format( opt.exp, utils.make_name_string(opt.data)) # Make data loader data_loader = data.make_data_loader(opt) loaded = data_loader.load_data(path) print(data_loader.sequences["train"]["total"].size(0)) data_loader.opt = opt data_loader.batch_size = opt.train.dynamic.bs print("Done.") text_encoder = TextEncoder(config.encoder_path, config.bpe_path) categories = data.conceptnet_data.conceptnet_relations special = [data.start_token, data.end_token] special += ["<{}>".format(cat) for cat in categories] if loaded: text_encoder.encoder = data_loader.vocab_encoder text_encoder.decoder = data_loader.vocab_decoder else: for special_token in special: text_encoder.decoder[len(encoder)] = special_token text_encoder.encoder[special_token] = len(encoder) data_loader.make_tensors(text_encoder, special) # Set max size of different parts of relation context_size_e1 = data_loader.max_e1 context_size_e2 = data_loader.max_e2 context_size_r = data_loader.max_r opt.data.maxr = context_size_r n_special = len(special) n_ctx = context_size_e1 + context_size_r + context_size_e2 n_vocab = len(text_encoder.encoder) + n_ctx print(data_loader.__dict__.keys()) opt.net.vSize = n_vocab # Build Model print("Building Model") model = models.make_model(opt, n_vocab, n_ctx, n_special, load=(opt.net.init == "pt")) print("Done.") print("Files will be logged at: {}".format( utils.make_name(opt, prefix="results/losses/", is_dir=True, eval_=True))) data_loader.reset_offsets("train", keys=["total"]) data.set_max_sizes(data_loader) # Push to GPU if config.gpu_mode: print("Pushing to GPU: {}".format(config.gpu_index)) cfg.device = config.gpu_index cfg.do_gpu = True torch.cuda.set_device(cfg.device) if config.multigpu: model = models.multi_gpu(model, config.gpu_indices).cuda() else: model.cuda(cfg.device) print("Done.") print("Training") optimizer = OpenAIAdam(model.parameters(), lr=opt.train.dynamic.lr, schedule=opt.train.static.lrsched, warmup=opt.train.static.lrwarm, t_total=meta.iterations, b1=opt.train.static.b1, b2=opt.train.static.b2, e=opt.train.static.e, l2=opt.train.static.l2, vector_l2=opt.train.static.vl2, max_grad_norm=opt.train.static.clip) trainer = train.make_trainer(opt, meta, data_loader, model, optimizer) print(data_loader.sequences["dev"]["total"].max()) trainer.set_generator(opt, model, data_loader) trainer.set_evaluator(opt, model, data_loader) trainer.run()
def main(num): # Generate configuration files depending on experiment being run #utils.generate_config_files("conceptnet", num) # Loads the correct configuration file config_file = "config/conceptnet/config_{}.json".format(num) print(config_file) # Read config file to option config = cfg.read_config(cfg.load_config(config_file)) opt, meta = cfg.get_parameters(config) # config.gpu_mode = torch.cuda.is_available() # Set the random seeds torch.manual_seed(opt.train.static.seed) random.seed(opt.train.static.seed) if config.gpu_mode: torch.cuda.manual_seed_all(opt.train.static.seed) # Load the data splits = ["train", "dev", "test"] opt.train.dynamic.epoch = 0 print("Loading Data") # Initialize path to pre-set data loader x = "data/conceptnet/processed/generation/rel_language-trainsize_100-devversion_12-maxe1_200-maxe2_200.pickle" path = x.format(opt.exp, utils.make_name_string(opt.data)) print(path) # Make data loader data_loader = data.make_data_loader(opt) loaded = data_loader.load_data(path) #print(data_loader.sequences["train"]["total"].size(0)) data_loader.opt = opt data_loader.batch_size = opt.train.dynamic.bs print("Done.") print(data_loader) #text_encoder = TextEncoder(config.encoder_path, config.bpe_path) text_encoder = GPT2Tokenizer.from_pretrained('gpt2') special_tokens = { "cls_token": "[CLS]", "unk_token": "[UNK]" } #, "mask": '["MASK"]',"separator": '["SEP"]', "start_of_sentence": '["SOS"]', "end_of_sentence": '["EOS"]'} text_encoder = GPT2Tokenizer.from_pretrained("gpt2", cls_token="[CLS]", unk_token="[UNK]", mask='["MASK"]', separator='["SEP"]', start_of_sentence='["SOS"]', end_of_sentence='["EOS"]') text_encoder.add_special_tokens(special_tokens) #categories = data.conceptnet_data.conceptnet_relations special = [data.start_token, data.end_token] #special += ["<{}>".format(cat) for cat in categories] if loaded: text_encoder.encoder = data_loader.vocab_encoder text_encoder.decoder = data_loader.vocab_decoder else: for special_token in special: text_encoder.decoder[len(encoder)] = special_token text_encoder.encoder[special_token] = len(encoder) data_loader.make_tensors(text_encoder, special) # Set max size of different parts of relation context_size_i1 = data_loader.max_input1 context_size_i2 = data_loader.max_input2 context_size_i3 = data_loader.max_input3 context_size_i4 = data_loader.max_input4 context_size_o1 = data_loader.max_output1 context_size_o2 = data_loader.max_output2 context_size_o3 = data_loader.max_output3 context_size_o4 = data_loader.max_output4 #opt.data.maxr = context_size_r n_special = len(special) n_ctx = context_size_i1 + context_size_i2 + context_size_i3 + context_size_i4 + context_size_o1 + context_size_o2 + context_size_o3 + context_size_o4 n_vocab = len(text_encoder.encoder) + n_ctx opt.net.vSize = n_vocab # Build Model print("Building Model") print(opt.net.init == "pt") model = models.make_model(opt, n_vocab, n_ctx, n_special) model.resize_token_embeddings(len(text_encoder)) model_knowledge = model_knowledge_story.make_model(opt, n_vocab, n_ctx, n_special) model_knowledge.resize_token_embeddings(len(text_encoder)) print("Done.") print("Files will be logged at: {}".format( utils.make_name(opt, prefix="results/losses/", is_dir=True, eval_=True))) data_loader.reset_offsets("train", keys=["total"]) data.set_max_sizes(data_loader) # Push to GPU if config.gpu_mode: print("Pushing to GPU: {}".format(config.gpu_index)) cfg.device = config.gpu_index cfg.do_gpu = True torch.cuda.set_device(cfg.device) if config.multigpu: #print("!!! I am here !!!") model = models.multi_gpu(model, config.gpu_indices).cuda() #model.to(f'cuda:{model.device_ids[0]}') model_knowledge = model_knowledge_story.multi_gpu( model_knowledge, config.gpu_indices).cuda() #model_knowledge.to(f'cuda:{model.device_ids[1]}') else: model.cuda(cfg.device) model_knowledge.cuda(cfg.device) print("Done.") print("Training") optimizer_m = OpenAIAdam(model.parameters(), lr=opt.train.dynamic.lr, schedule=opt.train.static.lrsched, warmup=opt.train.static.lrwarm, t_total=meta.iterations, b1=opt.train.static.b1, b2=opt.train.static.b2, e=opt.train.static.e, l2=opt.train.static.l2, vector_l2=opt.train.static.vl2, max_grad_norm=opt.train.static.clip) optimizer_k = Knowledge_Adam(model_knowledge.parameters(), lr=opt.train.dynamic.lr, schedule=opt.train.static.lrsched, warmup=opt.train.static.lrwarm, t_total=meta.iterations, b1=opt.train.static.b1, b2=opt.train.static.b2, e=opt.train.static.e, l2=opt.train.static.l2, vector_l2=opt.train.static.vl2, max_grad_norm=opt.train.static.clip) trainer = train.make_trainer(opt, meta, data_loader, model, model_knowledge, optimizer_m, optimizer_k) trainer.set_generator(opt, model, model_knowledge, data_loader) trainer.set_evaluator(opt, model, model_knowledge, data_loader) trainer.run()
cw /= num cw = 1 - torch.exp(-torch.sqrt(cw)) data_loader.class_weights[split] = cw # Build Model print("Building Model") model = baselines.ClassModel( opt.net, data_loader.vocabs["sentence"]._tokens, ent_vocab=data_loader.vocabs["entity"]._tokens, max_words=data_loader.max_words) print("Done.") print("Files will be logged at: {}".format( utils.make_name(opt, prefix="results/losses/", is_dir=True))) start = time.time() if config.gpu_mode: print("Pushing to GPU") cfg.device = config.gpu_index cfg.do_gpu = True torch.cuda.set_device(config.gpu_index) data_loader.cuda(cfg.device) for split in data_loader.class_weights: data_loader.class_weights[split] = \ data_loader.class_weights[split].cuda(cfg.device) print "Data to gpu took {} s".format(time.time() - start) model.cuda(cfg.device) print "Model to gpu took {} s".format(time.time() - start)
def save_train_losses(opt, iter_loss, ln="losses"): name = "{}/train.pickle".format( utils.make_name(opt, prefix="results/{}/".format(ln), is_dir=True)) print "Saving training losses to: {}".format(name) with open(name, "w") as f: pickle.dump(iter_loss, f)