def main(): predictions = ['London'] * len(open(args.eval_corpus_path).readlines()) total, correct = utils.evaluate_places(args.eval_corpus_path, predictions) if total > 0: print('Correct: {} out of {}: {}%'.format(correct, total, correct / total * 100)) else: print('Predictions written to {}; no targets provided'.format( args.outputs_path))
# Calculate the accuracy of a baseline that simply predicts "London" for every # example in the dev set. # Hint: Make use of existing code. # Your solution here should only be a few lines. import utils eval_corpus_path = "./birth_dev.tsv" with open(eval_corpus_path) as fin: predictions = ["London" for _ in fin] total, correct = utils.evaluate_places(eval_corpus_path, predictions) print('Correct: {} out of {}: {}%'.format(correct, total, correct / total * 100))
# Calculate the accuracy of a baseline that simply predicts "London" for every # example in the dev set. # Hint: Make use of existing code. # Your solution here should only be a few lines. import utils from tqdm import tqdm import torch correct = 0 total = 0 # with open(args.outputs_path, 'w') as fout: predictions = [] for line in tqdm(open('birth_dev.tsv')): # x = line.split('\t')[0] # x = x + '⁇' # x = torch.tensor([pretrain_dataset.stoi[s] for s in x], dtype=torch.long)[None,...].to(device) # pred = utils.sample(model, x, 32, sample=False)[0] # completion = ''.join([pretrain_dataset.itos[int(i)] for i in pred]) # pred = completion.split('⁇')[1] predictions.append("London") # fout.write(pred + '\n') total, correct = utils.evaluate_places('birth_dev.tsv', predictions) if total > 0: print('Correct: {} out of {}: {}%'.format(correct, total, correct / total * 100))
def main(): """ Don't change above here; write your code below """ if args.variant == 'vanilla': model = GPT(mconf) # TODO [part c]: Make some model here elif args.variant == 'synthesizer': # TODO [part g]: Make some other model here mconf.synthesizer = True model = GPT(mconf) # From here on, your code should be identical independent of which # variant (vanilla or synthesizer) has been chosen. if args.function == 'pretrain': assert args.pretrain_corpus_path is not None assert args.writing_params_path is not None # TODO [part f]: # - Given: # 1. A corpus specified in args.pretrain_corpus_path # 2. An output path args.writing_params_path for the model parameters # - Goals: # 1. Pretrain the model on this corpus # 2. Save the resulting model in args.writing_params_path # - Make sure to use the following hyperparameters for pretraining: # max_epochs=650 # batch_size=128 # learning_rate=6e-3 # lr_decay=True # warmup_tokens=512*20 # final_tokens=200*len(pretrain_dataset)*block_size # num_workers=4 # pretrain_text = open(args.pretrain_corpus_path, 'r', encoding='utf-8').read() # pretrain_dataset = tconf = TrainerConfig(max_epochs=650, batch_size=128, learning_rate=6e-3, lr_decay=True, warmup_token=512 * 20, final_tokens=200 * len(pretrain_dataset) * block_size, num_workers=4) trainer = Trainer(model, pretrain_dataset, None, tconf) trainer.train() torch.save(model.state_dict(), args.writing_params_path) elif args.function == 'finetune': assert args.writing_params_path is not None assert args.finetune_corpus_path is not None # TODO [part c] [part f]: # - Given: # 1. A finetuning corpus specified in args.finetune_corpus_path # 2. A path args.reading_params_path containing pretrained model # parameters, or None if finetuning without a pretrained model # 3. An output path args.writing_params_path for the model parameters # - Goals: # 1. If args.reading_params_path is specified, load these parameters # into the model # 2. Finetune the model on this corpus # 3. Save the resulting model in args.writing_params_path # - Make sure to use the following hyperparameters: # Hyperparameters for finetuning WITHOUT a pretrained model: # max_epochs=75 # batch_size=256 # learning_rate=6e-4 # lr_decay=True # warmup_tokens=512*20 # final_tokens=200*len(pretrain_dataset)*block_size # num_workers=4 # Hyperparameters for finetuning WITH a pretrained model: # max_epochs=10 # batch_size=256 # learning_rate=6e-4 # lr_decay=True # warmup_tokens=512*20 # final_tokens=200*len(pretrain_dataset)*block_size # num_workers=4 if args.reading_params_path is not None: model.load_state_dict(torch.load(args.reading_params_path)) tconf = TrainerConfig(max_epochs=75, batch_size=256, learning_rate=6e-4, lr_decay=True, warmup_tokens=512 * 20, final_tokens=200 * len(pretrain_dataset) * block_size, num_workers=4) text = open(args.finetune_corpus_path, 'r').read() train_dataset = dataset.NameDataset(pretrain_dataset, text) trainer = Trainer(model, train_dataset, None, tconf) trainer.train() # save to args.writing_params_path torch.save(model.state_dict(), args.writing_params_path) elif args.function == 'evaluate': assert args.outputs_path is not None assert args.reading_params_path is not None assert args.eval_corpus_path is not None model.load_state_dict(torch.load(args.reading_params_path)) model = model.to(device) correct = 0 total = 0 with open(args.outputs_path, 'w') as fout: predictions = [] for line in tqdm(open(args.eval_corpus_path)): x = line.split('\t')[0] x = x + '⁇' x = torch.tensor([pretrain_dataset.stoi[s] for s in x], dtype=torch.long)[None, ...].to(device) pred = utils.sample(model, x, 32, sample=False)[0] completion = ''.join( [pretrain_dataset.itos[int(i)] for i in pred]) pred = completion.split('⁇')[1] predictions.append(pred) fout.write(pred + '\n') total, correct = utils.evaluate_places(args.eval_corpus_path, predictions) if total > 0: print('Correct: {} out of {}: {}%'.format(correct, total, correct / total * 100)) else: print('Predictions written to {}; no targets provided'.format( args.outputs_path))
# Calculate the accuracy of a baseline that simply predicts "London" for every # example in the dev set. # Hint: Make use of existing code. # Your solution here should only be a few lines. import csv import utils eval_corpus_path = "birth_dev.tsv" file = open(eval_corpus_path) reader = csv.reader(file) num_lines = len(list(reader)) file.close() total, correct = utils.evaluate_places(eval_corpus_path, ["London"] * num_lines) if total > 0: print('Correct: {} out of {}: {}%'.format(correct, total, correct / total * 100))