def main():
predictions = ['London'] * len(open(args.eval_corpus_path).readlines())
total, correct = utils.evaluate_places(args.eval_corpus_path, predictions)
if total > 0:
print('Correct: {} out of {}: {}%'.format(correct, total,
correct / total * 100))
else:
print('Predictions written to {}; no targets provided'.format(
args.outputs_path))
# Calculate the accuracy of a baseline that simply predicts "London" for every
# example in the dev set.
# Hint: Make use of existing code.
# Your solution here should only be a few lines.
import utils
eval_corpus_path = "./birth_dev.tsv"
with open(eval_corpus_path) as fin:
predictions = ["London" for _ in fin]
total, correct = utils.evaluate_places(eval_corpus_path, predictions)
print('Correct: {} out of {}: {}%'.format(correct, total,
correct / total * 100))
# Calculate the accuracy of a baseline that simply predicts "London" for every
# example in the dev set.
# Hint: Make use of existing code.
# Your solution here should only be a few lines.
import utils
from tqdm import tqdm
import torch
correct = 0
total = 0
# with open(args.outputs_path, 'w') as fout:
predictions = []
for line in tqdm(open('birth_dev.tsv')):
# x = line.split('\t')[0]
# x = x + '⁇'
# x = torch.tensor([pretrain_dataset.stoi[s] for s in x], dtype=torch.long)[None,...].to(device)
# pred = utils.sample(model, x, 32, sample=False)[0]
# completion = ''.join([pretrain_dataset.itos[int(i)] for i in pred])
# pred = completion.split('⁇')[1]
predictions.append("London")
# fout.write(pred + '\n')
total, correct = utils.evaluate_places('birth_dev.tsv', predictions)
if total > 0:
print('Correct: {} out of {}: {}%'.format(correct, total,
correct / total * 100))
def main():
"""
Don't change above here; write your code below
"""
if args.variant == 'vanilla':
model = GPT(mconf) # TODO [part c]: Make some model here
elif args.variant == 'synthesizer':
# TODO [part g]: Make some other model here
mconf.synthesizer = True
model = GPT(mconf)
# From here on, your code should be identical independent of which
# variant (vanilla or synthesizer) has been chosen.
if args.function == 'pretrain':
assert args.pretrain_corpus_path is not None
assert args.writing_params_path is not None
# TODO [part f]:
# - Given:
# 1. A corpus specified in args.pretrain_corpus_path
# 2. An output path args.writing_params_path for the model parameters
# - Goals:
# 1. Pretrain the model on this corpus
# 2. Save the resulting model in args.writing_params_path
# - Make sure to use the following hyperparameters for pretraining:
# max_epochs=650
# batch_size=128
# learning_rate=6e-3
# lr_decay=True
# warmup_tokens=512*20
# final_tokens=200*len(pretrain_dataset)*block_size
# num_workers=4
# pretrain_text = open(args.pretrain_corpus_path, 'r', encoding='utf-8').read()
# pretrain_dataset =
tconf = TrainerConfig(max_epochs=650,
batch_size=128,
learning_rate=6e-3,
lr_decay=True,
warmup_token=512 * 20,
final_tokens=200 * len(pretrain_dataset) *
block_size,
num_workers=4)
trainer = Trainer(model, pretrain_dataset, None, tconf)
trainer.train()
torch.save(model.state_dict(), args.writing_params_path)
elif args.function == 'finetune':
assert args.writing_params_path is not None
assert args.finetune_corpus_path is not None
# TODO [part c] [part f]:
# - Given:
# 1. A finetuning corpus specified in args.finetune_corpus_path
# 2. A path args.reading_params_path containing pretrained model
# parameters, or None if finetuning without a pretrained model
# 3. An output path args.writing_params_path for the model parameters
# - Goals:
# 1. If args.reading_params_path is specified, load these parameters
# into the model
# 2. Finetune the model on this corpus
# 3. Save the resulting model in args.writing_params_path
# - Make sure to use the following hyperparameters:
# Hyperparameters for finetuning WITHOUT a pretrained model:
# max_epochs=75
# batch_size=256
# learning_rate=6e-4
# lr_decay=True
# warmup_tokens=512*20
# final_tokens=200*len(pretrain_dataset)*block_size
# num_workers=4
# Hyperparameters for finetuning WITH a pretrained model:
# max_epochs=10
# batch_size=256
# learning_rate=6e-4
# lr_decay=True
# warmup_tokens=512*20
# final_tokens=200*len(pretrain_dataset)*block_size
# num_workers=4
if args.reading_params_path is not None:
model.load_state_dict(torch.load(args.reading_params_path))
tconf = TrainerConfig(max_epochs=75,
batch_size=256,
learning_rate=6e-4,
lr_decay=True,
warmup_tokens=512 * 20,
final_tokens=200 * len(pretrain_dataset) *
block_size,
num_workers=4)
text = open(args.finetune_corpus_path, 'r').read()
train_dataset = dataset.NameDataset(pretrain_dataset, text)
trainer = Trainer(model, train_dataset, None, tconf)
trainer.train()
# save to args.writing_params_path
torch.save(model.state_dict(), args.writing_params_path)
elif args.function == 'evaluate':
assert args.outputs_path is not None
assert args.reading_params_path is not None
assert args.eval_corpus_path is not None
model.load_state_dict(torch.load(args.reading_params_path))
model = model.to(device)
correct = 0
total = 0
with open(args.outputs_path, 'w') as fout:
predictions = []
for line in tqdm(open(args.eval_corpus_path)):
x = line.split('\t')[0]
x = x + '⁇'
x = torch.tensor([pretrain_dataset.stoi[s] for s in x],
dtype=torch.long)[None, ...].to(device)
pred = utils.sample(model, x, 32, sample=False)[0]
completion = ''.join(
[pretrain_dataset.itos[int(i)] for i in pred])
pred = completion.split('⁇')[1]
predictions.append(pred)
fout.write(pred + '\n')
total, correct = utils.evaluate_places(args.eval_corpus_path,
predictions)
if total > 0:
print('Correct: {} out of {}: {}%'.format(correct, total,
correct / total * 100))
else:
print('Predictions written to {}; no targets provided'.format(
args.outputs_path))
# Calculate the accuracy of a baseline that simply predicts "London" for every
# example in the dev set.
# Hint: Make use of existing code.
# Your solution here should only be a few lines.
import csv
import utils
eval_corpus_path = "birth_dev.tsv"
file = open(eval_corpus_path)
reader = csv.reader(file)
num_lines = len(list(reader))
file.close()
total, correct = utils.evaluate_places(eval_corpus_path,
["London"] * num_lines)
if total > 0:
print('Correct: {} out of {}: {}%'.format(correct, total,
correct / total * 100))
