def main_targeted():
from args import args
parser = argparse.ArgumentParser()
parser.add_argument('--model', required=True)
parser.add_argument('--fold', required=True)
_args = parser.parse_args()
args.load_model_dir = _args.model
args.fold = _args.fold
out_dir = prepare_output_dir(args, 'results')
print('Generating [{}] targeted rawr data from [{}].'.format(args.fold, args.load_model_dir))
pkl_dir = os.path.join(out_dir, '{}.targeted.pkl'.format(args.fold))
print('Saving to {}'.format(pkl_dir))
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
train, dev, dev_y, embedding, opt = load_data(args)
data = {'train': train, 'dev': dev}
state_dict = torch.load(args.load_model_dir)['state_dict']
model = DocReaderModel(vars(opt), embedding, state_dict)
model.cuda()
batches = {}
batches['train'] = BatchGen(
[x[:8] for x in train], batch_size=30,
pos_size=args.pos_size, ner_size=args.ner_size,
gpu=args.cuda, evaluation=True)
batches['dev'] = BatchGen(dev, batch_size=30,
pos_size=args.pos_size, ner_size=args.ner_size,
gpu=args.cuda, evaluation=True)
all_reduced = []
all_removed = []
example_idx = 0
for batch_i, batch in enumerate(tqdm(batches[args.fold])):
# if batch_i > 10:
# break
n_examples = batch[1].shape[0]
answers, _, score_s, score_e, _, _ = model.predict(batch, get_all=True)
target_s = Variable(torch.max(score_s, 1)[1]).cuda()
target_e = Variable(torch.max(score_e, 1)[1]).cuda()
reduced, removed = get_targeted_rawr(
model, batch, answers, target_s, target_e, max_beam_size=5)
for i in range(n_examples):
idx = example_idx + i
assert batch[8][i] == data[args.fold][idx][7] # check if the spans match
all_reduced.append([])
for j, e in enumerate(reduced[i]):
x = list(data[args.fold][idx])
x[5] = e
all_reduced[-1].append(x)
all_removed.append(removed[i])
example_idx += n_examples
with open(pkl_dir, 'wb') as f:
ckp = {'reduced': all_reduced, 'removed': all_removed}
pickle.dump(ckp, f)
def test():
from args import conf
parser = argparse.ArgumentParser()
parser.add_argument('--baseline', required=True)
args = parser.parse_args()
# set random seed
random.seed(conf.seed)
torch.manual_seed(conf.seed)
if conf.cuda:
torch.cuda.manual_seed(conf.seed)
train, dev_x, dev_y, embedding, conf = load_data(conf)
checkpoint = torch.load(args.baseline)
# opt = checkpoint['config']
model = DocReaderModel(vars(conf), embedding, checkpoint['state_dict'])
model.cuda()
dev_batches = BatchGen(
dev_x, batch_size=conf.batch_size,
pos_size=conf.pos_size, ner_size=conf.ner_size,
gpu=conf.cuda, evaluation=True)
predictions = []
for batch in dev_batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
print(em, f1)
def main():
log.info('[program starts.]')
train, dev, dev_y, embedding, opt = load_data(vars(args))
log.info('[Data loaded.]')
if args.resume:
log.info('[loading previous model...]')
checkpoint = torch.load(os.path.join(model_dir, args.resume))
if args.resume_options:
opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = DocReaderModel(opt, embedding, state_dict)
epoch_0 = checkpoint['epoch'] + 1
for i in range(checkpoint['epoch']):
random.shuffle(list(range(len(train)))) # synchronize random seed
if args.reduce_lr:
lr_decay(model.optimizer, lr_decay=args.reduce_lr)
else:
model = DocReaderModel(opt, embedding)
epoch_0 = 1
if args.cuda:
model.cuda()
if args.resume:
batches = BatchGen(dev,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
log.info("[dev EM: {} F1: {}]".format(em, f1))
best_val_score = f1
else:
best_val_score = 0.0
for epoch in range(epoch_0, epoch_0 + args.epochs):
log.warn('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % args.log_per_updates == 0:
log.info(
'updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format(
model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) *
(len(batches) - i - 1)).split('.')[0]))
# eval
if epoch % args.eval_per_epoch == 0:
batches = BatchGen(dev,
batch_size=1,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
log.warn("dev EM: {} F1: {}".format(em, f1))
# save
if not args.save_last_only or epoch == epoch_0 + args.epochs - 1:
model_file = os.path.join(model_dir,
'checkpoint_epoch_{}.pt'.format(epoch))
model.save(model_file, epoch)
if f1 > best_val_score:
best_val_score = f1
copyfile(model_file, os.path.join(model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
def main():
log.info('[program starts.]')
train, dev, dev_y, embedding, opt = load_data(vars(args))
log.info('[Data loaded.]')
if args.resume:
log.info('[loading previous model...]')
checkpoint = torch.load(os.path.join(model_dir, args.resume))
if args.resume_options:
opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = DocReaderModel(opt, embedding, state_dict)
epoch_0 = checkpoint['epoch'] + 1
for i in range(checkpoint['epoch']):
random.shuffle(list(range(len(train)))) # synchronize random seed
if args.reduce_lr:
lr_decay(model.optimizer, lr_decay=args.reduce_lr)
else:
model = DocReaderModel(opt, embedding)
epoch_0 = 1
if args.cuda:
model.cuda()
if args.resume:
batches = BatchGen(dev, batch_size=1, evaluation=True, gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
log.info("[dev EM: {} F1: {}]".format(em, f1))
best_val_score = f1
else:
best_val_score = 0.0
for epoch in range(epoch_0, epoch_0 + args.epochs):
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % args.log_per_updates == 0:
# log.info('updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format(
# model.updates, model.train_loss.avg,
# str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0]))
log.info('epoch [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}] lr[{4:.4f}]'.format(
epoch, model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0],
model.optimizer.state_dict()['param_groups'][0]['lr']))
# eval
if epoch % args.eval_per_epoch == 0:
batches = BatchGen(dev, batch_size=1, evaluation=True, gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
log.warning("dev EM: {} F1: {}".format(em, f1))
# save
if not args.save_last_only or epoch == epoch_0 + args.epochs - 1:
model_file = os.path.join(model_dir, 'checkpoint_epoch_{}.pt'.format(epoch))
model.save(model_file, epoch)
if f1 > best_val_score:
best_val_score = f1
copyfile(
model_file,
os.path.join(model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
def main():
from args import args
# parser = argparse.ArgumentParser()
# parser.add_argument('--model', required=True)
# parser.add_argument('--train', required=True)
# parser.add_argument('--dev', required=True)
# args.load_model_dir = parser.parse_args().model
# args.ent_train_dir = parser.parse_args().train
# args.ent_dev_dir = parser.parse_args().dev
args.load_model_dir = '/scratch0/shifeng/rawr/drqa/original.pt'
args.ent_train_dir = 'results/20180217T172242.135276/train.pkl'
args.ent_dev_dir = 'pkls/original.rawr.dev.pkl'
args.other_train_dir = 'results/targeted_train_all.pkl'
out_dir = prepare_output_dir(args, '/scratch0/shifeng/rawr/drqa/')
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
fh = logging.FileHandler(os.path.join(out_dir, 'output.log'))
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter(fmt='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
log.addHandler(fh)
log.addHandler(ch)
log.info('===== {} ====='.format(out_dir))
with open(os.path.join(out_dir, 'args.pkl'), 'wb') as f:
pickle.dump(args, f)
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
log.info('loading regular data from {}'.format(args.data_file))
train_reg, dev_reg, dev_y, embedding, opt = load_data(args)
log.info('{} regular training examples'.format(len(train_reg)))
log.info('{} regular dev examples'.format(len(dev_reg)))
# log.info(opt)
''' load data for regularization '''
log.info('loading entropy training data from {}'.format(args.ent_train_dir))
with open(args.ent_train_dir, 'rb') as f:
train_ent = pickle.load(f)
if isinstance(train_ent, dict) and 'reduced' in train_ent:
train_ent = train_ent['reduced']
if isinstance(train_ent[0][0], list):
train_ent = list(itertools.chain(*train_ent))
log.info('loading targeted training data from {}'.format(args.other_train_dir))
with open(args.other_train_dir, 'rb') as f:
other_train_ent = pickle.load(f)
if isinstance(other_train_ent, dict) and 'reduced' in train_ent:
other_train_ent = other_train_ent['reduced']
if isinstance(other_train_ent[0][0], list):
other_train_ent = list(itertools.chain(*other_train_ent))
train_ent += other_train_ent
if args.filter_long > 0:
train_ent = [x for x in train_ent if len(x[5]) < args.filter_long]
log.info('loading entropy dev data from {}'.format(args.ent_train_dir))
with open(args.ent_dev_dir, 'rb') as f:
dev_ent = pickle.load(f)['reduced']
if isinstance(dev_ent[0], list):
# dev_ent = list(itertools.chain(*dev_ent))
dev_ent = [x[0] for x in dev_ent]
# if args.filter_long > 0:
# dev_ent = [x for x in dev_ent if len(x[5]) > args.filter_long]
log.info('{} entropy training examples'.format(len(train_ent)))
log.info('{} entropy dev examples'.format(len(dev_ent)))
log.info('loading model from {}'.format(args.load_model_dir))
checkpoint = torch.load(args.load_model_dir)
# opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = DocReaderModel(vars(opt), embedding, state_dict)
model.cuda()
''' initial evaluation '''
dev_reg_batches = BatchGen(
dev_reg, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size,
evaluation=True, gpu=args.cuda)
dev_ent_batches = BatchGen(
dev_ent, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size,
evaluation=True, gpu=args.cuda)
predictions = []
for batch in dev_reg_batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
ents, predictions_r = [], []
for batch in dev_ent_batches:
p, _, ss, se, _, _ = model.predict(batch, get_all=True)
ss = ss.cpu().numpy()
se = se.cpu().numpy()
ents.append(scipy.stats.entropy(ss.T).sum() + \
scipy.stats.entropy(se.T).sum())
predictions_r.extend(p)
ent = sum(ents) / len(ents)
em_r, f1_r = score(predictions_r, dev_y)
log.info("[dev EM: {:.5f} F1: {:.5f} Ent: {:.5f}]".format(em, f1, ent))
log.info("[dev EMR: {:.5f} F1R: {:.5f}]".format(em_r, f1_r))
best_f1_score = f1
''' interleaved training '''
train_ent_batches = BatchGen(
train_ent, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda)
len_train_ent_batches = len(train_ent_batches)
train_ent_batches = iter(train_ent_batches)
n_reg = 0
n_ent = 0
for epoch in range(args.epochs):
log.warning('Epoch {}'.format(epoch))
train_reg_batches = BatchGen(
train_reg, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda)
start = datetime.now()
for i_reg, reg_batch in enumerate(train_reg_batches):
model.update(reg_batch)
n_reg += 1
if n_reg > args.start_ent:
if i_reg % args.n_reg_per_ent == 0:
for j in range(args.n_ent_per_reg):
try:
model.update_entropy(next(train_ent_batches),
gamma=args.gamma)
n_ent += 1
except StopIteration:
n_ent = 0
train_ent_batches = iter(BatchGen(
train_ent, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size,
gpu=args.cuda))
if n_reg % args.n_report == 0:
log.info('epoch [{:2}] batch [{}, {}] loss[{:.5f}] entropy[{:.5f}]'.format(
epoch, i_reg, n_ent, model.train_loss.avg,
-model.entropy_loss.avg / args.gamma))
# if n_reg % args.n_eval == 0:
dev_reg_batches = BatchGen(
dev_reg, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size,
evaluation=True, gpu=args.cuda)
dev_ent_batches = BatchGen(
dev_ent, batch_size=args.batch_size,
pos_size=args.pos_size, ner_size=args.ner_size,
evaluation=True, gpu=args.cuda)
''' regular evaluation '''
predictions = []
for batch in dev_reg_batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
''' entropy evaluation '''
ents, predictions_r = [], []
for batch in dev_ent_batches:
p, _, ss, se, _, _ = model.predict(batch, get_all=True)
ss = ss.cpu().numpy()
se = se.cpu().numpy()
ents.append(scipy.stats.entropy(ss.T).sum() + \
scipy.stats.entropy(se.T).sum())
predictions_r.extend(p)
ent = sum(ents) / len(ents)
em_r, f1_r = score(predictions_r, dev_y)
log.info("dev EM: {:.5f} F1: {:.5f} Ent: {:.5f}".format(em, f1, ent))
log.info("[dev EMR: {:.5f} F1R: {:.5f}]".format(em_r, f1_r))
''' save best model '''
if f1 > best_f1_score:
best_f1_score = f1
model_file = os.path.join(out_dir, 'best_model.pt')
model.save(model_file, epoch)
log.info('[save best model F1: {:.5f}]'.format(best_f1_score))
''' save models '''
model_file = os.path.join(
out_dir, 'checkpoint_epoch_{}.pt'.format(epoch))
model.save(model_file, epoch)
log.info("[save model {}]".format(model_file))
def main():
if args.cuda:
checkpoint = torch.load(args.testmodel)
else:
checkpoint = torch.load(args.testmodel, map_location=lambda storage, loc: storage)
state_dict = checkpoint['state_dict']
opt = checkpoint['config']
dev_file = 'SQuAD/dev-v1.1.json'
dev = flatten_json(dev_file, 'dev')
with open('SQuAD/meta.msgpack', 'rb') as f:
meta = msgpack.load(f, encoding='utf8')
with open(opt['data_file'], 'rb') as f:
data = msgpack.load(f, encoding='utf8')
embedding = torch.Tensor(meta['glove_char_embedding'])
opt['cuda'] = args.cuda
dev = [x[:-1] for x in data['dev']]
dev_y = [x[-1] for x in data['dev']]
model = DocReaderModel(opt, embedding, state_dict)
if args.cuda:
model.cuda()
result_list = []
number_error = 0
error_id_list = []
id = 0
number_cases = len(dev)
batches = iter(BatchGen(opt, dev, batch_size=1, gpu=args.cuda, evaluation=True))
for id in range(number_cases):
input = next(batches)
predictions = model.predict(input)
predict_answer = predictions[0]
answer = dev_y[id]
em_score = _exact_match(predict_answer, answer)
f1_score = _f1_score(predict_answer, answer)
print('------ Case: {} ------:'.format(id))
print('Predictions list:{}'.format(predictions[0:5]))
print('Predict Answer: {}'.format(predict_answer))
print('True Answer: {}'.format(answer))
print('examt match: {}'.format(em_score))
print('f1 score: {}'.format(f1_score))
result = {}
result["case_id"]= id
result["predict_answer"]= predict_answer
result["true_answer"]= answer
result["em_score"]= em_score
result["f1_score"]= f1_score
result_list.append(result)
if f1_score < 0.5:
number_error +=1
error_id_list.append(id)
id += 1
print(opt)
return error_id_list, number_error/number_cases, result_list
def main():
from args import conf
parser = argparse.ArgumentParser()
parser.add_argument('--resume', default=False)
parser.add_argument('--resume-options', default=False)
args = parser.parse_args()
# set random seed
random.seed(conf.seed)
torch.manual_seed(conf.seed)
if conf.cuda:
torch.cuda.manual_seed(conf.seed)
# setup logger
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
fh = logging.FileHandler('main.log')
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter(
fmt='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
log.addHandler(fh)
log.addHandler(ch)
train, dev, dev_y, embedding, conf = load_data(conf)
log.info(conf)
log.info('[Data loaded.]')
if args.resume:
log.info('[loading previous model...]')
checkpoint = torch.load(args.resume)
if args.resume_options:
conf = checkpoint['config']
state_dict = checkpoint['state_dict']
model = DocReaderModel(vars(conf), embedding, state_dict)
epoch_0 = checkpoint['epoch'] + 1
for i in range(checkpoint['epoch']):
random.shuffle(list(range(len(train)))) # synchronize random seed
if conf.reduce_lr:
for param_group in model.optimizer.param_groups:
param_group['lr'] *= conf.lr_decay
log.info('[learning rate reduced by {}]'.format(conf.lr_decay))
else:
model = DocReaderModel(vars(conf), embedding)
epoch_0 = 1
if conf.cuda:
model.cuda()
if args.resume:
batches = BatchGen(
dev, batch_size=conf.batch_size,
pos_size=conf.pos_size, ner_size=conf.ner_size,
gpu=conf.cuda, evaluation=True)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
log.info("[dev EM: {} F1: {}]".format(em, f1))
best_val_score = f1
else:
best_val_score = 0.0
for epoch in range(epoch_0, epoch_0 + conf.epochs):
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(
train, batch_size=conf.batch_size,
pos_size=conf.pos_size, ner_size=conf.ner_size,
gpu=conf.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % conf.log_per_updates == 0:
log.info('epoch [{0:2}] updates[{1:6}] \
train loss[{2:.5f}] remaining[{3}]'.format(
epoch, model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0]))
# eval
if epoch % conf.eval_per_epoch == 0:
batches = BatchGen(
dev, batch_size=conf.batch_size,
pos_size=conf.pos_size, ner_size=conf.ner_size,
gpu=conf.cuda, evaluation=True)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
log.warning("dev EM: {} F1: {}".format(em, f1))
# save
if not conf.save_last_only or epoch == epoch_0 + conf.epochs - 1:
model_file = 'results/baseline_epoch_{}.pt'.format(epoch)
model.save(model_file, epoch)
if f1 > best_val_score:
best_val_score = f1
copyfile(
model_file,
os.path.join('results/baseline.pt'))
log.info('[new best model saved.]')
def main():
args, log = setup()
log.info('[Program starts. Loading data...]')
train, dev, dev_y, embedding, opt = load_data(vars(args))
log.info(opt)
log.info('[Data loaded.]')
if args.save_dawn_logs:
dawn_start = datetime.now()
log.info('dawn_entry: epoch\tf1Score\thours')
if args.resume:
log.info('[loading previous model...]')
checkpoint = torch.load(os.path.join(args.model_dir, args.resume))
if args.resume_options:
opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = DocReaderModel(opt, embedding, state_dict)
epoch_0 = checkpoint['epoch'] + 1
# synchronize random seed
random.setstate(checkpoint['random_state'])
torch.random.set_rng_state(checkpoint['torch_state'])
if args.cuda:
torch.cuda.set_rng_state(checkpoint['torch_cuda_state'])
if args.reduce_lr:
lr_decay(model.optimizer, lr_decay=args.reduce_lr)
log.info('[learning rate reduced by {}]'.format(args.reduce_lr))
batches = BatchGen(dev,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for i, batch in enumerate(batches):
predictions.extend(model.predict(batch))
log.debug('> evaluating [{}/{}]'.format(i, len(batches)))
em, f1 = score(predictions, dev_y)
log.info("[dev EM: {} F1: {}]".format(em, f1))
if math.fabs(em - checkpoint['em']) > 1e-3 or math.fabs(
f1 - checkpoint['f1']) > 1e-3:
log.info('Inconsistent: recorded EM: {} F1: {}'.format(
checkpoint['em'], checkpoint['f1']))
log.error(
'Error loading model: current code is inconsistent with code used to train the previous model.'
)
exit(1)
best_val_score = checkpoint['best_eval']
else:
model = DocReaderModel(opt, embedding)
epoch_0 = 1
best_val_score = 0.0
for epoch in range(epoch_0, epoch_0 + args.epochs):
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % args.log_per_updates == 0:
log.info(
'> epoch [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}]'
.format(
epoch, model.updates, model.train_loss.value,
str((datetime.now() - start) / (i + 1) *
(len(batches) - i - 1)).split('.')[0]))
log.debug('\n')
# eval
batches = BatchGen(dev,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for i, batch in enumerate(batches):
predictions.extend(model.predict(batch))
log.debug('> evaluating [{}/{}]'.format(i, len(batches)))
em, f1 = score(predictions, dev_y)
log.warning("dev EM: {} F1: {}".format(em, f1))
if args.save_dawn_logs:
time_diff = datetime.now() - dawn_start
log.warning("dawn_entry: {}\t{}\t{}".format(
epoch, f1 / 100.0, float(time_diff.total_seconds() / 3600.0)))
# save
if not args.save_last_only or epoch == epoch_0 + args.epochs - 1:
model_file = os.path.join(args.model_dir,
'checkpoint_epoch_{}.pt'.format(epoch))
model.save(model_file, epoch, [em, f1, best_val_score])
if f1 > best_val_score:
best_val_score = f1
copyfile(model_file,
os.path.join(args.model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
checkpoint = torch.load(args.model_file, map_location=lambda storage, loc: storage)
state_dict = checkpoint['state_dict']
opt = checkpoint['config']
with open('SQuAD/meta.msgpack', 'rb') as f:
meta = msgpack.load(f, encoding='utf8')
embedding = torch.Tensor(meta['embedding'])
opt['pretrained_words'] = True
opt['vocab_size'] = embedding.size(0)
opt['embedding_dim'] = embedding.size(1)
opt['pos_size'] = len(meta['vocab_tag'])
opt['ner_size'] = len(meta['vocab_ent'])
opt['cuda'] = args.cuda
BatchGen.pos_size = opt['pos_size']
BatchGen.ner_size = opt['ner_size']
model = DocReaderModel(opt, embedding, state_dict)
if args.cuda:
model.cuda()
w2id = {w: i for i, w in enumerate(meta['vocab'])}
tag2id = {w: i for i, w in enumerate(meta['vocab_tag'])}
ent2id = {w: i for i, w in enumerate(meta['vocab_ent'])}
init()
evidence = 'Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.'
question = 'When the Virgin Mary reputedly appeared to Saint Bernadette Soubirous ?'
start_time = time.time()
annotated = annotate(('interact-{}'.format(id_), evidence, question), meta['wv_cased'])
model_in = to_id(annotated, w2id, tag2id, ent2id)
model_in = next(iter(BatchGen([model_in], batch_size=1, gpu=args.cuda, evaluation=True)))
prediction = model.predict(model_in)[0]
def main():
from args import args
parser = argparse.ArgumentParser()
parser.add_argument('--model', required=True)
_args = parser.parse_args()
args.load_model_dir = _args.model
out_dir = prepare_output_dir(args, 'results')
print('Generating targeted rawr data from [{}].'.format(
args.load_model_dir))
print('Saving to {}'.format(out_dir))
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
train, dev, dev_y, embedding, opt = load_data(args)
data = {'train': train, 'dev': dev}
state_dict = torch.load(args.load_model_dir)['state_dict']
model = DocReaderModel(vars(opt), embedding, state_dict)
model.cuda()
# get answers and targets for original question
regular_train = BatchGen([x[:8] for x in train],
batch_size=30,
pos_size=args.pos_size,
ner_size=args.ner_size,
gpu=args.cuda,
evaluation=True)
all_answers, all_target_s, all_target_e = [], [], []
for i, batch in enumerate(tqdm(regular_train)):
# if i > 10:
# break
answers, _, score_s, score_e, _, _ = model.predict(batch, get_all=True)
target_s = np.argmax(score_s, 1).tolist()
target_e = np.argmax(score_e, 1).tolist()
all_answers += answers
all_target_s += target_s
all_target_e += target_e
all_train = zip(train, all_answers, all_target_s, all_target_s)
groups = defaultdict(list)
for x in all_train:
groups[x[0][0][:-2]].append(x)
train_with_other = []
other_answers = []
other_target_s = []
other_target_e = []
for group in groups.values():
data, answer, target_s, target_e = list(map(list, zip(*group)))
for i in range(len(data)):
for j in range(len(data)):
if answer[i] != answer[j]:
train_with_other.append(data[i])
other_answers.append(answer[j])
other_target_s.append(target_s[j])
other_target_e.append(target_e[j])
other_train = BatchGen([x[:8] for x in train_with_other],
batch_size=30,
pos_size=args.pos_size,
ner_size=args.ner_size,
gpu=args.cuda,
evaluation=True)
targeted_train = []
start = 0
for i, batch in enumerate(tqdm(other_train)):
batch_size = batch[5].shape[0]
end = start + batch_size
# if i >= 2500:
# break
# if i < 2500 or i == 2530:
# start = end
# continue
# if i >=5000:
# break
# if i < 5000:
# start = end
# continue
# if i >= 7500:
# break
if i < 7500:
start = end
continue
ts = Variable(torch.LongTensor(other_target_s[start:end])).cuda()
te = Variable(torch.LongTensor(other_target_e[start:end])).cuda()
ans = other_answers[start:end]
reduced, _ = get_targeted_rawr(model, batch, ans, ts, te)
for j in range(batch_size):
if len(reduced[j]) == 0:
continue
for r in reduced[j]:
x = train_with_other[start + j]
x[5] = r
targeted_train.append(x)
start = end
print(len(targeted_train))
out_path = os.path.join(out_dir, 'targeted_other.train.pkl')
with open(out_path, 'wb') as f:
pickle.dump(targeted_train, f)
def main():
log.info('[program starts.]')
train, dev, dev_y, embedding, opt = load_data(vars(args))
log.info('[Data loaded.]')
run_name = str(time.time())
graph = tf.Graph()
with graph.as_default():
log.info('[Loading graph.]')
model = DocReaderModel(opt, embedding)
log.info('[Graph loaded.]')
sv = tf.train.Supervisor(logdir=opt["model_dir"],
save_model_secs=opt["save_model_secs"])
with sv.managed_session() as sess:
log.info('[Begin training.]')
step = 0
test_count = 0
epoch = 0
while not sv.should_stop() and epoch < args.epoches:
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, opt=opt)
start = datetime.now()
for i, batch in enumerate(batches):
if sv.should_stop():
log.warning("Supervisor: should_stop")
break
t_step = time.time()
step, _, loss, preds, y_true, learn_rate = model.train(
batch, sess)
em, f1 = score(preds, y_true)
log.warning("train EM: {} F1: {} in {} seconds".format(
em, f1,
time.time() - t_step))
sendStatElastic({
"phase": "train",
"name": "DrQA",
"run_name": run_name,
"step": int(step),
"precision": float(em),
"f1": float(f1),
"loss": float(loss),
"epoch": epoch,
"learning_rate": float(learn_rate)
})
if i % args.log_per_updates == 0:
log.info('updates[{}] remaining[{}]'.format(
step,
str((datetime.now() - start) / (i + 1) *
(len(batches) - i - 1)).split('.')[0]))
# eval
if step - test_count * args.eval_per_step > args.eval_per_step:
te_batches = BatchGen(dev,
batch_size=args.batch_size,
opt=opt,
evaluation=True)
predictions = []
for batch in te_batches:
predictions.extend(model.test(batch, sess))
em, f1 = score(predictions, dev_y)
sendStatElastic({
"phase": "test",
"name": "DrQA",
"run_name": run_name,
"step": float(step),
"precision": float(em),
"f1": float(f1),
"epoch": epoch
})
log.warning("dev EM: {} F1: {}".format(em, f1))
test_count = step // args.eval_per_step
epoch += 1
def main():
log.info('[program starts.]')
train, dev, dev_y, embedding, opt = load_data(vars(args))
log.info('[Data loaded. ]')
if args.resume:
log.info('[loading previous model...]')
checkpoint = torch.load(os.path.join(model_dir, args.resume))
if args.resume_options:
opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = DocReaderModel(opt, embedding, state_dict)
epoch_0 = checkpoint['epoch'] + 1
for i in range(checkpoint['epoch']):
random.shuffle(list(range(len(train)))) # synchronize random seed
if args.reduce_lr:
lr_decay(model.optimizer, lr_decay=args.reduce_lr)
else:
model = DocReaderModel(opt, embedding)
epoch_0 = 1
if args.cuda:
model.cuda()
############################
point_em = []
point_f1 = []
point_loss = []
stacked_epoch = []
train_loss_list = []
############################
if args.resume:
batches = BatchGen(dev, batch_size=1, evaluation=True, gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
############################################################################################################
for i in range(epoch_0 - 1):
stacked_epoch.append(i + 1) #stack epoch from 1 to (epoch_0 - 1)
for em_resume in open(em_dir + "/until_epoch_" + str(epoch_0 - 1) +
"_em.txt").read().strip().split('\n'):
em_resume = float(em_resume)
point_em.append(em_resume)
for f1_resume in open(f1_dir + "/until_epoch_" + str(epoch_0 - 1) +
"_f1.txt").read().strip().split('\n'):
f1_resume = float(f1_resume)
point_f1.append(f1_resume)
for loss_resume in open(loss_dir + "/until_epoch_" + str(epoch_0 - 1) +
"_loss.txt").read().strip().split('\n'):
loss_resume = float(loss_resume)
point_loss.append(loss_resume)
save_each_plot(stacked_epoch, point_em, EM_graph_dir, 'EM')
save_each_plot(stacked_epoch, point_f1, F1_graph_dir, 'F1')
save_each_plot(stacked_epoch, point_loss, Loss_graph_dir, 'Loss')
############################################################################################################
log.info("[dev EM: {} F1: {}]".format(em, f1))
best_val_score = f1
else:
best_val_score = 0.0
for epoch in range(epoch_0, epoch_0 + args.epochs):
log.warn('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % args.log_per_updates == 0:
log.info(
'updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format(
model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) *
(len(batches) - i - 1)).split('.')[0]))
train_loss_list.append(
model.train_loss.avg
) # ########## <- add train_loss of all batches
train_loss_avg = np.sum(train_loss_list) / len(
train_loss_list) # ######## <- added.
print(train_loss_avg)
# eval
if epoch % args.eval_per_epoch == 0:
batches = BatchGen(dev,
batch_size=1,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_y)
######################################################################################
stacked_epoch.append(epoch)
point_em.append(em)
point_f1.append(f1)
point_loss.append(train_loss_avg)
print("train_loss:")
print(model.train_loss.avg)
with open(em_dir + "/until_epoch_" + str(epoch) + "_em.txt",
"wb") as f:
np.savetxt(f, point_em)
print("until_epoch_" + str(epoch) + "em.txt saved.")
with open(f1_dir + "/until_epoch_" + str(epoch) + "_f1.txt",
"wb") as f:
np.savetxt(f, point_f1)
print("until_epoch_" + str(epoch) + "f1.txt saved.")
with open(loss_dir + "/until_epoch_" + str(epoch) + "_loss.txt",
"wb") as f:
np.savetxt(f, point_loss)
print("until_epoch_" + str(epoch) + "loss.txt saved.")
save_each_plot(stacked_epoch, point_em, EM_graph_dir, 'EM')
save_each_plot(stacked_epoch, point_f1, F1_graph_dir, 'F1')
save_each_plot(stacked_epoch, point_loss, Loss_graph_dir, 'Loss')
######################################################################################
log.warn("dev EM: {} F1: {}".format(em, f1))
# save
if not args.save_last_only or epoch == epoch_0 + args.epochs - 1:
model_file = os.path.join(model_dir,
'checkpoint_epoch_{}.pt'.format(epoch))
model.save(model_file, epoch)
if f1 > best_val_score:
best_val_score = f1
copyfile(model_file, os.path.join(model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
#############################################################################################
# After processing for all epoch, save the plot of all saved graphs(all_previous + present)
save_all_model_plot('../result/data/em_data', 'EM', EM_graph_dir)
save_all_model_plot('../result/data/f1_data', 'F1', F1_graph_dir)
save_all_model_plot('../result/data/loss_data', 'Loss', Loss_graph_dir)
def main():
from args import conf, rawr_conf
parser = argparse.ArgumentParser()
parser.add_argument('--model', default='results/baseline.pt')
parser.add_argument('--fold', required=True)
args = parser.parse_args()
out_dir = prepare_output_dir(conf, 'results', 'rawr')
pkl_dir = os.path.join(out_dir, '{}.pkl'.format(args.fold))
print('Generating [{}] rawr data from [{}].'.format(args.fold, args.model))
print('Saving to {}'.format(pkl_dir))
random.seed(conf.seed)
torch.manual_seed(conf.seed)
if conf.cuda:
torch.cuda.manual_seed(conf.seed)
with open('data/meta.msgpack', 'rb') as f:
vocab = msgpack.load(f, encoding='utf-8')['vocab']
train, dev, dev_y, embedding, opt = load_data(conf)
data = {'train': train, 'dev': dev}
state_dict = torch.load(args.model)['state_dict']
model = DocReaderModel(vars(opt), embedding, state_dict)
model.cuda()
batches = {}
batches['train'] = BatchGen([x[:8] for x in train],
batch_size=30,
pos_size=conf.pos_size,
ner_size=conf.ner_size,
gpu=conf.cuda,
evaluation=True)
batches['dev'] = BatchGen(dev,
batch_size=30,
pos_size=conf.pos_size,
ner_size=conf.ner_size,
gpu=conf.cuda,
evaluation=True)
checkpoint = []
example_idx = 0
for batch_i, batch in enumerate(tqdm(batches[args.fold])):
n_examples = batch[1].shape[0]
# original predictions
r = model.predict(batch, get_all=True)
target = r[0]
original_score_s = r[2].cpu().numpy()
original_score_e = r[3].cpu().numpy()
original_index_s = r[4]
original_index_e = r[5]
reduced, removed = get_rawr(model,
batch,
max_beam_size=rawr_conf.max_beam_size)
for i in range(n_examples):
beam_size = len(reduced[i])
rq = torch.LongTensor(reduced[i])
mask = torch.ByteTensor(np.zeros_like(rq))
test_batch = [batch_repeat(x[i], beam_size) for x in batch[:5]]
test_batch += [rq, mask]
test_batch += [batch_repeat(x[i], beam_size) for x in batch[7:]]
output = model.predict(test_batch, get_all=True)
preds = output[0]
reduced_score_s = output[2].cpu().numpy()
reduced_score_e = output[3].cpu().numpy()
reduced_index_s = output[4]
reduced_index_e = output[5]
idx = example_idx + i
assert batch[8][i] == data[args.fold][idx][
7] # check if the spans match
if args.fold == 'train':
indices = data['train'][idx][i][7]
start, end = data['train'][i][-2], data['train'][i][-1]
start, end = indices[start][0], indices[end][1]
label = [train[i][6][start:end]]
else:
label = dev_y[idx]
og = {
'batch':
data[args.fold][idx],
'label':
label,
'score_e':
original_score_e[i],
'score_s':
original_score_s[i],
'index_e':
original_index_e[i],
'index_s':
original_index_s[i],
'prediction':
target[i],
'context_readable':
batch[7][i],
'question_readable':
' '.join(vocab[x] for x in batch[5][i] if x != 0),
}
rd = []
for j, e in enumerate(reduced[i]):
x = list(data[args.fold][idx])
x[5] = e
rd.append({
'batch':
x,
'label':
label,
'removed_indices':
removed[i][j],
'context_readable':
batch[7][i],
'question_readable':
' '.join(vocab[x] for x in reduced[i][j] if x != 0),
'score_e':
reduced_score_e[j],
'score_s':
reduced_score_s[j],
'index_e':
reduced_index_e[j],
'index_s':
reduced_index_s[j],
'prediction':
preds[j]
})
checkpoint.append({'original': og, 'reduced': rd})
example_idx += n_examples
with open(pkl_dir, 'wb') as f:
pickle.dump(checkpoint, f)
