def main_targeted(): from args import args parser = argparse.ArgumentParser() parser.add_argument('--model', required=True) parser.add_argument('--fold', required=True) _args = parser.parse_args() args.load_model_dir = _args.model args.fold = _args.fold out_dir = prepare_output_dir(args, 'results') print('Generating [{}] targeted rawr data from [{}].'.format(args.fold, args.load_model_dir)) pkl_dir = os.path.join(out_dir, '{}.targeted.pkl'.format(args.fold)) print('Saving to {}'.format(pkl_dir)) random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) train, dev, dev_y, embedding, opt = load_data(args) data = {'train': train, 'dev': dev} state_dict = torch.load(args.load_model_dir)['state_dict'] model = DocReaderModel(vars(opt), embedding, state_dict) model.cuda() batches = {} batches['train'] = BatchGen( [x[:8] for x in train], batch_size=30, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda, evaluation=True) batches['dev'] = BatchGen(dev, batch_size=30, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda, evaluation=True) all_reduced = [] all_removed = [] example_idx = 0 for batch_i, batch in enumerate(tqdm(batches[args.fold])): # if batch_i > 10: # break n_examples = batch[1].shape[0] answers, _, score_s, score_e, _, _ = model.predict(batch, get_all=True) target_s = Variable(torch.max(score_s, 1)[1]).cuda() target_e = Variable(torch.max(score_e, 1)[1]).cuda() reduced, removed = get_targeted_rawr( model, batch, answers, target_s, target_e, max_beam_size=5) for i in range(n_examples): idx = example_idx + i assert batch[8][i] == data[args.fold][idx][7] # check if the spans match all_reduced.append([]) for j, e in enumerate(reduced[i]): x = list(data[args.fold][idx]) x[5] = e all_reduced[-1].append(x) all_removed.append(removed[i]) example_idx += n_examples with open(pkl_dir, 'wb') as f: ckp = {'reduced': all_reduced, 'removed': all_removed} pickle.dump(ckp, f)
def test(): from args import conf parser = argparse.ArgumentParser() parser.add_argument('--baseline', required=True) args = parser.parse_args() # set random seed random.seed(conf.seed) torch.manual_seed(conf.seed) if conf.cuda: torch.cuda.manual_seed(conf.seed) train, dev_x, dev_y, embedding, conf = load_data(conf) checkpoint = torch.load(args.baseline) # opt = checkpoint['config'] model = DocReaderModel(vars(conf), embedding, checkpoint['state_dict']) model.cuda() dev_batches = BatchGen( dev_x, batch_size=conf.batch_size, pos_size=conf.pos_size, ner_size=conf.ner_size, gpu=conf.cuda, evaluation=True) predictions = [] for batch in dev_batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) print(em, f1)
def main(): log.info('[program starts.]') train, dev, dev_y, embedding, opt = load_data(vars(args)) log.info('[Data loaded.]') if args.resume: log.info('[loading previous model...]') checkpoint = torch.load(os.path.join(model_dir, args.resume)) if args.resume_options: opt = checkpoint['config'] state_dict = checkpoint['state_dict'] model = DocReaderModel(opt, embedding, state_dict) epoch_0 = checkpoint['epoch'] + 1 for i in range(checkpoint['epoch']): random.shuffle(list(range(len(train)))) # synchronize random seed if args.reduce_lr: lr_decay(model.optimizer, lr_decay=args.reduce_lr) else: model = DocReaderModel(opt, embedding) epoch_0 = 1 if args.cuda: model.cuda() if args.resume: batches = BatchGen(dev, batch_size=args.batch_size, evaluation=True, gpu=args.cuda) predictions = [] for batch in batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) log.info("[dev EM: {} F1: {}]".format(em, f1)) best_val_score = f1 else: best_val_score = 0.0 for epoch in range(epoch_0, epoch_0 + args.epochs): log.warn('Epoch {}'.format(epoch)) # train batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda) start = datetime.now() for i, batch in enumerate(batches): model.update(batch) if i % args.log_per_updates == 0: log.info( 'updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format( model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0])) # eval if epoch % args.eval_per_epoch == 0: batches = BatchGen(dev, batch_size=1, evaluation=True, gpu=args.cuda) predictions = [] for batch in batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) log.warn("dev EM: {} F1: {}".format(em, f1)) # save if not args.save_last_only or epoch == epoch_0 + args.epochs - 1: model_file = os.path.join(model_dir, 'checkpoint_epoch_{}.pt'.format(epoch)) model.save(model_file, epoch) if f1 > best_val_score: best_val_score = f1 copyfile(model_file, os.path.join(model_dir, 'best_model.pt')) log.info('[new best model saved.]')
def main(): from args import args # parser = argparse.ArgumentParser() # parser.add_argument('--model', required=True) # parser.add_argument('--train', required=True) # parser.add_argument('--dev', required=True) # args.load_model_dir = parser.parse_args().model # args.ent_train_dir = parser.parse_args().train # args.ent_dev_dir = parser.parse_args().dev args.load_model_dir = '/scratch0/shifeng/rawr/drqa/original.pt' args.ent_train_dir = 'results/20180217T172242.135276/train.pkl' args.ent_dev_dir = 'pkls/original.rawr.dev.pkl' args.other_train_dir = 'results/targeted_train_all.pkl' out_dir = prepare_output_dir(args, '/scratch0/shifeng/rawr/drqa/') log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) fh = logging.FileHandler(os.path.join(out_dir, 'output.log')) fh.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) formatter = logging.Formatter(fmt='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S') fh.setFormatter(formatter) ch.setFormatter(formatter) log.addHandler(fh) log.addHandler(ch) log.info('===== {} ====='.format(out_dir)) with open(os.path.join(out_dir, 'args.pkl'), 'wb') as f: pickle.dump(args, f) random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) log.info('loading regular data from {}'.format(args.data_file)) train_reg, dev_reg, dev_y, embedding, opt = load_data(args) log.info('{} regular training examples'.format(len(train_reg))) log.info('{} regular dev examples'.format(len(dev_reg))) # log.info(opt) ''' load data for regularization ''' log.info('loading entropy training data from {}'.format(args.ent_train_dir)) with open(args.ent_train_dir, 'rb') as f: train_ent = pickle.load(f) if isinstance(train_ent, dict) and 'reduced' in train_ent: train_ent = train_ent['reduced'] if isinstance(train_ent[0][0], list): train_ent = list(itertools.chain(*train_ent)) log.info('loading targeted training data from {}'.format(args.other_train_dir)) with open(args.other_train_dir, 'rb') as f: other_train_ent = pickle.load(f) if isinstance(other_train_ent, dict) and 'reduced' in train_ent: other_train_ent = other_train_ent['reduced'] if isinstance(other_train_ent[0][0], list): other_train_ent = list(itertools.chain(*other_train_ent)) train_ent += other_train_ent if args.filter_long > 0: train_ent = [x for x in train_ent if len(x[5]) < args.filter_long] log.info('loading entropy dev data from {}'.format(args.ent_train_dir)) with open(args.ent_dev_dir, 'rb') as f: dev_ent = pickle.load(f)['reduced'] if isinstance(dev_ent[0], list): # dev_ent = list(itertools.chain(*dev_ent)) dev_ent = [x[0] for x in dev_ent] # if args.filter_long > 0: # dev_ent = [x for x in dev_ent if len(x[5]) > args.filter_long] log.info('{} entropy training examples'.format(len(train_ent))) log.info('{} entropy dev examples'.format(len(dev_ent))) log.info('loading model from {}'.format(args.load_model_dir)) checkpoint = torch.load(args.load_model_dir) # opt = checkpoint['config'] state_dict = checkpoint['state_dict'] model = DocReaderModel(vars(opt), embedding, state_dict) model.cuda() ''' initial evaluation ''' dev_reg_batches = BatchGen( dev_reg, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, evaluation=True, gpu=args.cuda) dev_ent_batches = BatchGen( dev_ent, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, evaluation=True, gpu=args.cuda) predictions = [] for batch in dev_reg_batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) ents, predictions_r = [], [] for batch in dev_ent_batches: p, _, ss, se, _, _ = model.predict(batch, get_all=True) ss = ss.cpu().numpy() se = se.cpu().numpy() ents.append(scipy.stats.entropy(ss.T).sum() + \ scipy.stats.entropy(se.T).sum()) predictions_r.extend(p) ent = sum(ents) / len(ents) em_r, f1_r = score(predictions_r, dev_y) log.info("[dev EM: {:.5f} F1: {:.5f} Ent: {:.5f}]".format(em, f1, ent)) log.info("[dev EMR: {:.5f} F1R: {:.5f}]".format(em_r, f1_r)) best_f1_score = f1 ''' interleaved training ''' train_ent_batches = BatchGen( train_ent, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda) len_train_ent_batches = len(train_ent_batches) train_ent_batches = iter(train_ent_batches) n_reg = 0 n_ent = 0 for epoch in range(args.epochs): log.warning('Epoch {}'.format(epoch)) train_reg_batches = BatchGen( train_reg, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda) start = datetime.now() for i_reg, reg_batch in enumerate(train_reg_batches): model.update(reg_batch) n_reg += 1 if n_reg > args.start_ent: if i_reg % args.n_reg_per_ent == 0: for j in range(args.n_ent_per_reg): try: model.update_entropy(next(train_ent_batches), gamma=args.gamma) n_ent += 1 except StopIteration: n_ent = 0 train_ent_batches = iter(BatchGen( train_ent, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda)) if n_reg % args.n_report == 0: log.info('epoch [{:2}] batch [{}, {}] loss[{:.5f}] entropy[{:.5f}]'.format( epoch, i_reg, n_ent, model.train_loss.avg, -model.entropy_loss.avg / args.gamma)) # if n_reg % args.n_eval == 0: dev_reg_batches = BatchGen( dev_reg, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, evaluation=True, gpu=args.cuda) dev_ent_batches = BatchGen( dev_ent, batch_size=args.batch_size, pos_size=args.pos_size, ner_size=args.ner_size, evaluation=True, gpu=args.cuda) ''' regular evaluation ''' predictions = [] for batch in dev_reg_batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) ''' entropy evaluation ''' ents, predictions_r = [], [] for batch in dev_ent_batches: p, _, ss, se, _, _ = model.predict(batch, get_all=True) ss = ss.cpu().numpy() se = se.cpu().numpy() ents.append(scipy.stats.entropy(ss.T).sum() + \ scipy.stats.entropy(se.T).sum()) predictions_r.extend(p) ent = sum(ents) / len(ents) em_r, f1_r = score(predictions_r, dev_y) log.info("dev EM: {:.5f} F1: {:.5f} Ent: {:.5f}".format(em, f1, ent)) log.info("[dev EMR: {:.5f} F1R: {:.5f}]".format(em_r, f1_r)) ''' save best model ''' if f1 > best_f1_score: best_f1_score = f1 model_file = os.path.join(out_dir, 'best_model.pt') model.save(model_file, epoch) log.info('[save best model F1: {:.5f}]'.format(best_f1_score)) ''' save models ''' model_file = os.path.join( out_dir, 'checkpoint_epoch_{}.pt'.format(epoch)) model.save(model_file, epoch) log.info("[save model {}]".format(model_file))
def main(): if args.cuda: checkpoint = torch.load(args.testmodel) else: checkpoint = torch.load(args.testmodel, map_location=lambda storage, loc: storage) state_dict = checkpoint['state_dict'] opt = checkpoint['config'] dev_file = 'SQuAD/dev-v1.1.json' dev = flatten_json(dev_file, 'dev') with open('SQuAD/meta.msgpack', 'rb') as f: meta = msgpack.load(f, encoding='utf8') with open(opt['data_file'], 'rb') as f: data = msgpack.load(f, encoding='utf8') embedding = torch.Tensor(meta['glove_char_embedding']) opt['cuda'] = args.cuda dev = [x[:-1] for x in data['dev']] dev_y = [x[-1] for x in data['dev']] model = DocReaderModel(opt, embedding, state_dict) if args.cuda: model.cuda() result_list = [] number_error = 0 error_id_list = [] id = 0 number_cases = len(dev) batches = iter(BatchGen(opt, dev, batch_size=1, gpu=args.cuda, evaluation=True)) for id in range(number_cases): input = next(batches) predictions = model.predict(input) predict_answer = predictions[0] answer = dev_y[id] em_score = _exact_match(predict_answer, answer) f1_score = _f1_score(predict_answer, answer) print('------ Case: {} ------:'.format(id)) print('Predictions list:{}'.format(predictions[0:5])) print('Predict Answer: {}'.format(predict_answer)) print('True Answer: {}'.format(answer)) print('examt match: {}'.format(em_score)) print('f1 score: {}'.format(f1_score)) result = {} result["case_id"]= id result["predict_answer"]= predict_answer result["true_answer"]= answer result["em_score"]= em_score result["f1_score"]= f1_score result_list.append(result) if f1_score < 0.5: number_error +=1 error_id_list.append(id) id += 1 print(opt) return error_id_list, number_error/number_cases, result_list
def main(): from args import conf parser = argparse.ArgumentParser() parser.add_argument('--resume', default=False) parser.add_argument('--resume-options', default=False) args = parser.parse_args() # set random seed random.seed(conf.seed) torch.manual_seed(conf.seed) if conf.cuda: torch.cuda.manual_seed(conf.seed) # setup logger log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) fh = logging.FileHandler('main.log') fh.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) formatter = logging.Formatter( fmt='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S') fh.setFormatter(formatter) ch.setFormatter(formatter) log.addHandler(fh) log.addHandler(ch) train, dev, dev_y, embedding, conf = load_data(conf) log.info(conf) log.info('[Data loaded.]') if args.resume: log.info('[loading previous model...]') checkpoint = torch.load(args.resume) if args.resume_options: conf = checkpoint['config'] state_dict = checkpoint['state_dict'] model = DocReaderModel(vars(conf), embedding, state_dict) epoch_0 = checkpoint['epoch'] + 1 for i in range(checkpoint['epoch']): random.shuffle(list(range(len(train)))) # synchronize random seed if conf.reduce_lr: for param_group in model.optimizer.param_groups: param_group['lr'] *= conf.lr_decay log.info('[learning rate reduced by {}]'.format(conf.lr_decay)) else: model = DocReaderModel(vars(conf), embedding) epoch_0 = 1 if conf.cuda: model.cuda() if args.resume: batches = BatchGen( dev, batch_size=conf.batch_size, pos_size=conf.pos_size, ner_size=conf.ner_size, gpu=conf.cuda, evaluation=True) predictions = [] for batch in batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) log.info("[dev EM: {} F1: {}]".format(em, f1)) best_val_score = f1 else: best_val_score = 0.0 for epoch in range(epoch_0, epoch_0 + conf.epochs): log.warning('Epoch {}'.format(epoch)) # train batches = BatchGen( train, batch_size=conf.batch_size, pos_size=conf.pos_size, ner_size=conf.ner_size, gpu=conf.cuda) start = datetime.now() for i, batch in enumerate(batches): model.update(batch) if i % conf.log_per_updates == 0: log.info('epoch [{0:2}] updates[{1:6}] \ train loss[{2:.5f}] remaining[{3}]'.format( epoch, model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0])) # eval if epoch % conf.eval_per_epoch == 0: batches = BatchGen( dev, batch_size=conf.batch_size, pos_size=conf.pos_size, ner_size=conf.ner_size, gpu=conf.cuda, evaluation=True) predictions = [] for batch in batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) log.warning("dev EM: {} F1: {}".format(em, f1)) # save if not conf.save_last_only or epoch == epoch_0 + conf.epochs - 1: model_file = 'results/baseline_epoch_{}.pt'.format(epoch) model.save(model_file, epoch) if f1 > best_val_score: best_val_score = f1 copyfile( model_file, os.path.join('results/baseline.pt')) log.info('[new best model saved.]')
def main(): args, log = setup() log.info('[Program starts. Loading data...]') train, dev, dev_y, embedding, opt = load_data(vars(args)) log.info(opt) log.info('[Data loaded.]') if args.save_dawn_logs: dawn_start = datetime.now() log.info('dawn_entry: epoch\tf1Score\thours') if args.resume: log.info('[loading previous model...]') checkpoint = torch.load(os.path.join(args.model_dir, args.resume)) if args.resume_options: opt = checkpoint['config'] state_dict = checkpoint['state_dict'] model = DocReaderModel(opt, embedding, state_dict) epoch_0 = checkpoint['epoch'] + 1 # synchronize random seed random.setstate(checkpoint['random_state']) torch.random.set_rng_state(checkpoint['torch_state']) if args.cuda: torch.cuda.set_rng_state(checkpoint['torch_cuda_state']) if args.reduce_lr: lr_decay(model.optimizer, lr_decay=args.reduce_lr) log.info('[learning rate reduced by {}]'.format(args.reduce_lr)) batches = BatchGen(dev, batch_size=args.batch_size, evaluation=True, gpu=args.cuda) predictions = [] for i, batch in enumerate(batches): predictions.extend(model.predict(batch)) log.debug('> evaluating [{}/{}]'.format(i, len(batches))) em, f1 = score(predictions, dev_y) log.info("[dev EM: {} F1: {}]".format(em, f1)) if math.fabs(em - checkpoint['em']) > 1e-3 or math.fabs( f1 - checkpoint['f1']) > 1e-3: log.info('Inconsistent: recorded EM: {} F1: {}'.format( checkpoint['em'], checkpoint['f1'])) log.error( 'Error loading model: current code is inconsistent with code used to train the previous model.' ) exit(1) best_val_score = checkpoint['best_eval'] else: model = DocReaderModel(opt, embedding) epoch_0 = 1 best_val_score = 0.0 for epoch in range(epoch_0, epoch_0 + args.epochs): log.warning('Epoch {}'.format(epoch)) # train batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda) start = datetime.now() for i, batch in enumerate(batches): model.update(batch) if i % args.log_per_updates == 0: log.info( '> epoch [{0:2}] updates[{1:6}] train loss[{2:.5f}] remaining[{3}]' .format( epoch, model.updates, model.train_loss.value, str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0])) log.debug('\n') # eval batches = BatchGen(dev, batch_size=args.batch_size, evaluation=True, gpu=args.cuda) predictions = [] for i, batch in enumerate(batches): predictions.extend(model.predict(batch)) log.debug('> evaluating [{}/{}]'.format(i, len(batches))) em, f1 = score(predictions, dev_y) log.warning("dev EM: {} F1: {}".format(em, f1)) if args.save_dawn_logs: time_diff = datetime.now() - dawn_start log.warning("dawn_entry: {}\t{}\t{}".format( epoch, f1 / 100.0, float(time_diff.total_seconds() / 3600.0))) # save if not args.save_last_only or epoch == epoch_0 + args.epochs - 1: model_file = os.path.join(args.model_dir, 'checkpoint_epoch_{}.pt'.format(epoch)) model.save(model_file, epoch, [em, f1, best_val_score]) if f1 > best_val_score: best_val_score = f1 copyfile(model_file, os.path.join(args.model_dir, 'best_model.pt')) log.info('[new best model saved.]')
checkpoint = torch.load(args.model_file, map_location=lambda storage, loc: storage) state_dict = checkpoint['state_dict'] opt = checkpoint['config'] with open('SQuAD/meta.msgpack', 'rb') as f: meta = msgpack.load(f, encoding='utf8') embedding = torch.Tensor(meta['embedding']) opt['pretrained_words'] = True opt['vocab_size'] = embedding.size(0) opt['embedding_dim'] = embedding.size(1) opt['pos_size'] = len(meta['vocab_tag']) opt['ner_size'] = len(meta['vocab_ent']) opt['cuda'] = args.cuda BatchGen.pos_size = opt['pos_size'] BatchGen.ner_size = opt['ner_size'] model = DocReaderModel(opt, embedding, state_dict) if args.cuda: model.cuda() w2id = {w: i for i, w in enumerate(meta['vocab'])} tag2id = {w: i for i, w in enumerate(meta['vocab_tag'])} ent2id = {w: i for i, w in enumerate(meta['vocab_ent'])} init() evidence = 'Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.' question = 'When the Virgin Mary reputedly appeared to Saint Bernadette Soubirous ?' start_time = time.time() annotated = annotate(('interact-{}'.format(id_), evidence, question), meta['wv_cased']) model_in = to_id(annotated, w2id, tag2id, ent2id) model_in = next(iter(BatchGen([model_in], batch_size=1, gpu=args.cuda, evaluation=True))) prediction = model.predict(model_in)[0]
def main(): from args import args parser = argparse.ArgumentParser() parser.add_argument('--model', required=True) _args = parser.parse_args() args.load_model_dir = _args.model out_dir = prepare_output_dir(args, 'results') print('Generating targeted rawr data from [{}].'.format( args.load_model_dir)) print('Saving to {}'.format(out_dir)) random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) train, dev, dev_y, embedding, opt = load_data(args) data = {'train': train, 'dev': dev} state_dict = torch.load(args.load_model_dir)['state_dict'] model = DocReaderModel(vars(opt), embedding, state_dict) model.cuda() # get answers and targets for original question regular_train = BatchGen([x[:8] for x in train], batch_size=30, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda, evaluation=True) all_answers, all_target_s, all_target_e = [], [], [] for i, batch in enumerate(tqdm(regular_train)): # if i > 10: # break answers, _, score_s, score_e, _, _ = model.predict(batch, get_all=True) target_s = np.argmax(score_s, 1).tolist() target_e = np.argmax(score_e, 1).tolist() all_answers += answers all_target_s += target_s all_target_e += target_e all_train = zip(train, all_answers, all_target_s, all_target_s) groups = defaultdict(list) for x in all_train: groups[x[0][0][:-2]].append(x) train_with_other = [] other_answers = [] other_target_s = [] other_target_e = [] for group in groups.values(): data, answer, target_s, target_e = list(map(list, zip(*group))) for i in range(len(data)): for j in range(len(data)): if answer[i] != answer[j]: train_with_other.append(data[i]) other_answers.append(answer[j]) other_target_s.append(target_s[j]) other_target_e.append(target_e[j]) other_train = BatchGen([x[:8] for x in train_with_other], batch_size=30, pos_size=args.pos_size, ner_size=args.ner_size, gpu=args.cuda, evaluation=True) targeted_train = [] start = 0 for i, batch in enumerate(tqdm(other_train)): batch_size = batch[5].shape[0] end = start + batch_size # if i >= 2500: # break # if i < 2500 or i == 2530: # start = end # continue # if i >=5000: # break # if i < 5000: # start = end # continue # if i >= 7500: # break if i < 7500: start = end continue ts = Variable(torch.LongTensor(other_target_s[start:end])).cuda() te = Variable(torch.LongTensor(other_target_e[start:end])).cuda() ans = other_answers[start:end] reduced, _ = get_targeted_rawr(model, batch, ans, ts, te) for j in range(batch_size): if len(reduced[j]) == 0: continue for r in reduced[j]: x = train_with_other[start + j] x[5] = r targeted_train.append(x) start = end print(len(targeted_train)) out_path = os.path.join(out_dir, 'targeted_other.train.pkl') with open(out_path, 'wb') as f: pickle.dump(targeted_train, f)
def main(): log.info('[program starts.]') train, dev, dev_y, embedding, opt = load_data(vars(args)) log.info('[Data loaded.]') run_name = str(time.time()) graph = tf.Graph() with graph.as_default(): log.info('[Loading graph.]') model = DocReaderModel(opt, embedding) log.info('[Graph loaded.]') sv = tf.train.Supervisor(logdir=opt["model_dir"], save_model_secs=opt["save_model_secs"]) with sv.managed_session() as sess: log.info('[Begin training.]') step = 0 test_count = 0 epoch = 0 while not sv.should_stop() and epoch < args.epoches: log.warning('Epoch {}'.format(epoch)) # train batches = BatchGen(train, batch_size=args.batch_size, opt=opt) start = datetime.now() for i, batch in enumerate(batches): if sv.should_stop(): log.warning("Supervisor: should_stop") break t_step = time.time() step, _, loss, preds, y_true, learn_rate = model.train( batch, sess) em, f1 = score(preds, y_true) log.warning("train EM: {} F1: {} in {} seconds".format( em, f1, time.time() - t_step)) sendStatElastic({ "phase": "train", "name": "DrQA", "run_name": run_name, "step": int(step), "precision": float(em), "f1": float(f1), "loss": float(loss), "epoch": epoch, "learning_rate": float(learn_rate) }) if i % args.log_per_updates == 0: log.info('updates[{}] remaining[{}]'.format( step, str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0])) # eval if step - test_count * args.eval_per_step > args.eval_per_step: te_batches = BatchGen(dev, batch_size=args.batch_size, opt=opt, evaluation=True) predictions = [] for batch in te_batches: predictions.extend(model.test(batch, sess)) em, f1 = score(predictions, dev_y) sendStatElastic({ "phase": "test", "name": "DrQA", "run_name": run_name, "step": float(step), "precision": float(em), "f1": float(f1), "epoch": epoch }) log.warning("dev EM: {} F1: {}".format(em, f1)) test_count = step // args.eval_per_step epoch += 1
def main(): log.info('[program starts.]') train, dev, dev_y, embedding, opt = load_data(vars(args)) log.info('[Data loaded. ]') if args.resume: log.info('[loading previous model...]') checkpoint = torch.load(os.path.join(model_dir, args.resume)) if args.resume_options: opt = checkpoint['config'] state_dict = checkpoint['state_dict'] model = DocReaderModel(opt, embedding, state_dict) epoch_0 = checkpoint['epoch'] + 1 for i in range(checkpoint['epoch']): random.shuffle(list(range(len(train)))) # synchronize random seed if args.reduce_lr: lr_decay(model.optimizer, lr_decay=args.reduce_lr) else: model = DocReaderModel(opt, embedding) epoch_0 = 1 if args.cuda: model.cuda() ############################ point_em = [] point_f1 = [] point_loss = [] stacked_epoch = [] train_loss_list = [] ############################ if args.resume: batches = BatchGen(dev, batch_size=1, evaluation=True, gpu=args.cuda) predictions = [] for batch in batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) ############################################################################################################ for i in range(epoch_0 - 1): stacked_epoch.append(i + 1) #stack epoch from 1 to (epoch_0 - 1) for em_resume in open(em_dir + "/until_epoch_" + str(epoch_0 - 1) + "_em.txt").read().strip().split('\n'): em_resume = float(em_resume) point_em.append(em_resume) for f1_resume in open(f1_dir + "/until_epoch_" + str(epoch_0 - 1) + "_f1.txt").read().strip().split('\n'): f1_resume = float(f1_resume) point_f1.append(f1_resume) for loss_resume in open(loss_dir + "/until_epoch_" + str(epoch_0 - 1) + "_loss.txt").read().strip().split('\n'): loss_resume = float(loss_resume) point_loss.append(loss_resume) save_each_plot(stacked_epoch, point_em, EM_graph_dir, 'EM') save_each_plot(stacked_epoch, point_f1, F1_graph_dir, 'F1') save_each_plot(stacked_epoch, point_loss, Loss_graph_dir, 'Loss') ############################################################################################################ log.info("[dev EM: {} F1: {}]".format(em, f1)) best_val_score = f1 else: best_val_score = 0.0 for epoch in range(epoch_0, epoch_0 + args.epochs): log.warn('Epoch {}'.format(epoch)) # train batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda) start = datetime.now() for i, batch in enumerate(batches): model.update(batch) if i % args.log_per_updates == 0: log.info( 'updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format( model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(batches) - i - 1)).split('.')[0])) train_loss_list.append( model.train_loss.avg ) # ########## <- add train_loss of all batches train_loss_avg = np.sum(train_loss_list) / len( train_loss_list) # ######## <- added. print(train_loss_avg) # eval if epoch % args.eval_per_epoch == 0: batches = BatchGen(dev, batch_size=1, evaluation=True, gpu=args.cuda) predictions = [] for batch in batches: predictions.extend(model.predict(batch)) em, f1 = score(predictions, dev_y) ###################################################################################### stacked_epoch.append(epoch) point_em.append(em) point_f1.append(f1) point_loss.append(train_loss_avg) print("train_loss:") print(model.train_loss.avg) with open(em_dir + "/until_epoch_" + str(epoch) + "_em.txt", "wb") as f: np.savetxt(f, point_em) print("until_epoch_" + str(epoch) + "em.txt saved.") with open(f1_dir + "/until_epoch_" + str(epoch) + "_f1.txt", "wb") as f: np.savetxt(f, point_f1) print("until_epoch_" + str(epoch) + "f1.txt saved.") with open(loss_dir + "/until_epoch_" + str(epoch) + "_loss.txt", "wb") as f: np.savetxt(f, point_loss) print("until_epoch_" + str(epoch) + "loss.txt saved.") save_each_plot(stacked_epoch, point_em, EM_graph_dir, 'EM') save_each_plot(stacked_epoch, point_f1, F1_graph_dir, 'F1') save_each_plot(stacked_epoch, point_loss, Loss_graph_dir, 'Loss') ###################################################################################### log.warn("dev EM: {} F1: {}".format(em, f1)) # save if not args.save_last_only or epoch == epoch_0 + args.epochs - 1: model_file = os.path.join(model_dir, 'checkpoint_epoch_{}.pt'.format(epoch)) model.save(model_file, epoch) if f1 > best_val_score: best_val_score = f1 copyfile(model_file, os.path.join(model_dir, 'best_model.pt')) log.info('[new best model saved.]') ############################################################################################# # After processing for all epoch, save the plot of all saved graphs(all_previous + present) save_all_model_plot('../result/data/em_data', 'EM', EM_graph_dir) save_all_model_plot('../result/data/f1_data', 'F1', F1_graph_dir) save_all_model_plot('../result/data/loss_data', 'Loss', Loss_graph_dir)
def main(): from args import conf, rawr_conf parser = argparse.ArgumentParser() parser.add_argument('--model', default='results/baseline.pt') parser.add_argument('--fold', required=True) args = parser.parse_args() out_dir = prepare_output_dir(conf, 'results', 'rawr') pkl_dir = os.path.join(out_dir, '{}.pkl'.format(args.fold)) print('Generating [{}] rawr data from [{}].'.format(args.fold, args.model)) print('Saving to {}'.format(pkl_dir)) random.seed(conf.seed) torch.manual_seed(conf.seed) if conf.cuda: torch.cuda.manual_seed(conf.seed) with open('data/meta.msgpack', 'rb') as f: vocab = msgpack.load(f, encoding='utf-8')['vocab'] train, dev, dev_y, embedding, opt = load_data(conf) data = {'train': train, 'dev': dev} state_dict = torch.load(args.model)['state_dict'] model = DocReaderModel(vars(opt), embedding, state_dict) model.cuda() batches = {} batches['train'] = BatchGen([x[:8] for x in train], batch_size=30, pos_size=conf.pos_size, ner_size=conf.ner_size, gpu=conf.cuda, evaluation=True) batches['dev'] = BatchGen(dev, batch_size=30, pos_size=conf.pos_size, ner_size=conf.ner_size, gpu=conf.cuda, evaluation=True) checkpoint = [] example_idx = 0 for batch_i, batch in enumerate(tqdm(batches[args.fold])): n_examples = batch[1].shape[0] # original predictions r = model.predict(batch, get_all=True) target = r[0] original_score_s = r[2].cpu().numpy() original_score_e = r[3].cpu().numpy() original_index_s = r[4] original_index_e = r[5] reduced, removed = get_rawr(model, batch, max_beam_size=rawr_conf.max_beam_size) for i in range(n_examples): beam_size = len(reduced[i]) rq = torch.LongTensor(reduced[i]) mask = torch.ByteTensor(np.zeros_like(rq)) test_batch = [batch_repeat(x[i], beam_size) for x in batch[:5]] test_batch += [rq, mask] test_batch += [batch_repeat(x[i], beam_size) for x in batch[7:]] output = model.predict(test_batch, get_all=True) preds = output[0] reduced_score_s = output[2].cpu().numpy() reduced_score_e = output[3].cpu().numpy() reduced_index_s = output[4] reduced_index_e = output[5] idx = example_idx + i assert batch[8][i] == data[args.fold][idx][ 7] # check if the spans match if args.fold == 'train': indices = data['train'][idx][i][7] start, end = data['train'][i][-2], data['train'][i][-1] start, end = indices[start][0], indices[end][1] label = [train[i][6][start:end]] else: label = dev_y[idx] og = { 'batch': data[args.fold][idx], 'label': label, 'score_e': original_score_e[i], 'score_s': original_score_s[i], 'index_e': original_index_e[i], 'index_s': original_index_s[i], 'prediction': target[i], 'context_readable': batch[7][i], 'question_readable': ' '.join(vocab[x] for x in batch[5][i] if x != 0), } rd = [] for j, e in enumerate(reduced[i]): x = list(data[args.fold][idx]) x[5] = e rd.append({ 'batch': x, 'label': label, 'removed_indices': removed[i][j], 'context_readable': batch[7][i], 'question_readable': ' '.join(vocab[x] for x in reduced[i][j] if x != 0), 'score_e': reduced_score_e[j], 'score_s': reduced_score_s[j], 'index_e': reduced_index_e[j], 'index_s': reduced_index_s[j], 'prediction': preds[j] }) checkpoint.append({'original': og, 'reduced': rd}) example_idx += n_examples with open(pkl_dir, 'wb') as f: pickle.dump(checkpoint, f)