vocab += sorted([t for t in counter_c.keys() - counter_q.keys() if t in glove_vocab],
key=counter.get, reverse=True)
total = sum(counter.values())
matched = sum(counter[t] for t in vocab)
log.info('vocab {1}/{0} OOV {2}/{3} ({4:.4f}%)'.format(
len(counter), len(vocab), (total - matched), total, (total - matched) / total * 100))
vocab.insert(0, "<PAD>")
vocab.insert(1, "<UNK>")
vocab.insert(2, "<S>")
vocab.insert(3, "</S>")
return vocab
# vocab
tr_vocab = build_train_vocab(trQ_tokens, trC_tokens)
trC_ids = token2id(trC_tokens, tr_vocab, unk_id=1)
trQ_ids = token2id(trQ_tokens, tr_vocab, unk_id=1)
trQ_tokens = [["<S>"] + doc + ["</S>"] for doc in trQ_tokens]
trQ_ids = [[2] + qsent + [3] for qsent in trQ_ids]
# print(trQ_ids[:10])
# tags
vocab_tag = [''] + list(nlp.tagger.labels)
trC_tag_ids = token2id(trC_tags, vocab_tag)
# entities
vocab_ent = list(set([ent for sent in trC_ents for ent in sent]))
trC_ent_ids = token2id(trC_ents, vocab_ent, unk_id=0)
log.info('Found {} POS tags.'.format(len(vocab_tag)))
log.info('Found {} entity tags: {}'.format(len(vocab_ent), vocab_ent))
log.info('vocabulary for training is built.')
def preprocess_eval_data(filename, output_msgpack):
EvalData = process_jsonlines(filename)
filename = os.path.basename(filename)
log.info(filename + ' flattened.')
EvalDataP_iter = (pre_proc(p) for p in EvalData.P)
EvalDataH_iter = (pre_proc(h) for h in EvalData.H)
EvalDataP_docs = [
doc for doc in nlp.pipe(
EvalDataP_iter, batch_size=64, n_threads=args.threads)
]
EvalDataH_docs = [
doc for doc in nlp.pipe(
EvalDataH_iter, batch_size=64, n_threads=args.threads)
]
# tokens
EvalDataP_tokens = [[normalize_text(w.text) for w in doc]
for doc in EvalDataP_docs]
EvalDataH_tokens = [[normalize_text(w.text) for w in doc]
for doc in EvalDataH_docs]
log.info('All tokens for ' + filename + ' are obtained.')
# features
EvalDataP_tags, EvalDataP_ents, EvalDataP_features = feature_gen(
EvalDataP_docs, EvalDataH_docs)
EvalDataH_tags, EvalDataH_ents, EvalDataH_features = feature_gen(
EvalDataH_docs, EvalDataP_docs)
log.info('features for ' + filename + ' is generated.')
def build_EvalData_vocab(A, B): # most vocabulary comes from tr_vocab
existing_vocab = set(tr_vocab)
new_vocab = list(
set([
w for doc in A + B for w in doc
if w not in existing_vocab and w in glove_vocab
]))
vocab = tr_vocab + new_vocab
log.info('train vocab {0}, total vocab {1}'.format(
len(tr_vocab), len(vocab)))
return vocab
# vocab
EvalData_vocab = build_EvalData_vocab(
EvalDataP_tokens,
EvalDataH_tokens) # tr_vocab is a subset of EvalData_vocab
EvalDataP_ids = token2id(EvalDataP_tokens, EvalData_vocab, unk_id=1)
EvalDataH_ids = token2id(EvalDataH_tokens, EvalData_vocab, unk_id=1)
# tags
EvalDataP_tag_ids = token2id(EvalDataP_tags, vocab_tag)
EvalDataH_tag_ids = token2id(EvalDataH_tags,
vocab_tag) # vocab_tag same as training
# entities
EvalDataP_ent_ids = token2id(EvalDataP_ents,
vocab_ent) # vocab_ent same as training
EvalDataH_ent_ids = token2id(EvalDataH_ents,
vocab_ent) # vocab_ent same as training
log.info('vocabulary for ' + filename + ' is built.')
EvalData_embedding = build_embedding(
wv_file, EvalData_vocab,
wv_dim) # tr_embedding is a submatrix of EvalData_embedding
log.info('got embedding matrix for ' + filename)
result = {
'premise_ids': EvalDataP_ids,
'premise_features': EvalDataP_features, # exact match, tf
'premise_tags': EvalDataP_tag_ids, # POS tagging
'premise_ents': EvalDataP_ent_ids, # Entity recognition
'hypothesis_ids': EvalDataH_ids,
'hypothesis_features': EvalDataH_features, # exact match, tf
'hypothesis_tags': EvalDataH_tag_ids, # POS tagging
'hypothesis_ents': EvalDataH_ent_ids, # Entity recognition
'vocab': EvalData_vocab,
'embedding': EvalData_embedding.tolist(),
'answers': EvalData.label
}
with open(output_msgpack, 'wb') as f:
msgpack.dump(result, f)
log.info('saved ' + output_msgpack + ' to disk.')
set([
w for doc in questions + contexts for w in doc
if w not in existing_vocab and w in glove_vocab
]))
vocab = tr_vocab + new_vocab
log.info('train vocab {0}, total vocab {1}'.format(len(tr_vocab),
len(vocab)))
return vocab
# tokens
testC_tokens = [[normalize_text(w.text) for w in doc] for doc in testC_docs]
testQ_tokens = [[normalize_text(w.text) for w in doc] for doc in testQ_docs]
test_vocab = build_test_vocab(
testQ_tokens, testC_tokens) # tr_vocab is a subset of test_vocab
testC_ids = token2id(testC_tokens, test_vocab, unk_id=1)
testQ_ids = token2id(testQ_tokens, test_vocab, unk_id=1)
# tags
vocab_tag = list(nlp.tagger.tag_names)
testC_tag_ids = token2id(testC_tags, vocab_tag) # vocab_tag same as training
# entities
vocab_ent = [''] + nlp.entity.cfg[u'actions']['1']
testC_ent_ids = token2id(testC_ents, vocab_ent) # vocab_ent same as training
log.info('vocabulary for test is built.')
test_embedding = build_embedding(wv_file, test_vocab, wv_dim)
# tr_embedding is a submatrix of test_embedding
log.info('got embedding matrix for test.')
# don't store row name in csv
test.to_csv('SQuAD/test.csv', index=False, encoding='utf8')
key=counter.get,
reverse=True)
total = sum(counter.values())
matched = sum(counter[t] for t in vocab)
log.info('vocab {1}/{0} OOV {2}/{3} ({4:.4f}%)'.format(
len(counter), len(vocab), (total - matched), total,
(total - matched) / total * 100))
vocab.insert(0, "<PAD>")
vocab.insert(1, "<UNK>")
return vocab
# vocab
tr_vocab = build_train_vocab(trH_tokens, trP_tokens)
trP_ids = token2id(trP_tokens, tr_vocab, unk_id=1)
trH_ids = token2id(trH_tokens, tr_vocab, unk_id=1)
# tags
vocab_tag = list(nlp.tagger.labels)
trP_tag_ids = token2id(trP_tags, vocab_tag)
trH_tag_ids = token2id(trH_tags, vocab_tag)
# entities
vocab_ent = [
'', 'PERSON', 'CARDINAL', 'ORG', 'GPE', 'FAC', 'MONEY', 'NORP', 'DATE',
'TIME', 'ORDINAL', 'PERCENT', 'PRODUCT', 'LANGUAGE', 'LOC', 'QUANTITY',
'WORK_OF_ART', 'EVENT', 'LAW'
]
# vocab_ent = [''] + nlp.entity.move_names
matched = sum(counter[t] for t in vocab)
log.info('vocab {1}/{0} OOV {2}/{3} ({4:.4f}%)'.format(
len(counter), len(vocab), (total - matched), total,
(total - matched) / total * 100))
vocab.insert(0, "<PAD>")
vocab.insert(1, "<UNK>")
vocab.insert(2, "<S>")
vocab.insert(3, "</S>")
return vocab
# vocab
tr_vocab = build_train_vocab(
trQ_tokens, trC_tokens) #不在glove词典中的词就作为unk看待了 ,#tr_vocab实际上是一个list
trC_ids = token2id(trC_tokens, tr_vocab, unk_id=1)
trQ_ids = token2id(trQ_tokens, tr_vocab, unk_id=1) #转化为id表示了
trQ_tokens = [["<S>"] + doc + ["</S>"] for doc in trQ_tokens]
trQ_ids = [[2] + qsent + [3] for qsent in trQ_ids]
print("trQ_ids = ")
print(trQ_ids[:10])
# tags
a = list(nlp.tagger.labels)
# print("a = ",a)
# print(len(a))
vocab_tag = [''] + list(nlp.tagger.labels)
# 这些tag的意义还不知道，有50个tag，不可能是人工写规则，即便是使用，也是在后端用word embedding的方式，弄一个tag embedding
# print("vocab_tag = ",vocab_tag)
# print(len(vocab_tag))
# exit(789)
# trc_features:list of 每个word为一个元组(match_origin, match_lower, match_lemma, term_freq)
trC_tags, trC_ents, trC_features = feature_gen(trC_docs, train.context_idx,
trQ_docs, args.no_match)
log.info('features for training is generated: {}, {}, {}'.format(
len(trC_tags), len(trC_ents), len(trC_features)))
# tr_vocab = build_train_vocab(trQ_tokens, trC_tokens)
# trC_ids = token2id(trC_tokens, tr_vocab, unk_id=1)
# trQ_ids = token2id(trQ_tokens, tr_vocab, unk_id=1)
trC_ids = tokens2ids(trC_docs_forbert)
trQ_ids = tokens2ids(trQ_docs_forbert)
log.info('build index has been built by bert tokenization')
# tags
vocab_tag = [''] + list(nlp.tagger.labels)
trC_tag_ids = token2id(trC_tags, vocab_tag)
# entities
vocab_ent = list(set([ent for sent in trC_ents for ent in sent]))
trC_ent_ids = token2id(trC_ents, vocab_ent, unk_id=0)
log.info('Found {} POS tags.'.format(len(vocab_tag)))
log.info('Found {} entity tags: {}'.format(len(vocab_ent), vocab_ent))
# don't store row name in csv
# train.to_csv('QuAC_data/train.csv', index=False, encoding='utf8')
prev_CID, first_question = -1, []
for i, CID in enumerate(train.context_idx):
if not (CID == prev_CID):
first_question.append(i)
prev_CID = CID
def preprocess_data(dev_file):
dev, dev_context = flatten_json(dev_file, proc_dev)
dev = pd.DataFrame(dev,
columns=[
'context_idx', 'question', 'answer', 'answer_start',
'answer_end', 'answer_choice', 'all_answer', 'qid'
])
print('dev json data flattened.')
devC_iter = (pre_proc(c) for c in dev_context)
devQ_iter = (pre_proc(q) for q in dev.question)
nlp = spacy.load('en', disable=['parser'])
devC_docs = [
doc for doc in nlp.pipe(
devC_iter, batch_size=64, n_threads=multiprocessing.cpu_count())
]
devQ_docs = [
doc for doc in nlp.pipe(
devQ_iter, batch_size=64, n_threads=multiprocessing.cpu_count())
]
del nlp
devC_tokens = [[normalize_text(w.text) for w in doc] for doc in devC_docs]
devQ_tokens = [[normalize_text(w.text) for w in doc] for doc in devQ_docs]
devC_unnorm_tokens = [[w.text for w in doc] for doc in devC_docs]
print('All tokens for dev are obtained.')
dev_context_span = [
get_context_span(a, b) for a, b in zip(dev_context, devC_unnorm_tokens)
]
print('context span for dev is generated.')
ans_st_token_ls, ans_end_token_ls = [], []
for ans_st, ans_end, idx in zip(dev.answer_start, dev.answer_end,
dev.context_idx):
ans_st_token, ans_end_token = find_answer_span(dev_context_span[idx],
ans_st, ans_end)
ans_st_token_ls.append(ans_st_token)
ans_end_token_ls.append(ans_end_token)
dev['answer_start_token'], dev[
'answer_end_token'] = ans_st_token_ls, ans_end_token_ls
initial_len = len(dev)
dev.dropna(inplace=True) # modify self DataFrame
print('drop {0}/{1} inconsistent samples.'.format(initial_len - len(dev),
initial_len))
print('answer span for dev is generated.')
devC_tags, devC_ents, devC_features = feature_gen(devC_docs,
dev.context_idx,
devQ_docs, False)
print('features for dev is generated: {}, {}, {}'.format(
len(devC_tags), len(devC_ents), len(devC_features)))
dev_vocab = build_dev_vocab(
devQ_tokens, devC_tokens) # tr_vocab is a subset of dev_vocab
devC_ids = token2id(devC_tokens, dev_vocab, unk_id=1)
devQ_ids = token2id(devQ_tokens, dev_vocab, unk_id=1)
devQ_tokens = [["<S>"] + doc + ["</S>"] for doc in devQ_tokens]
devQ_ids = [[2] + qsent + [3] for qsent in devQ_ids]
# BERT stuff
devC_bert_tokens = tokenize(devC_tokens)
devC_bert_ids = [bert_tokens_to_ids(x) for x in devC_bert_tokens]
devQ_bert_tokens = tokenize(devQ_tokens)
devQ_bert_ids = [bert_tokens_to_ids(x) for x in devQ_bert_tokens]
devC_bert_spans = [
calc_bert_spans(b, t) for b, t in zip(devC_bert_tokens, devC_tokens)
]
devQ_bert_spans = [
calc_bert_spans(b, t) for b, t in zip(devQ_bert_tokens, devQ_tokens)
]
vocab_tag = pickle.load(open('./vocab_tag.pkl', 'rb'))
vocab_ent = pickle.load(open('./vocab_ent.pkl', 'rb'))
devC_tag_ids = token2id(devC_tags, vocab_tag) # vocab_tag same as training
# entities
devC_ent_ids = token2id(devC_ents, vocab_ent,
unk_id=0) # vocab_ent same as training
print('vocabulary for dev is built.')
dev_embedding = build_embedding('glove/glove.840B.300d.txt', dev_vocab,
300)
meta = {'vocab': dev_vocab, 'embedding': dev_embedding.tolist()}
prev_CID, first_question = -1, []
for i, CID in enumerate(dev.context_idx):
if not (CID == prev_CID):
first_question.append(i)
prev_CID = CID
result = {
'qids': dev.qid.tolist(),
'question_ids': devQ_ids,
'context_ids': devC_ids,
'context_features': devC_features, # exact match, tf
'context_tags': devC_tag_ids, # POS tagging
'context_ents': devC_ent_ids, # Entity recognition
'context': dev_context,
'context_span': dev_context_span,
'1st_question': first_question,
'question_CID': dev.context_idx.tolist(),
'question': dev.question.tolist(),
'answer': dev.answer.tolist(),
'answer_start': dev.answer_start_token.tolist(),
'answer_end': dev.answer_end_token.tolist(),
'answer_choice': dev.answer_choice.tolist(),
'all_answer': dev.all_answer.tolist(),
'context_tokenized': devC_tokens,
'question_tokenized': devQ_tokens,
'context_bertidx': devC_bert_ids,
'context_bert_spans': devC_bert_spans,
'question_bertidx': devQ_bert_ids,
'question_bert_spans': devQ_bert_spans
}
return meta, result
def build_test_data(opt, dev_file, vocab):
# random.seed(args.seed)
# np.random.seed(args.seed)
# logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG,
# datefmt='%m/%d/%Y %I:%M:%S')
log = logging.getLogger(__name__)
# tags
vocab_tag = [''] + list(nlp.tagger.labels)
# entities
# log.info('start data preparing... (using {} threads)'.format(args.threads))
# glove_vocab = load_glove_vocab(wv_file, wv_dim) # return a "set" of vocabulary
# log.info('glove loaded.')
def proc_dev(ith, article):
rows = []
context = article['story']
for j, (question, answers) in enumerate(
zip(article['questions'], article['answers'])):
gold_answer = answers['input_text']
span_answer = answers['span_text']
answer, char_i, char_j = free_text_to_span(gold_answer,
span_answer)
answer_choice = 0 if answer == '__NA__' else \
1 if answer == '__YES__' else \
2 if answer == '__NO__' else \
3 # Not a yes/no question
if answer_choice == 3:
answer_start = answers['span_start'] + char_i
answer_end = answers['span_start'] + char_j
else:
answer_start, answer_end = -1, -1
rationale = answers['span_text']
rationale_start = answers['span_start']
rationale_end = answers['span_end']
q_text = question['input_text']
if j > 0:
q_text = article['answers'][j -
1]['input_text'] + " // " + q_text
rows.append(
(ith, q_text, answer, answer_start, answer_end, rationale,
rationale_start, rationale_end, answer_choice))
return rows, context
dev, dev_context = flatten_json(dev_file, proc_dev)
dev = pd.DataFrame(dev,
columns=[
'context_idx', 'question', 'answer', 'answer_start',
'answer_end', 'rationale', 'rationale_start',
'rationale_end', 'answer_choice'
])
# log.info('dev json data flattened.')
# print(dev)
devC_iter = (pre_proc(c) for c in dev_context)
devQ_iter = (pre_proc(q) for q in dev.question)
devC_docs = [
doc
for doc in nlp.pipe(devC_iter, batch_size=64, n_threads=args.threads)
]
devQ_docs = [
doc
for doc in nlp.pipe(devQ_iter, batch_size=64, n_threads=args.threads)
]
# tokens
devC_tokens = [[re.sub(r'_', ' ', normalize_text(w.text)) for w in doc]
for doc in devC_docs]
devQ_tokens = [[re.sub(r'_', ' ', normalize_text(w.text)) for w in doc]
for doc in devQ_docs]
devC_unnorm_tokens = [[re.sub(r'_', ' ', w.text) for w in doc]
for doc in devC_docs]
# log.info('All tokens for dev are obtained.')
dev_context_span = [
get_context_span(a, b) for a, b in zip(dev_context, devC_unnorm_tokens)
]
# log.info('context span for dev is generated.')
ans_st_token_ls, ans_end_token_ls = [], []
for ans_st, ans_end, idx in zip(dev.answer_start, dev.answer_end,
dev.context_idx):
ans_st_token, ans_end_token = find_answer_span(dev_context_span[idx],
ans_st, ans_end)
ans_st_token_ls.append(ans_st_token)
ans_end_token_ls.append(ans_end_token)
ration_st_token_ls, ration_end_token_ls = [], []
for ration_st, ration_end, idx in zip(dev.rationale_start,
dev.rationale_end, dev.context_idx):
ration_st_token, ration_end_token = find_answer_span(
dev_context_span[idx], ration_st, ration_end)
ration_st_token_ls.append(ration_st_token)
ration_end_token_ls.append(ration_end_token)
dev['answer_start_token'], dev[
'answer_end_token'] = ans_st_token_ls, ans_end_token_ls
dev['rationale_start_token'], dev[
'rationale_end_token'] = ration_st_token_ls, ration_end_token_ls
initial_len = len(dev)
dev.dropna(inplace=True) # modify self DataFrame
# log.info('drop {0}/{1} inconsistent samples.'.format(initial_len - len(dev), initial_len))
# log.info('answer span for dev is generated.')
# features
devC_tags, devC_ents, devC_features = feature_gen(devC_docs,
dev.context_idx,
devQ_docs, args.no_match)
# log.info('features for dev is generated: {}, {}, {}'.format(len(devC_tags), len(devC_ents), len(devC_features)))
vocab_ent = list(set([ent for sent in devC_ents for ent in sent]))
# vocab
dev_vocab = vocab # tr_vocab is a subset of dev_vocab
devC_ids = token2id(devC_tokens, dev_vocab, unk_id=1)
devQ_ids = token2id(devQ_tokens, dev_vocab, unk_id=1)
devQ_tokens = [["<S>"] + doc + ["</S>"] for doc in devQ_tokens]
devQ_ids = [[2] + qsent + [3] for qsent in devQ_ids]
# print(devQ_ids[:10])
# tags
devC_tag_ids = token2id(devC_tags, vocab_tag) # vocab_tag same as training
# entities
devC_ent_ids = token2id(devC_ents, vocab_ent,
unk_id=0) # vocab_ent same as training
# log.info('vocabulary for dev is built.')
prev_CID, first_question = -1, []
for i, CID in enumerate(dev.context_idx):
if not (CID == prev_CID):
first_question.append(i)
prev_CID = CID
data = {
'question_ids': devQ_ids,
'context_ids': devC_ids,
'context_features': devC_features, # exact match, tf
'context_tags': devC_tag_ids, # POS tagging
'context_ents': devC_ent_ids, # Entity recognition
'context': dev_context,
'context_span': dev_context_span,
'1st_question': first_question,
'question_CID': dev.context_idx.tolist(),
'question': dev.question.tolist(),
'answer': dev.answer.tolist(),
'answer_start': dev.answer_start_token.tolist(),
'answer_end': dev.answer_end_token.tolist(),
'rationale_start': dev.rationale_start_token.tolist(),
'rationale_end': dev.rationale_end_token.tolist(),
'answer_choice': dev.answer_choice.tolist(),
'context_tokenized': devC_tokens,
'question_tokenized': devQ_tokens
}
# with open('CoQA/test_data.msgpack', 'wb') as f:
# msgpack.dump(result, f)
# log.info('saved test to disk.')
dev = {
'context':
list(
zip(data['context_ids'], data['context_tags'],
data['context_ents'], data['context'], data['context_span'],
data['1st_question'], data['context_tokenized'])),
'qa':
list(
zip(data['question_CID'], data['question_ids'],
data['context_features'], data['answer_start'],
data['answer_end'], data['rationale_start'],
data['rationale_end'], data['answer_choice'], data['question'],
data['answer'], data['question_tokenized']))
}
print("test_data built")
# embedding = torch.Tensor(meta['embedding'])
return dev
