forked from jstarc/deep_reasoning
/
paraphrase.py
65 lines (52 loc) · 1.61 KB
/
paraphrase.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import load_data
import pickle
GLOVE = 'data/snli_vectors_300.txt'
PPDB = 'data/ppdb-2.0-s-all'
def test():
result = set()
with open(PPDB) as ppdb:
count = 0
for line in ppdb:
example = line[:-1].split(' ||| ')
p = example[1]
h = example[2]
rel = example[-1]
result.add(rel)
if rel == 'Equivalence' and p in glove:
print p, h
count += 1
if count % 100000 == 0:
break
return result
def load_ppdb_data(glove):
result = {}
with open(PPDB) as ppdb:
count = 0
for line in ppdb:
example = line[:-1].split(' ||| ')
p = example[1]
h = example[2]
rel = example[-1]
if rel == 'Equivalence':
if not (p.startswith(h) or h.startswith(p)):
if p in glove:
add_pair(result, p, h)
if h in glove:
add_pair(result, h, p)
count += 1
if count % 100000 == 0:
print count
return result
def dump_parap(filename, data):
with open(filename, 'wb') as f:
pickle.dump(data, f)
def load_parap(filename):
with open(filename, 'rb') as f:
return pickle.load(f)
def add_pair(dct, p, h):
if p not in dct:
dct[p] = set()
dct[p].add(h)
if __name__ == "__main__":
glove = load_data.import_glove(GLOVE)
rep = load_ppdb_data(glove)