def testSmallTree(self):
tree = ImmutableParentedTree.fromstring(
"(ROOT (S (NP (PRP I)) (VP (VB like) (NP (NN fish)))))")
graph = nltk_tree_to_graph(tree)
self.assertEqual((8, 8), graph.shape)
self.compare_graph_to_correct(graph, [(0, 1), (1, 2), (2, 3), (1, 4),
(4, 5), (4, 6), (6, 7)])
def _sentence_graph_from_ptb_str(ptb_str, num_tokens):
# We need to have num_tokens provided here, or else we won't know for
# sure how big the graph should be. (There can be tokens missing from
# the graph, and even if there aren't it would take more processing
# than it's worth to find the max node index in the PTB tree.)
tree = ImmutableParentedTree.fromstring(ptb_str)
edge_graph = lil_matrix((num_tokens, num_tokens), dtype='float')
edge_labels = {}
excluded_edges = []
def convert_node(parent_index, node):
# Node index is whatever's after the last underscore.
node_label = node.label()
node_index = int(node_label[node_label.rindex('_') + 1:])
edge_label = node[0] # 0th child is always edge label
if edge_label in StanfordParsedSentence.DEPTH_EXCLUDED_EDGE_LABELS:
excluded_edges.append((parent_index, node_index))
else:
edge_graph[parent_index, node_index] = 1.0
edge_labels[parent_index, node_index] = edge_label
for child in node[
2:]: # Skip edge label (child 0) & POS (child 1).
convert_node(node_index, child)
for root_child in tree:
convert_node(0, root_child) # initial parent index is 0 for root
return edge_graph.tocsr(), edge_labels, excluded_edges
def __init__(self, tokenized_text, tagged_lemmas, penn_tree, edges,
document_text):
'''
`tokenized_text` and `tagged_lemmas` are the token and lemma strings
from the parser.
`edges` is a list of edge strings from the parser.
`document_text` is an instance of
`util.streams.CharacterTrackingStreamWrapper`. (Built-in stream types
will *not* work.)
'''
self.next_sentence = None
self.previous_sentence = None
# TODO: move much of the initialization functionality, particularly
# aligning tokens to text, into the reader class.
self.tokens = []
self.edge_labels = {} # maps (n1_index, n2_index) tuples to labels
try:
self.source_file_path = document_text.name
except AttributeError:
self.source_file_path = None
# Declare a few variables that will be overwritten later, just so that
# it's easy to tell what's in an instance of this class.
self.edge_graph = csr_matrix((0, 0), dtype='float')
self.document_char_offset = 0
self.original_text = ''
self.__depths = np.array([])
self.path_predecessors = np.array([[]])
self.path_costs = np.array([[]])
token_strings, tag_strings = self.__get_token_strings(
tokenized_text, tagged_lemmas)
copy_node_indices = self.__create_tokens(token_strings, tag_strings)
self.__align_tokens_to_text(document_text)
self.__create_edges(edges, copy_node_indices)
if FLAGS.use_constituency_parse:
self.constituency_tree = ImmutableParentedTree.fromstring(
penn_tree)
self.constituency_graph = nltk_tree_to_graph(
self.constituency_tree)
self.constituent_heads = collins_find_heads(self.constituency_tree)
else:
self.constituency_tree = None
self.constituency_graph = None
self.constituent_heads = None
def testLargerTree(self):
tree_str = ('''
(ROOT
(S
(NP
(NP (RB nearly) (DT every) (NN session))
(PP (IN since) (NP (NNP November))))
(VP
(VBZ have)
(VP
(VBN be)
(VP
(VBN adjourn)
(SBAR
(IN because)
(S
(NP (QP (RB as) (JJ few) (IN as) (CD 65))
(NNS member))
(VP
(VBD make)
(S (NP (PRP it)) (VP (TO to) (VP (VB work)))))))
(, ,)
(SBAR
(RB even)
(IN as)
(S
(NP
(NP (PRP they))
(CC and)
(NP (DT the) (NNS absentee)))
(VP
(VBD earn)
(NP (NNS salary) (CC and) (NNS benefit))
(PP
(IN worth)
(NP (QP (RB about) ($ $) (CD 120,000))))))))))
(. .)))
''')
tree = ImmutableParentedTree.fromstring(tree_str)
heads = collins_find_heads(tree)
self.assertIs(get_head(heads, tree[0][1]), tree[0][1][1][1][0])
self.assertIs(get_head(heads, tree[0]), tree[0][1][1][1][0])
def test_single_word_case():
tree = ImmutableParentedTree.fromstring("""(ROOT
(S
(NP
(NP (JJ Strong) (NN programming) (NNS skills))
(PP (IN in)
(NP (NNP Java))))
(VP (VBP are)
(VP (VBN required)))
(. .)))
""")
words = ["Java"]
expected = TreeRegexp('NP',
[TreeRegexp('NP',
[TreeRegexp('JJ', ['Strong']), TreeRegexp('NN', ['programming']), TreeRegexp('NNS', ['skills'])]),
TreeRegexp('PP', [TreeRegexp('IN', ['in']), TreeRegexp('NP', [MatchAllNode()])])])
actual = produce_patterns(tree, words)
assert_equal(len(actual), 1)
assert_equal(actual[0], expected)
def test_parallel_words():
tree = ImmutableParentedTree.fromstring("""
(ROOT
(NP
(NP
(NP
(QP (IN At) (JJS least) (CD 3))
(NNS years) (NN experience))
(PP (IN in)
(NP (NN programming))))
(PP (IN in)
(NP (NNP Java)
(CC and)
(NNP C++)))))
""")
words = ["Java", "C++", "Python", "Closure"]
expected = TreeRegexp('PP', [TreeRegexp('IN', ['in']), TreeRegexp('NP', [MatchAllNode(), TreeRegexp('CC', ['and']), MatchAllNode()])])
actual = produce_patterns(tree, words)
assert_equal(len(actual), 1)
assert_equal(actual[0], expected)
def testSmallImmutableTree(self):
tree = ImmutableParentedTree.fromstring(
"(ROOT (S (NP (PRP I)) (VP (VB like) (NP (NN fish))) (. .)))")
self._testSmallTree(tree)
