Python TreeBank Examples

Example #1

File: tmp.py Project: windweller/nlp-class

from treeBank import TreeBank

tb_train = TreeBank.from_file("./data/train.trees")
word_freq = tb_train.count_word_freq()
vocab_train = set(
    map(lambda x: x[0], filter(lambda x: x[1] > 1, word_freq.items())))
rare_train = set(
    map(lambda x: x[0], filter(lambda x: x[1] == 1, word_freq.items())))
#vocab_train = tb_train.collect_vocabulary()

tb_dev = TreeBank.from_file("./data/test.trees")
vocab_dev = tb_dev.collect_vocabulary()

diff = vocab_dev.difference(vocab_train)
for w in diff:
    print(w)

print(rare_train.intersection(vocab_dev))

Example #2

def train_on(filename):
    tb = TreeBank.from_file(filename)
    tb.normalize()
    rule_set = tb.collect_rule_set(vertical_markov=True,
                                   vertical_markov_factor=0.8)
    return rule_set

Example #3

File: q4-7.py Project: TongZZZ/nlp-class

def train_on(filename):
    tb = TreeBank.from_file(filename)
    tb.normalize()
    rule_set = tb.collect_rule_set(vertical_markov=True, vertical_markov_factor=0.8)
    return rule_set

Example #4

File: q2.py Project: windweller/nlp-class

#!/usr/bin/env python

from treeBank import TreeBank

train_words = TreeBank.from_file('./data/train.trees').collect_vocabulary()

dev_words = TreeBank.from_file('./data/dev.trees').collect_vocabulary()

diff = dev_words.difference(train_words)
print(diff)
print(len(diff))

Example #5

File: tmp.py Project: TongZZZ/nlp-class

from treeBank import TreeBank

tb_train = TreeBank.from_file("./data/train.trees")
word_freq = tb_train.count_word_freq()
vocab_train = set(map(lambda x: x[0], filter(lambda x: x[1] > 1, word_freq.items())))
rare_train = set(map(lambda x: x[0], filter(lambda x: x[1] == 1, word_freq.items())))
#vocab_train = tb_train.collect_vocabulary()


tb_dev = TreeBank.from_file("./data/test.trees")
vocab_dev = tb_dev.collect_vocabulary()

diff = vocab_dev.difference(vocab_train)
for w in diff:
    print(w)

print(rare_train.intersection(vocab_dev))

Example #6

#!/usr/bin/env python

from treeBank import TreeBank

ans = max([
    len(node.children) for t in TreeBank.from_file('./data/train.trees').trees
    for node in t.bottom_up()
])

print(ans)

Example #7

File: q1.py Project: TongZZZ/nlp-class

#!/usr/bin/env python

from treeBank import TreeBank

ans = max([len(node.children)
           for t in TreeBank.from_file('./data/train.trees').trees
           for node in t.bottom_up()])

print(ans)

Example #8

#!/usr/bin/env python

from treeBank import TreeBank

tb = TreeBank.from_file("./data/train.trees")
tb.normalize()

print(tb.collect_rule_set().size)