def pcfg_demo():
"""
A demonstration showing how C{WeightedGrammar}s can be created and used.
"""
from nltk.corpus import treebank
from nltk import treetransforms
from nltk import induce_pcfg
from nltk.parse import pchart
pcfg_prods = toy_pcfg1.productions()
pcfg_prod = pcfg_prods[2]
print 'A PCFG production:', ` pcfg_prod `
print ' pcfg_prod.lhs() =>', ` pcfg_prod.lhs() `
print ' pcfg_prod.rhs() =>', ` pcfg_prod.rhs() `
print ' pcfg_prod.prob() =>', ` pcfg_prod.prob() `
print
grammar = toy_pcfg2
print 'A PCFG grammar:', ` grammar `
print ' grammar.start() =>', ` grammar.start() `
print ' grammar.productions() =>',
# Use string.replace(...) is to line-wrap the output.
print ` grammar.productions() `.replace(',', ',\n' + ' ' * 26)
print
print 'Coverage of input words by a grammar:'
print grammar.covers(['a', 'boy'])
print grammar.covers(['a', 'girl'])
# extract productions from three trees and induce the PCFG
print "Induce PCFG grammar from treebank data:"
productions = []
for item in treebank.items[:2]:
for tree in treebank.parsed_sents(item):
# perform optional tree transformations, e.g.:
tree.collapse_unary(collapsePOS=False)
tree.chomsky_normal_form(horzMarkov=2)
productions += tree.productions()
S = Nonterminal('S')
grammar = induce_pcfg(S, productions)
print grammar
print
print "Parse sentence using induced grammar:"
parser = pchart.InsideChartParser(grammar)
parser.trace(3)
# doesn't work as tokens are different:
#sent = treebank.tokenized('wsj_0001.mrg')[0]
sent = treebank.parsed_sents('wsj_0001.mrg')[0].leaves()
print sent
for parse in parser.nbest_parse(sent):
print parse
def pcfg_demo():
"""
A demonstration showing how a ``PCFG`` can be created and used.
"""
from nltk.corpus import treebank
from nltk import treetransforms
from nltk import induce_pcfg
from nltk.parse import pchart
# pcfg_prods = toy_pcfg1.productions()
#
# pcfg_prod = pcfg_prods[2]
# print('A PCFG production:', repr(pcfg_prod))
# print(' pcfg_prod.lhs() =>', repr(pcfg_prod.lhs()))
# print(' pcfg_prod.rhs() =>', repr(pcfg_prod.rhs()))
# print(' pcfg_prod.prob() =>', repr(pcfg_prod.prob()))
# print()
#
# grammar = toy_pcfg2
# print('A PCFG grammar:', repr(grammar))
# print(' grammar.start() =>', repr(grammar.start()))
# print ' grammar.productions() =>',
# # Use .replace(...) is to line-wrap the output.
# print(repr(grammar.productions()).replace(',', ',\n' + ' ' * 26))
# print()
# extract productions from three trees and induce the PCFG
print("Induce PCFG grammar from treebank data:")
productions = []
item = treebank._fileids[0]
for tree in treebank.parsed_sents(item)[:3]:
# perform optional tree transformations, e.g.:
tree.collapse_unary(collapsePOS=False)
tree.chomsky_normal_form(horzMarkov=2)
productions += tree.productions()
# S = Nonterminal('S')
# grammar = induce_pcfg(S, productions)
print(productions)
print()
print("Parse sentence using induced grammar:")
parser = pchart.InsideChartParser(grammar)
parser.trace(3)
# doesn't work as tokens are different:
# sent = treebank.tokenized('wsj_0001.mrg')[0]
sent = treebank.parsed_sents(item)[0].leaves()
print(sent)
for parse in parser.parse(sent):
print(parse)
def ex7():
"""
Using NLTK's own library to generate the probabilities.
"""
productions = [
p for tree in treebank.parsed_sents() for p in tree.productions()
]
pcfg = induce_pcfg(Nonterminal("S"), productions)
# print(pcfg.productions())
parser = pchart.InsideChartParser(pcfg, beam_size=800)
for sent in sentences1:
parsed = list(parser.parse(sent.split()))
print("Parsing sent: {}".format(sent))
print(parsed[0])
def PCFG_Section():
toy_pcfg1 = PCFG.fromstring("""
S -> NP VP [1.0]
NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
Det -> 'the' [0.8] | 'my' [0.2]
N -> 'man' [0.5] | 'telescope' [0.5]
VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
V -> 'ate' [0.35] | 'saw' [0.65]
PP -> P NP [1.0]
P -> 'with' [0.61] | 'under' [0.39]
""")
pcfg_prods = toy_pcfg1.productions()
pcfg_prod = pcfg_prods[2]
print('A PCFG production:', pcfg_prod)
print('pcfg_prod.lhs() =>', pcfg_prod.lhs())
print('pcfg_prod.rhs() =>', pcfg_prod.rhs())
print('pcfg_prod.prob() =>', pcfg_prod.prob())
# extract productions from three trees and induce the PCFG
print("Induce PCFG grammar from treebank data:")
productions = []
for item in treebank.fileids()[:2]:
for tree in treebank.parsed_sents(item):
# print(" ".join(tree.leaves()))
# perform optional tree transformations, e.g.:
# tree.collapse_unary(collapsePOS = False)# Remove branches A-B-C into A-B+C
# tree.chomsky_normal_form(horzMarkov = 2)# Remove A->(B,C,D) into A->B,C+D->D
prods = tree.productions()
# print(prods[0].prob())
productions += prods
S = Nonterminal('S')
grammar = induce_pcfg(S, productions)
# print(grammar) # This is a PCFG
### Parsing section below ###
print("\nParse sentence using induced grammar:")
parser = pchart.InsideChartParser(grammar)
parser.trace(1)
sent = treebank.parsed_sents('wsj_0001.mrg')[0]
print(sent.prob())
def ex8():
"""
Parsing unseen sentences (not in the training corpus)
"""
legal_sentences = [
"If there is any conflict between the terms in the General Terms and the Additional Terms, then the Additional Terms govern .",
"You may have additional rights under the law .",
"We do not seek to limit those rights where it is prohibited to do so by law ."
]
tokenized = [[TreebankWordTokenizer().tokenize(sent)]
for sent in legal_sentences] # sentences2
productions = [
p for tree in treebank.parsed_sents() for p in tree.productions()
]
pcfg = induce_pcfg(Nonterminal("S"), productions)
parser = pchart.InsideChartParser(pcfg, beam_size=500)
for sent in tokenized:
print("Parsing sent: {}".format(sent[0]))
parsed = list(parser.parse(sent[0]))
print(parsed)
def demo(choice=None, draw_parses=None, print_parses=None):
"""
A demonstration of the probabilistic parsers. The user is
prompted to select which demo to run, and how many parses should
be found; and then each parser is run on the same demo, and a
summary of the results are displayed.
"""
import sys, time
from nltk import tokenize
from nltk.parse import pchart
# Define two demos. Each demo has a sentence and a grammar.
toy_pcfg1 = PCFG.fromstring("""
S -> NP VP [1.0]
NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
Det -> 'the' [0.8] | 'my' [0.2]
N -> 'man' [0.5] | 'telescope' [0.5]
VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
V -> 'ate' [0.35] | 'saw' [0.65]
PP -> P NP [1.0]
P -> 'with' [0.61] | 'under' [0.39]
""")
toy_pcfg2 = PCFG.fromstring("""
S -> NP VP [1.0]
VP -> V NP [.59]
VP -> V [.40]
VP -> VP PP [.01]
NP -> Det N [.41]
NP -> Name [.28]
NP -> NP PP [.31]
PP -> P NP [1.0]
V -> 'saw' [.21]
V -> 'ate' [.51]
V -> 'ran' [.28]
N -> 'boy' [.11]
N -> 'cookie' [.12]
N -> 'table' [.13]
N -> 'telescope' [.14]
N -> 'hill' [.5]
Name -> 'Jack' [.52]
Name -> 'Bob' [.48]
P -> 'with' [.61]
P -> 'under' [.39]
Det -> 'the' [.41]
Det -> 'a' [.31]
Det -> 'my' [.28]
""")
demos = [('I saw John with my telescope', toy_pcfg1),
('the boy saw Jack with Bob under the table with a telescope',
toy_pcfg2)]
if choice is None:
# Ask the user which demo they want to use.
print()
for i in range(len(demos)):
print('%3s: %s' % (i + 1, demos[i][0]))
print(' %r' % demos[i][1])
print()
print('Which demo (%d-%d)? ' % (1, len(demos)), end=' ')
choice = int(sys.stdin.readline().strip()) - 1
try:
sent, grammar = demos[choice]
except:
print('Bad sentence number')
return
# Tokenize the sentence.
tokens = sent.split()
# Define a list of parsers. We'll use all parsers.
parsers = [
pchart.InsideChartParser(grammar),
pchart.RandomChartParser(grammar),
pchart.UnsortedChartParser(grammar),
pchart.LongestChartParser(grammar),
pchart.InsideChartParser(grammar,
beam_size=len(tokens) + 1) # was BeamParser
]
# Run the parsers on the tokenized sentence.
times = []
average_p = []
num_parses = []
all_parses = {}
for parser in parsers:
print('\ns: %s\nparser: %s\ngrammar: %s' % (sent, parser, grammar))
parser.trace(3)
t = time.time()
parses = list(parser.parse(tokens))
times.append(time.time() - t)
p = (reduce(lambda a, b: a + b.prob(), parses, 0) /
len(parses) if parses else 0)
average_p.append(p)
num_parses.append(len(parses))
for p in parses:
all_parses[p.freeze()] = 1
# Print some summary statistics
print()
print(
' Parser Beam | Time (secs) # Parses Average P(parse)')
print(
'------------------------+------------------------------------------')
for i in range(len(parsers)):
print('%18s %4d |%11.4f%11d%19.14f' %
(parsers[i].__class__.__name__, parsers[i].beam_size, times[i],
num_parses[i], average_p[i]))
parses = all_parses.keys()
if parses: p = reduce(lambda a, b: a + b.prob(), parses, 0) / len(parses)
else: p = 0
print(
'------------------------+------------------------------------------')
print('%18s |%11s%11d%19.14f' %
('(All Parses)', 'n/a', len(parses), p))
if draw_parses is None:
# Ask the user if we should draw the parses.
print()
print('Draw parses (y/n)? ', end=' ')
draw_parses = sys.stdin.readline().strip().lower().startswith('y')
if draw_parses:
from nltk.draw.tree import draw_trees
print(' please wait...')
draw_trees(*parses)
if print_parses is None:
# Ask the user if we should print the parses.
print()
print('Print parses (y/n)? ', end=' ')
print_parses = sys.stdin.readline().strip().lower().startswith('y')
if print_parses:
for parse in parses:
print(parse)
def main(sentences, grammarfile, pcfg_grammar, algo, output, \
to_keeps, percent_discard, beam=0):
grammar = nltk.data.load("file:%s" %(grammarfile))
chart_parser = ChartParser(grammar,strategy=EARLEY_STRATEGY,trace=0)
f = open(pcfg_grammar)
pcfgrammar = f.read()
f.close()
if algo == "viterbi":
pcfg_parser = nltk.ViterbiParser(nltk.parse_pcfg(pcfgrammar))
elif algo == "inside":
pcfg_parser = pchart.InsideChartParser(nltk.parse_pcfg(pcfgrammar),\
beam_size=beam)
elif algo == "random":
pcfg_parser = pchart.RandomChartParser(nltk.parse_pcfg(pcfgrammar),\
beam_size=beam)
elif algo == "longest":
pcfg_parser = pchart.LongestChartParser(nltk.parse_pcfg(pcfgrammar),\
beam_size=beam)
elif algo == "unsorted":
pcfg_parser = pchart.UnsortedChartParser(nltk.parse_pcfg(pcfgrammar),\
beam_size=beam)
elif algo == "chart":
pass
else:
print "unrecognized algorithm: %s" %(algo)
return 1
forest = []
for sentence in sentences:
parsed_sent = sentence.split()
print "parsed_sent: %s" %(parsed_sent)
start = datetime.now()
if algo == "chart":
trees = chart_parser.nbest_parse(parsed_sent)
else:
trees = pcfg_parser.nbest_parse(parsed_sent)
end = datetime.now()
elapsed = end - start
print "parsing time elapsed: %s" %(elapsed)
print "parsing time elapsed: %d us" %(elapsed.microseconds)
if (len(trees) == 0):
print "failed to parse: %s" %(sentence)
return 1;
forest.append(trees)
all_productions = grammar.productions()
# randomly shuffle the productions
all_productions = all_productions[0:len(all_productions)]
random.shuffle(all_productions)
random.shuffle(all_productions)
status = 0
for keep in to_keeps:
for discard in percent_discard:
status += create_pruned_grammar(forest, all_productions, keep,\
discard, output)
return status
def demo():
"""
A demonstration of the probabilistic parsers. The user is
prompted to select which demo to run, and how many parses should
be found; and then each parser is run on the same demo, and a
summary of the results are displayed.
"""
import sys, time
from nltk import tokenize, toy_pcfg1, toy_pcfg2
from nltk.parse import pchart
# Define two demos. Each demo has a sentence and a grammar.
demos = [('I saw John with my telescope', toy_pcfg1),
('the boy saw Jack with Bob under the table with a telescope',
toy_pcfg2)]
# Ask the user which demo they want to use.
print
for i in range(len(demos)):
print '%3s: %s' % (i + 1, demos[i][0])
print ' %r' % demos[i][1]
print
print 'Which demo (%d-%d)? ' % (1, len(demos)),
try:
snum = int(sys.stdin.readline().strip()) - 1
sent, grammar = demos[snum]
except:
print 'Bad sentence number'
return
# Tokenize the sentence.
tokens = sent.split()
# Define a list of parsers. We'll use all parsers.
parsers = [
pchart.InsideChartParser(grammar),
pchart.RandomChartParser(grammar),
pchart.UnsortedChartParser(grammar),
pchart.LongestChartParser(grammar),
pchart.InsideChartParser(grammar,
beam_size=len(tokens) + 1) # was BeamParser
]
# Run the parsers on the tokenized sentence.
times = []
average_p = []
num_parses = []
all_parses = {}
for parser in parsers:
print '\ns: %s\nparser: %s\ngrammar: %s' % (sent, parser, grammar)
parser.trace(3)
t = time.time()
parses = parser.nbest_parse(tokens)
times.append(time.time() - t)
if parses:
p = reduce(lambda a, b: a + b.prob(), parses, 0) / len(parses)
else:
p = 0
average_p.append(p)
num_parses.append(len(parses))
for p in parses:
all_parses[p.freeze()] = 1
# Print some summary statistics
print
print ' Parser Beam | Time (secs) # Parses Average P(parse)'
print '------------------------+------------------------------------------'
for i in range(len(parsers)):
print '%18s %4d |%11.4f%11d%19.14f' % (parsers[i].__class__.__name__,
parsers[i].beam_size, times[i],
num_parses[i], average_p[i])
parses = all_parses.keys()
if parses: p = reduce(lambda a, b: a + b.prob(), parses, 0) / len(parses)
else: p = 0
print '------------------------+------------------------------------------'
print '%18s |%11s%11d%19.14f' % ('(All Parses)', 'n/a', len(parses),
p)
# Ask the user if we should draw the parses.
print
print 'Draw parses (y/n)? ',
if sys.stdin.readline().strip().lower().startswith('y'):
from nltk.draw.tree import draw_trees
print ' please wait...'
draw_trees(*parses)
# Ask the user if we should print the parses.
print
print 'Print parses (y/n)? ',
if sys.stdin.readline().strip().lower().startswith('y'):
for parse in parses:
print parse
S = nltk.Nonterminal("S")
grammar = nltk.induce_pcfg(S, treeProductions)
### Extracting PCFG to a text file
#grammar_PCFG = str(grammar)
#file = open('/Users/mayapetranova/Documents/QMUL/NLP/assignment_2/6/PCFG.txt', 'w')
#file.write(grammar_PCFG)
#file.close()
########################
## Q u e s t i o n 6b ##
########################
sentence = "show me the meals on the flight from Phoenix".split()
parser = pchart.InsideChartParser(grammar)
for tp in parser.parse(sentence):
print(tp)
## For Drawing the CFG trees, uncomment lines 59-80
#q6s1 = Tree.fromstring('(S(IVP(IVerb show)(NP (Pronoun me))(NP (Det the) (Nominal (Noun meals)))(PP(Preposition on)(NP(Det the)(Nominal(Nominal (Noun flight))(PP(Preposition from)(NP (Proper_Noun Phoenix))))))))')
#TreeView(q6s1)._cframe.print_to_file('q6s1.ps')
#q6s2 = Tree.fromstring('(S(IVP(IVerb show)(NP (Pronoun me))(NP(Det the)(Nominal(Nominal (Noun meals))(PP(Preposition on)(NP (Det the) (Nominal (Noun flight))))))(PP (Preposition from) (NP (Proper_Noun Phoenix)))))')
#TreeView(q6s2)._cframe.print_to_file('q6s2.ps')
#q6s3 = Tree.fromstring('(S(IVP(IVerb show)(NP (Pronoun me))(NP(NP (Det the) (Nominal (Noun meals)))(PP(Preposition on)(NP (Det the) (Nominal (Noun flight)))))(PP (Preposition from) (NP (Proper_Noun Phoenix)))))')
#TreeView(q6s3)._cframe.print_to_file('q6s3.ps')
#q6s4 = Tree.fromstring('(S(IVP(IVerb show)(NP (Pronoun me))(NP (Det the) (Nominal (Noun meals)))(PP(Preposition on)(NP(NP (Det the) (Nominal (Noun flight)))(PP (Preposition from) (NP (Proper_Noun Phoenix)))))))')
#TreeView(q6s4)._cframe.print_to_file('q6s4.ps')
#q6s5 = Tree.fromstring('(S(IVP(IVerb show)(NP (Pronoun me))(NP(Det the)(Nominal(Nominal(Nominal (Noun meals))(PP(Preposition on)(NP (Det the) (Nominal (Noun flight)))))(PP (Preposition from) (NP (Proper_Noun Phoenix)))))))')
#TreeView(q6s5)._cframe.print_to_file('q6s5.ps')
#q6s6 = Tree.fromstring('(S(IVP(IVerb show)(NP (Pronoun me))(NP(Det the)(Nominal(Nominal (Noun meals))(PP(Preposition on)(NP(Det the)(Nominal(Nominal (Noun flight))(PP(Preposition from)(NP (Proper_Noun Phoenix))))))))))')
TEN -> 'eighty' [0.125]
TEN -> 'ninety' [0.125]
CD -> 'zero' [0.1]
CD -> 'one' [0.1]
CD -> 'two' [0.1]
CD -> 'three' [0.1]
CD -> 'four' [0.1]
CD -> 'five' [0.1]
CD -> 'six' [0.1]
CD -> 'seven' [0.1]
CD -> 'eight' [0.1]
CD -> 'nine' [0.1]
""")
PARSER = pchart.InsideChartParser(GRAMMAR)
LEMMATIZER = WordNetLemmatizer()
def add(one, two):
"""
returns one + two
"""
return one + two
def subtract(one, two):
"""
returns one - two
"""
return one - two
