grammar = None # Grammar object that includes lexicon and

# production rules

queue = None # Queue object on which new edges are stacked

chart = None # Chart object in which edges are stored for the

# final parse generation

sentence_length = 0

will_print_chart = True # Set to false if you want to deactivate printing of the found parses

def __init__(self, grammar):

self.grammar = Grammar(grammar)

def parse(self, sentence, number_of_parses, strategy):

'''

Parse the input sentence

This is the central method to be called from outside.

'''

### Preprocessing ###

# Tokenize input sentence

tokens = self.tokenize(sentence)

self.sentence_length = len(tokens)

# Check for unknown tokens

unknown_words = self.get_unknown_words(tokens)

if unknown_words:

# TODO: Run fallback solutions to fix unknown words, else

# raise exception

raise ParseException("Sentence contains unknown words (%s). Please try again!" % ', '.join(unknown_words))

### Main steps ###

# (1) Initialize empty chart and queue

self.initialize_chart()

self.initialize_queue(strategy)

# (2) For every token, create a complete edge and push it to

# the queue

self.init_rule(tokens)

# Iteration counter for evaluation purposes

iters = 0

# (3) Repeat until no more edges are added

# or sufficient number of parses has been found:

while not self.queue.is_empty() and not self.enough_parses_found(number_of_parses):

iters = iters + 1

# (3.1) Add next element on queue to the chart

edge = self.queue.get_next_edge()

self.chart.add_edge(edge)

# (3.2) If input edge is complete,

# apply predict rule and fundamental rule.

# If input edge is incomplete,

# apply fundamental rule only

if edge.is_complete():

self.predict_rule(edge)

self.fundamental_rule(edge)

# (3.3) For alt search strategy, run search rule

# if input edge is a complete parse

# or last element of priority queue

if strategy == 'altsearch':

if ( ( (not self.queue.is_priority_active()) # Case 1: Complete parse was added to chart

and edge.get_prod_rule().get_lhs() == 'S'

and edge.is_complete()

and edge.get_start() == 0

and edge.get_end() == self.sentence_length )

or (self.queue.is_priority_active() # Case 2: Priority queue emptied

and self.queue.is_priority_empty() ) ):

self.search_rule(edge)

# 4) Display generated parses

s_edges = self.chart.get_s_edges()

print '%s parses found after %s iterations:' % (len(s_edges),iters)

if self.will_print_chart :

self.display_parses()

else:

for s_edge in s_edges:

print 'Found s-edge: %s' % s_edge

def tokenize(self, sentence):

'''

Separate a sentence into a list of tokens and return the list.

Currently this simply splits at each whitespace character with no

special preprocessing

'''

return sentence.split()

def get_unknown_words(self, tokens):

'''

Check list of tokens for unknown words by consulting the

lexicon and return them

'''

lexicon = self.grammar.get_lexicon()

unknown_words = [token for token in tokens if token not in lexicon]

return unknown_words

def initialize_chart(self):

'''

Initialize chart

Size of chart will be sentence_length+1 in both dimensions

'''

self.chart = Chart(self.sentence_length)

def initialize_queue(self, strategy):

'''

Initialize queue according to the parsing strategy chosen by

the user

'''

if strategy == 'fifo':

self.queue = Queue()

elif strategy == 'bestfirst':

self.queue = BestFirstQueue()

elif strategy == 'altsearch':

self.queue = AltSearchQueue(self.sentence_length+1)

else:

raise QueueException('Invalid strategy (%s). Please try again and choose a strategy from the following set: {fifo, bestfirst, altsearch}' % strategy)

def init_rule(self, tokens):

'''

Generate initial edges for all given tokens and add them to

the queue

Formal definition:

For every word w_i add the edge [w_i -> . , (i, i+1)]

'''

node = -1 # Position between tokens of sentence

# (0 is start of sentence)

for token in tokens:

node += 1

rule = ProductionRule(token, [], 1.0)

edge = Edge(node, node+1, rule, 0, [])

self.queue.add_edge(edge)

def enough_parses_found(self, number_of_parses):

'''

Check if enough parses have been found for the input sentence

Return True if the number of complete S edges that the chart

contains is >= the number of parses that the user wants, else

False

'''

return False if (number_of_parses == -1 or len(self.chart.get_s_edges()) < number_of_parses) else True

def predict_rule(self, complete_edge):

'''

If the LHS of a complete edge can be the first RHS element of

a production rule, create a self-loop edge with that rule and

push it to the queue

Input: Complete edge

Push to queue: Incomplete self-loop edges

Formal definition:

For each complete edge [A -> alpha . , (i, j)]

and each production rule B -> A beta,

add the self-loop edge [B -> . A beta , (i, i)]

'''

start = complete_edge.get_start()

lhs = complete_edge.get_prod_rule().get_lhs()

parent_rules = self.grammar.get_possible_parent_rules(lhs)

for parent_rule in parent_rules:

new_edge = Edge(start, start, parent_rule, 0, [])

if not self.queue.has_edge(new_edge) and not self.chart.has_edge(new_edge):

self.queue.add_edge(new_edge)

def fundamental_rule(self, input_edge):

'''

If an incomplete edge can be advanced by a complete edge,

create a new edge with the advanced dot.

Create new edges (which can be complete or incomplete) by

"advancing the dot", i.e. by matching incomplete edges with

appropriate complete ones:

(1) If the input edge is incomplete, find all complete edges

- whose start node equals the end node of the input edge

- whose LHS matches the RHS element

that the input edge is currently looking for.

If the input edge is complete, find all incomplete edges

- whose end node equals the start node of the input edge

- whose dot can be advanced by pairing them with the input

edge.

(2) From every pairing, create a new edge with the dot

advanced over the RHS element that has just been found.

(3) Push that edge to the queue IFF it does not exist already,

i.e. if it has not been added to the chart or the queue

before. This constraint keeps the parser from entering an

infinite loop when using left-recursive grammar rules.

Input: Single edge

Push to queue: Complete and incomplete edges

Formal definition:

If the chart contains the edges [A -> alpha . B beta, (i, j)]

and [B -> gamma . , (j, k)]

then add a new edge [A -> alpha B . beta, (i, k)].

'''

if input_edge.is_complete():

j = input_edge.get_start()

incomplete_edges = [edge for edge \

in self.chart.get_edges_ending_at(j) \

if not edge.is_complete()]

complete_edges = [input_edge]

else:

j = input_edge.get_end()

incomplete_edges = [input_edge]

complete_edges = [edge for edge \

in self.chart.get_edges_starting_at(j) \

if edge.is_complete()]

### New Edges ###

for incomp_edge in incomplete_edges:

# Prepare info from incomplete edge that is necessary to ...

prod_rule = incomp_edge.get_prod_rule()

dot = incomp_edge.get_dot()

next_missing_dtr = prod_rule.get_rhs_element(dot)

for comp_edge in complete_edges:

# ... check for compatibility with complete edges:

if next_missing_dtr == comp_edge.get_prod_rule().get_lhs():

# Prepare additional info from incomplete edge

i = incomp_edge.get_start()

known_dtrs = incomp_edge.get_known_dtrs()

# Prepare info from complete edge

k = comp_edge.get_end()

# Combine info from both edges,

# and use it to create new edge

new_dtrs = known_dtrs + [comp_edge]

new_edge = Edge(i, k, prod_rule, dot+1, new_dtrs)

# Add new edge to queue

if not self.queue.has_edge(new_edge) and not self.chart.has_edge(new_edge):

self.queue.add_edge(new_edge)

def search_rule(self, s_edge):

'Scans queue for priority edges. See project report for detailed explanation'

self.queue.activate_priority_queue()

# Check for further complete s-edges on queue

new_s_edge = self.queue.get_next_particular_edge('S', 0, self.sentence_length)

if not new_s_edge == None: # Further s-edge was found

self.queue.add_edge(new_s_edge)

else: # No further s-edge remained on queue

s_edges = self.chart.get_s_edges()

# Sort s-edges by likelihood

sorted_edges = []

for s_edge in s_edges:

pos = bisect([edge.get_prob() for edge in sorted_edges], s_edge)

sorted_edges.insert(pos, s_edge)

# Check s-edges until alternative parses are found

while len(s_edges) > 0 and self.queue.is_empty:

s_edge = s_edges.pop()

dtrs = s_edge.get_known_dtrs()

# Iterate through depths until alternatives are found

# or tree is exhausted

while len(dtrs) > 0:

# print "---"

for dtr in dtrs:

lhs = dtr.get_prod_rule().get_lhs()

start = dtr.get_start()

end = dtr.get_end()

alt_dtr = self.queue.get_next_particular_edge(lhs, start, end)

while not alt_dtr == None:

self.queue.add_edge(alt_dtr)

alt_dtr = self.queue.get_next_particular_edge(lhs, start, end)

if self.queue.is_empty:

# If no alt edge was found on this level,

# check all elements of next lower level.

for dtr in dtrs:

mthrs = dtrs

dtrs = []

for mthr in mthrs:

dtrs.extend(mthr.get_known_dtrs())

def display_parses(self):

'''

Display parse trees for all successful parses

'''

s_edges = self.chart.get_s_edges()

if len(s_edges) == 0:

raise ParseException("No parse could be found.")

for s_edge in s_edges:

parse_string = self.build_parse_string_from_edge(s_edge, 'S')

print self.add_indentation_to_parse_string(parse_string) + '\t' + str(s_edge.get_prob())

def build_parse_string_from_edge(self, edge, root):

'''

Recursively work your way down through the known daughters of

the input edge; return a bracketed structure representing the

parse tree.

In order to obtain a complete structure, this

method needs to be called with a string representing

the appropriate tree root (as the second argument)

'''

if not edge.get_known_dtrs() == []:

for dtr in edge.get_known_dtrs():

root += ' [ ' + dtr.get_prod_rule().get_lhs() + self.build_parse_string_from_edge(dtr, '') + ' ]'

return root

def add_indentation_to_parse_string(self, parse_string):

'''

Convert flat string representation of parse to appropriately

indented structure

'''

parse_string = '[ ' + parse_string + ' ]'

indented_string = ''

level = -1

for char in parse_string:

if char == '[':

level += 1

indented_string += '\n' + '\t'*level + char

elif char == ']':

level -= 1

indented_string += char

else:

indented_string += char