示例#1
0
    def test_closures_ofLRValue(self):
        a1 = set([
            LR0(self.StartExtendedSymbol, 0, 0),
            LR0('S', 0, 0),
            LR0('S', 1, 0),
            LR0('R', 0, 0),
            LR0('R', 1, 0),
            LR0('L', 0, 0),
            LR0('L', 1, 0)
        ])

        a2 = set([
            LR0('S', 0, 0),
            LR0('S', 1, 0),
            LR0('R', 0, 0),
            LR0('R', 1, 0),
            LR0('R', 1, 1),
            LR0('L', 0, 0),
            LR0('L', 0, 1),
            LR0('L', 1, 0)
        ])

        self.assertTrue(a1 == closure(
            set([LR0(self.StartExtendedSymbol, 0, 0)]), self.lrvalue))
        self.assertTrue(
            a2 == closure(set([LR0('L', 0, 1), LR0('R', 1, 1)]), self.lrvalue))
示例#2
0
    def test_closures_ofLRValue_with_actions(self):
        a1 = set([
            LR0(self.StartExtendedSymbol, 0, 0),
            LR0('S', 0, 0),
            LR0('S', 1, 0),
            LR0('R', 0, 0),
            LR0('R', 1, 0),
            LR0('L', 0, 0),
            LR0('L', 1, 0)
        ])

        a2 = set([
            LR0(self.lrvalue_with_actions.ACTION_INSIDE % (3, '<lambda>'), 0,
                0),
            LR0('R', 0, 0),
            LR0('R', 1, 0),
            LR0('R', 1, 1),
            LR0('L', 0, 0),
            LR0('L', 0, 1),
            LR0('L', 1, 0)
        ])

        self.assertTrue(
            a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]),
                          self.lrvalue_with_actions))
        self.assertTrue(a2 == closure(set([LR0(
            'L', 0, 1), LR0('R', 1, 1)]), self.lrvalue_with_actions))
示例#3
0
文件: builder.py 项目: eldipa/dragon
def generate_spontaneously_lookaheads(grammar, start_item, to_id):
   '''Builds a LR0 table using as a seed the start_item and then tries to 
      determinate what terminals are lookahead of each item (in which case, 
      these lookaheads are spontaneously generated), building initially the
      LALR items (they are very similar to the LR0 item but can be modified
      adding to him lookaheads, see the documentation of LALR class in the 
      item module).
      '''

   goto_table = collections.defaultdict(dict)
   
   lalr_start_item = LALR(
         start_item.sym_production, 
         start_item.alternative, 
         start_item.position)
   lalr_start_item.add_new(grammar.EOF)
   kernels_lalr = kernel_collection(grammar, lalr_start_item)

   ids_kernes_lalr = dict()
   for kernels in kernels_lalr:
      closure_lalr = closure(kernels, grammar)
      ids_kernes_lalr[to_id[closure_lalr]] = kernels
      populate_goto_table_from_state(grammar, closure_lalr, goto_table, to_id)

   for id_, kernels in ids_kernes_lalr.iteritems():
      for item_lalr in kernels:
         closure_lr1 = closure(set([LR1(
            item_lalr.sym_production, 
            item_lalr.alternative, 
            item_lalr.position, 
            grammar.PROBE)]), grammar)

         for item_lr1 in closure_lr1:
            next_symbol = item_lr1.next_symbol(grammar)
            if not next_symbol:
               continue
            item_lr1_shifted = item_lr1.item_shifted(grammar)
               
            assert id_ in goto_table
            goto_set_id = goto_table[id_][next_symbol]
            goto_set = ids_kernes_lalr[goto_set_id]

            item_lalr_from_shifted = LALR(
                  item_lr1_shifted.sym_production, 
                  item_lr1_shifted.alternative, 
                  item_lr1_shifted.position)
            assert item_lalr_from_shifted in goto_set
            
            singleton = (goto_set - (goto_set - {item_lalr_from_shifted}))
            assert len(singleton) == 1
            
            item_lalr_hidden = set(singleton).pop()

            if item_lr1.lookahead != grammar.PROBE:
               item_lalr_hidden.add_new(item_lr1.lookahead)

            else:
               item_lalr.subscribe(item_lalr_hidden)

   return kernels_lalr, goto_table
示例#4
0
def generate_spontaneously_lookaheads(grammar, start_item, to_id):
    '''Builds a LR0 table using as a seed the start_item and then tries to 
      determinate what terminals are lookahead of each item (in which case, 
      these lookaheads are spontaneously generated), building initially the
      LALR items (they are very similar to the LR0 item but can be modified
      adding to him lookaheads, see the documentation of LALR class in the 
      item module).
      '''

    goto_table = collections.defaultdict(dict)

    lalr_start_item = LALR(start_item.sym_production, start_item.alternative,
                           start_item.position)
    lalr_start_item.add_new(grammar.EOF)
    kernels_lalr = kernel_collection(grammar, lalr_start_item)

    ids_kernes_lalr = dict()
    for kernels in kernels_lalr:
        closure_lalr = closure(kernels, grammar)
        ids_kernes_lalr[to_id[closure_lalr]] = kernels
        populate_goto_table_from_state(grammar, closure_lalr, goto_table,
                                       to_id)

    for id_, kernels in ids_kernes_lalr.iteritems():
        for item_lalr in kernels:
            closure_lr1 = closure(
                set([
                    LR1(item_lalr.sym_production, item_lalr.alternative,
                        item_lalr.position, grammar.PROBE)
                ]), grammar)

            for item_lr1 in closure_lr1:
                next_symbol = item_lr1.next_symbol(grammar)
                if not next_symbol:
                    continue
                item_lr1_shifted = item_lr1.item_shifted(grammar)

                assert id_ in goto_table
                goto_set_id = goto_table[id_][next_symbol]
                goto_set = ids_kernes_lalr[goto_set_id]

                item_lalr_from_shifted = LALR(item_lr1_shifted.sym_production,
                                              item_lr1_shifted.alternative,
                                              item_lr1_shifted.position)
                assert item_lalr_from_shifted in goto_set

                singleton = (goto_set - (goto_set - {item_lalr_from_shifted}))
                assert len(singleton) == 1

                item_lalr_hidden = set(singleton).pop()

                if item_lr1.lookahead != grammar.PROBE:
                    item_lalr_hidden.add_new(item_lr1.lookahead)

                else:
                    item_lalr.subscribe(item_lalr_hidden)

    return kernels_lalr, goto_table
示例#5
0
   def test_closures_ofSome(self):
      a1 = set([LR0(self.StartExtendedSymbol, 0, 0),
            LR0('S', 0, 0),
            LR0('C', 0, 0), LR0('C', 1, 0)])
      
      a2 = set([
            LR0('S', 0, 1), 
            LR0('C', 0, 0), LR0('C', 1, 0), LR0('C', 1, 1)])

      self.assertTrue(a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]), self.some))
      self.assertTrue(a2 == closure(set([LR0('S', 0, 1), LR0('C', 1, 1)]), self.some))
示例#6
0
   def test_closures_ofLRValue_with_actions(self):
      a1 = set([LR0(self.StartExtendedSymbol, 0, 0),
            LR0('S', 0, 0), LR0('S', 1, 0), 
            LR0('R', 0, 0), LR0('R', 1, 0),
            LR0('L', 0, 0), LR0('L', 1, 0)])
      
      a2 = set([
            LR0(self.lrvalue_with_actions.ACTION_INSIDE % (3, '<lambda>'), 0, 0), 
            LR0('R', 0, 0), LR0('R', 1, 0), LR0('R', 1, 1),
            LR0('L', 0, 0), LR0('L', 0, 1), LR0('L', 1, 0)])

      self.assertTrue(a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]), self.lrvalue_with_actions))
      self.assertTrue(a2 == closure(set([LR0('L', 0, 1), LR0('R', 1, 1)]), self.lrvalue_with_actions))
示例#7
0
   def test_closures_ofLRValue(self):
      a1 = set([LR0(self.StartExtendedSymbol, 0, 0),
            LR0('S', 0, 0), LR0('S', 1, 0), 
            LR0('R', 0, 0), LR0('R', 1, 0),
            LR0('L', 0, 0), LR0('L', 1, 0)])
      
      a2 = set([
            LR0('S', 0, 0), LR0('S', 1, 0), 
            LR0('R', 0, 0), LR0('R', 1, 0), LR0('R', 1, 1),
            LR0('L', 0, 0), LR0('L', 0, 1), LR0('L', 1, 0)])

      self.assertTrue(a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]), self.lrvalue))
      self.assertTrue(a2 == closure(set([LR0('L', 0, 1), LR0('R', 1, 1)]), self.lrvalue))
示例#8
0
   def test_closures_ofArith(self):
      a1 = set([LR0(self.StartExtendedSymbol, 0, 0),
            LR0('E', 0, 0), LR0('E', 1, 0), 
            LR0('T', 0, 0), LR0('T', 1, 0),
            LR0('F', 0, 0), LR0('F', 1, 0)])
      
      a2 = set([LR0('F', 0, 1),
            LR0('E', 0, 0), LR0('E', 1, 0), 
            LR0('T', 0, 0), LR0('T', 1, 0),
            LR0('F', 0, 0), LR0('F', 1, 0)])

      self.assertTrue(a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]), self.arith))
      self.assertTrue(a2 == closure(set([LR0('F', 0, 1)]), self.arith))
示例#9
0
    def test_closures_ofSome(self):
        a1 = set([
            LR0(self.StartExtendedSymbol, 0, 0),
            LR0('S', 0, 0),
            LR0('C', 0, 0),
            LR0('C', 1, 0)
        ])

        a2 = set(
            [LR0('S', 0, 1),
             LR0('C', 0, 0),
             LR0('C', 1, 0),
             LR0('C', 1, 1)])

        self.assertTrue(a1 == closure(
            set([LR0(self.StartExtendedSymbol, 0, 0)]), self.some))
        self.assertTrue(
            a2 == closure(set([LR0('S', 0, 1), LR0('C', 1, 1)]), self.some))
示例#10
0
    def test_closures_ofArith(self):
        a1 = set([
            LR0(self.StartExtendedSymbol, 0, 0),
            LR0('E', 0, 0),
            LR0('E', 1, 0),
            LR0('T', 0, 0),
            LR0('T', 1, 0),
            LR0('F', 0, 0),
            LR0('F', 1, 0)
        ])

        a2 = set([
            LR0('F', 0, 1),
            LR0('E', 0, 0),
            LR0('E', 1, 0),
            LR0('T', 0, 0),
            LR0('T', 1, 0),
            LR0('F', 0, 0),
            LR0('F', 1, 0)
        ])

        self.assertTrue(a1 == closure(
            set([LR0(self.StartExtendedSymbol, 0, 0)]), self.arith))
        self.assertTrue(a2 == closure(set([LR0('F', 0, 1)]), self.arith))
示例#11
0
    def test_action_table(self):
        states = [
            closure(kernel, self.grammar) for kernel in self.kernel_states
        ]

        expected_terminal_shift = [
            {'NL', 'NUM'},
            {},
            {'NL', 'NUM'},
            {},
            {'NL', 'NUM'},
            {},
            {},
            {},
            {'NUM', '+', '-'},
            {},
            {},
        ]

        for state, terminals in zip(states, expected_terminal_shift):
            keys = self.action_table[hash(state)].keys()
            keys = filter(
                lambda k: "Shift" in str(self.action_table[hash(state)][k]),
                keys)
            self.assertTrue(frozenset(keys) == frozenset(terminals))

        expected_terminal_reduce = [
            {},
            {},
            {self.grammar.EOF},  #input
            {self.grammar.EOF, 'NL', 'NUM'},  #line
            {},
            {'NL', '+', '-', 'NUM'},  #expr
            {self.grammar.EOF},  #input
            {self.grammar.EOF, 'NL', 'NUM'},  #line
            {},
            {'NL', '+', '-', 'NUM'},  #expr
            {'NL', '+', '-', 'NUM'},  #expr
        ]

        for state, terminals in zip(states, expected_terminal_reduce):
            keys = self.action_table[hash(state)].keys()
            keys = filter(
                lambda k: "Reduce" in str(self.action_table[hash(state)][k]),
                keys)
            self.assertTrue(frozenset(keys) == frozenset(terminals))
示例#12
0
文件: builder.py 项目: eldipa/dragon
def build_parsing_table(grammar, start_item, handle_shift_reduce = True,
      disable_mapping = False):
   '''Builds the Action and Goto tables for be used by a driver returning
      these tables and the id of the start state, where the driver will use
      as a point of start to parse.
      
      See the documentation of handler_conflict function for more info with
      respect the handle_shift_reduce parameter.

      The start_item can be an instance of Item (see the module item).
      This works well with LR0 and LR1 items, but with LALR items, the
      algorithm used is the build_parsing_table_lalr (see that function, in
      this module)

      If 'disable_mapping' is True, the internal states are identified by
      its hash. This is only useful for testing and should not be modified
      in the normal case.

      Preconditions: the grammar must be already processed.'''
   if isinstance(start_item, LALR):
      return build_parsing_table_lalr(grammar, start_item, handle_shift_reduce)

   action_table = collections.defaultdict(dict)
   goto_table = collections.defaultdict(dict)
   kernels = kernel_collection(grammar, start_item)
   start_set_hash = None

   to_id = UserFriendlyMapping(disable_mapping)

   for kernel in kernels:
      state_set = closure(kernel, grammar)
      
      populate_goto_table_from_state(grammar, state_set, goto_table, to_id)
      populate_action_table_from_state(
            grammar, 
            state_set, 
            action_table, 
            handle_shift_reduce,
            to_id)
      
      if not start_set_hash:
         for item in state_set:
            if item == start_item:
               start_set_hash = to_id[state_set]

   return dict(action_table), dict(goto_table), start_set_hash
示例#13
0
def build_parsing_table(grammar,
                        start_item,
                        handle_shift_reduce=True,
                        disable_mapping=False):
    '''Builds the Action and Goto tables for be used by a driver returning
      these tables and the id of the start state, where the driver will use
      as a point of start to parse.
      
      See the documentation of handler_conflict function for more info with
      respect the handle_shift_reduce parameter.

      The start_item can be an instance of Item (see the module item).
      This works well with LR0 and LR1 items, but with LALR items, the
      algorithm used is the build_parsing_table_lalr (see that function, in
      this module)

      If 'disable_mapping' is True, the internal states are identified by
      its hash. This is only useful for testing and should not be modified
      in the normal case.

      Preconditions: the grammar must be already processed.'''
    if isinstance(start_item, LALR):
        return build_parsing_table_lalr(grammar, start_item,
                                        handle_shift_reduce)

    action_table = collections.defaultdict(dict)
    goto_table = collections.defaultdict(dict)
    kernels = kernel_collection(grammar, start_item)
    start_set_hash = None

    to_id = UserFriendlyMapping(disable_mapping)

    for kernel in kernels:
        state_set = closure(kernel, grammar)

        populate_goto_table_from_state(grammar, state_set, goto_table, to_id)
        populate_action_table_from_state(grammar, state_set, action_table,
                                         handle_shift_reduce, to_id)

        if not start_set_hash:
            for item in state_set:
                if item == start_item:
                    start_set_hash = to_id[state_set]

    return dict(action_table), dict(goto_table), start_set_hash
示例#14
0
文件: builder.py 项目: eldipa/dragon
def build_parsing_table_lalr(grammar, start_item, handle_shift_reduce = True,
      disable_mapping=False):
   '''Builds a LALR table, first builds a LR0 table using as a seed the 
      start_item and then tries to determinate what terminals are lookahead of 
      each item (in which case, these lookaheads are spontaneously generated)
      In a second stage, the spontaneously generated terminals are propagated
      from one item to other.
      When no more terminals propagated, the algorithm builds a kernel
      of items LALR.
      With this, the function returns the action and goto tables used by the
      driver.

      See the documentation of handler_conflict function for more info with
      respect the handle_shift_reduce parameter.
      
      If 'disable_mapping' is True, the internal states are identified by
      its hash. This is only useful for testing and should not be modified
      in the normal case.
   '''

   to_id = UserFriendlyMapping(disable_mapping)
   action_table = collections.defaultdict(dict)
   start_set_hash = None
   
   kernels_lalr, goto_table = generate_spontaneously_lookaheads(
         grammar, 
         start_item,
         to_id)
   propagate_lookaheads(kernels_lalr)
   for kernel in kernels_lalr:
      state_set = closure(kernel, grammar)
      populate_action_table_from_state(
            grammar, 
            state_set, 
            action_table, 
            handle_shift_reduce,
            to_id)
      
      if not start_set_hash:
         for item in state_set:
            if item == start_item:
               start_set_hash = to_id[state_set]
               break
   
   return dict(action_table), dict(goto_table), start_set_hash
示例#15
0
def build_parsing_table_lalr(grammar,
                             start_item,
                             handle_shift_reduce=True,
                             disable_mapping=False):
    '''Builds a LALR table, first builds a LR0 table using as a seed the 
      start_item and then tries to determinate what terminals are lookahead of 
      each item (in which case, these lookaheads are spontaneously generated)
      In a second stage, the spontaneously generated terminals are propagated
      from one item to other.
      When no more terminals propagated, the algorithm builds a kernel
      of items LALR.
      With this, the function returns the action and goto tables used by the
      driver.

      See the documentation of handler_conflict function for more info with
      respect the handle_shift_reduce parameter.
      
      If 'disable_mapping' is True, the internal states are identified by
      its hash. This is only useful for testing and should not be modified
      in the normal case.
   '''

    to_id = UserFriendlyMapping(disable_mapping)
    action_table = collections.defaultdict(dict)
    start_set_hash = None

    kernels_lalr, goto_table = generate_spontaneously_lookaheads(
        grammar, start_item, to_id)
    propagate_lookaheads(kernels_lalr)
    for kernel in kernels_lalr:
        state_set = closure(kernel, grammar)
        populate_action_table_from_state(grammar, state_set, action_table,
                                         handle_shift_reduce, to_id)

        if not start_set_hash:
            for item in state_set:
                if item == start_item:
                    start_set_hash = to_id[state_set]
                    break

    return dict(action_table), dict(goto_table), start_set_hash
示例#16
0
   def test_action_table(self):
      states = [closure(kernel, self.grammar) for kernel in self.kernel_states]
      
      expected_terminal_shift = [
            {'NL', 'NUM'},
            {},
            {'NL', 'NUM'},
            {},
            {'NL', 'NUM'},
            {},
            {},
            {},
            {'NUM', '+', '-'},
            {},
            {},
            ]
   
      for state, terminals in zip(states, expected_terminal_shift):
         keys = self.action_table[hash(state)].keys()
         keys = filter(lambda k: "Shift" in str(self.action_table[hash(state)][k]), keys) 
         self.assertTrue(frozenset(keys) == frozenset(terminals))

      expected_terminal_reduce = [
            {},
            {},
            {self.grammar.EOF}, #input
            {self.grammar.EOF, 'NL', 'NUM'}, #line
            {},
            {'NL', '+', '-', 'NUM'}, #expr
            {self.grammar.EOF}, #input
            {self.grammar.EOF, 'NL', 'NUM'}, #line
            {},
            {'NL', '+', '-', 'NUM'}, #expr
            {'NL', '+', '-', 'NUM'}, #expr
            ]
      
      for state, terminals in zip(states, expected_terminal_reduce):
         keys = self.action_table[hash(state)].keys()
         keys = filter(lambda k: "Reduce" in str(self.action_table[hash(state)][k]), keys) 
         self.assertTrue(frozenset(keys) == frozenset(terminals))