Esempio n. 1
0
	def __create_key_fsa(self, dict):
		"""
		Create an FSA from the keys in a map.
		"""
		def add_key(fsa, key):
			def add_new_transition(fsa, start, label, end, tag):
				for x in fsa.transitions_from(start):
					if x == (label, end, tag):
						return False
				fsa.add_transition(start, label, end, tag)

			for j in range(len(key), -1, -1):
				if j > 0 and fsa.has_state(key[:j]):
					break
			s = key + self._separator
			for i in range(j, len(s)):
				c = s[i]
				if i == len(s) - 1: #last step
					end = fsa.get_initial()
					tag = key
				else:
					end = s[:i + 1]
					tag = None
				add_new_transition(fsa, s[:i], c, end, tag)
		fsa = FSA()
		fsa.add_state("")
		fsa.set_initial("")
		fsa.set_final("")
		for k, v in dict.items():
			if not isinstance(v, list):
				raise TypeError(v)
			add_key(fsa, k)
		return fsa
 def parse_table(name, table, subsets):
     lines = table.split('\n')
     if len(lines) < 4:
         raise ValueError,\
         "Rule %s has too few lines to be an FSA table." % name
     pairs1 = lines[1].strip().split()
     pairs2 = lines[2].strip().split()
     if len(pairs1) != len(pairs2):
         raise ValueError,\
         "Rule %s has pair definitions that don't line up." % name
     pairs = [KimmoPair(p1, p2) for p1, p2 in zip(pairs1, pairs2)]
     finals = []
     fsa = FSA()
     for line in lines[3:]:
         line = line.strip()
         if not line: continue
         groups = re.match(r'(\w+)(\.|:)\s*(.*)', line)
         if groups is None:
             raise ValueError,\
             "Can't parse this line of the state table for rule %s:\n%s"\
             % (name, line)
         state, char, morestates = groups.groups()
         if fsa.start() == 0: fsa.set_start(state)
         if char == ':': finals.append(state)
         fsa.add_state(state)
         morestates = morestates.split()
         if len(morestates) != len(pairs):
             raise ValueError,\
             "Rule %s has a row of the wrong length:\n%s\ngot %d items, should be %d"\
             % (name, line, len(morestates), len(pairs))
         for pair, nextstate in zip(pairs, morestates):
             fsa.insert_safe(state, pair, nextstate)
     fsa.set_final(finals)
     return KimmoFSARule(name, fsa, subsets)
Esempio n. 3
0
 def parse_table(name, table, subsets):
     lines = table.split('\n')
     if len(lines) < 4:
         raise ValueError,\
         "Rule %s has too few lines to be an FSA table." % name
     pairs1 = lines[1].strip().split()
     pairs2 = lines[2].strip().split()
     if len(pairs1) != len(pairs2):
         raise ValueError,\
         "Rule %s has pair definitions that don't line up." % name
     pairs = [KimmoPair(p1, p2) for p1, p2 in zip(pairs1, pairs2)]
     finals = []
     fsa = FSA()
     for line in lines[3:]:
         line = line.strip()
         if not line: continue
         groups = re.match(r'(\w+)(\.|:)\s*(.*)', line)
         if groups is None:
             raise ValueError,\
             "Can't parse this line of the state table for rule %s:\n%s"\
             % (name, line)
         state, char, morestates = groups.groups()
         if fsa.start() == 0: fsa.set_start(state)
         if char == ':': finals.append(state)
         fsa.add_state(state)
         morestates = morestates.split()
         if len(morestates) != len(pairs):
             raise ValueError,\
             "Rule %s has a row of the wrong length:\n%s\ngot %d items, should be %d"\
             % (name, line, len(morestates), len(pairs))
         for pair, nextstate in zip(pairs, morestates):
             fsa.insert_safe(state, pair, nextstate)
     fsa.set_final(finals)
     return KimmoFSARule(name, fsa, subsets)
    def _parse_context(self, tokens, i, reverse):
        (j, tree) = self._parse_list(tokens, i)
        if j == i: return (i, None)

        sigma = set()
        self._collect_alphabet(tree, sigma)
        fsa = FSA(sigma)
        final_state = self._build_fsa(fsa, fsa.start(), tree, reverse)
        fsa.set_final([final_state])
        #fsa.pp()
        dfa = fsa.dfa()
        #dfa.pp()
        dfa.prune()
        #dfa.pp()
        return (j, dfa)
Esempio n. 5
0
    def _parse_context(self, tokens, i, reverse):
        (j, tree) = self._parse_list(tokens, i)
        if j == i: return (i, None)

        sigma = set()
        self._collect_alphabet(tree, sigma)
        fsa = FSA(sigma)
        final_state = self._build_fsa(fsa, fsa.start(), tree, reverse)
        fsa.set_final([final_state])
        #fsa.pp()
        dfa = fsa.dfa()
        #dfa.pp()
        dfa.prune()
        #dfa.pp()
        return (j, dfa)
Esempio n. 6
0
	def compile(self, force = False):
		"""
		Compile the set of rules into a Deterministic State Automaton (DFSA).

		If the grammar has never been compiled or it has been modified after last compilation, it will be translated into an FSA.
		The result will be kept available until the rules are modified or the grammar reset.

		The algorithm calls recursively the L{bnf.NormalExpression.insert_transitions} method.

		If the C{force} flag is off and the grammar was already compiled and was not updated, the old result is taken with no recompiling.

		@see: L{Finite State Automaton<fsa.FSA>}
		@param force: Recompile grammar even if it has already been validated and compiled.
		@type force: bool
		@raise GrammarError: If anomalies are encountered while precompiling.
		@return: A parser for the grammar.
		@rtype: fsa.Parser

		"""

		if force or not self.__valid and self.__compiled is None:
			self.__valid = False

			self.browse()
			nfa = FSA()
			initial = nfa.add_state()
			nfa.set_initial(initial)
			final = nfa.add_state()
			nfa.set_final(final)

			s = self.__rules[self.start]
			s.insert_transitions(self, nfa, initial, final, ())
			#rewriting to save memory
			nfa = nfa.reduced()
			nfa = nfa.minimized()
			self.__compiled = _GrammarParser(nfa)
			self.__valid = True
		return self.__compiled