def __create_key_fsa(self, dict):
"""
Create an FSA from the keys in a map.
"""
def add_key(fsa, key):
def add_new_transition(fsa, start, label, end, tag):
for x in fsa.transitions_from(start):
if x == (label, end, tag):
return False
fsa.add_transition(start, label, end, tag)
for j in range(len(key), -1, -1):
if j > 0 and fsa.has_state(key[:j]):
break
s = key + self._separator
for i in range(j, len(s)):
c = s[i]
if i == len(s) - 1: #last step
end = fsa.get_initial()
tag = key
else:
end = s[:i + 1]
tag = None
add_new_transition(fsa, s[:i], c, end, tag)
fsa = FSA()
fsa.add_state("")
fsa.set_initial("")
fsa.set_final("")
for k, v in dict.items():
if not isinstance(v, list):
raise TypeError(v)
add_key(fsa, k)
return fsa
def parse_table(name, table, subsets):
lines = table.split('\n')
if len(lines) < 4:
raise ValueError,\
"Rule %s has too few lines to be an FSA table." % name
pairs1 = lines[1].strip().split()
pairs2 = lines[2].strip().split()
if len(pairs1) != len(pairs2):
raise ValueError,\
"Rule %s has pair definitions that don't line up." % name
pairs = [KimmoPair(p1, p2) for p1, p2 in zip(pairs1, pairs2)]
finals = []
fsa = FSA()
for line in lines[3:]:
line = line.strip()
if not line: continue
groups = re.match(r'(\w+)(\.|:)\s*(.*)', line)
if groups is None:
raise ValueError,\
"Can't parse this line of the state table for rule %s:\n%s"\
% (name, line)
state, char, morestates = groups.groups()
if fsa.start() == 0: fsa.set_start(state)
if char == ':': finals.append(state)
fsa.add_state(state)
morestates = morestates.split()
if len(morestates) != len(pairs):
raise ValueError,\
"Rule %s has a row of the wrong length:\n%s\ngot %d items, should be %d"\
% (name, line, len(morestates), len(pairs))
for pair, nextstate in zip(pairs, morestates):
fsa.insert_safe(state, pair, nextstate)
fsa.set_final(finals)
return KimmoFSARule(name, fsa, subsets)
def parse_table(name, table, subsets):
lines = table.split('\n')
if len(lines) < 4:
raise ValueError,\
"Rule %s has too few lines to be an FSA table." % name
pairs1 = lines[1].strip().split()
pairs2 = lines[2].strip().split()
if len(pairs1) != len(pairs2):
raise ValueError,\
"Rule %s has pair definitions that don't line up." % name
pairs = [KimmoPair(p1, p2) for p1, p2 in zip(pairs1, pairs2)]
finals = []
fsa = FSA()
for line in lines[3:]:
line = line.strip()
if not line: continue
groups = re.match(r'(\w+)(\.|:)\s*(.*)', line)
if groups is None:
raise ValueError,\
"Can't parse this line of the state table for rule %s:\n%s"\
% (name, line)
state, char, morestates = groups.groups()
if fsa.start() == 0: fsa.set_start(state)
if char == ':': finals.append(state)
fsa.add_state(state)
morestates = morestates.split()
if len(morestates) != len(pairs):
raise ValueError,\
"Rule %s has a row of the wrong length:\n%s\ngot %d items, should be %d"\
% (name, line, len(morestates), len(pairs))
for pair, nextstate in zip(pairs, morestates):
fsa.insert_safe(state, pair, nextstate)
fsa.set_final(finals)
return KimmoFSARule(name, fsa, subsets)
def _parse_context(self, tokens, i, reverse):
(j, tree) = self._parse_list(tokens, i)
if j == i: return (i, None)
sigma = set()
self._collect_alphabet(tree, sigma)
fsa = FSA(sigma)
final_state = self._build_fsa(fsa, fsa.start(), tree, reverse)
fsa.set_final([final_state])
#fsa.pp()
dfa = fsa.dfa()
#dfa.pp()
dfa.prune()
#dfa.pp()
return (j, dfa)
def _parse_context(self, tokens, i, reverse):
(j, tree) = self._parse_list(tokens, i)
if j == i: return (i, None)
sigma = set()
self._collect_alphabet(tree, sigma)
fsa = FSA(sigma)
final_state = self._build_fsa(fsa, fsa.start(), tree, reverse)
fsa.set_final([final_state])
#fsa.pp()
dfa = fsa.dfa()
#dfa.pp()
dfa.prune()
#dfa.pp()
return (j, dfa)
def compile(self, force = False):
"""
Compile the set of rules into a Deterministic State Automaton (DFSA).
If the grammar has never been compiled or it has been modified after last compilation, it will be translated into an FSA.
The result will be kept available until the rules are modified or the grammar reset.
The algorithm calls recursively the L{bnf.NormalExpression.insert_transitions} method.
If the C{force} flag is off and the grammar was already compiled and was not updated, the old result is taken with no recompiling.
@see: L{Finite State Automaton<fsa.FSA>}
@param force: Recompile grammar even if it has already been validated and compiled.
@type force: bool
@raise GrammarError: If anomalies are encountered while precompiling.
@return: A parser for the grammar.
@rtype: fsa.Parser
"""
if force or not self.__valid and self.__compiled is None:
self.__valid = False
self.browse()
nfa = FSA()
initial = nfa.add_state()
nfa.set_initial(initial)
final = nfa.add_state()
nfa.set_final(final)
s = self.__rules[self.start]
s.insert_transitions(self, nfa, initial, final, ())
#rewriting to save memory
nfa = nfa.reduced()
nfa = nfa.minimized()
self.__compiled = _GrammarParser(nfa)
self.__valid = True
return self.__compiled
