def addDFAState(self, configs:ATNConfigSet) -> DFAState:
proposed = DFAState(configs=configs)
firstConfigWithRuleStopState = None
for c in configs:
if isinstance(c.state, RuleStopState):
firstConfigWithRuleStopState = c
break
if firstConfigWithRuleStopState is not None:
proposed.isAcceptState = True
proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor
proposed.prediction = self.atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex]
dfa = self.decisionToDFA[self.mode]
existing = dfa.states.get(proposed, None)
if existing is not None:
return existing
newState = proposed
newState.stateNumber = len(dfa.states)
configs.setReadonly(True)
newState.configs = configs
dfa.states[newState] = newState
return newState
def addDFAState(self, configs: ATNConfigSet) -> DFAState:
# the lexer evaluates predicates on-the-fly; by this point configs
# should not contain any configurations with unevaluated predicates.
assert not configs.hasSemanticContext
proposed = DFAState(configs=configs)
firstConfigWithRuleStopState = None
for c in configs:
if isinstance(c.state, RuleStopState):
firstConfigWithRuleStopState = c
break
if firstConfigWithRuleStopState is not None:
proposed.isAcceptState = True
proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor
proposed.prediction = self.atn.ruleToTokenType[
firstConfigWithRuleStopState.state.ruleIndex]
dfa = self.decisionToDFA[self.mode]
existing = dfa.states.get(proposed, None)
if existing is not None:
return existing
newState = proposed
newState.stateNumber = len(dfa.states)
configs.setReadonly(True)
newState.configs = configs
dfa.states[newState] = newState
return newState
def addDFAState(self, configs):
proposed = DFAState(configs=configs)
firstConfigWithRuleStopState = next(
(cfg for cfg in configs if isinstance(cfg.state, RuleStopState)),
None)
if firstConfigWithRuleStopState is not None:
proposed.isAcceptState = True
proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor
proposed.prediction = self.atn.ruleToTokenType[
firstConfigWithRuleStopState.state.ruleIndex]
dfa = self.decisionToDFA[self.mode]
existing = dfa.states.get(proposed, None)
if existing is not None:
return existing
newState = proposed
newState.stateNumber = len(dfa.states)
configs.setReadonly(True)
newState.configs = configs
dfa.states[newState] = newState
return newState
def addDFAState(self, configs):
# the lexer evaluates predicates on-the-fly; by this point configs
# should not contain any configurations with unevaluated predicates.
assert not configs.hasSemanticContext
proposed = DFAState(configs=configs)
firstConfigWithRuleStopState = None
for c in configs:
if isinstance(c.state, RuleStopState):
firstConfigWithRuleStopState = c
break
if firstConfigWithRuleStopState is not None:
proposed.isAcceptState = True
proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor
proposed.prediction = self.atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex]
dfa = self.decisionToDFA[self.mode]
existing = dfa.states.get(proposed, None)
if existing is not None:
return existing
newState = proposed
newState.stateNumber = len(dfa.states)
configs.setReadonly(True)
newState.configs = configs
dfa.states[newState] = newState
return newState
def setPrecedenceDfa(self, precedenceDfa):
if self.precedenceDfa != precedenceDfa:
self._states = dict()
if precedenceDfa:
precedenceState = DFAState(configs=ATNConfigSet())
precedenceState.edges = []
precedenceState.isAcceptState = False
precedenceState.requiresFullContext = False
self.s0 = precedenceState
else:
self.s0 = None
self.precedenceDfa = precedenceDfa
def setPrecedenceDfa(self, precedenceDfa):
if self.precedenceDfa != precedenceDfa:
self._states = dict()
if precedenceDfa:
precedenceState = DFAState(configs=ATNConfigSet())
precedenceState.edges = []
precedenceState.isAcceptState = False
precedenceState.requiresFullContext = False
self.s0 = precedenceState
else:
self.s0 = None
self.precedenceDfa = precedenceDfa
class ATNSimulator(object):
# Must distinguish between missing edge and edge we know leads nowhere#/
ERROR = DFAState(0x7FFFFFFF, ATNConfigSet())
# The context cache maps all PredictionContext objects that are ==
# to a single cached copy. This cache is shared across all contexts
# in all ATNConfigs in all DFA states. We rebuild each ATNConfigSet
# to use only cached nodes/graphs in addDFAState(). We don't want to
# fill this during closure() since there are lots of contexts that
# pop up but are not used ever again. It also greatly slows down closure().
#
# <p>This cache makes a huge difference in memory and a little bit in speed.
# For the Java grammar on java.*, it dropped the memory requirements
# at the end from 25M to 16M. We don't store any of the full context
# graphs in the DFA because they are limited to local context only,
# but apparently there's a lot of repetition there as well. We optimize
# the config contexts before storing the config set in the DFA states
# by literally rebuilding them with cached subgraphs only.</p>
#
# <p>I tried a cache for use during closure operations, that was
# whacked after each adaptivePredict(). It cost a little bit
# more time I think and doesn't save on the overall footprint
# so it's not worth the complexity.</p>
#/
def __init__(self, atn, sharedContextCache):
self.atn = atn
self.sharedContextCache = sharedContextCache
def getCachedContext(self, context):
if self.sharedContextCache is None:
return context
visited = dict()
return getCachedPredictionContext(context, self.sharedContextCache,
visited)
def get_s0(dfa, parser_atn_simulator, outerContext):
if dfa.precedenceDfa:
# the start state for a precedence DFA depends on the current
# parser precedence, and is provided by a DFA method.
s0 = dfa.getPrecedenceStartState(
parser_atn_simulator.parser.getPrecedence())
else:
# the start state for a "regular" DFA is just s0
s0 = dfa.s0
if s0 is None:
if outerContext is None:
outerContext = ParserRuleContext.EMPTY
if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions:
print("predictATN decision " + str(dfa.decision) +
" exec LA(1)==" +
parser_atn_simulator.getLookaheadName(input) +
", outerContext=" + outerContext.toString(
parser_atn_simulator.parser.literalNames, None))
fullCtx = False
s0_closure = parser_atn_simulator.computeStartState(
dfa.atnStartState, ParserRuleContext.EMPTY, fullCtx)
if dfa.precedenceDfa:
# If this is a precedence DFA, we use applyPrecedenceFilter
# to convert the computed start state to a precedence start
# state. We then use DFA.setPrecedenceStartState to set the
# appropriate start state for the precedence level rather
# than simply setting DFA.s0.
#
dfa.s0.configs = s0_closure # not used for prediction but useful to know start configs anyway
s0_closure = parser_atn_simulator.applyPrecedenceFilter(s0_closure)
s0 = parser_atn_simulator.addDFAState(dfa,
DFAState(configs=s0_closure))
dfa.setPrecedenceStartState(
parser_atn_simulator.parser.getPrecedence(), s0)
else:
s0 = parser_atn_simulator.addDFAState(dfa,
DFAState(configs=s0_closure))
dfa.s0 = s0
return s0
def __init__(self, atnStartState, decision=0):
# From which ATN state did we create this DFA?
self.atnStartState = atnStartState
self.decision = decision
# A set of all DFA states. Use {@link Map} so we can get old state back
# ({@link Set} only allows you to see if it's there).
self._states = dict()
self.s0 = None
# {@code true} if this DFA is for a precedence decision; otherwise,
# {@code false}. This is the backing field for {@link #isPrecedenceDfa},
# {@link #setPrecedenceDfa}.
self.precedenceDfa = False
if isinstance(atnStartState, StarLoopEntryState):
if atnStartState.isPrecedenceDecision:
self.precedenceDfa = True
precedenceState = DFAState(configs=ATNConfigSet())
precedenceState.edges = []
precedenceState.isAcceptState = False
precedenceState.requiresFullContext = False
self.s0 = precedenceState
def __init__(self, atnStartState, decision=0):
# From which ATN state did we create this DFA?
self.atnStartState = atnStartState
self.decision = decision
# A set of all DFA states. Use {@link Map} so we can get old state back
# ({@link Set} only allows you to see if it's there).
self._states = dict()
self.s0 = None
# {@code true} if this DFA is for a precedence decision; otherwise,
# {@code false}. This is the backing field for {@link #isPrecedenceDfa},
# {@link #setPrecedenceDfa}.
self.precedenceDfa = False
if isinstance(atnStartState, StarLoopEntryState):
if atnStartState.isPrecedenceDecision:
self.precedenceDfa = True
precedenceState = DFAState(configs=ATNConfigSet())
precedenceState.edges = []
precedenceState.isAcceptState = False
precedenceState.requiresFullContext = False
self.s0 = precedenceState
def addDFAEdge(self,
from_: DFAState,
tk: int,
to: DFAState = None,
cfgs: ATNConfigSet = None) -> DFAState:
if to is None and cfgs is not None:
# leading to this call, ATNConfigSet.hasSemanticContext is used as a
# marker indicating dynamic predicate evaluation makes this edge
# dependent on the specific input sequence, so the static edge in the
# DFA should be omitted. The target DFAState is still created since
# execATN has the ability to resynchronize with the DFA state cache
# following the predicate evaluation step.
#
# TJP notes: next time through the DFA, we see a pred again and eval.
# If that gets us to a previously created (but dangling) DFA
# state, we can continue in pure DFA mode from there.
#/
suppressEdge = cfgs.hasSemanticContext
cfgs.hasSemanticContext = False
to = self.addDFAState(cfgs)
if suppressEdge:
return to
# add the edge
if tk < self.MIN_DFA_EDGE or tk > self.MAX_DFA_EDGE:
# Only track edges within the DFA bounds
return to
if self.debug:
print("EDGE " + str(from_) + " -> " + str(to) + " upon " + chr(tk))
if from_.edges is None:
# make room for tokens 1..n and -1 masquerading as index 0
from_.edges = [None] * (self.MAX_DFA_EDGE - self.MIN_DFA_EDGE + 1)
from_.edges[tk - self.MIN_DFA_EDGE] = to # connect
return to
def addDFAEdge(self, from_:DFAState, tk:int, to:DFAState=None, cfgs:ATNConfigSet=None) -> DFAState:
if to is None and cfgs is not None:
# leading to this call, ATNConfigSet.hasSemanticContext is used as a
# marker indicating dynamic predicate evaluation makes this edge
# dependent on the specific input sequence, so the static edge in the
# DFA should be omitted. The target DFAState is still created since
# execATN has the ability to resynchronize with the DFA state cache
# following the predicate evaluation step.
#
# TJP notes: next time through the DFA, we see a pred again and eval.
# If that gets us to a previously created (but dangling) DFA
# state, we can continue in pure DFA mode from there.
#/
suppressEdge = cfgs.hasSemanticContext
cfgs.hasSemanticContext = False
to = self.addDFAState(cfgs)
if suppressEdge:
return to
# add the edge
if tk < self.MIN_DFA_EDGE or tk > self.MAX_DFA_EDGE:
# Only track edges within the DFA bounds
return to
if LexerATNSimulator.debug:
print("EDGE " + str(from_) + " -> " + str(to) + " upon "+ chr(tk))
if from_.edges is None:
# make room for tokens 1..n and -1 masquerading as index 0
from_.edges = [ None ] * (self.MAX_DFA_EDGE - self.MIN_DFA_EDGE + 1)
from_.edges[tk - self.MIN_DFA_EDGE] = to # connect
return to
newState.configs = configs
dfa.states[newState] = newState
return newState
def getDFA(self, mode: int):
return self.decisionToDFA[mode]
# Get the text matched so far for the current token.
def getText(self, input: InputStream):
# index is first lookahead char, don't include.
return input.getText(self.startIndex, input.index - 1)
def consume(self, input: InputStream):
curChar = input.LA(1)
if curChar == ord('\n'):
self.line += 1
self.column = 0
else:
self.column += 1
input.consume()
def getTokenName(self, t: int):
if t == -1:
return "EOF"
else:
return "'" + chr(t) + "'"
LexerATNSimulator.ERROR = DFAState(0x7FFFFFFF, ATNConfigSet())
del Lexer