def addDFAState(self, configs:ATNConfigSet) -> DFAState: proposed = DFAState(configs=configs) firstConfigWithRuleStopState = None for c in configs: if isinstance(c.state, RuleStopState): firstConfigWithRuleStopState = c break if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor proposed.prediction = self.atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex] dfa = self.decisionToDFA[self.mode] existing = dfa.states.get(proposed, None) if existing is not None: return existing newState = proposed newState.stateNumber = len(dfa.states) configs.setReadonly(True) newState.configs = configs dfa.states[newState] = newState return newState
def addDFAState(self, configs: ATNConfigSet) -> DFAState: # the lexer evaluates predicates on-the-fly; by this point configs # should not contain any configurations with unevaluated predicates. assert not configs.hasSemanticContext proposed = DFAState(configs=configs) firstConfigWithRuleStopState = None for c in configs: if isinstance(c.state, RuleStopState): firstConfigWithRuleStopState = c break if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor proposed.prediction = self.atn.ruleToTokenType[ firstConfigWithRuleStopState.state.ruleIndex] dfa = self.decisionToDFA[self.mode] existing = dfa.states.get(proposed, None) if existing is not None: return existing newState = proposed newState.stateNumber = len(dfa.states) configs.setReadonly(True) newState.configs = configs dfa.states[newState] = newState return newState
def addDFAState(self, configs): proposed = DFAState(configs=configs) firstConfigWithRuleStopState = next( (cfg for cfg in configs if isinstance(cfg.state, RuleStopState)), None) if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor proposed.prediction = self.atn.ruleToTokenType[ firstConfigWithRuleStopState.state.ruleIndex] dfa = self.decisionToDFA[self.mode] existing = dfa.states.get(proposed, None) if existing is not None: return existing newState = proposed newState.stateNumber = len(dfa.states) configs.setReadonly(True) newState.configs = configs dfa.states[newState] = newState return newState
def addDFAState(self, configs): # the lexer evaluates predicates on-the-fly; by this point configs # should not contain any configurations with unevaluated predicates. assert not configs.hasSemanticContext proposed = DFAState(configs=configs) firstConfigWithRuleStopState = None for c in configs: if isinstance(c.state, RuleStopState): firstConfigWithRuleStopState = c break if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True proposed.lexerActionExecutor = firstConfigWithRuleStopState.lexerActionExecutor proposed.prediction = self.atn.ruleToTokenType[firstConfigWithRuleStopState.state.ruleIndex] dfa = self.decisionToDFA[self.mode] existing = dfa.states.get(proposed, None) if existing is not None: return existing newState = proposed newState.stateNumber = len(dfa.states) configs.setReadonly(True) newState.configs = configs dfa.states[newState] = newState return newState
def setPrecedenceDfa(self, precedenceDfa): if self.precedenceDfa != precedenceDfa: self._states = dict() if precedenceDfa: precedenceState = DFAState(configs=ATNConfigSet()) precedenceState.edges = [] precedenceState.isAcceptState = False precedenceState.requiresFullContext = False self.s0 = precedenceState else: self.s0 = None self.precedenceDfa = precedenceDfa
class ATNSimulator(object): # Must distinguish between missing edge and edge we know leads nowhere#/ ERROR = DFAState(0x7FFFFFFF, ATNConfigSet()) # The context cache maps all PredictionContext objects that are == # to a single cached copy. This cache is shared across all contexts # in all ATNConfigs in all DFA states. We rebuild each ATNConfigSet # to use only cached nodes/graphs in addDFAState(). We don't want to # fill this during closure() since there are lots of contexts that # pop up but are not used ever again. It also greatly slows down closure(). # # <p>This cache makes a huge difference in memory and a little bit in speed. # For the Java grammar on java.*, it dropped the memory requirements # at the end from 25M to 16M. We don't store any of the full context # graphs in the DFA because they are limited to local context only, # but apparently there's a lot of repetition there as well. We optimize # the config contexts before storing the config set in the DFA states # by literally rebuilding them with cached subgraphs only.</p> # # <p>I tried a cache for use during closure operations, that was # whacked after each adaptivePredict(). It cost a little bit # more time I think and doesn't save on the overall footprint # so it's not worth the complexity.</p> #/ def __init__(self, atn, sharedContextCache): self.atn = atn self.sharedContextCache = sharedContextCache def getCachedContext(self, context): if self.sharedContextCache is None: return context visited = dict() return getCachedPredictionContext(context, self.sharedContextCache, visited)
def get_s0(dfa, parser_atn_simulator, outerContext): if dfa.precedenceDfa: # the start state for a precedence DFA depends on the current # parser precedence, and is provided by a DFA method. s0 = dfa.getPrecedenceStartState( parser_atn_simulator.parser.getPrecedence()) else: # the start state for a "regular" DFA is just s0 s0 = dfa.s0 if s0 is None: if outerContext is None: outerContext = ParserRuleContext.EMPTY if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("predictATN decision " + str(dfa.decision) + " exec LA(1)==" + parser_atn_simulator.getLookaheadName(input) + ", outerContext=" + outerContext.toString( parser_atn_simulator.parser.literalNames, None)) fullCtx = False s0_closure = parser_atn_simulator.computeStartState( dfa.atnStartState, ParserRuleContext.EMPTY, fullCtx) if dfa.precedenceDfa: # If this is a precedence DFA, we use applyPrecedenceFilter # to convert the computed start state to a precedence start # state. We then use DFA.setPrecedenceStartState to set the # appropriate start state for the precedence level rather # than simply setting DFA.s0. # dfa.s0.configs = s0_closure # not used for prediction but useful to know start configs anyway s0_closure = parser_atn_simulator.applyPrecedenceFilter(s0_closure) s0 = parser_atn_simulator.addDFAState(dfa, DFAState(configs=s0_closure)) dfa.setPrecedenceStartState( parser_atn_simulator.parser.getPrecedence(), s0) else: s0 = parser_atn_simulator.addDFAState(dfa, DFAState(configs=s0_closure)) dfa.s0 = s0 return s0
def __init__(self, atnStartState, decision=0): # From which ATN state did we create this DFA? self.atnStartState = atnStartState self.decision = decision # A set of all DFA states. Use {@link Map} so we can get old state back # ({@link Set} only allows you to see if it's there). self._states = dict() self.s0 = None # {@code true} if this DFA is for a precedence decision; otherwise, # {@code false}. This is the backing field for {@link #isPrecedenceDfa}, # {@link #setPrecedenceDfa}. self.precedenceDfa = False if isinstance(atnStartState, StarLoopEntryState): if atnStartState.isPrecedenceDecision: self.precedenceDfa = True precedenceState = DFAState(configs=ATNConfigSet()) precedenceState.edges = [] precedenceState.isAcceptState = False precedenceState.requiresFullContext = False self.s0 = precedenceState
def addDFAEdge(self, from_: DFAState, tk: int, to: DFAState = None, cfgs: ATNConfigSet = None) -> DFAState: if to is None and cfgs is not None: # leading to this call, ATNConfigSet.hasSemanticContext is used as a # marker indicating dynamic predicate evaluation makes this edge # dependent on the specific input sequence, so the static edge in the # DFA should be omitted. The target DFAState is still created since # execATN has the ability to resynchronize with the DFA state cache # following the predicate evaluation step. # # TJP notes: next time through the DFA, we see a pred again and eval. # If that gets us to a previously created (but dangling) DFA # state, we can continue in pure DFA mode from there. #/ suppressEdge = cfgs.hasSemanticContext cfgs.hasSemanticContext = False to = self.addDFAState(cfgs) if suppressEdge: return to # add the edge if tk < self.MIN_DFA_EDGE or tk > self.MAX_DFA_EDGE: # Only track edges within the DFA bounds return to if self.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon " + chr(tk)) if from_.edges is None: # make room for tokens 1..n and -1 masquerading as index 0 from_.edges = [None] * (self.MAX_DFA_EDGE - self.MIN_DFA_EDGE + 1) from_.edges[tk - self.MIN_DFA_EDGE] = to # connect return to
def addDFAEdge(self, from_:DFAState, tk:int, to:DFAState=None, cfgs:ATNConfigSet=None) -> DFAState: if to is None and cfgs is not None: # leading to this call, ATNConfigSet.hasSemanticContext is used as a # marker indicating dynamic predicate evaluation makes this edge # dependent on the specific input sequence, so the static edge in the # DFA should be omitted. The target DFAState is still created since # execATN has the ability to resynchronize with the DFA state cache # following the predicate evaluation step. # # TJP notes: next time through the DFA, we see a pred again and eval. # If that gets us to a previously created (but dangling) DFA # state, we can continue in pure DFA mode from there. #/ suppressEdge = cfgs.hasSemanticContext cfgs.hasSemanticContext = False to = self.addDFAState(cfgs) if suppressEdge: return to # add the edge if tk < self.MIN_DFA_EDGE or tk > self.MAX_DFA_EDGE: # Only track edges within the DFA bounds return to if LexerATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon "+ chr(tk)) if from_.edges is None: # make room for tokens 1..n and -1 masquerading as index 0 from_.edges = [ None ] * (self.MAX_DFA_EDGE - self.MIN_DFA_EDGE + 1) from_.edges[tk - self.MIN_DFA_EDGE] = to # connect return to
newState.configs = configs dfa.states[newState] = newState return newState def getDFA(self, mode: int): return self.decisionToDFA[mode] # Get the text matched so far for the current token. def getText(self, input: InputStream): # index is first lookahead char, don't include. return input.getText(self.startIndex, input.index - 1) def consume(self, input: InputStream): curChar = input.LA(1) if curChar == ord('\n'): self.line += 1 self.column = 0 else: self.column += 1 input.consume() def getTokenName(self, t: int): if t == -1: return "EOF" else: return "'" + chr(t) + "'" LexerATNSimulator.ERROR = DFAState(0x7FFFFFFF, ATNConfigSet()) del Lexer