def closure(self, input: InputStream, config: LexerATNConfig, configs: ATNConfigSet, currentAltReachedAcceptState: bool, speculative: bool, treatEofAsEpsilon: bool): if self.debug: print("closure(" + config.toString(self.recog, True) + ")") if isinstance(config.state, RuleStopState): if self.debug: if self.recog is not None: print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config) else: print("closure at rule stop %s\n", config) if config.context is None or config.context.hasEmptyPath(): if config.context is None or config.context.isEmpty(): configs.add(config) return True else: configs.add( LexerATNConfig(state=config.state, config=config, context=PredictionContext.EMPTY)) currentAltReachedAcceptState = True if config.context is not None and not config.context.isEmpty(): for i in range(0, len(config.context)): if config.context.getReturnState( i) != PredictionContext.EMPTY_RETURN_STATE: newContext = config.context.getParent( i) # "pop" return state returnState = self.atn.states[ config.context.getReturnState(i)] c = LexerATNConfig(state=returnState, config=config, context=newContext) currentAltReachedAcceptState = self.closure( input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon) return currentAltReachedAcceptState # optimization if not config.state.epsilonOnlyTransitions: if not currentAltReachedAcceptState or not config.passedThroughNonGreedyDecision: configs.add(config) for t in config.state.transitions: c = self.getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon) if c is not None: currentAltReachedAcceptState = self.closure( input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon) return currentAltReachedAcceptState
def getReachableConfigSet(self, input: InputStream, closure: ATNConfigSet, reach: ATNConfigSet, t: int): # this is used to skip processing for configs which have a lower priority # than a config that already reached an accept state for the same rule skipAlt = ATN.INVALID_ALT_NUMBER for cfg in closure: currentAltReachedAcceptState = (cfg.alt == skipAlt) if currentAltReachedAcceptState and cfg.passedThroughNonGreedyDecision: continue if self.debug: print("testing %s at %s\n", self.getTokenName(t), cfg.toString(self.recog, True)) for trans in cfg.state.transitions: # for each transition target = self.getReachableTarget(trans, t) if target is not None: lexerActionExecutor = cfg.lexerActionExecutor if lexerActionExecutor is not None: lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch( input.index - self.startIndex) treatEofAsEpsilon = (t == Token.EOF) config = LexerATNConfig( state=target, lexerActionExecutor=lexerActionExecutor, config=cfg) if self.closure(input, config, reach, currentAltReachedAcceptState, True, treatEofAsEpsilon): # any remaining configs for this alt have a lower priority than # the one that just reached an accept state. skipAlt = cfg.alt
def computeStartState(self, input: InputStream, p: ATNState): initialContext = PredictionContext.EMPTY configs = OrderedATNConfigSet() for i in range(0, len(p.transitions)): target = p.transitions[i].target c = LexerATNConfig(state=target, alt=i + 1, context=initialContext) self.closure(input, c, configs, False, False, False) return configs
def closure( self, input: InputStream, config: LexerATNConfig, configs: ATNConfigSet, currentAltReachedAcceptState: bool, speculative: bool, treatEofAsEpsilon: bool, ): if self.debug: print("closure(" + config.toString(self.recog, True) + ")") if isinstance(config.state, RuleStopState): if self.debug: if self.recog is not None: print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config) else: print("closure at rule stop %s\n", config) if config.context is None or config.context.hasEmptyPath(): if config.context is None or config.context.isEmpty(): configs.add(config) return True else: configs.add(LexerATNConfig(state=config.state, config=config, context=PredictionContext.EMPTY)) currentAltReachedAcceptState = True if config.context is not None and not config.context.isEmpty(): for i in range(0, len(config.context)): if config.context.getReturnState(i) != PredictionContext.EMPTY_RETURN_STATE: newContext = config.context.getParent(i) # "pop" return state returnState = self.atn.states[config.context.getReturnState(i)] c = LexerATNConfig(state=returnState, config=config, context=newContext) currentAltReachedAcceptState = self.closure( input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon ) return currentAltReachedAcceptState # optimization if not config.state.epsilonOnlyTransitions: if not currentAltReachedAcceptState or not config.passedThroughNonGreedyDecision: configs.add(config) for t in config.state.transitions: c = self.getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon) if c is not None: currentAltReachedAcceptState = self.closure( input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon ) return currentAltReachedAcceptState
def getEpsilonTarget(self, input: InputStream, config: LexerATNConfig, t: Transition, configs: ATNConfigSet, speculative: bool, treatEofAsEpsilon: bool): c = None if t.serializationType == Transition.RULE: newContext = SingletonPredictionContext.create( config.context, t.followState.stateNumber) c = LexerATNConfig(state=t.target, config=config, context=newContext) elif t.serializationType == Transition.PRECEDENCE: raise UnsupportedOperationException( "Precedence predicates are not supported in lexers.") elif t.serializationType == Transition.PREDICATE: # Track traversing semantic predicates. If we traverse, # we cannot add a DFA state for this "reach" computation # because the DFA would not test the predicate again in the # future. Rather than creating collections of semantic predicates # like v3 and testing them on prediction, v4 will test them on the # fly all the time using the ATN not the DFA. This is slower but # semantically it's not used that often. One of the key elements to # this predicate mechanism is not adding DFA states that see # predicates immediately afterwards in the ATN. For example, # a : ID {p1}? | ID {p2}? ; # should create the start state for rule 'a' (to save start state # competition), but should not create target of ID state. The # collection of ATN states the following ID references includes # states reached by traversing predicates. Since this is when we # test them, we cannot cash the DFA state target of ID. if self.debug: print("EVAL rule " + str(t.ruleIndex) + ":" + str(t.predIndex)) configs.hasSemanticContext = True if self.evaluatePredicate(input, t.ruleIndex, t.predIndex, speculative): c = LexerATNConfig(state=t.target, config=config) elif t.serializationType == Transition.ACTION: if config.context is None or config.context.hasEmptyPath(): # execute actions anywhere in the start rule for a token. # # TODO: if the entry rule is invoked recursively, some # actions may be executed during the recursive call. The # problem can appear when hasEmptyPath() is true but # isEmpty() is false. In this case, the config needs to be # split into two contexts - one with just the empty path # and another with everything but the empty path. # Unfortunately, the current algorithm does not allow # getEpsilonTarget to return two configurations, so # additional modifications are needed before we can support # the split operation. lexerActionExecutor = LexerActionExecutor.append( config.lexerActionExecutor, self.atn.lexerActions[t.actionIndex]) c = LexerATNConfig(state=t.target, config=config, lexerActionExecutor=lexerActionExecutor) else: # ignore actions in referenced rules c = LexerATNConfig(state=t.target, config=config) elif t.serializationType == Transition.EPSILON: c = LexerATNConfig(state=t.target, config=config) elif t.serializationType in [ Transition.ATOM, Transition.RANGE, Transition.SET ]: if treatEofAsEpsilon: if t.matches(Token.EOF, 0, 0xFFFF): c = LexerATNConfig(state=t.target, config=config) return c