def closure(self, input: InputStream, config: LexerATNConfig,
configs: ATNConfigSet, currentAltReachedAcceptState: bool,
speculative: bool, treatEofAsEpsilon: bool):
if self.debug:
print("closure(" + config.toString(self.recog, True) + ")")
if isinstance(config.state, RuleStopState):
if self.debug:
if self.recog is not None:
print("closure at %s rule stop %s\n",
self.recog.getRuleNames()[config.state.ruleIndex],
config)
else:
print("closure at rule stop %s\n", config)
if config.context is None or config.context.hasEmptyPath():
if config.context is None or config.context.isEmpty():
configs.add(config)
return True
else:
configs.add(
LexerATNConfig(state=config.state,
config=config,
context=PredictionContext.EMPTY))
currentAltReachedAcceptState = True
if config.context is not None and not config.context.isEmpty():
for i in range(0, len(config.context)):
if config.context.getReturnState(
i) != PredictionContext.EMPTY_RETURN_STATE:
newContext = config.context.getParent(
i) # "pop" return state
returnState = self.atn.states[
config.context.getReturnState(i)]
c = LexerATNConfig(state=returnState,
config=config,
context=newContext)
currentAltReachedAcceptState = self.closure(
input, c, configs, currentAltReachedAcceptState,
speculative, treatEofAsEpsilon)
return currentAltReachedAcceptState
# optimization
if not config.state.epsilonOnlyTransitions:
if not currentAltReachedAcceptState or not config.passedThroughNonGreedyDecision:
configs.add(config)
for t in config.state.transitions:
c = self.getEpsilonTarget(input, config, t, configs, speculative,
treatEofAsEpsilon)
if c is not None:
currentAltReachedAcceptState = self.closure(
input, c, configs, currentAltReachedAcceptState,
speculative, treatEofAsEpsilon)
return currentAltReachedAcceptState
def getReachableConfigSet(self, input: InputStream, closure: ATNConfigSet,
reach: ATNConfigSet, t: int):
# this is used to skip processing for configs which have a lower priority
# than a config that already reached an accept state for the same rule
skipAlt = ATN.INVALID_ALT_NUMBER
for cfg in closure:
currentAltReachedAcceptState = (cfg.alt == skipAlt)
if currentAltReachedAcceptState and cfg.passedThroughNonGreedyDecision:
continue
if self.debug:
print("testing %s at %s\n", self.getTokenName(t),
cfg.toString(self.recog, True))
for trans in cfg.state.transitions: # for each transition
target = self.getReachableTarget(trans, t)
if target is not None:
lexerActionExecutor = cfg.lexerActionExecutor
if lexerActionExecutor is not None:
lexerActionExecutor = lexerActionExecutor.fixOffsetBeforeMatch(
input.index - self.startIndex)
treatEofAsEpsilon = (t == Token.EOF)
config = LexerATNConfig(
state=target,
lexerActionExecutor=lexerActionExecutor,
config=cfg)
if self.closure(input, config, reach,
currentAltReachedAcceptState, True,
treatEofAsEpsilon):
# any remaining configs for this alt have a lower priority than
# the one that just reached an accept state.
skipAlt = cfg.alt
def computeStartState(self, input: InputStream, p: ATNState):
initialContext = PredictionContext.EMPTY
configs = OrderedATNConfigSet()
for i in range(0, len(p.transitions)):
target = p.transitions[i].target
c = LexerATNConfig(state=target, alt=i + 1, context=initialContext)
self.closure(input, c, configs, False, False, False)
return configs
def closure(
self,
input: InputStream,
config: LexerATNConfig,
configs: ATNConfigSet,
currentAltReachedAcceptState: bool,
speculative: bool,
treatEofAsEpsilon: bool,
):
if self.debug:
print("closure(" + config.toString(self.recog, True) + ")")
if isinstance(config.state, RuleStopState):
if self.debug:
if self.recog is not None:
print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config)
else:
print("closure at rule stop %s\n", config)
if config.context is None or config.context.hasEmptyPath():
if config.context is None or config.context.isEmpty():
configs.add(config)
return True
else:
configs.add(LexerATNConfig(state=config.state, config=config, context=PredictionContext.EMPTY))
currentAltReachedAcceptState = True
if config.context is not None and not config.context.isEmpty():
for i in range(0, len(config.context)):
if config.context.getReturnState(i) != PredictionContext.EMPTY_RETURN_STATE:
newContext = config.context.getParent(i) # "pop" return state
returnState = self.atn.states[config.context.getReturnState(i)]
c = LexerATNConfig(state=returnState, config=config, context=newContext)
currentAltReachedAcceptState = self.closure(
input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon
)
return currentAltReachedAcceptState
# optimization
if not config.state.epsilonOnlyTransitions:
if not currentAltReachedAcceptState or not config.passedThroughNonGreedyDecision:
configs.add(config)
for t in config.state.transitions:
c = self.getEpsilonTarget(input, config, t, configs, speculative, treatEofAsEpsilon)
if c is not None:
currentAltReachedAcceptState = self.closure(
input, c, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon
)
return currentAltReachedAcceptState
def getEpsilonTarget(self, input: InputStream, config: LexerATNConfig,
t: Transition, configs: ATNConfigSet,
speculative: bool, treatEofAsEpsilon: bool):
c = None
if t.serializationType == Transition.RULE:
newContext = SingletonPredictionContext.create(
config.context, t.followState.stateNumber)
c = LexerATNConfig(state=t.target,
config=config,
context=newContext)
elif t.serializationType == Transition.PRECEDENCE:
raise UnsupportedOperationException(
"Precedence predicates are not supported in lexers.")
elif t.serializationType == Transition.PREDICATE:
# Track traversing semantic predicates. If we traverse,
# we cannot add a DFA state for this "reach" computation
# because the DFA would not test the predicate again in the
# future. Rather than creating collections of semantic predicates
# like v3 and testing them on prediction, v4 will test them on the
# fly all the time using the ATN not the DFA. This is slower but
# semantically it's not used that often. One of the key elements to
# this predicate mechanism is not adding DFA states that see
# predicates immediately afterwards in the ATN. For example,
# a : ID {p1}? | ID {p2}? ;
# should create the start state for rule 'a' (to save start state
# competition), but should not create target of ID state. The
# collection of ATN states the following ID references includes
# states reached by traversing predicates. Since this is when we
# test them, we cannot cash the DFA state target of ID.
if self.debug:
print("EVAL rule " + str(t.ruleIndex) + ":" + str(t.predIndex))
configs.hasSemanticContext = True
if self.evaluatePredicate(input, t.ruleIndex, t.predIndex,
speculative):
c = LexerATNConfig(state=t.target, config=config)
elif t.serializationType == Transition.ACTION:
if config.context is None or config.context.hasEmptyPath():
# execute actions anywhere in the start rule for a token.
#
# TODO: if the entry rule is invoked recursively, some
# actions may be executed during the recursive call. The
# problem can appear when hasEmptyPath() is true but
# isEmpty() is false. In this case, the config needs to be
# split into two contexts - one with just the empty path
# and another with everything but the empty path.
# Unfortunately, the current algorithm does not allow
# getEpsilonTarget to return two configurations, so
# additional modifications are needed before we can support
# the split operation.
lexerActionExecutor = LexerActionExecutor.append(
config.lexerActionExecutor,
self.atn.lexerActions[t.actionIndex])
c = LexerATNConfig(state=t.target,
config=config,
lexerActionExecutor=lexerActionExecutor)
else:
# ignore actions in referenced rules
c = LexerATNConfig(state=t.target, config=config)
elif t.serializationType == Transition.EPSILON:
c = LexerATNConfig(state=t.target, config=config)
elif t.serializationType in [
Transition.ATOM, Transition.RANGE, Transition.SET
]:
if treatEofAsEpsilon:
if t.matches(Token.EOF, 0, 0xFFFF):
c = LexerATNConfig(state=t.target, config=config)
return c
