Ejemplo n.º 1
0
    def getEpsilonTarget(self, input: InputStream, config: LexerATNConfig,
                         t: Transition, configs: ATNConfigSet,
                         speculative: bool, treatEofAsEpsilon: bool):
        c = None
        if t.serializationType == Transition.RULE:
            newContext = SingletonPredictionContext.create(
                config.context, t.followState.stateNumber)
            c = LexerATNConfig(state=t.target,
                               config=config,
                               context=newContext)

        elif t.serializationType == Transition.PRECEDENCE:
            raise UnsupportedOperationException(
                "Precedence predicates are not supported in lexers.")

        elif t.serializationType == Transition.PREDICATE:
            #  Track traversing semantic predicates. If we traverse,
            # we cannot add a DFA state for this "reach" computation
            # because the DFA would not test the predicate again in the
            # future. Rather than creating collections of semantic predicates
            # like v3 and testing them on prediction, v4 will test them on the
            # fly all the time using the ATN not the DFA. This is slower but
            # semantically it's not used that often. One of the key elements to
            # this predicate mechanism is not adding DFA states that see
            # predicates immediately afterwards in the ATN. For example,

            # a : ID {p1}? | ID {p2}? ;

            # should create the start state for rule 'a' (to save start state
            # competition), but should not create target of ID state. The
            # collection of ATN states the following ID references includes
            # states reached by traversing predicates. Since this is when we
            # test them, we cannot cash the DFA state target of ID.

            if self.debug:
                print("EVAL rule " + str(t.ruleIndex) + ":" + str(t.predIndex))
            configs.hasSemanticContext = True
            if self.evaluatePredicate(input, t.ruleIndex, t.predIndex,
                                      speculative):
                c = LexerATNConfig(state=t.target, config=config)

        elif t.serializationType == Transition.ACTION:
            if config.context is None or config.context.hasEmptyPath():
                # execute actions anywhere in the start rule for a token.
                #
                # TODO: if the entry rule is invoked recursively, some
                # actions may be executed during the recursive call. The
                # problem can appear when hasEmptyPath() is true but
                # isEmpty() is false. In this case, the config needs to be
                # split into two contexts - one with just the empty path
                # and another with everything but the empty path.
                # Unfortunately, the current algorithm does not allow
                # getEpsilonTarget to return two configurations, so
                # additional modifications are needed before we can support
                # the split operation.
                lexerActionExecutor = LexerActionExecutor.append(
                    config.lexerActionExecutor,
                    self.atn.lexerActions[t.actionIndex])
                c = LexerATNConfig(state=t.target,
                                   config=config,
                                   lexerActionExecutor=lexerActionExecutor)

            else:
                # ignore actions in referenced rules
                c = LexerATNConfig(state=t.target, config=config)

        elif t.serializationType == Transition.EPSILON:
            c = LexerATNConfig(state=t.target, config=config)

        elif t.serializationType in [
                Transition.ATOM, Transition.RANGE, Transition.SET
        ]:
            if treatEofAsEpsilon:
                if t.matches(Token.EOF, 0, 0xFFFF):
                    c = LexerATNConfig(state=t.target, config=config)

        return c
Ejemplo n.º 2
0
    def _LOOK(self, s, stopState , ctx, look, lookBusy, \
                     calledRuleStack, seeThruPreds, addEOF):
        c = ATNConfig(s, 0, ctx)

        if c in lookBusy:
            return
        lookBusy.add(c)

        if s == stopState:
            if ctx is None:
                look.addOne(Token.EPSILON)
                return
            elif ctx.isEmpty() and addEOF:
                look.addOne(Token.EOF)
                return

        if isinstance(s, RuleStopState ):
            if ctx is None:
                look.addOne(Token.EPSILON)
                return
            elif ctx.isEmpty() and addEOF:
                look.addOne(Token.EOF)
                return

            if ctx != PredictionContext.EMPTY:
                # run thru all possible stack tops in ctx
                for i in range(0, len(ctx)):
                    returnState = self.atn.states[ctx.getReturnState(i)]
                    removed = returnState.ruleIndex in calledRuleStack
                    try:
                        calledRuleStack.discard(returnState.ruleIndex)
                        self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
                    finally:
                        if removed:
                            calledRuleStack.add(returnState.ruleIndex)
                return

        for t in s.transitions:
            if type(t) == RuleTransition:
                if t.target.ruleIndex in calledRuleStack:
                    continue

                newContext = SingletonPredictionContext.create(ctx, t.followState.stateNumber)

                try:
                    calledRuleStack.add(t.target.ruleIndex)
                    self._LOOK(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
                finally:
                    calledRuleStack.remove(t.target.ruleIndex)
            elif isinstance(t, AbstractPredicateTransition ):
                if seeThruPreds:
                    self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
                else:
                    look.addOne(self.HIT_PRED)
            elif t.isEpsilon:
                self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
            elif type(t) == WildcardTransition:
                look.addRange( Interval(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) )
            else:
                set = t.label
                if set is not None:
                    if isinstance(t, NotSetTransition):
                        set = set.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
                    look.addSet(set)
Ejemplo n.º 3
0
    def getEpsilonTarget(self, input:InputStream, config:LexerATNConfig, t:Transition, configs:ATNConfigSet,
                                           speculative:bool, treatEofAsEpsilon:bool):
        c = None
        if t.serializationType==Transition.RULE:
                newContext = SingletonPredictionContext.create(config.context, t.followState.stateNumber)
                c = LexerATNConfig(state=t.target, config=config, context=newContext)

        elif t.serializationType==Transition.PRECEDENCE:
                raise UnsupportedOperationException("Precedence predicates are not supported in lexers.")

        elif t.serializationType==Transition.PREDICATE:
                #  Track traversing semantic predicates. If we traverse,
                # we cannot add a DFA state for this "reach" computation
                # because the DFA would not test the predicate again in the
                # future. Rather than creating collections of semantic predicates
                # like v3 and testing them on prediction, v4 will test them on the
                # fly all the time using the ATN not the DFA. This is slower but
                # semantically it's not used that often. One of the key elements to
                # this predicate mechanism is not adding DFA states that see
                # predicates immediately afterwards in the ATN. For example,

                # a : ID {p1}? | ID {p2}? ;

                # should create the start state for rule 'a' (to save start state
                # competition), but should not create target of ID state. The
                # collection of ATN states the following ID references includes
                # states reached by traversing predicates. Since this is when we
                # test them, we cannot cash the DFA state target of ID.

                if LexerATNSimulator.debug:
                    print("EVAL rule "+ str(t.ruleIndex) + ":" + str(t.predIndex))
                configs.hasSemanticContext = True
                if self.evaluatePredicate(input, t.ruleIndex, t.predIndex, speculative):
                    c = LexerATNConfig(state=t.target, config=config)

        elif t.serializationType==Transition.ACTION:
                if config.context is None or config.context.hasEmptyPath():
                    # execute actions anywhere in the start rule for a token.
                    #
                    # TODO: if the entry rule is invoked recursively, some
                    # actions may be executed during the recursive call. The
                    # problem can appear when hasEmptyPath() is true but
                    # isEmpty() is false. In this case, the config needs to be
                    # split into two contexts - one with just the empty path
                    # and another with everything but the empty path.
                    # Unfortunately, the current algorithm does not allow
                    # getEpsilonTarget to return two configurations, so
                    # additional modifications are needed before we can support
                    # the split operation.
                    lexerActionExecutor = LexerActionExecutor.append(config.lexerActionExecutor,
                                    self.atn.lexerActions[t.actionIndex])
                    c = LexerATNConfig(state=t.target, config=config, lexerActionExecutor=lexerActionExecutor)

                else:
                    # ignore actions in referenced rules
                    c = LexerATNConfig(state=t.target, config=config)

        elif t.serializationType==Transition.EPSILON:
            c = LexerATNConfig(state=t.target, config=config)

        elif t.serializationType in [ Transition.ATOM, Transition.RANGE, Transition.SET ]:
            if treatEofAsEpsilon:
                if t.matches(Token.EOF, 0, 0xFFFF):
                    c = LexerATNConfig(state=t.target, config=config)

        return c