Ejemplo n.º 1
0
def require_data(TState, TheAnalyzer):
    """Requires all variables which are necessary to implement the TemplateState.

       NOTE: The target schemes are required from inside 'require_scheme_variable()' 
             in quex.engine.generator.mega_state.core.py
    """
    variable_db.require("state_key")
    get_label("$state-router", U=True)  # Ensure reference of state router
Ejemplo n.º 2
0
def require_data(TState, TheAnalyzer):
    """Requires all variables which are necessary to implement the TemplateState.

       NOTE: The target schemes are required from inside 'require_scheme_variable()' 
             in quex.engine.generator.mega_state.core.py
    """
    variable_db.require("state_key")
    get_label("$state-router", U=True) # Ensure reference of state router
Ejemplo n.º 3
0
def __header_definitions(LanguageDB, OnAfterMatchStr):

    txt = __header_definitions_txt
    txt = txt.replace("$$GOTO_START_PREPARATION$$", get_label("$re-start", U=True))

    if len(OnAfterMatchStr) != 0: txt += __return_with_on_after_match
    else:                         txt += __return_without_on_after_match
    return txt
Ejemplo n.º 4
0
def drop_out_scheme_implementation(txt, TheState, TheAnalyzer, StateKeyString, DebugString):
    """DropOut Section:

       The drop out section is the place where we come if the transition map
       does not trigger to another state. We also land here if the reload fails.
       The routing to the different drop-outs of the related states happens by 
       means of a switch statement, e.g.
       
       _4711: /* Address of the drop out */
           switch( state_key ) {
           case 0:
                 ... drop out of state 815 ...
           case 1: 
                 ... drop out of state 541 ...
           }

       The switch statement is not necessary if all drop outs are the same, 
       of course.
    """
    LanguageDB = Setup.language_db
    # (*) Central Label for the Templates Drop Out
    #     (The rules for having or not having a label here are complicated, 
    #      so rely on the label's usage database.)
    txt.append("%s:\n" % get_label("$drop-out", TheState.index))
    txt.append("    %s\n" % DebugString) 

    def implement_prototype(StateIndices, TheAnalyzer):
        # There **must** be at least one element, at this point in time
        assert len(StateIndices) != 0
        prototype_i = StateIndices.__iter__().next()
        prototype   = TheAnalyzer.state_db[prototype_i]
        result      = []
        drop_out_coder.do(result, prototype, TheAnalyzer, \
                          DefineLabelF=False, MentionStateIndexF=False)
        return result


    # (*) Drop Out Section(s)
    if TheState.drop_out.uniform_f:
        # uniform drop outs => no 'switch-case' required
        txt.extend(implement_prototype(TheState.implemented_state_index_list(), TheAnalyzer))
        return

    # non-uniform drop outs => route by 'state_key'
    case_list = []
    for drop_out, state_index_set in TheState.drop_out.iteritems():
        # state keys related to drop out
        state_key_list = map(lambda i: TheState.map_state_index_to_state_key(i), state_index_set)
        # drop out action
        # Implement drop-out for each state key. 'state_key_list' combines
        # states that implement the same drop-out behavior. Same drop-outs
        # are implemented only once.
        case_list.append( (state_key_list, implement_prototype(state_index_set, TheAnalyzer)) )

    case_txt = LanguageDB.SELECTION(StateKeyString, case_list)
    LanguageDB.INDENT(case_txt)
    txt.extend(case_txt)
Ejemplo n.º 5
0
def __header_definitions(LanguageDB, OnAfterMatchStr):

    txt = __header_definitions_txt
    txt = txt.replace("$$GOTO_START_PREPARATION$$",
                      get_label("$re-start", U=True))

    if len(OnAfterMatchStr) != 0: txt += __return_with_on_after_match
    else: txt += __return_without_on_after_match
    return txt
Ejemplo n.º 6
0
def drop_out_scheme_implementation(txt, TheState, TheAnalyzer, StateKeyString, DebugString):
    """DropOut Section:

       The drop out section is the place where we come if the transition map
       does not trigger to another state. We also land here if the reload fails.
       The routing to the different drop-outs of the related states happens by 
       means of a switch statement, e.g.
       
       _4711: /* Address of the drop out */
           switch( state_key ) {
           case 0:
                 ... drop out of state 815 ...
           case 1: 
                 ... drop out of state 541 ...
           }

       The switch statement is not necessary if all drop outs are the same, 
       of course.
    """
    LanguageDB = Setup.language_db
    # (*) Central Label for the Templates Drop Out
    #     (The rules for having or not having a label here are complicated, 
    #      so rely on the label's usage database.)
    txt.append("%s:\n" % get_label("$drop-out", TheState.index))
    txt.append("    %s\n" % DebugString) 

    def implement_prototype(StateIndices, TheAnalyzer):
        # There **must** be at least one element, at this point in time
        assert len(StateIndices) != 0
        prototype_i = StateIndices.__iter__().next()
        prototype   = TheAnalyzer.state_db[prototype_i]
        result      = []
        drop_out_coder.do(result, prototype, TheAnalyzer, \
                          DefineLabelF=False, MentionStateIndexF=False)
        return result


    # (*) Drop Out Section(s)
    if TheState.drop_out.uniform_f:
        # uniform drop outs => no 'switch-case' required
        txt.extend(implement_prototype(TheState.implemented_state_index_list(), TheAnalyzer))
        return

    # non-uniform drop outs => route by 'state_key'
    case_list = []
    for drop_out, state_index_set in TheState.drop_out.iteritems():
        # state keys related to drop out
        state_key_list = map(lambda i: TheState.map_state_index_to_state_key(i), state_index_set)
        # drop out action
        # Implement drop-out for each state key. 'state_key_list' combines
        # states that implement the same drop-out behavior. Same drop-outs
        # are implemented only once.
        case_list.append( (state_key_list, implement_prototype(state_index_set, TheAnalyzer)) )

    case_txt = LanguageDB.SELECTION(StateKeyString, case_list)
    LanguageDB.INDENT(case_txt)
    txt.extend(case_txt)
Ejemplo n.º 7
0
    def GOTO_RELOAD(self, StateIndex, InitStateIndexF, EngineType):
        """On reload a special section is entered that tries to reload data. Reload
           has two possible results:
           
           -- Data has been loaded: Now, a new input character can be determined
              and the current transition map can be reentered. For convenience, 
              'RELOAD' expects to jump to right before the place where the input
              pointer is adapted.

           -- No data available to be loaded: Then the current state's drop-out
              section must be entered. The forward init state immediate jumps
              to 'end of stream'.

           Thus: The reload behavior can be determined based on **one** state index.
                 The related drop-out label can be determined here.
        """
        direction = {
            E_EngineTypes.FORWARD: "FORWARD",
            E_EngineTypes.BACKWARD_PRE_CONTEXT: "BACKWARD",
            E_EngineTypes.BACKWARD_INPUT_POSITION: None,
            E_EngineTypes.INDENTATION_COUNTER: "FORWARD",
            # There is never a reload on backward input position detection.
            # The lexeme to parse must lie inside the borders!
        }[EngineType]
        assert direction is not None, \
               "There is no reload during BACKWARD_INPUT_POSITION detection."

        # 'DoorIndex == 0' is the entry into the state without any actions.
        on_success = get_address("$entry",
                                 entry_action.DoorID(StateIndex, DoorIndex=0),
                                 U=True)
        if InitStateIndexF and EngineType == E_EngineTypes.FORWARD:
            on_fail = get_address("$terminal-EOF", U=True)
        else:
            on_fail = get_address("$drop-out", StateIndex, U=True, R=True)

        get_label("$state-router", U=True)  # Mark as 'referenced'
        get_label("$reload-%s" % direction, U=True)  # ...
        return "QUEX_GOTO_RELOAD_%s(%s, %s);" % (direction, on_success,
                                                 on_fail)
Ejemplo n.º 8
0
    def GOTO_RELOAD(self, StateIndex, InitStateIndexF, EngineType):
        """On reload a special section is entered that tries to reload data. Reload
           has two possible results:
           
           -- Data has been loaded: Now, a new input character can be determined
              and the current transition map can be reentered. For convenience, 
              'RELOAD' expects to jump to right before the place where the input
              pointer is adapted.

           -- No data available to be loaded: Then the current state's drop-out
              section must be entered. The forward init state immediate jumps
              to 'end of stream'.

           Thus: The reload behavior can be determined based on **one** state index.
                 The related drop-out label can be determined here.
        """
        direction = { 
            E_EngineTypes.FORWARD:              "FORWARD",
            E_EngineTypes.BACKWARD_PRE_CONTEXT: "BACKWARD",
            E_EngineTypes.BACKWARD_INPUT_POSITION: None,
            E_EngineTypes.INDENTATION_COUNTER:  "FORWARD",
            # There is never a reload on backward input position detection.
            # The lexeme to parse must lie inside the borders!
        }[EngineType]
        assert direction is not None, \
               "There is no reload during BACKWARD_INPUT_POSITION detection."

        # 'DoorIndex == 0' is the entry into the state without any actions.
        on_success = get_address("$entry", entry_action.DoorID(StateIndex, DoorIndex=0), U=True)
        if InitStateIndexF and EngineType == E_EngineTypes.FORWARD:
            on_fail = get_address("$terminal-EOF", U=True) 
        else:
            on_fail = get_address("$drop-out", StateIndex, U=True, R=True) 

        get_label("$state-router", U=True)            # Mark as 'referenced'
        get_label("$reload-%s" % direction, U=True)   # ...
        return "QUEX_GOTO_RELOAD_%s(%s, %s);" % (direction, on_success, on_fail)
Ejemplo n.º 9
0
def get_terminal_code(AcceptanceID, pattern_action_info, LanguageDB):
    pattern = pattern_action_info.pattern()
    #
    action_code = pattern_action_info.action().get_code()

    # (*) The 'normal' terminal state can also be reached by the terminal
    #     router and, thus, **must** restore the acceptance input position. This is so,
    #     because when the 'goto last_acceptance' is triggered the 'last_acceptance'
    #     may lay backwards and needs to be restored.
    result = []
    for letter in pattern_action_info.pattern_string():
        if letter in ['\\', '"', '\n', '\t', '\r', '\a', '\v']:
            result.append("\\")
        result.append(letter)

    safe_pattern = "".join(result)

    input_position_search_backward_str = ""
    if pattern.input_position_search_backward_sm is not None:
        # Pseudo Ambiguous Post Contexts:
        # (Retrieving the input position for the next run)
        # -- Requires that the end of the core pattern is to be searched! One
        #    cannot simply restore some stored input position.
        # -- The pseudo-ambiguous post condition is translated into a 'normal'
        #    pattern. However, after a match a backward detection of the end
        #    of the core pattern is done. Here, we first need to go to the point
        #    where the 'normal' pattern ended, then we can do a backward detection.

        bipd_id = pattern.input_position_search_backward_sm.get_id()
        bipd_str = "    goto %s;\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_DETECTOR(
            bipd_id)
        # After having finished the analyzis, enter the terminal code, here.
        bipd_str += "%s:\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_RETURN(
            bipd_id)
        input_position_search_backward_str = bipd_str

    txt = [
        "\nTERMINAL_%i:\n" % AcceptanceID, input_position_search_backward_str,
        "    __quex_debug(\"* terminal %i:   %s\\n\");\n" %
        (AcceptanceID, safe_pattern), action_code, "\n",
        "    goto %s;\n" % get_label("$re-start", U=True)
    ]

    return txt
Ejemplo n.º 10
0
def get_terminal_code(AcceptanceID, pattern_action_info, LanguageDB):
    pattern     = pattern_action_info.pattern()
    #
    action_code = pattern_action_info.action().get_code()
        
    # (*) The 'normal' terminal state can also be reached by the terminal
    #     router and, thus, **must** restore the acceptance input position. This is so, 
    #     because when the 'goto last_acceptance' is triggered the 'last_acceptance'
    #     may lay backwards and needs to be restored.
    result      = []
    for letter in pattern_action_info.pattern_string():
        if letter in ['\\', '"', '\n', '\t', '\r', '\a', '\v']:
            result.append("\\")
        result.append(letter)

    safe_pattern = "".join(result)

    input_position_search_backward_str = ""
    if pattern.input_position_search_backward_sm is not None:
        # Pseudo Ambiguous Post Contexts:
        # (Retrieving the input position for the next run)
        # -- Requires that the end of the core pattern is to be searched! One 
        #    cannot simply restore some stored input position.
        # -- The pseudo-ambiguous post condition is translated into a 'normal'
        #    pattern. However, after a match a backward detection of the end
        #    of the core pattern is done. Here, we first need to go to the point
        #    where the 'normal' pattern ended, then we can do a backward detection.

        bipd_id   = pattern.input_position_search_backward_sm.get_id()
        bipd_str  = "    goto %s;\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_DETECTOR(bipd_id)
        # After having finished the analyzis, enter the terminal code, here.
        bipd_str += "%s:\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_RETURN(bipd_id) 
        input_position_search_backward_str = bipd_str


    txt = [
            "\nTERMINAL_%i:\n" % AcceptanceID,
            input_position_search_backward_str,
            "    __quex_debug(\"* terminal %i:   %s\\n\");\n" % (AcceptanceID, safe_pattern),
            action_code, "\n",
            "    goto %s;\n" % get_label("$re-start", U=True)
    ]

    return txt
Ejemplo n.º 11
0
def get_info(StateIndexList):
    # In some strange cases, a 'dummy' state router is required so that 
    # 'goto __STATE_ROUTER;' does not reference a non-existing label. Then,
    # we return an empty text array.
    if len(StateIndexList) == 0: return []

    # Make sure, that for every state the 'drop-out' state is also mentioned
    result = [None] * len(StateIndexList)
    for i, index in enumerate(StateIndexList):
        assert type(index) != str
        if index >= 0:
            # Transition to state entry
            code = "goto %s; " % get_label_of_address(index)
            result[i] = (index, code)
        else:
            # Transition to a templates 'drop-out'
            code = "goto " + get_label("$drop-out", - index) + "; "
            result[i] = (get_address("$drop-out", - index), code)
    return result
Ejemplo n.º 12
0
def get_info(StateIndexList):
    # In some strange cases, a 'dummy' state router is required so that
    # 'goto __STATE_ROUTER;' does not reference a non-existing label. Then,
    # we return an empty text array.
    if len(StateIndexList) == 0: return []

    # Make sure, that for every state the 'drop-out' state is also mentioned
    result = [None] * len(StateIndexList)
    for i, index in enumerate(StateIndexList):
        assert type(index) != str
        if index >= 0:
            # Transition to state entry
            code = "goto %s; " % get_label_of_address(index)
            result[i] = (index, code)
        else:
            # Transition to a templates 'drop-out'
            code = "goto " + get_label("$drop-out", -index) + "; "
            result[i] = (get_address("$drop-out", -index), code)
    return result
Ejemplo n.º 13
0
def __terminal_states(StateMachineName, action_db, OnFailureAction, EndOfStreamAction, 
                      PreConditionIDList, LanguageDB, VariableDB, OnAfterMatchStr, LexemeNullObjectName):
    """NOTE: During backward-lexing, for a pre-condition, there is not need for terminal
             states, since only the flag 'pre-condition fulfilled is raised.
    """      

    # (*) specific terminal states of patterns (entered from acceptance states)
    specific_terminal_states = []
    for pattern_id, pattern_action_info in action_db.items():
        if pattern_id in PreConditionIDList: continue
        code = get_terminal_code(pattern_id, pattern_action_info, LanguageDB)
        specific_terminal_states.extend(code)

    delete_pre_context_flags = []
    for pre_context_sm_id in PreConditionIDList:
        delete_pre_context_flags.append("    ")
        delete_pre_context_flags.append(LanguageDB.ASSIGN("pre_context_%s_fulfilled_f" % __nice(pre_context_sm_id), 0))

    # If there is at least a single terminal, the the 're-entry' preparation must be accomplished
    if len(action_db) != 0: get_label("$re-start", U=True)

    #  -- execute 'on_failure' pattern action 
    #  -- goto initial state    
    end_of_stream_code_action_str = EndOfStreamAction.action().get_code()

    # -- FAILURE ACTION: Under 'normal' circumstances the on_failure action is simply to be executed
    #                    since the 'get_forward()' incremented the 'current' pointer.
    #                    HOWEVER, when end of file has been reached the 'current' pointer has to
    #                    be reset so that the initial state can drop out on the buffer limit code
    #                    and then transit to the end of file action.
    # NOTE: It is possible that 'miss' happens after a chain of characters appeared. In any case the input
    #       pointer must be setup right after the lexeme start. This way, the lexer becomes a new chance as
    #       soon as possible.
    on_failure = __terminal_on_failure_prolog(LanguageDB)
    msg        = OnFailureAction.action().get_code()

    on_failure.append(msg)

    prolog = blue_print(__terminal_state_prolog,
                        [
                          ["$$LEXEME_LENGTH$$",      LanguageDB.LEXEME_LENGTH()],
                          ["$$INPUT_P$$",            LanguageDB.INPUT_P()],
                          ["$$LEXEME_NULL_OBJECT$$", LexemeNullObjectName],
                        ]
                       )

    router = Address("$terminal-router", None,
                  [
                      blue_print(__terminal_router_prolog_str,
                      [
                       ["$$TERMINAL_FAILURE-REF$$",         "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")],
                       ["$$TERMINAL_FAILURE$$",             get_label("$terminal-FAILURE")],
                      ]),
                      # DO NOT 'U=True' for the state router. This is done automatically if 
                      # 'goto reload' is used. 
                      get_label("$state-router"), ";",
                      __terminal_router_epilog_str, 
                  ])
                     
    epilog = blue_print(__terminal_state_epilog, 
             [
              ["$$FAILURE_ACTION$$",             "".join(on_failure)],
              ["$$END_OF_STREAM_ACTION$$",       end_of_stream_code_action_str],
              ["$$TERMINAL_END_OF_STREAM-DEF$$", get_label("$terminal-EOF")],
              ["$$TERMINAL_FAILURE-DEF$$",       get_label("$terminal-FAILURE")],
              ["$$STATE_MACHINE_NAME$$",         StateMachineName],
              ["$$GOTO_START_PREPARATION$$",     get_label("$re-start", U=True)],
             ])


    reset_last_acceptance_str = ""
    if VariableDB.has_key("last_acceptance"):
        reset_last_acceptance_str = "last_acceptance = $$TERMINAL_FAILURE-REF$$; /* TERMINAL: FAILURE */"

    return_preparation = ""
    if OnAfterMatchStr != "":
        return_preparation = blue_print(__return_preparation_str,
                                        [["$$ON_AFTER_MATCH$$",  OnAfterMatchStr]])

    reentry_preparation = blue_print(__reentry_preparation_str,
                          [["$$REENTRY_PREPARATION$$",                    get_label("$re-start")],
                           ["$$DELETE_PRE_CONDITION_FULLFILLED_FLAGS$$",  "".join(delete_pre_context_flags)],
                           ["$$GOTO_START$$",                             get_label("$start", U=True)],
                           ["$$ON_AFTER_MATCH$$",                         OnAfterMatchStr],
                           ["$$RESET_LAST_ACCEPTANCE$$",                  reset_last_acceptance_str],
                           ["$$COMMENT_ON_POST_CONTEXT_INITIALIZATION$$", comment_on_post_context_position_init_str],
                           ["$$TERMINAL_FAILURE-REF$$",                   "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")],
                          ])

    txt = []
    txt.append(router)
    txt.append(prolog)
    txt.extend(specific_terminal_states)
    txt.append(epilog)
    txt.append(return_preparation)
    txt.append(reentry_preparation)

    return txt
Ejemplo n.º 14
0
def __terminal_states(StateMachineName, action_db, OnFailureAction,
                      EndOfStreamAction, PreConditionIDList, LanguageDB,
                      VariableDB, OnAfterMatchStr, LexemeNullObjectName):
    """NOTE: During backward-lexing, for a pre-condition, there is not need for terminal
             states, since only the flag 'pre-condition fulfilled is raised.
    """

    # (*) specific terminal states of patterns (entered from acceptance states)
    specific_terminal_states = []
    for pattern_id, pattern_action_info in action_db.items():
        if pattern_id in PreConditionIDList: continue
        code = get_terminal_code(pattern_id, pattern_action_info, LanguageDB)
        specific_terminal_states.extend(code)

    delete_pre_context_flags = []
    for pre_context_sm_id in PreConditionIDList:
        delete_pre_context_flags.append("    ")
        delete_pre_context_flags.append(
            LanguageDB.ASSIGN(
                "pre_context_%s_fulfilled_f" % __nice(pre_context_sm_id), 0))

    # If there is at least a single terminal, the the 're-entry' preparation must be accomplished
    if len(action_db) != 0: get_label("$re-start", U=True)

    #  -- execute 'on_failure' pattern action
    #  -- goto initial state
    end_of_stream_code_action_str = EndOfStreamAction.action().get_code()

    # -- FAILURE ACTION: Under 'normal' circumstances the on_failure action is simply to be executed
    #                    since the 'get_forward()' incremented the 'current' pointer.
    #                    HOWEVER, when end of file has been reached the 'current' pointer has to
    #                    be reset so that the initial state can drop out on the buffer limit code
    #                    and then transit to the end of file action.
    # NOTE: It is possible that 'miss' happens after a chain of characters appeared. In any case the input
    #       pointer must be setup right after the lexeme start. This way, the lexer becomes a new chance as
    #       soon as possible.
    on_failure = __terminal_on_failure_prolog(LanguageDB)
    msg = OnFailureAction.action().get_code()

    on_failure.append(msg)

    prolog = blue_print(__terminal_state_prolog, [
        ["$$LEXEME_LENGTH$$", LanguageDB.LEXEME_LENGTH()],
        ["$$INPUT_P$$", LanguageDB.INPUT_P()],
        ["$$LEXEME_NULL_OBJECT$$", LexemeNullObjectName],
    ])

    router = Address(
        "$terminal-router",
        None,
        [
            blue_print(__terminal_router_prolog_str, [
                [
                    "$$TERMINAL_FAILURE-REF$$",
                    "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")
                ],
                ["$$TERMINAL_FAILURE$$",
                 get_label("$terminal-FAILURE")],
            ]),
            # DO NOT 'U=True' for the state router. This is done automatically if
            # 'goto reload' is used.
            get_label("$state-router"),
            ";",
            __terminal_router_epilog_str,
        ])

    epilog = blue_print(__terminal_state_epilog, [
        ["$$FAILURE_ACTION$$", "".join(on_failure)],
        ["$$END_OF_STREAM_ACTION$$", end_of_stream_code_action_str],
        ["$$TERMINAL_END_OF_STREAM-DEF$$",
         get_label("$terminal-EOF")],
        ["$$TERMINAL_FAILURE-DEF$$",
         get_label("$terminal-FAILURE")],
        ["$$STATE_MACHINE_NAME$$", StateMachineName],
        ["$$GOTO_START_PREPARATION$$",
         get_label("$re-start", U=True)],
    ])

    reset_last_acceptance_str = ""
    if VariableDB.has_key("last_acceptance"):
        reset_last_acceptance_str = "last_acceptance = $$TERMINAL_FAILURE-REF$$; /* TERMINAL: FAILURE */"

    return_preparation = ""
    if OnAfterMatchStr != "":
        return_preparation = blue_print(
            __return_preparation_str,
            [["$$ON_AFTER_MATCH$$", OnAfterMatchStr]])

    reentry_preparation = blue_print(__reentry_preparation_str, [
        ["$$REENTRY_PREPARATION$$",
         get_label("$re-start")],
        [
            "$$DELETE_PRE_CONDITION_FULLFILLED_FLAGS$$",
            "".join(delete_pre_context_flags)
        ],
        ["$$GOTO_START$$", get_label("$start", U=True)],
        ["$$ON_AFTER_MATCH$$", OnAfterMatchStr],
        ["$$RESET_LAST_ACCEPTANCE$$", reset_last_acceptance_str],
        [
            "$$COMMENT_ON_POST_CONTEXT_INITIALIZATION$$",
            comment_on_post_context_position_init_str
        ],
        [
            "$$TERMINAL_FAILURE-REF$$",
            "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")
        ],
    ])

    txt = []
    txt.append(router)
    txt.append(prolog)
    txt.extend(specific_terminal_states)
    txt.append(epilog)
    txt.append(return_preparation)
    txt.append(reentry_preparation)

    return txt
Ejemplo n.º 15
0
 def GOTO_DROP_OUT(self, StateIndex):
     return "goto %s;" % get_label("$drop-out", StateIndex, U=True, R=True)
Ejemplo n.º 16
0
def __analyzer_function(StateMachineName,
                        Setup,
                        variable_definitions,
                        function_body,
                        ModeNameList=[]):
    """EngineClassName = name of the structure that contains the engine state.
                         if a mode of a complete quex environment is created, this
                         is the mode name. otherwise, any name can be chosen. 
       SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
                           if a stand-alone lexical engine is required (without the
                           complete mode-handling framework of quex).
    """
    LanguageDB = Setup.language_db
    SingleModeAnalyzerF = Setup.single_mode_analyzer_f

    txt = [
        "#include <quex/code_base/temporary_macros_on>\n",
        __function_signature.replace("$$STATE_MACHINE_NAME$$",
                                     StateMachineName),
    ]

    txt.extend(variable_definitions)

    if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
        L = max(map(lambda name: len(name), ModeNameList))
        for name in ModeNameList:
            txt.append("#   define %s%s    (QUEX_NAME(%s))\n" %
                       (name, " " * (L - len(name)), name))

    txt.extend([
        comment_on_post_context_position_init_str,
        "#   if    defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
        "       || defined(QUEX_OPTION_ASSERTS)\n",
        "    me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
        "#   endif\n",
    ])

    txt.append(get_label("$start") + ":\n")

    # -- entry to the actual function body
    txt.append("    %s\n" % LanguageDB.LEXEME_START_SET())
    txt.append("    QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n")

    txt.extend(function_body)

    # -- prevent the warning 'unused variable'
    txt.append(
        "\n"                                                                                              \
        "    /* Prevent compiler warning 'unused variable': use variables once in a part of the code*/\n" \
        "    /* that is never reached (and deleted by the compiler anyway).*/\n")

    # Mode Names are defined as macros, so the following is not necessary.
    # for mode_name in ModeNameList:
    #    txt.append("    (void)%s;\n" % mode_name)
    txt.append(                                                             \
        "    (void)QUEX_LEXEME_NULL;\n"                                     \
        "    (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n"                 \
        "    QUEX_ERROR_EXIT(\"Unreachable code has been reached.\\n\");\n")

    ## This was once we did not know ... if there was a goto to the initial state or not.
    ## txt += "        goto %s;\n" % label.get(StateMachineName, InitialStateIndex)
    if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
        L = max(map(lambda name: len(name), ModeNameList))
        for name in ModeNameList:
            txt.append("#   undef %s\n" % name)

    txt.append("#   undef self\n")
    txt.append("}\n")

    txt.append("#include <quex/code_base/temporary_macros_off>\n")
    return txt
Ejemplo n.º 17
0
def get_skipper(OpenerSequence,
                CloserSequence,
                Mode=None,
                IndentationCounterTerminalID=None,
                OnSkipRangeOpenStr=""):
    assert OpenerSequence.__class__ == list
    assert len(OpenerSequence) >= 1
    assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
    assert CloserSequence.__class__ == list
    assert len(CloserSequence) >= 1
    assert map(type, CloserSequence) == [int] * len(CloserSequence)
    assert OpenerSequence != CloserSequence

    LanguageDB = Setup.language_db

    skipper_index = sm_index.get()

    opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
    opener_length = len(OpenerSequence)
    closer_str, closer_comment_str = get_character_sequence(CloserSequence)
    closer_length = len(CloserSequence)

    if not end_delimiter_is_subset_of_indentation_counter_newline(
            Mode, CloserSequence):
        goto_after_end_of_skipping_str = LanguageDB.GOTO(
            E_StateIndices.ANALYZER_REENTRY)
    else:
        # If there is indentation counting involved, then the counter's terminal id must
        # be determined at this place.
        assert IndentationCounterTerminalID is not None
        # If the ending delimiter is a subset of what the 'newline' pattern triggers
        # in indentation counting => move on to the indentation counter.
        goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(
            IndentationCounterTerminalID)

    if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
    else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)

    local_variable_db = {}
    variable_db.enter(local_variable_db,
                      "reference_p",
                      Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
    # variable_db.enter(local_variable_db, "text_end")
    variable_db.enter(local_variable_db, "counter")
    variable_db.enter(local_variable_db,
                      "Skipper%i_Opener",
                      "{ %s }" % opener_str,
                      ElementN=opener_length,
                      Index=skipper_index)
    variable_db.enter(local_variable_db,
                      "Skipper%i_OpenerEnd",
                      "Skipper%i_Opener + (ptrdiff_t)%i" %
                      (skipper_index, opener_length),
                      Index=skipper_index)
    variable_db.enter(local_variable_db,
                      "Skipper%i_Opener_it",
                      "0x0",
                      Index=skipper_index)
    variable_db.enter(local_variable_db,
                      "Skipper%i_Closer",
                      "{ %s }" % closer_str,
                      ElementN=closer_length,
                      Index=skipper_index)
    variable_db.enter(local_variable_db,
                      "Skipper%i_CloserEnd",
                      "Skipper%i_Closer + (ptrdiff_t)%i" %
                      (skipper_index, closer_length),
                      Index=skipper_index)
    variable_db.enter(local_variable_db,
                      "Skipper%i_Closer_it",
                      "0x0",
                      Index=skipper_index)

    reference_p_def = "    __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"

    reference_p_def = "    __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
    before_reload   = "    __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
                      "                                - reference_p));\n"
    after_reload = "        __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"

    if CloserSequence[-1] == ord('\n'):
        end_procedure = "       __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
        end_procedure += "       __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
    else:
        end_procedure = "        __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
                        "                                    - reference_p));\n"

    code_str = blue_print(
        template_str,
        [
            ["$$SKIPPER_INDEX$$", __nice(skipper_index)],
            #
            ["$$OPENER_LENGTH$$", "%i" % opener_length],
            ["$$INPUT_P_INCREMENT$$",
             LanguageDB.INPUT_P_INCREMENT()],
            ["$$INPUT_P_DECREMENT$$",
             LanguageDB.INPUT_P_DECREMENT()],
            ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
            [
                "$$IF_INPUT_EQUAL_DELIMITER_0$$",
                LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
            ],
            ["$$ENDIF$$", LanguageDB.END_IF()],
            ["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
            ["$$RELOAD$$", get_label("$reload", skipper_index)],
            ["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
            ["$$GOTO_RELOAD$$",
             get_label("$reload", skipper_index)],
            [
                "$$INPUT_P_TO_LEXEME_START$$",
                LanguageDB.INPUT_P_TO_LEXEME_START()
            ],
            # When things were skipped, no change to acceptance flags or modes has
            # happend. One can jump immediately to the start without re-entry preparation.
            ["$$GOTO_ENTRY$$",
             LanguageDB.GOTO(skipper_index)],
            ["$$MARK_LEXEME_START$$",
             LanguageDB.LEXEME_START_SET()],
            ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
            #
            ["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
            ["$$LC_COUNT_IN_LOOP$$",
             line_column_counter_in_loop()],
            ["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
            ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
            ["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
        ])

    return code_str, local_variable_db
Ejemplo n.º 18
0
def get_skipper(OpenerSequence, CloserSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
    assert OpenerSequence.__class__  == list
    assert len(OpenerSequence)       >= 1
    assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
    assert CloserSequence.__class__  == list
    assert len(CloserSequence)       >= 1
    assert map(type, CloserSequence) == [int] * len(CloserSequence)
    assert OpenerSequence != CloserSequence

    LanguageDB    = Setup.language_db

    skipper_index = sm_index.get()

    opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
    opener_length = len(OpenerSequence)
    closer_str, closer_comment_str = get_character_sequence(CloserSequence)
    closer_length = len(CloserSequence)

    if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, CloserSequence):
        goto_after_end_of_skipping_str = LanguageDB.GOTO(E_StateIndices.ANALYZER_REENTRY)
    else:
        # If there is indentation counting involved, then the counter's terminal id must
        # be determined at this place.
        assert IndentationCounterTerminalID is not None
        # If the ending delimiter is a subset of what the 'newline' pattern triggers 
        # in indentation counting => move on to the indentation counter.
        goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(IndentationCounterTerminalID)

    if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
    else:                        on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)

    local_variable_db = {}
    variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
    # variable_db.enter(local_variable_db, "text_end")
    variable_db.enter(local_variable_db, "counter")
    variable_db.enter(local_variable_db, "Skipper%i_Opener",    "{ %s }" % opener_str, ElementN=opener_length, 
                                         Index = skipper_index)
    variable_db.enter(local_variable_db, "Skipper%i_OpenerEnd", 
                                         "Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length),
                                         Index = skipper_index) 
    variable_db.enter(local_variable_db, "Skipper%i_Opener_it", "0x0", 
                                         Index = skipper_index) 
    variable_db.enter(local_variable_db, "Skipper%i_Closer",    "{ %s }" % closer_str, ElementN=closer_length, 
                                         Index = skipper_index) 
    variable_db.enter(local_variable_db, "Skipper%i_CloserEnd", 
                                         "Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length),
                                         Index = skipper_index) 
    variable_db.enter(local_variable_db, "Skipper%i_Closer_it", "0x0", 
                                         Index = skipper_index) 

   
    reference_p_def = "    __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"

    reference_p_def = "    __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
    before_reload   = "    __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
                      "                                - reference_p));\n" 
    after_reload    = "        __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"

    if CloserSequence[-1] == ord('\n'):
        end_procedure  = "       __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
        end_procedure += "       __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
    else:
        end_procedure = "        __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
                        "                                    - reference_p));\n" 

    code_str = blue_print(template_str,
                          [
                           ["$$SKIPPER_INDEX$$",   __nice(skipper_index)],
                           #
                           ["$$OPENER_LENGTH$$",                  "%i" % opener_length],
                           ["$$INPUT_P_INCREMENT$$",              LanguageDB.INPUT_P_INCREMENT()],
                           ["$$INPUT_P_DECREMENT$$",              LanguageDB.INPUT_P_DECREMENT()],
                           ["$$INPUT_GET$$",                      LanguageDB.ACCESS_INPUT()],
                           ["$$IF_INPUT_EQUAL_DELIMITER_0$$",     LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
                           ["$$ENDIF$$",                          LanguageDB.END_IF()],
                           ["$$ENTRY$$",                          LanguageDB.LABEL(skipper_index)],
                           ["$$RELOAD$$",                         get_label("$reload", skipper_index)],
                           ["$$GOTO_AFTER_END_OF_SKIPPING$$",     goto_after_end_of_skipping_str], 
                           ["$$GOTO_RELOAD$$",                    get_label("$reload", skipper_index)],
                           ["$$INPUT_P_TO_LEXEME_START$$",        LanguageDB.INPUT_P_TO_LEXEME_START()],
                           # When things were skipped, no change to acceptance flags or modes has
                           # happend. One can jump immediately to the start without re-entry preparation.
                           ["$$GOTO_ENTRY$$",                     LanguageDB.GOTO(skipper_index)],
                           ["$$MARK_LEXEME_START$$",              LanguageDB.LEXEME_START_SET()],
                           ["$$ON_SKIP_RANGE_OPEN$$",             on_skip_range_open_str],
                           #
                           ["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
                           ["$$LC_COUNT_IN_LOOP$$",                     line_column_counter_in_loop()],
                           ["$$LC_COUNT_END_PROCEDURE$$",               end_procedure],
                           ["$$LC_COUNT_BEFORE_RELOAD$$",               before_reload],
                           ["$$LC_COUNT_AFTER_RELOAD$$",                after_reload],
                          ])

    return code_str, local_variable_db
Ejemplo n.º 19
0
def __parse_option(fh, new_mode):
    def get_pattern_object(SM):
        if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
        else: result = SM
        result = hopcroft.do(result, CreateNewStateMachineF=False)
        return Pattern(result, AllowStateMachineTrafoF=True)

    identifier = read_option_start(fh)
    if identifier is None: return False

    verify_word_in_list(identifier, mode_option_info_db.keys(), "mode option",
                        fh.name, get_current_line_info_number(fh))

    if identifier == "skip":
        # A skipper 'eats' characters at the beginning of a pattern that belong
        # to a specified set of characters. A useful application is most probably
        # the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
        # implement a very effective way to skip these regions.
        pattern_str, trigger_set = regular_expression.parse_character_set(
            fh, PatternStringF=True)
        skip_whitespace(fh)

        if fh.read(1) != ">":
            error_msg("missing closing '>' for mode option '%s'." % identifier,
                      fh)

        if trigger_set.is_empty():
            error_msg("Empty trigger set for skipper." % identifier, fh)

        # TriggerSet skipping is implemented the following way: As soon as one element of the
        # trigger set appears, the state machine enters the 'trigger set skipper section'.
        # Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
        # NOTE: The correspondent CodeFragment for skipping is created in 'implement_skippers(...)'
        pattern_sm = StateMachine()
        pattern_sm.add_transition(pattern_sm.init_state_index,
                                  trigger_set,
                                  AcceptanceF=True)

        # Skipper code is to be generated later
        action = GeneratedCode(skip_character_set.do,
                               FileName=fh.name,
                               LineN=get_current_line_info_number(fh))
        action.data["character_set"] = trigger_set

        new_mode.add_match(pattern_str,
                           action,
                           get_pattern_object(pattern_sm),
                           Comment=E_SpecialPatterns.SKIP)

        return True

    elif identifier in ["skip_range", "skip_nested_range"]:
        # A non-nesting skipper can contain a full fledged regular expression as opener,
        # since it only effects the trigger. Not so the nested range skipper-see below.

        # -- opener
        skip_whitespace(fh)
        if identifier == "skip_nested_range":
            # Nested range state machines only accept 'strings' not state machines
            opener_str, opener_sequence = __parse_string(
                fh, "Opener pattern for 'skip_nested_range'")
            opener_sm = StateMachine.from_sequence(opener_sequence)
        else:
            opener_str, opener_pattern = regular_expression.parse(fh)
            opener_sm = opener_pattern.sm
            # For 'range skipping' the opener sequence is not needed, only the opener state
            # machine is webbed into the pattern matching state machine.
            opener_sequence = None

        skip_whitespace(fh)

        # -- closer
        closer_str, closer_sequence = __parse_string(
            fh, "Closing pattern for 'skip_range' or 'skip_nested_range'")
        skip_whitespace(fh)
        if fh.read(1) != ">":
            error_msg("missing closing '>' for mode option '%s'" % identifier,
                      fh)

        # Skipper code is to be generated later
        generator_function, comment = {
            "skip_range": (skip_range.do, E_SpecialPatterns.SKIP_RANGE),
            "skip_nested_range":
            (skip_nested_range.do, E_SpecialPatterns.SKIP_NESTED_RANGE),
        }[identifier]
        action = GeneratedCode(generator_function,
                               FileName=fh.name,
                               LineN=get_current_line_info_number(fh))

        action.data["opener_sequence"] = opener_sequence
        action.data["closer_sequence"] = closer_sequence
        action.data["mode_name"] = new_mode.name

        new_mode.add_match(opener_str,
                           action,
                           get_pattern_object(opener_sm),
                           Comment=comment)

        return True

    elif identifier == "indentation":
        value = indentation_setup.do(fh)

        # Enter 'Newline' and 'Suppressed Newline' as matches into the engine.
        # Similar to skippers, the indentation count is then triggered by the newline.
        # -- Suppressed Newline = Suppressor followed by Newline,
        #    then newline does not trigger indentation counting.
        suppressed_newline_pattern_str = ""
        if value.newline_suppressor_state_machine.get() is not None:
            suppressed_newline_pattern_str = \
                  "(" + value.newline_suppressor_state_machine.pattern_string() + ")" \
                + "(" + value.newline_state_machine.pattern_string() + ")"

            suppressed_newline_sm = \
                sequentialize.do([value.newline_suppressor_state_machine.get(),
                                  value.newline_state_machine.get()])

            FileName = value.newline_suppressor_state_machine.file_name
            LineN = value.newline_suppressor_state_machine.line_n
            # Go back to start.
            code = UserCodeFragment("goto %s;" % get_label("$start", U=True),
                                    FileName, LineN)

            new_mode.add_match(
                suppressed_newline_pattern_str,
                code,
                get_pattern_object(suppressed_newline_sm),
                Comment=E_SpecialPatterns.SUPPRESSED_INDENTATION_NEWLINE)

        # When there is an empty line, then there shall be no indentation count on it.
        # Here comes the trick:
        #
        #      Let               newline
        #      be defined as:    newline ([space]* newline])*
        #
        # This way empty lines are eating away before the indentation count is activated.

        # -- 'space'
        x0 = StateMachine()
        x0.add_transition(x0.init_state_index,
                          value.indentation_count_character_set(),
                          AcceptanceF=True)
        # -- '[space]*'
        x1 = repeat.do(x0)
        # -- '[space]* newline'
        x2 = sequentialize.do([x1, value.newline_state_machine.get()])
        # -- '([space]* newline)*'
        x3 = repeat.do(x2)
        # -- 'newline ([space]* newline)*'
        x4 = sequentialize.do([value.newline_state_machine.get(), x3])
        # -- nfa to dfa; hopcroft optimization
        sm = beautifier.do(x4)

        FileName = value.newline_state_machine.file_name
        LineN = value.newline_state_machine.line_n
        action = GeneratedCode(indentation_counter.do, FileName, LineN)

        action.data["indentation_setup"] = value

        new_mode.add_match(value.newline_state_machine.pattern_string(),
                           action,
                           get_pattern_object(sm),
                           Comment=E_SpecialPatterns.INDENTATION_NEWLINE)

        # Announce the mode to which the setup belongs
        value.set_containing_mode_name(new_mode.name)
    else:
        value = read_option_value(fh)

    # The 'verify_word_in_list()' call must have ensured that the following holds
    assert mode_option_info_db.has_key(identifier)

    # Is the option of the appropriate value?
    option_info = mode_option_info_db[identifier]
    if option_info.domain is not None and value not in option_info.domain:
        error_msg("Tried to set value '%s' for option '%s'. " % (value, identifier) + \
                  "Though, possible for this option are only: %s." % repr(option_info.domain)[1:-1], fh)

    # Finally, set the option
    new_mode.add_option(identifier, value)

    return True
Ejemplo n.º 20
0
def get_skipper(EndSequence,
                Mode=None,
                IndentationCounterTerminalID=None,
                OnSkipRangeOpenStr=""):
    assert type(EndSequence) == list
    assert len(EndSequence) >= 1
    assert map(type, EndSequence) == [int] * len(EndSequence)

    local_variable_db = {}

    global template_str

    LanguageDB = Setup.language_db

    # Name the $$SKIPPER$$
    skipper_index = sm_index.get()

    # Determine the $$DELIMITER$$
    delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
    delimiter_length = len(EndSequence)

    tmp = []
    LanguageDB.COMMENT(
        tmp, "                         Delimiter: %s" % delimiter_comment_str)
    delimiter_comment_str = "".join(tmp)
    # Determine the check for the tail of the delimiter
    delimiter_remainder_test_str = ""
    if len(EndSequence) != 1:
        txt = ""
        i = 0
        for letter in EndSequence[1:]:
            i += 1
            txt += "    %s\n" % LanguageDB.ASSIGN(
                "input", LanguageDB.INPUT_P_DEREFERENCE(i - 1))
            txt += "    %s" % LanguageDB.IF_INPUT(
                "!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i)
            txt += "         %s" % LanguageDB.GOTO(skipper_index)
            txt += "    %s" % LanguageDB.END_IF()
        delimiter_remainder_test_str = txt

    if not end_delimiter_is_subset_of_indentation_counter_newline(
            Mode, EndSequence):
        goto_after_end_of_skipping_str = LanguageDB.GOTO(
            E_StateIndices.ANALYZER_REENTRY)
    else:
        # If there is indentation counting involved, then the counter's terminal id must
        # be determined at this place.
        assert IndentationCounterTerminalID is not None
        # If the ending delimiter is a subset of what the 'newline' pattern triggers
        # in indentation counting => move on to the indentation counter.
        goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(
            IndentationCounterTerminalID)

    if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
    else: on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)

    # The main part
    code_str = blue_print(
        template_str,
        [
            ["$$DELIMITER_COMMENT$$", delimiter_comment_str],
            ["$$INPUT_P_INCREMENT$$",
             LanguageDB.INPUT_P_INCREMENT()],
            ["$$INPUT_P_DECREMENT$$",
             LanguageDB.INPUT_P_DECREMENT()],
            ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
            [
                "$$IF_INPUT_EQUAL_DELIMITER_0$$",
                LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
            ],
            ["$$ENDIF$$", LanguageDB.END_IF()],
            ["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
            ["$$RELOAD$$", get_label("$reload", skipper_index)],
            ["$$GOTO_ENTRY$$",
             LanguageDB.GOTO(skipper_index)],
            [
                "$$INPUT_P_TO_LEXEME_START$$",
                LanguageDB.INPUT_P_TO_LEXEME_START()
            ],
            # When things were skipped, no change to acceptance flags or modes has
            # happend. One can jump immediately to the start without re-entry preparation.
            ["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
            ["$$MARK_LEXEME_START$$",
             LanguageDB.LEXEME_START_SET()],
            ["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
            ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
        ])

    # Line and column number counting
    code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)

    # The finishing touch
    code_str = blue_print(
        code_str,
        [["$$SKIPPER_INDEX$$", __nice(skipper_index)],
         ["$$GOTO_RELOAD$$",
          get_label("$reload", skipper_index)]])

    if reference_p_f:
        variable_db.enter(local_variable_db,
                          "reference_p",
                          Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")

    variable_db.enter(local_variable_db,
                      "Skipper%i",
                      "{ %s }" % delimiter_str,
                      delimiter_length,
                      Index=skipper_index)
    variable_db.enter(local_variable_db,
                      "Skipper%iL",
                      "%i" % delimiter_length,
                      Index=skipper_index)
    variable_db.enter(local_variable_db, "text_end")
    return code_str, local_variable_db
Ejemplo n.º 21
0
 def GOTO_DROP_OUT(self, StateIndex):
     return "goto %s;" % get_label("$drop-out", StateIndex, U=True, R=True)
Ejemplo n.º 22
0
def do(Data):
    """The generated code is very similar to the 'skipper' code. It is to be executed
       as soon as a 'real' newline arrived. Then it skips whitespace until the next 
       non-whitepace (also newline may trigger a 'stop'). 

       Dependent on the setup the indentation is determined.
    """
    IndentationSetup = Data["indentation_setup"]
    assert IndentationSetup.__class__.__name__ == "IndentationSetup"


    LanguageDB = Setup.language_db
    Mode = None
    if IndentationSetup.containing_mode_name() != "":
        Mode = blackboard.mode_db[IndentationSetup.containing_mode_name()]

    counter_index = sm_index.get()
    
    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.

    trigger_map = []
    # If the indentation consists only of spaces, than it is 'uniform' ...
    if IndentationSetup.has_only_single_spaces():
        # Count indentation/column at end of run;
        # simply: current position - reference_p

        character_set = IndentationSetup.space_db.values()[0]
        for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
            trigger_map.append([interval, counter_index])

        # Reference Pointer: Define Variable, Initialize, determine how to subtact.
        end_procedure = \
        "    me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n" 
    else:
        # Count the indentation/column during the 'run'

        # Add the space counters
        for count, character_set in IndentationSetup.space_db.items():
            for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
                trigger_map.append([interval, IndentationCounter("space", count, counter_index)])

        # Add the grid counters
        for count, character_set in IndentationSetup.grid_db.items():
            for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
                trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])

        # Reference Pointer: Not required.
        #                    No subtraction 'current_position - reference_p'.
        #                    (however, we pass 'reference_p' to indentation handler)
        end_procedure = "" 

    # Bad character detection
    if IndentationSetup.bad_character_set.get().is_empty() == False:
        for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
            trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])

    # Since we do not use a 'TransitionMap', there are some things we need 
    # to do by hand.
    arrange_trigger_map(trigger_map)

    local_variable_db = { "reference_p" : 
                          Variable("reference_p", 
                                   "QUEX_TYPE_CHARACTER_POSITION", 
                                   None, 
                                   "(QUEX_TYPE_CHARACTER_POSITION)0x0")
    }
    init_reference_p  = "    reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n" + \
                        "    me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"

    iteration_code = []
    transition_block.do(iteration_code, trigger_map, 
                        StateIndex     = counter_index, 
                        EngineType     = E_EngineTypes.INDENTATION_COUNTER,
                        GotoReload_Str = "goto %s;" % get_label("$reload", counter_index))

    tmp = []
    LanguageDB.COMMENT(tmp, "Skip whitespace at line begin; count indentation.")
    comment_str = "".join(tmp)

    # NOTE: Line and column number counting is off
    #       -- No newline can occur
    #       -- column number = indentation at the end of the process

    end_procedure += "    __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
    if Mode is None or Mode.default_indentation_handler_sufficient():
        end_procedure += "    QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
    else:
        # Definition of '%s_on_indentation' in mode_classes.py.
        end_procedure += "    QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" \
                         % Mode.name

    # The finishing touch
    prolog = blue_print(prolog_txt,
                         [
                           ["$$DELIMITER_COMMENT$$",              comment_str],
                           ["$$INIT_REFERENCE_POINTER$$",         init_reference_p],
                           ["$$COUNTER_INDEX$$",                  repr(counter_index)],
                           ["$$INPUT_GET$$",                      LanguageDB.ACCESS_INPUT()],
                         ])

    # The finishing touch
    epilog = blue_print(epilog_txt,
                      [
                       ["$$INPUT_P_INCREMENT$$",              LanguageDB.INPUT_P_INCREMENT()],
                       ["$$INPUT_P_DECREMENT$$",              LanguageDB.INPUT_P_DECREMENT()],
                       ["$$IF_INPUT_EQUAL_DELIMITER_0$$",     LanguageDB.IF_INPUT("==", "SkipDelimiter$$COUNTER_INDEX$$[0]")],
                       ["$$ENDIF$$",                          LanguageDB.END_IF()],
                       ["$$LOOP_REENTRANCE$$",                LanguageDB.LABEL(counter_index)], 
                       ["$$IF_INPUT_EQUAL_BUFFER_LIMIT_CODE$$",  LanguageDB.IF_INPUT("==", LanguageDB.BUFFER_LIMIT_CODE)],
                       ["$$RELOAD$$",                         get_label("$reload", counter_index)],
                       ["$$COUNTER_INDEX$$",                  repr(counter_index)],
                       ["$$GOTO_TERMINAL_EOF$$",              get_label("$terminal-EOF", U=True)],
                       ["$$LEXEME_START_SET_TO_REF$$",        LanguageDB.LEXEME_START_SET("reference_p")],
                       # When things were skipped, no change to acceptance flags or modes has
                       # happend. One can jump immediately to the start without re-entry preparation.
                       ["$$GOTO_START$$",                     get_label("$start", U=True)], 
                       ["$$END_PROCEDURE$$",                  end_procedure],
                       ["$$BAD_CHARACTER_HANDLING$$",         get_bad_character_handler(Mode, IndentationSetup, counter_index)],
                      ])

    txt = [prolog]
    txt.extend(iteration_code)
    # txt.append(Address("$drop-out", counter_index))
    txt.append("\n")
    txt.append(epilog)

    return txt, local_variable_db
Ejemplo n.º 23
0
def do(Data):
    """The generated code is very similar to the 'skipper' code. It is to be executed
       as soon as a 'real' newline arrived. Then it skips whitespace until the next 
       non-whitepace (also newline may trigger a 'stop'). 

       Dependent on the setup the indentation is determined.
    """
    IndentationSetup = Data["indentation_setup"]
    assert IndentationSetup.__class__.__name__ == "IndentationSetup"

    LanguageDB = Setup.language_db
    Mode = None
    if IndentationSetup.containing_mode_name() != "":
        Mode = blackboard.mode_db[IndentationSetup.containing_mode_name()]

    counter_index = sm_index.get()

    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.

    trigger_map = []
    # If the indentation consists only of spaces, than it is 'uniform' ...
    if IndentationSetup.has_only_single_spaces():
        # Count indentation/column at end of run;
        # simply: current position - reference_p

        character_set = IndentationSetup.space_db.values()[0]
        for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
            trigger_map.append([interval, counter_index])

        # Reference Pointer: Define Variable, Initialize, determine how to subtact.
        end_procedure = (
            "    me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n"
        )
    else:
        # Count the indentation/column during the 'run'

        # Add the space counters
        for count, character_set in IndentationSetup.space_db.items():
            for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
                trigger_map.append([interval, IndentationCounter("space", count, counter_index)])

        # Add the grid counters
        for count, character_set in IndentationSetup.grid_db.items():
            for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
                trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])

        # Reference Pointer: Not required.
        #                    No subtraction 'current_position - reference_p'.
        #                    (however, we pass 'reference_p' to indentation handler)
        end_procedure = ""

    # Bad character detection
    if IndentationSetup.bad_character_set.get().is_empty() == False:
        for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
            trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])

    # Since we do not use a 'TransitionMap', there are some things we need
    # to do by hand.
    arrange_trigger_map(trigger_map)

    local_variable_db = {
        "reference_p": Variable(
            "reference_p", "QUEX_TYPE_CHARACTER_POSITION", None, "(QUEX_TYPE_CHARACTER_POSITION)0x0"
        )
    }
    init_reference_p = (
        "    reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n"
        + "    me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"
    )

    iteration_code = []
    transition_block.do(
        iteration_code,
        trigger_map,
        StateIndex=counter_index,
        EngineType=E_EngineTypes.INDENTATION_COUNTER,
        GotoReload_Str="goto %s;" % get_label("$reload", counter_index),
    )

    tmp = []
    LanguageDB.COMMENT(tmp, "Skip whitespace at line begin; count indentation.")
    comment_str = "".join(tmp)

    # NOTE: Line and column number counting is off
    #       -- No newline can occur
    #       -- column number = indentation at the end of the process

    end_procedure += "    __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
    if Mode is None or Mode.default_indentation_handler_sufficient():
        end_procedure += "    QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
    else:
        # Definition of '%s_on_indentation' in mode_classes.py.
        end_procedure += "    QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" % Mode.name

    # The finishing touch
    prolog = blue_print(
        prolog_txt,
        [
            ["$$DELIMITER_COMMENT$$", comment_str],
            ["$$INIT_REFERENCE_POINTER$$", init_reference_p],
            ["$$COUNTER_INDEX$$", repr(counter_index)],
            ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
        ],
    )

    # The finishing touch
    epilog = blue_print(
        epilog_txt,
        [
            ["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
            ["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
            ["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "SkipDelimiter$$COUNTER_INDEX$$[0]")],
            ["$$ENDIF$$", LanguageDB.END_IF()],
            ["$$LOOP_REENTRANCE$$", LanguageDB.LABEL(counter_index)],
            ["$$IF_INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB.IF_INPUT("==", LanguageDB.BUFFER_LIMIT_CODE)],
            ["$$RELOAD$$", get_label("$reload", counter_index)],
            ["$$COUNTER_INDEX$$", repr(counter_index)],
            ["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
            ["$$LEXEME_START_SET_TO_REF$$", LanguageDB.LEXEME_START_SET("reference_p")],
            # When things were skipped, no change to acceptance flags or modes has
            # happend. One can jump immediately to the start without re-entry preparation.
            ["$$GOTO_START$$", get_label("$start", U=True)],
            ["$$END_PROCEDURE$$", end_procedure],
            ["$$BAD_CHARACTER_HANDLING$$", get_bad_character_handler(Mode, IndentationSetup, counter_index)],
        ],
    )

    txt = [prolog]
    txt.extend(iteration_code)
    # txt.append(Address("$drop-out", counter_index))
    txt.append("\n")
    txt.append(epilog)

    return txt, local_variable_db
Ejemplo n.º 24
0
def get_skipper(TriggerSet):
    """This function implements simple 'skipping' in the sense of passing by
       characters that belong to a given set of characters--the TriggerSet.
    """
    global template_str
    assert TriggerSet.__class__.__name__ == "NumberSet"
    assert not TriggerSet.is_empty()

    LanguageDB = Setup.language_db

    skipper_index = sm_index.get()

    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.
    transition_map = TransitionMap(
    )  # (don't worry about 'drop-out-ranges' etc.)
    transition_map.add_transition(TriggerSet, skipper_index)
    # On buffer limit code, the skipper must transit to a dedicated reloader

    goto_loop_str = [
        LanguageDB.INPUT_P_INCREMENT(),
        " goto _%s_LOOP;\n" % skipper_index
    ]
    trigger_map = transition_map.get_trigger_map()
    for i, info in enumerate(trigger_map):
        interval, target = info
        if target == E_StateIndices.DROP_OUT: continue
        trigger_map[i] = (interval, TextTransitionCode(goto_loop_str))

    iteration_code = []
    transition_block.do(iteration_code,
                        trigger_map,
                        skipper_index,
                        E_EngineTypes.ELSE,
                        GotoReload_Str="goto %s;" %
                        get_label("$reload", skipper_index))

    tmp = []
    LanguageDB.COMMENT(tmp,
                       "Skip any character in " + TriggerSet.get_utf8_string())
    comment_str = "".join(tmp)

    # Line and column number counting
    prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
    epilog = __lc_counting_replacements(epilog_txt, TriggerSet)

    prolog = blue_print(prolog, [
        ["$$DELIMITER_COMMENT$$", comment_str],
        ["$$SKIPPER_INDEX$$", "%i" % skipper_index],
        ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
    ])

    epilog = blue_print(
        epilog,
        [
            ["$$INPUT_P_INCREMENT$$",
             LanguageDB.INPUT_P_INCREMENT()],
            ["$$INPUT_P_DECREMENT$$",
             LanguageDB.INPUT_P_DECREMENT()],
            [
                "$$IF_INPUT_EQUAL_DELIMITER_0$$",
                LanguageDB.IF_INPUT("==", "SkipDelimiter$$SKIPPER_INDEX$$[0]")
            ],
            ["$$ENDIF$$", LanguageDB.END_IF()],
            ["$$LOOP_REENTRANCE$$",
             LanguageDB.LABEL(skipper_index)],
            ["$$BUFFER_LIMIT_CODE$$", LanguageDB.BUFFER_LIMIT_CODE],
            ["$$RELOAD$$", get_label("$reload", skipper_index)],
            [
                "$$DROP_OUT_DIRECT$$",
                get_label("$drop-out", skipper_index, U=True)
            ],
            ["$$SKIPPER_INDEX$$", "%i" % skipper_index],
            [
                "$$LABEL_TERMINAL_EOF$$",
                "QUEX_LABEL(%i)" % get_address("$terminal-EOF", U=True)
            ],
            [
                "$$LABEL_REF_AFTER_RELOAD$$",
                "QUEX_LABEL(%i)" %
                get_address("$skipper-reload", skipper_index, U=True)
            ],
            ["$$GOTO_TERMINAL_EOF$$",
             get_label("$terminal-EOF", U=True)],
            [
                "$$LABEL_AFTER_RELOAD$$",
                get_label("$skipper-reload", skipper_index)
            ],
            # When things were skipped, no change to acceptance flags or modes has
            # happend. One can jump immediately to the start without re-entry preparation.
            ["$$GOTO_START$$", get_label("$start", U=True)],
            ["$$MARK_LEXEME_START$$",
             LanguageDB.LEXEME_START_SET()],
        ])

    code = [prolog]
    code.extend(iteration_code)
    code.append(epilog)

    local_variable_db = {}
    variable_db.enter(local_variable_db,
                      "reference_p",
                      Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")

    return code, local_variable_db
Ejemplo n.º 25
0
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
    assert type(EndSequence) == list
    assert len(EndSequence) >= 1
    assert map(type, EndSequence) == [int] * len(EndSequence)

    local_variable_db = {}

    global template_str

    LanguageDB = Setup.language_db

    # Name the $$SKIPPER$$
    skipper_index = sm_index.get()

    # Determine the $$DELIMITER$$
    delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
    delimiter_length = len(EndSequence)

    tmp = []
    LanguageDB.COMMENT(tmp, "                         Delimiter: %s" % delimiter_comment_str)
    delimiter_comment_str = "".join(tmp)
    # Determine the check for the tail of the delimiter
    delimiter_remainder_test_str = ""
    if len(EndSequence) != 1:
        txt = ""
        i = 0
        for letter in EndSequence[1:]:
            i += 1
            txt += "    %s\n" % LanguageDB.ASSIGN("input", LanguageDB.INPUT_P_DEREFERENCE(i - 1))
            txt += "    %s" % LanguageDB.IF_INPUT("!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i)
            txt += "         %s" % LanguageDB.GOTO(skipper_index)
            txt += "    %s" % LanguageDB.END_IF()
        delimiter_remainder_test_str = txt

    if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence):
        goto_after_end_of_skipping_str = LanguageDB.GOTO(E_StateIndices.ANALYZER_REENTRY)
    else:
        # If there is indentation counting involved, then the counter's terminal id must
        # be determined at this place.
        assert IndentationCounterTerminalID is not None
        # If the ending delimiter is a subset of what the 'newline' pattern triggers
        # in indentation counting => move on to the indentation counter.
        goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(IndentationCounterTerminalID)

    if OnSkipRangeOpenStr != "":
        on_skip_range_open_str = OnSkipRangeOpenStr
    else:
        on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)

    # The main part
    code_str = blue_print(
        template_str,
        [
            ["$$DELIMITER_COMMENT$$", delimiter_comment_str],
            ["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
            ["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
            ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
            ["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
            ["$$ENDIF$$", LanguageDB.END_IF()],
            ["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
            ["$$RELOAD$$", get_label("$reload", skipper_index)],
            ["$$GOTO_ENTRY$$", LanguageDB.GOTO(skipper_index)],
            ["$$INPUT_P_TO_LEXEME_START$$", LanguageDB.INPUT_P_TO_LEXEME_START()],
            # When things were skipped, no change to acceptance flags or modes has
            # happend. One can jump immediately to the start without re-entry preparation.
            ["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
            ["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()],
            ["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
            ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
        ],
    )

    # Line and column number counting
    code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)

    # The finishing touch
    code_str = blue_print(
        code_str,
        [["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]],
    )

    if reference_p_f:
        variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")

    variable_db.enter(local_variable_db, "Skipper%i", "{ %s }" % delimiter_str, delimiter_length, Index=skipper_index)
    variable_db.enter(local_variable_db, "Skipper%iL", "%i" % delimiter_length, Index=skipper_index)
    variable_db.enter(local_variable_db, "text_end")
    return code_str, local_variable_db
Ejemplo n.º 26
0
def __analyzer_function(StateMachineName, Setup,
                        variable_definitions, function_body, ModeNameList=[]):
    """EngineClassName = name of the structure that contains the engine state.
                         if a mode of a complete quex environment is created, this
                         is the mode name. otherwise, any name can be chosen. 
       SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
                           if a stand-alone lexical engine is required (without the
                           complete mode-handling framework of quex).
    """              
    LanguageDB          = Setup.language_db
    SingleModeAnalyzerF = Setup.single_mode_analyzer_f

    txt = [
            "#include <quex/code_base/temporary_macros_on>\n",
            __function_signature.replace("$$STATE_MACHINE_NAME$$", StateMachineName),
    ]

    txt.extend(variable_definitions)

    if len(ModeNameList) != 0 and not SingleModeAnalyzerF: 
        L = max(map(lambda name: len(name), ModeNameList))
        for name in ModeNameList:
            txt.append("#   define %s%s    (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name))

    txt.extend([
        comment_on_post_context_position_init_str,
        "#   if    defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
        "       || defined(QUEX_OPTION_ASSERTS)\n",
        "    me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
        "#   endif\n",
    ])

    txt.append(get_label("$start") + ":\n")

    # -- entry to the actual function body
    txt.append("    %s\n" % LanguageDB.LEXEME_START_SET())
    txt.append("    QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n")
    
    txt.extend(function_body)

    # -- prevent the warning 'unused variable'
    txt.append( 
        "\n"                                                                                              \
        "    /* Prevent compiler warning 'unused variable': use variables once in a part of the code*/\n" \
        "    /* that is never reached (and deleted by the compiler anyway).*/\n")

    # Mode Names are defined as macros, so the following is not necessary.
    # for mode_name in ModeNameList:
    #    txt.append("    (void)%s;\n" % mode_name)
    txt.append(                                                             \
        "    (void)QUEX_LEXEME_NULL;\n"                                     \
        "    (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n"                 \
        "    QUEX_ERROR_EXIT(\"Unreachable code has been reached.\\n\");\n") 

    ## This was once we did not know ... if there was a goto to the initial state or not.
    ## txt += "        goto %s;\n" % label.get(StateMachineName, InitialStateIndex)
    if len(ModeNameList) != 0 and not SingleModeAnalyzerF: 
        L = max(map(lambda name: len(name), ModeNameList))
        for name in ModeNameList:
            txt.append("#   undef %s\n" % name)

    txt.append("#   undef self\n")
    txt.append("}\n")

    txt.append("#include <quex/code_base/temporary_macros_off>\n")
    return txt
Ejemplo n.º 27
0
def __parse_option(fh, new_mode):
    def get_pattern_object(SM):
        if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
        else:                         result = SM
        result = hopcroft.do(result, CreateNewStateMachineF=False)
        return Pattern(result, AllowStateMachineTrafoF=True)

    identifier = read_option_start(fh)
    if identifier is None: return False

    verify_word_in_list(identifier, mode_option_info_db.keys(),
                        "mode option", fh.name, get_current_line_info_number(fh))

    if identifier == "skip":
        # A skipper 'eats' characters at the beginning of a pattern that belong
        # to a specified set of characters. A useful application is most probably
        # the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
        # implement a very effective way to skip these regions.
        pattern_str, trigger_set = regular_expression.parse_character_set(fh, PatternStringF=True)
        skip_whitespace(fh)

        if fh.read(1) != ">":
            error_msg("missing closing '>' for mode option '%s'." % identifier, fh)

        if trigger_set.is_empty():
            error_msg("Empty trigger set for skipper." % identifier, fh)

        # TriggerSet skipping is implemented the following way: As soon as one element of the 
        # trigger set appears, the state machine enters the 'trigger set skipper section'.
        # Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
        # NOTE: The correspondent CodeFragment for skipping is created in 'implement_skippers(...)'
        pattern_sm  = StateMachine()
        pattern_sm.add_transition(pattern_sm.init_state_index, trigger_set, AcceptanceF=True)

        # Skipper code is to be generated later
        action = GeneratedCode(skip_character_set.do, 
                               FileName = fh.name, 
                               LineN    = get_current_line_info_number(fh))
        action.data["character_set"] = trigger_set

        new_mode.add_match(pattern_str, action, get_pattern_object(pattern_sm), 
                           Comment=E_SpecialPatterns.SKIP)

        return True

    elif identifier in ["skip_range", "skip_nested_range"]:
        # A non-nesting skipper can contain a full fledged regular expression as opener,
        # since it only effects the trigger. Not so the nested range skipper-see below.

        # -- opener
        skip_whitespace(fh)
        if identifier == "skip_nested_range":
            # Nested range state machines only accept 'strings' not state machines
            opener_str, opener_sequence = __parse_string(fh, "Opener pattern for 'skip_nested_range'")
            opener_sm = StateMachine.from_sequence(opener_sequence)
        else:
            opener_str, opener_pattern = regular_expression.parse(fh)
            opener_sm = opener_pattern.sm
            # For 'range skipping' the opener sequence is not needed, only the opener state
            # machine is webbed into the pattern matching state machine.
            opener_sequence       = None

        skip_whitespace(fh)

        # -- closer
        closer_str, closer_sequence = __parse_string(fh, "Closing pattern for 'skip_range' or 'skip_nested_range'")
        skip_whitespace(fh)
        if fh.read(1) != ">":
            error_msg("missing closing '>' for mode option '%s'" % identifier, fh)

        # Skipper code is to be generated later
        generator_function, comment = { 
                "skip_range":        (skip_range.do,        E_SpecialPatterns.SKIP_RANGE),
                "skip_nested_range": (skip_nested_range.do, E_SpecialPatterns.SKIP_NESTED_RANGE),
        }[identifier]
        action = GeneratedCode(generator_function,
                               FileName = fh.name, 
                               LineN    = get_current_line_info_number(fh))

        action.data["opener_sequence"] = opener_sequence
        action.data["closer_sequence"] = closer_sequence
        action.data["mode_name"]       = new_mode.name

        new_mode.add_match(opener_str, action, get_pattern_object(opener_sm), Comment=comment)

        return True
        
    elif identifier == "indentation":
        value = indentation_setup.do(fh)

        # Enter 'Newline' and 'Suppressed Newline' as matches into the engine.
        # Similar to skippers, the indentation count is then triggered by the newline.
        # -- Suppressed Newline = Suppressor followed by Newline,
        #    then newline does not trigger indentation counting.
        suppressed_newline_pattern_str = ""
        if value.newline_suppressor_state_machine.get() is not None:
            suppressed_newline_pattern_str = \
                  "(" + value.newline_suppressor_state_machine.pattern_string() + ")" \
                + "(" + value.newline_state_machine.pattern_string() + ")"
                                           
            suppressed_newline_sm = \
                sequentialize.do([value.newline_suppressor_state_machine.get(),
                                  value.newline_state_machine.get()])
                 
            FileName = value.newline_suppressor_state_machine.file_name
            LineN    = value.newline_suppressor_state_machine.line_n
            # Go back to start.
            code = UserCodeFragment("goto %s;" % get_label("$start", U=True), FileName, LineN)

            new_mode.add_match(suppressed_newline_pattern_str, code, 
                               get_pattern_object(suppressed_newline_sm),
                               Comment=E_SpecialPatterns.SUPPRESSED_INDENTATION_NEWLINE)

        # When there is an empty line, then there shall be no indentation count on it.
        # Here comes the trick: 
        #
        #      Let               newline         
        #      be defined as:    newline ([space]* newline])*
        # 
        # This way empty lines are eating away before the indentation count is activated.

        # -- 'space'
        x0 = StateMachine()
        x0.add_transition(x0.init_state_index, value.indentation_count_character_set(), 
                          AcceptanceF=True)
        # -- '[space]*'
        x1 = repeat.do(x0)
        # -- '[space]* newline'
        x2 = sequentialize.do([x1, value.newline_state_machine.get()])
        # -- '([space]* newline)*'
        x3 = repeat.do(x2)
        # -- 'newline ([space]* newline)*'
        x4 = sequentialize.do([value.newline_state_machine.get(), x3])
        # -- nfa to dfa; hopcroft optimization
        sm = beautifier.do(x4)

        FileName = value.newline_state_machine.file_name
        LineN    = value.newline_state_machine.line_n
        action   = GeneratedCode(indentation_counter.do, FileName, LineN)

        action.data["indentation_setup"] = value

        new_mode.add_match(value.newline_state_machine.pattern_string(), action, 
                           get_pattern_object(sm), 
                           Comment=E_SpecialPatterns.INDENTATION_NEWLINE)

        # Announce the mode to which the setup belongs
        value.set_containing_mode_name(new_mode.name)
    else:
        value = read_option_value(fh)

    # The 'verify_word_in_list()' call must have ensured that the following holds
    assert mode_option_info_db.has_key(identifier)

    # Is the option of the appropriate value?
    option_info = mode_option_info_db[identifier]
    if option_info.domain is not None and value not in option_info.domain:
        error_msg("Tried to set value '%s' for option '%s'. " % (value, identifier) + \
                  "Though, possible for this option are only: %s." % repr(option_info.domain)[1:-1], fh)

    # Finally, set the option
    new_mode.add_option(identifier, value)

    return True
Ejemplo n.º 28
0
def get_skipper(TriggerSet):
    """This function implements simple 'skipping' in the sense of passing by
       characters that belong to a given set of characters--the TriggerSet.
    """
    global template_str
    assert TriggerSet.__class__.__name__ == "NumberSet"
    assert not TriggerSet.is_empty()

    LanguageDB = Setup.language_db

    skipper_index   = sm_index.get()

    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.
    transition_map = TransitionMap() # (don't worry about 'drop-out-ranges' etc.)
    transition_map.add_transition(TriggerSet, skipper_index)
    # On buffer limit code, the skipper must transit to a dedicated reloader

    goto_loop_str = [LanguageDB.INPUT_P_INCREMENT(), " goto _%s_LOOP;\n" % skipper_index]
    trigger_map = transition_map.get_trigger_map()
    for i, info in enumerate(trigger_map):
        interval, target = info
        if target == E_StateIndices.DROP_OUT: continue
        trigger_map[i] = (interval, TextTransitionCode(goto_loop_str))

    iteration_code = []
    transition_block.do(iteration_code, 
                        trigger_map, 
                        skipper_index, 
                        E_EngineTypes.ELSE,
                        GotoReload_Str="goto %s;" % get_label("$reload", skipper_index))

    tmp = []
    LanguageDB.COMMENT(tmp, "Skip any character in " + TriggerSet.get_utf8_string())
    comment_str = "".join(tmp)

    # Line and column number counting
    prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
    epilog = __lc_counting_replacements(epilog_txt, TriggerSet)

    prolog = blue_print(prolog,
                        [
                         ["$$DELIMITER_COMMENT$$",              comment_str],
                         ["$$SKIPPER_INDEX$$",                  "%i" % skipper_index],
                         ["$$INPUT_GET$$",                      LanguageDB.ACCESS_INPUT()],
                        ])

    epilog = blue_print(epilog,
                        [
                         ["$$INPUT_P_INCREMENT$$",              LanguageDB.INPUT_P_INCREMENT()],
                         ["$$INPUT_P_DECREMENT$$",              LanguageDB.INPUT_P_DECREMENT()],
                         ["$$IF_INPUT_EQUAL_DELIMITER_0$$",     LanguageDB.IF_INPUT("==", "SkipDelimiter$$SKIPPER_INDEX$$[0]")],
                         ["$$ENDIF$$",                          LanguageDB.END_IF()],
                         ["$$LOOP_REENTRANCE$$",                LanguageDB.LABEL(skipper_index)],
                         ["$$BUFFER_LIMIT_CODE$$",              LanguageDB.BUFFER_LIMIT_CODE],
                         ["$$RELOAD$$",                         get_label("$reload", skipper_index)],
                         ["$$DROP_OUT_DIRECT$$",                get_label("$drop-out", skipper_index, U=True)],
                         ["$$SKIPPER_INDEX$$",                  "%i" % skipper_index],
                         ["$$LABEL_TERMINAL_EOF$$",             "QUEX_LABEL(%i)" % get_address("$terminal-EOF", U=True)],
                         ["$$LABEL_REF_AFTER_RELOAD$$",         "QUEX_LABEL(%i)" % get_address("$skipper-reload", skipper_index, U=True)],
                         ["$$GOTO_TERMINAL_EOF$$",              get_label("$terminal-EOF", U=True)],
                         ["$$LABEL_AFTER_RELOAD$$",             get_label("$skipper-reload", skipper_index)],
                         # When things were skipped, no change to acceptance flags or modes has
                         # happend. One can jump immediately to the start without re-entry preparation.
                         ["$$GOTO_START$$",                     get_label("$start", U=True)], 
                         ["$$MARK_LEXEME_START$$",              LanguageDB.LEXEME_START_SET()],
                        ])

    code = [ prolog ]
    code.extend(iteration_code)
    code.append(epilog)

    local_variable_db = {}
    variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")

    return code, local_variable_db