コード例 #1
0
ファイル: template_coder.py プロジェクト: jirkamarsik/quex
def __template_state(txt, TheTemplate, SMD):
    """Generate the template state that 'hosts' the templated states.
    """
    state = TheTemplate
    state_index = TheTemplate.core().state_index
    TriggerMap = state.transitions().get_trigger_map()

    if TheTemplate.uniform_state_entries_f():
        txt.extend(input_block.do(state_index, False, SMD))
        txt.extend(
            acceptance_info.do(state,
                               state_index,
                               SMD,
                               ForceSaveLastAcceptanceF=True))
    else:
        label_str = "    __quex_assert_no_passage();\n" + \
                    get_label("$entry", state_index) + ":\n"
        txt.append(label_str)

    state_index_str = None
    if not TheTemplate.uniform_state_entries_f():
        # Templates that need to implement more than one state need to return to
        # dedicated state entries, if the state entries are not uniform.
        state_index_str = "template_%i_map_state_key_to_state_index[template_state_key]" % state_index

    txt.extend(
        transition_block.do(TriggerMap,
                            state_index,
                            SMD,
                            ReturnToState_Str=state_index_str))

    txt.extend(drop_out.do(state, state_index, SMD))
コード例 #2
0
ファイル: template_coder.py プロジェクト: jirkamarsik/quex
def __template_state(txt, TheTemplate, SMD):
    """Generate the template state that 'hosts' the templated states.
    """
    state       = TheTemplate
    state_index = TheTemplate.core().state_index
    TriggerMap  = state.transitions().get_trigger_map()


    if TheTemplate.uniform_state_entries_f():
        txt.extend(input_block.do(state_index, False, SMD))
        txt.extend(acceptance_info.do(state, state_index, SMD, ForceSaveLastAcceptanceF=True))
    else:
        label_str = "    __quex_assert_no_passage();\n" + \
                    get_label("$entry", state_index) + ":\n"
        txt.append(label_str)

    state_index_str = None
    if not TheTemplate.uniform_state_entries_f():
        # Templates that need to implement more than one state need to return to
        # dedicated state entries, if the state entries are not uniform.
        state_index_str = "template_%i_map_state_key_to_state_index[template_state_key]" % state_index

    txt.extend(transition_block.do(TriggerMap, state_index, SMD, ReturnToState_Str=state_index_str))

    txt.extend(drop_out.do(state, state_index, SMD))
コード例 #3
0
ファイル: core.py プロジェクト: jirkamarsik/quex
def do(state, StateIdx, SMD=False):
    """Produces code for all state transitions. Programming language is determined
       by 'Language'.
    """    
    assert isinstance(state, State)
    assert SMD.__class__.__name__   == "StateMachineDecorator"
    assert len(state.transitions().get_epsilon_target_state_index_list()) == 0, \
           "Epsilon transition contained target states: state machine was not made a DFA!\n" + \
           "Epsilon target states = " + repr(state.transitions().get_epsilon_target_state_index_list())
    InitStateF = StateIdx == SMD.sm().init_state_index

    LanguageDB = Setup.language_db

    # (*) Dead End States 
    #     i.e. states with no further transitions.
    dead_end_state_info = SMD.dead_end_state_db().get(StateIdx)
    if dead_end_state_info != None:
        state_stub = __dead_end_state_stub(dead_end_state_info, SMD)
        # Some states do not need 'stubs' to terminal since they are straight
        # forward transitions to the terminal.
        if len(state_stub) == 0: return []
        return [ get_label("$entry", StateIdx), ":\n",
                "    ", LanguageDB["$debug-state"](StateIdx, SMD.forward_lexing_f())
               ] + state_stub 
        

    # (*) Normal States
    TriggerMap = state.transitions().get_trigger_map()
    assert TriggerMap != []  # Only dead end states have empty trigger maps.
    #                        # => Here, the trigger map cannot be empty.

    txt = []
    txt.extend(input_block.do(StateIdx, InitStateF, SMD))
    txt.extend(acceptance_info.do(state, StateIdx, SMD))
    txt.extend(transition_block.do(TriggerMap, StateIdx, SMD))
    txt.extend(drop_out.do(state, StateIdx, SMD))
    txt.extend(get_epilog(StateIdx, InitStateF, SMD))
    
    return txt 
コード例 #4
0
ファイル: core.py プロジェクト: jirkamarsik/quex
def do(state, StateIdx, SMD=False):
    """Produces code for all state transitions. Programming language is determined
       by 'Language'.
    """
    assert isinstance(state, State)
    assert SMD.__class__.__name__ == "StateMachineDecorator"
    assert len(state.transitions().get_epsilon_target_state_index_list()) == 0, \
           "Epsilon transition contained target states: state machine was not made a DFA!\n" + \
           "Epsilon target states = " + repr(state.transitions().get_epsilon_target_state_index_list())
    InitStateF = StateIdx == SMD.sm().init_state_index

    LanguageDB = Setup.language_db

    # (*) Dead End States
    #     i.e. states with no further transitions.
    dead_end_state_info = SMD.dead_end_state_db().get(StateIdx)
    if dead_end_state_info != None:
        state_stub = __dead_end_state_stub(dead_end_state_info, SMD)
        # Some states do not need 'stubs' to terminal since they are straight
        # forward transitions to the terminal.
        if len(state_stub) == 0: return []
        return [
            get_label("$entry", StateIdx), ":\n", "    ",
            LanguageDB["$debug-state"](StateIdx, SMD.forward_lexing_f())
        ] + state_stub

    # (*) Normal States
    TriggerMap = state.transitions().get_trigger_map()
    assert TriggerMap != []  # Only dead end states have empty trigger maps.
    #                        # => Here, the trigger map cannot be empty.

    txt = []
    txt.extend(input_block.do(StateIdx, InitStateF, SMD))
    txt.extend(acceptance_info.do(state, StateIdx, SMD))
    txt.extend(transition_block.do(TriggerMap, StateIdx, SMD))
    txt.extend(drop_out.do(state, StateIdx, SMD))
    txt.extend(get_epilog(StateIdx, InitStateF, SMD))

    return txt
コード例 #5
0
ファイル: paths_coder.py プロジェクト: jirkamarsik/quex
def __path_walker(txt, PathWalker, SMD):
    """Generates the path walker, that walks along the character sequence.
    """
    PathList     = PathWalker.path_list()
    Skeleton     = PathList[0].skeleton()
    PathWalkerID = PathWalker.core().state_index

    if PathWalker.uniform_state_entries_f():
        # (1) Input Block (get the new character)
        txt.extend(input_block.do(PathWalkerID, False, SMD))
        # (2) Acceptance information/Store Input positions
        txt.extend(acceptance_info.do(PathWalker, PathWalkerID, SMD, ForceSaveLastAcceptanceF=True))

    else:
        txt.append("    __quex_assert_no_passage();\n")
        txt.append(get_label("$entry", PathWalkerID) + ":\n")
        txt.append("    __quex_debug(\"path walker %i\");\n" % PathWalkerID)

    # (3) Transition Map

    # (3.1) The comparison with the path's current character
    #       If terminating zero is reached, the path's end state is entered.
    if PathWalker.uniform_state_entries_f():
        next_state = [ "goto %s;\n" % get_label_of_address(PathWalkerID, U=True) ]
        end_state = __end_state_router(PathWalker, SMD)
    else:
        next_state = [ __state_router(PathWalker, SMD) ]
        end_state  = ["        "] + next_state

    txt.append("    ")
    txt.append(LanguageDB["$if =="]("*path_iterator"))
    txt.append("        ")
    txt.append(LanguageDB["$increment"]("path_iterator"))
    txt.append("\n")
    txt.append("        ")
    txt.extend(next_state)
    txt.append("    ")
    txt.append(LanguageDB["$elseif"] \
               + LanguageDB["$=="]("*path_iterator", "QUEX_SETTING_PATH_TERMINATION_CODE") \
               + LanguageDB["$then"])
    txt.extend(end_state)
    txt.append("    ")
    txt.append(LanguageDB["$endif"])
    txt.append("\n")

    # (3.2) Transition map of the 'skeleton'        
    trigger_map = PathWalker.transitions().get_trigger_map()
    if len(trigger_map) == 0:
        # (This happens, for example, if there are only keywords and no 
        #  'overlaying' identifier pattern.)

        # Even if the skeleton/trigger map is empty there must be something
        # that catches the 'buffer limit code'. 
        # => Define an 'all drop out' trigger_map and then,
        # => Adapt the trigger map, so that the 'buffer limit' is an 
        #    isolated single interval.
        trigger_map = [ (Interval(-sys.maxint, sys.maxint), None) ]

    state_index_str = None
    if not PathWalker.uniform_state_entries_f():
        state_index_str = "path_walker_%i_state[path_iterator - path_walker_%i_base]" % (PathWalkerID, PathWalkerID)

    txt.extend(transition_block.do(trigger_map, PathWalkerID, SMD, ReturnToState_Str=state_index_str))

    # (4) The drop out (nothing matched)
    #     (Path iterator has not been increased yet)

    txt.extend(drop_out.do(PathWalker, PathWalkerID, SMD))

    return 
コード例 #6
0
def do(Data):
    """The generated code is very similar to the 'skipper' code. It is to be executed
       as soon as a 'real' newline arrived. Then it skips whitespace until the next 
       non-whitepace (also newline may trigger a 'stop'). 

       Dependent on the setup the indentation is determined.
    """
    IndentationSetup = Data["indentation_setup"]
    assert IndentationSetup.__class__.__name__ == "IndentationSetup"


    LanguageDB = Setup.language_db
    Mode = None
    if IndentationSetup.containing_mode_name() != "":
        Mode = lexer_mode.mode_db[IndentationSetup.containing_mode_name()]

    counter_index = sm_index.get()
    
    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.

    trigger_map = []
    # If the indentation consists only of spaces, than it is 'uniform' ...
    if IndentationSetup.has_only_single_spaces():
        # Count indentation/column at end of run;
        # simply: current position - reference_p

        character_set = IndentationSetup.space_db.values()[0]
        for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
            trigger_map.append([interval, counter_index])

        # Reference Pointer: Define Variable, Initialize, determine how to subtact.
        end_procedure = \
        "    me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n" 
    else:
        # Count the indentation/column during the 'run'

        # Add the space counters
        for count, character_set in IndentationSetup.space_db.items():
            for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
                trigger_map.append([interval, IndentationCounter("space", count, counter_index)])

        # Add the grid counters
        for count, character_set in IndentationSetup.grid_db.items():
            for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
                trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])

        # Reference Pointer: Not required.
        #                    No subtraction 'current_position - reference_p'.
        #                    (however, we pass 'reference_p' to indentation handler)
        end_procedure = "" 

    # Bad character detection
    if IndentationSetup.bad_character_set.get().is_empty() == False:
        for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
            trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])

    # Since we do not use a 'TransitionMap', there are some things we need 
    # to do by hand.
    arrange_trigger_map(trigger_map)

    local_variable_db = { "reference_p" : 
                          Variable("reference_p", 
                                   "QUEX_TYPE_CHARACTER_POSITION", 
                                   None, 
                                   "(QUEX_TYPE_CHARACTER_POSITION)0x0")
    }
    init_reference_p  = "    reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n" + \
                        "    me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"

    iteration_code = transition_block.do(trigger_map, 
                                         counter_index, 
                                         DSM=None, 
                                         GotoReload_Str="goto %s;" % get_label("$reload", counter_index))

    comment_str    = LanguageDB["$comment"]("Skip whitespace at line begin; count indentation.")

    # NOTE: Line and column number counting is off
    #       -- No newline can occur
    #       -- column number = indentation at the end of the process

    end_procedure += "    __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
    if Mode == None or Mode.default_indentation_handler_sufficient():
        end_procedure += "    QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
    else:
        # Definition of '%s_on_indentation' in mode_classes.py.
        end_procedure += "    QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" \
                         % Mode.name

    # The finishing touch
    prolog = blue_print(prolog_txt,
                         [
                           ["$$DELIMITER_COMMENT$$",              comment_str],
                           ["$$INIT_REFERENCE_POINTER$$",         init_reference_p],
                           ["$$COUNTER_INDEX$$",                  repr(counter_index)],
                           ["$$INPUT_GET$$",                      LanguageDB["$input/get"]],
                         ])

    # The finishing touch
    epilog = blue_print(epilog_txt,
                      [
                       ["$$INPUT_P_INCREMENT$$",              LanguageDB["$input/increment"]],
                       ["$$INPUT_P_DECREMENT$$",              LanguageDB["$input/decrement"]],
                       ["$$IF_INPUT_EQUAL_DELIMITER_0$$",     LanguageDB["$if =="]("SkipDelimiter$$COUNTER_INDEX$$[0]")],
                       ["$$ENDIF$$",                          LanguageDB["$endif"]],
                       ["$$LOOP_REENTRANCE$$",                get_label("$entry",  counter_index)],
                       ["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$",  LanguageDB["$BLC"]],
                       ["$$RELOAD$$",                         get_label("$reload", counter_index)],
                       ["$$COUNTER_INDEX$$",                  repr(counter_index)],
                       ["$$GOTO_TERMINAL_EOF$$",              get_label("$terminal-EOF", U=True)],
                       # When things were skipped, no change to acceptance flags or modes has
                       # happend. One can jump immediately to the start without re-entry preparation.
                       ["$$GOTO_START$$",                     get_label("$start", U=True)], 
                       ["$$END_PROCEDURE$$",                  end_procedure],
                       ["$$BAD_CHARACTER_HANDLING$$",         get_bad_character_handler(Mode, IndentationSetup, counter_index)],
                      ])

    txt = [prolog]
    txt.extend(iteration_code)
    # txt.append(Address("$drop-out", counter_index))
    txt.append("\n")
    txt.append(epilog)

    return txt, local_variable_db
コード例 #7
0
ファイル: character_set.py プロジェクト: jirkamarsik/quex
def get_skipper(TriggerSet):
    """This function implements simple 'skipping' in the sense of passing by
       characters that belong to a given set of characters--the TriggerSet.
    """
    global template_str
    assert TriggerSet.__class__.__name__ == "NumberSet"
    assert not TriggerSet.is_empty()

    LanguageDB = Setup.language_db

    skipper_index = sm_index.get()
    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.
    transition_map = TransitionMap() # (don't worry about 'drop-out-ranges' etc.)
    transition_map.add_transition(TriggerSet, skipper_index)
    # On buffer limit code, the skipper must transit to a dedicated reloader

    iteration_code = transition_block.do(transition_map.get_trigger_map(), 
                                         skipper_index, 
                                         DSM=None, 
                                         GotoReload_Str="goto %s;" % get_label("$reload", skipper_index))

    comment_str = LanguageDB["$comment"]("Skip any character in " + TriggerSet.get_utf8_string())

    # Line and column number counting
    prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
    epilog = __lc_counting_replacements(epilog_txt, TriggerSet)

    prolog = blue_print(prolog,
                        [
                         ["$$DELIMITER_COMMENT$$",              comment_str],
                         ["$$SKIPPER_INDEX$$",                  "%i" % skipper_index],
                         ["$$INPUT_GET$$",                      LanguageDB["$input/get"]],
                        ])

    epilog = blue_print(epilog,
                        [
                         ["$$INPUT_P_INCREMENT$$",              LanguageDB["$input/increment"]],
                         ["$$INPUT_P_DECREMENT$$",              LanguageDB["$input/decrement"]],
                         ["$$IF_INPUT_EQUAL_DELIMITER_0$$",     LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")],
                         ["$$ENDIF$$",                          LanguageDB["$endif"]],
                         ["$$LOOP_REENTRANCE$$",                get_label("$entry", skipper_index)],
                         ["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$",  LanguageDB["$BLC"]],
                         ["$$RELOAD$$",                         get_label("$reload", skipper_index)],
                         ["$$DROP_OUT_DIRECT$$",                get_label("$drop-out", skipper_index, U=True)],
                         ["$$SKIPPER_INDEX$$",                  "%i" % skipper_index],
                         ["$$GOTO_TERMINAL_EOF$$",              get_label("$terminal-EOF", U=True)],
                         # When things were skipped, no change to acceptance flags or modes has
                         # happend. One can jump immediately to the start without re-entry preparation.
                         ["$$GOTO_START$$",                     get_label("$start", U=True)], 
                         ["$$MARK_LEXEME_START$$",              LanguageDB["$mark-lexeme-start"]],
                        ])

    code = [ prolog ]
    code.extend(iteration_code)
    code.append(epilog)

    local_variable_db = {}
    local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
                     Variable("reference_p", 
                              "QUEX_TYPE_CHARACTER_POSITION", 
                              None,
                              "(QUEX_TYPE_CHARACTER_POSITION)0x0",
                              "QUEX_OPTION_COLUMN_NUMBER_COUNTING") 

    return code, local_variable_db
コード例 #8
0
ファイル: character_set.py プロジェクト: jirkamarsik/quex
def get_skipper(TriggerSet):
    """This function implements simple 'skipping' in the sense of passing by
       characters that belong to a given set of characters--the TriggerSet.
    """
    global template_str
    assert TriggerSet.__class__.__name__ == "NumberSet"
    assert not TriggerSet.is_empty()

    LanguageDB = Setup.language_db

    skipper_index = sm_index.get()
    # Mini trigger map:  [ trigger set ] --> loop start
    # That means: As long as characters of the trigger set appear, we go to the loop start.
    transition_map = TransitionMap(
    )  # (don't worry about 'drop-out-ranges' etc.)
    transition_map.add_transition(TriggerSet, skipper_index)
    # On buffer limit code, the skipper must transit to a dedicated reloader

    iteration_code = transition_block.do(transition_map.get_trigger_map(),
                                         skipper_index,
                                         DSM=None,
                                         GotoReload_Str="goto %s;" %
                                         get_label("$reload", skipper_index))

    comment_str = LanguageDB["$comment"]("Skip any character in " +
                                         TriggerSet.get_utf8_string())

    # Line and column number counting
    prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
    epilog = __lc_counting_replacements(epilog_txt, TriggerSet)

    prolog = blue_print(prolog, [
        ["$$DELIMITER_COMMENT$$", comment_str],
        ["$$SKIPPER_INDEX$$", "%i" % skipper_index],
        ["$$INPUT_GET$$", LanguageDB["$input/get"]],
    ])

    epilog = blue_print(
        epilog,
        [
            ["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
            ["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
            [
                "$$IF_INPUT_EQUAL_DELIMITER_0$$",
                LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")
            ],
            ["$$ENDIF$$", LanguageDB["$endif"]],
            ["$$LOOP_REENTRANCE$$",
             get_label("$entry", skipper_index)],
            ["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
            ["$$RELOAD$$", get_label("$reload", skipper_index)],
            [
                "$$DROP_OUT_DIRECT$$",
                get_label("$drop-out", skipper_index, U=True)
            ],
            ["$$SKIPPER_INDEX$$", "%i" % skipper_index],
            ["$$GOTO_TERMINAL_EOF$$",
             get_label("$terminal-EOF", U=True)],
            # When things were skipped, no change to acceptance flags or modes has
            # happend. One can jump immediately to the start without re-entry preparation.
            ["$$GOTO_START$$", get_label("$start", U=True)],
            ["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
        ])

    code = [prolog]
    code.extend(iteration_code)
    code.append(epilog)

    local_variable_db = {}
    local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
                     Variable("reference_p",
                              "QUEX_TYPE_CHARACTER_POSITION",
                              None,
                              "(QUEX_TYPE_CHARACTER_POSITION)0x0",
                              "QUEX_OPTION_COLUMN_NUMBER_COUNTING")

    return code, local_variable_db