def __template_state(txt, TheTemplate, SMD):
"""Generate the template state that 'hosts' the templated states.
"""
state = TheTemplate
state_index = TheTemplate.core().state_index
TriggerMap = state.transitions().get_trigger_map()
if TheTemplate.uniform_state_entries_f():
txt.extend(input_block.do(state_index, False, SMD))
txt.extend(
acceptance_info.do(state,
state_index,
SMD,
ForceSaveLastAcceptanceF=True))
else:
label_str = " __quex_assert_no_passage();\n" + \
get_label("$entry", state_index) + ":\n"
txt.append(label_str)
state_index_str = None
if not TheTemplate.uniform_state_entries_f():
# Templates that need to implement more than one state need to return to
# dedicated state entries, if the state entries are not uniform.
state_index_str = "template_%i_map_state_key_to_state_index[template_state_key]" % state_index
txt.extend(
transition_block.do(TriggerMap,
state_index,
SMD,
ReturnToState_Str=state_index_str))
txt.extend(drop_out.do(state, state_index, SMD))
def __template_state(txt, TheTemplate, SMD):
"""Generate the template state that 'hosts' the templated states.
"""
state = TheTemplate
state_index = TheTemplate.core().state_index
TriggerMap = state.transitions().get_trigger_map()
if TheTemplate.uniform_state_entries_f():
txt.extend(input_block.do(state_index, False, SMD))
txt.extend(acceptance_info.do(state, state_index, SMD, ForceSaveLastAcceptanceF=True))
else:
label_str = " __quex_assert_no_passage();\n" + \
get_label("$entry", state_index) + ":\n"
txt.append(label_str)
state_index_str = None
if not TheTemplate.uniform_state_entries_f():
# Templates that need to implement more than one state need to return to
# dedicated state entries, if the state entries are not uniform.
state_index_str = "template_%i_map_state_key_to_state_index[template_state_key]" % state_index
txt.extend(transition_block.do(TriggerMap, state_index, SMD, ReturnToState_Str=state_index_str))
txt.extend(drop_out.do(state, state_index, SMD))
def do(state, StateIdx, SMD=False):
"""Produces code for all state transitions. Programming language is determined
by 'Language'.
"""
assert isinstance(state, State)
assert SMD.__class__.__name__ == "StateMachineDecorator"
assert len(state.transitions().get_epsilon_target_state_index_list()) == 0, \
"Epsilon transition contained target states: state machine was not made a DFA!\n" + \
"Epsilon target states = " + repr(state.transitions().get_epsilon_target_state_index_list())
InitStateF = StateIdx == SMD.sm().init_state_index
LanguageDB = Setup.language_db
# (*) Dead End States
# i.e. states with no further transitions.
dead_end_state_info = SMD.dead_end_state_db().get(StateIdx)
if dead_end_state_info != None:
state_stub = __dead_end_state_stub(dead_end_state_info, SMD)
# Some states do not need 'stubs' to terminal since they are straight
# forward transitions to the terminal.
if len(state_stub) == 0: return []
return [ get_label("$entry", StateIdx), ":\n",
" ", LanguageDB["$debug-state"](StateIdx, SMD.forward_lexing_f())
] + state_stub
# (*) Normal States
TriggerMap = state.transitions().get_trigger_map()
assert TriggerMap != [] # Only dead end states have empty trigger maps.
# # => Here, the trigger map cannot be empty.
txt = []
txt.extend(input_block.do(StateIdx, InitStateF, SMD))
txt.extend(acceptance_info.do(state, StateIdx, SMD))
txt.extend(transition_block.do(TriggerMap, StateIdx, SMD))
txt.extend(drop_out.do(state, StateIdx, SMD))
txt.extend(get_epilog(StateIdx, InitStateF, SMD))
return txt
def do(state, StateIdx, SMD=False):
"""Produces code for all state transitions. Programming language is determined
by 'Language'.
"""
assert isinstance(state, State)
assert SMD.__class__.__name__ == "StateMachineDecorator"
assert len(state.transitions().get_epsilon_target_state_index_list()) == 0, \
"Epsilon transition contained target states: state machine was not made a DFA!\n" + \
"Epsilon target states = " + repr(state.transitions().get_epsilon_target_state_index_list())
InitStateF = StateIdx == SMD.sm().init_state_index
LanguageDB = Setup.language_db
# (*) Dead End States
# i.e. states with no further transitions.
dead_end_state_info = SMD.dead_end_state_db().get(StateIdx)
if dead_end_state_info != None:
state_stub = __dead_end_state_stub(dead_end_state_info, SMD)
# Some states do not need 'stubs' to terminal since they are straight
# forward transitions to the terminal.
if len(state_stub) == 0: return []
return [
get_label("$entry", StateIdx), ":\n", " ",
LanguageDB["$debug-state"](StateIdx, SMD.forward_lexing_f())
] + state_stub
# (*) Normal States
TriggerMap = state.transitions().get_trigger_map()
assert TriggerMap != [] # Only dead end states have empty trigger maps.
# # => Here, the trigger map cannot be empty.
txt = []
txt.extend(input_block.do(StateIdx, InitStateF, SMD))
txt.extend(acceptance_info.do(state, StateIdx, SMD))
txt.extend(transition_block.do(TriggerMap, StateIdx, SMD))
txt.extend(drop_out.do(state, StateIdx, SMD))
txt.extend(get_epilog(StateIdx, InitStateF, SMD))
return txt
def __path_walker(txt, PathWalker, SMD):
"""Generates the path walker, that walks along the character sequence.
"""
PathList = PathWalker.path_list()
Skeleton = PathList[0].skeleton()
PathWalkerID = PathWalker.core().state_index
if PathWalker.uniform_state_entries_f():
# (1) Input Block (get the new character)
txt.extend(input_block.do(PathWalkerID, False, SMD))
# (2) Acceptance information/Store Input positions
txt.extend(acceptance_info.do(PathWalker, PathWalkerID, SMD, ForceSaveLastAcceptanceF=True))
else:
txt.append(" __quex_assert_no_passage();\n")
txt.append(get_label("$entry", PathWalkerID) + ":\n")
txt.append(" __quex_debug(\"path walker %i\");\n" % PathWalkerID)
# (3) Transition Map
# (3.1) The comparison with the path's current character
# If terminating zero is reached, the path's end state is entered.
if PathWalker.uniform_state_entries_f():
next_state = [ "goto %s;\n" % get_label_of_address(PathWalkerID, U=True) ]
end_state = __end_state_router(PathWalker, SMD)
else:
next_state = [ __state_router(PathWalker, SMD) ]
end_state = [" "] + next_state
txt.append(" ")
txt.append(LanguageDB["$if =="]("*path_iterator"))
txt.append(" ")
txt.append(LanguageDB["$increment"]("path_iterator"))
txt.append("\n")
txt.append(" ")
txt.extend(next_state)
txt.append(" ")
txt.append(LanguageDB["$elseif"] \
+ LanguageDB["$=="]("*path_iterator", "QUEX_SETTING_PATH_TERMINATION_CODE") \
+ LanguageDB["$then"])
txt.extend(end_state)
txt.append(" ")
txt.append(LanguageDB["$endif"])
txt.append("\n")
# (3.2) Transition map of the 'skeleton'
trigger_map = PathWalker.transitions().get_trigger_map()
if len(trigger_map) == 0:
# (This happens, for example, if there are only keywords and no
# 'overlaying' identifier pattern.)
# Even if the skeleton/trigger map is empty there must be something
# that catches the 'buffer limit code'.
# => Define an 'all drop out' trigger_map and then,
# => Adapt the trigger map, so that the 'buffer limit' is an
# isolated single interval.
trigger_map = [ (Interval(-sys.maxint, sys.maxint), None) ]
state_index_str = None
if not PathWalker.uniform_state_entries_f():
state_index_str = "path_walker_%i_state[path_iterator - path_walker_%i_base]" % (PathWalkerID, PathWalkerID)
txt.extend(transition_block.do(trigger_map, PathWalkerID, SMD, ReturnToState_Str=state_index_str))
# (4) The drop out (nothing matched)
# (Path iterator has not been increased yet)
txt.extend(drop_out.do(PathWalker, PathWalkerID, SMD))
return
def do(Data):
"""The generated code is very similar to the 'skipper' code. It is to be executed
as soon as a 'real' newline arrived. Then it skips whitespace until the next
non-whitepace (also newline may trigger a 'stop').
Dependent on the setup the indentation is determined.
"""
IndentationSetup = Data["indentation_setup"]
assert IndentationSetup.__class__.__name__ == "IndentationSetup"
LanguageDB = Setup.language_db
Mode = None
if IndentationSetup.containing_mode_name() != "":
Mode = lexer_mode.mode_db[IndentationSetup.containing_mode_name()]
counter_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
trigger_map = []
# If the indentation consists only of spaces, than it is 'uniform' ...
if IndentationSetup.has_only_single_spaces():
# Count indentation/column at end of run;
# simply: current position - reference_p
character_set = IndentationSetup.space_db.values()[0]
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, counter_index])
# Reference Pointer: Define Variable, Initialize, determine how to subtact.
end_procedure = \
" me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n"
else:
# Count the indentation/column during the 'run'
# Add the space counters
for count, character_set in IndentationSetup.space_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("space", count, counter_index)])
# Add the grid counters
for count, character_set in IndentationSetup.grid_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])
# Reference Pointer: Not required.
# No subtraction 'current_position - reference_p'.
# (however, we pass 'reference_p' to indentation handler)
end_procedure = ""
# Bad character detection
if IndentationSetup.bad_character_set.get().is_empty() == False:
for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])
# Since we do not use a 'TransitionMap', there are some things we need
# to do by hand.
arrange_trigger_map(trigger_map)
local_variable_db = { "reference_p" :
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0")
}
init_reference_p = " reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n" + \
" me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"
iteration_code = transition_block.do(trigger_map,
counter_index,
DSM=None,
GotoReload_Str="goto %s;" % get_label("$reload", counter_index))
comment_str = LanguageDB["$comment"]("Skip whitespace at line begin; count indentation.")
# NOTE: Line and column number counting is off
# -- No newline can occur
# -- column number = indentation at the end of the process
end_procedure += " __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
if Mode == None or Mode.default_indentation_handler_sufficient():
end_procedure += " QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
else:
# Definition of '%s_on_indentation' in mode_classes.py.
end_procedure += " QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" \
% Mode.name
# The finishing touch
prolog = blue_print(prolog_txt,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$INIT_REFERENCE_POINTER$$", init_reference_p],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
])
# The finishing touch
epilog = blue_print(epilog_txt,
[
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("SkipDelimiter$$COUNTER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_REENTRANCE$$", get_label("$entry", counter_index)],
["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
["$$RELOAD$$", get_label("$reload", counter_index)],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$END_PROCEDURE$$", end_procedure],
["$$BAD_CHARACTER_HANDLING$$", get_bad_character_handler(Mode, IndentationSetup, counter_index)],
])
txt = [prolog]
txt.extend(iteration_code)
# txt.append(Address("$drop-out", counter_index))
txt.append("\n")
txt.append(epilog)
return txt, local_variable_db
def get_skipper(TriggerSet):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
global template_str
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap() # (don't worry about 'drop-out-ranges' etc.)
transition_map.add_transition(TriggerSet, skipper_index)
# On buffer limit code, the skipper must transit to a dedicated reloader
iteration_code = transition_block.do(transition_map.get_trigger_map(),
skipper_index,
DSM=None,
GotoReload_Str="goto %s;" % get_label("$reload", skipper_index))
comment_str = LanguageDB["$comment"]("Skip any character in " + TriggerSet.get_utf8_string())
# Line and column number counting
prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
epilog = __lc_counting_replacements(epilog_txt, TriggerSet)
prolog = blue_print(prolog,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
])
epilog = blue_print(epilog,
[
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_REENTRANCE$$", get_label("$entry", skipper_index)],
["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$DROP_OUT_DIRECT$$", get_label("$drop-out", skipper_index, U=True)],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
])
code = [ prolog ]
code.extend(iteration_code)
code.append(epilog)
local_variable_db = {}
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code, local_variable_db
def get_skipper(TriggerSet):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
global template_str
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap(
) # (don't worry about 'drop-out-ranges' etc.)
transition_map.add_transition(TriggerSet, skipper_index)
# On buffer limit code, the skipper must transit to a dedicated reloader
iteration_code = transition_block.do(transition_map.get_trigger_map(),
skipper_index,
DSM=None,
GotoReload_Str="goto %s;" %
get_label("$reload", skipper_index))
comment_str = LanguageDB["$comment"]("Skip any character in " +
TriggerSet.get_utf8_string())
# Line and column number counting
prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
epilog = __lc_counting_replacements(epilog_txt, TriggerSet)
prolog = blue_print(prolog, [
["$$DELIMITER_COMMENT$$", comment_str],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
])
epilog = blue_print(
epilog,
[
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_REENTRANCE$$",
get_label("$entry", skipper_index)],
["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
["$$RELOAD$$", get_label("$reload", skipper_index)],
[
"$$DROP_OUT_DIRECT$$",
get_label("$drop-out", skipper_index, U=True)
],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$GOTO_TERMINAL_EOF$$",
get_label("$terminal-EOF", U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
])
code = [prolog]
code.extend(iteration_code)
code.append(epilog)
local_variable_db = {}
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code, local_variable_db