def require_data(TState, TheAnalyzer):
"""Requires all variables which are necessary to implement the TemplateState.
NOTE: The target schemes are required from inside 'require_scheme_variable()'
in quex.engine.generator.mega_state.core.py
"""
variable_db.require("state_key")
get_label("$state-router", U=True) # Ensure reference of state router
def require_data(TState, TheAnalyzer):
"""Requires all variables which are necessary to implement the TemplateState.
NOTE: The target schemes are required from inside 'require_scheme_variable()'
in quex.engine.generator.mega_state.core.py
"""
variable_db.require("state_key")
get_label("$state-router", U=True) # Ensure reference of state router
def __header_definitions(LanguageDB, OnAfterMatchStr):
txt = __header_definitions_txt
txt = txt.replace("$$GOTO_START_PREPARATION$$", get_label("$re-start", U=True))
if len(OnAfterMatchStr) != 0: txt += __return_with_on_after_match
else: txt += __return_without_on_after_match
return txt
def drop_out_scheme_implementation(txt, TheState, TheAnalyzer, StateKeyString, DebugString):
"""DropOut Section:
The drop out section is the place where we come if the transition map
does not trigger to another state. We also land here if the reload fails.
The routing to the different drop-outs of the related states happens by
means of a switch statement, e.g.
_4711: /* Address of the drop out */
switch( state_key ) {
case 0:
... drop out of state 815 ...
case 1:
... drop out of state 541 ...
}
The switch statement is not necessary if all drop outs are the same,
of course.
"""
LanguageDB = Setup.language_db
# (*) Central Label for the Templates Drop Out
# (The rules for having or not having a label here are complicated,
# so rely on the label's usage database.)
txt.append("%s:\n" % get_label("$drop-out", TheState.index))
txt.append(" %s\n" % DebugString)
def implement_prototype(StateIndices, TheAnalyzer):
# There **must** be at least one element, at this point in time
assert len(StateIndices) != 0
prototype_i = StateIndices.__iter__().next()
prototype = TheAnalyzer.state_db[prototype_i]
result = []
drop_out_coder.do(result, prototype, TheAnalyzer, \
DefineLabelF=False, MentionStateIndexF=False)
return result
# (*) Drop Out Section(s)
if TheState.drop_out.uniform_f:
# uniform drop outs => no 'switch-case' required
txt.extend(implement_prototype(TheState.implemented_state_index_list(), TheAnalyzer))
return
# non-uniform drop outs => route by 'state_key'
case_list = []
for drop_out, state_index_set in TheState.drop_out.iteritems():
# state keys related to drop out
state_key_list = map(lambda i: TheState.map_state_index_to_state_key(i), state_index_set)
# drop out action
# Implement drop-out for each state key. 'state_key_list' combines
# states that implement the same drop-out behavior. Same drop-outs
# are implemented only once.
case_list.append( (state_key_list, implement_prototype(state_index_set, TheAnalyzer)) )
case_txt = LanguageDB.SELECTION(StateKeyString, case_list)
LanguageDB.INDENT(case_txt)
txt.extend(case_txt)
def __header_definitions(LanguageDB, OnAfterMatchStr):
txt = __header_definitions_txt
txt = txt.replace("$$GOTO_START_PREPARATION$$",
get_label("$re-start", U=True))
if len(OnAfterMatchStr) != 0: txt += __return_with_on_after_match
else: txt += __return_without_on_after_match
return txt
def drop_out_scheme_implementation(txt, TheState, TheAnalyzer, StateKeyString, DebugString):
"""DropOut Section:
The drop out section is the place where we come if the transition map
does not trigger to another state. We also land here if the reload fails.
The routing to the different drop-outs of the related states happens by
means of a switch statement, e.g.
_4711: /* Address of the drop out */
switch( state_key ) {
case 0:
... drop out of state 815 ...
case 1:
... drop out of state 541 ...
}
The switch statement is not necessary if all drop outs are the same,
of course.
"""
LanguageDB = Setup.language_db
# (*) Central Label for the Templates Drop Out
# (The rules for having or not having a label here are complicated,
# so rely on the label's usage database.)
txt.append("%s:\n" % get_label("$drop-out", TheState.index))
txt.append(" %s\n" % DebugString)
def implement_prototype(StateIndices, TheAnalyzer):
# There **must** be at least one element, at this point in time
assert len(StateIndices) != 0
prototype_i = StateIndices.__iter__().next()
prototype = TheAnalyzer.state_db[prototype_i]
result = []
drop_out_coder.do(result, prototype, TheAnalyzer, \
DefineLabelF=False, MentionStateIndexF=False)
return result
# (*) Drop Out Section(s)
if TheState.drop_out.uniform_f:
# uniform drop outs => no 'switch-case' required
txt.extend(implement_prototype(TheState.implemented_state_index_list(), TheAnalyzer))
return
# non-uniform drop outs => route by 'state_key'
case_list = []
for drop_out, state_index_set in TheState.drop_out.iteritems():
# state keys related to drop out
state_key_list = map(lambda i: TheState.map_state_index_to_state_key(i), state_index_set)
# drop out action
# Implement drop-out for each state key. 'state_key_list' combines
# states that implement the same drop-out behavior. Same drop-outs
# are implemented only once.
case_list.append( (state_key_list, implement_prototype(state_index_set, TheAnalyzer)) )
case_txt = LanguageDB.SELECTION(StateKeyString, case_list)
LanguageDB.INDENT(case_txt)
txt.extend(case_txt)
def GOTO_RELOAD(self, StateIndex, InitStateIndexF, EngineType):
"""On reload a special section is entered that tries to reload data. Reload
has two possible results:
-- Data has been loaded: Now, a new input character can be determined
and the current transition map can be reentered. For convenience,
'RELOAD' expects to jump to right before the place where the input
pointer is adapted.
-- No data available to be loaded: Then the current state's drop-out
section must be entered. The forward init state immediate jumps
to 'end of stream'.
Thus: The reload behavior can be determined based on **one** state index.
The related drop-out label can be determined here.
"""
direction = {
E_EngineTypes.FORWARD: "FORWARD",
E_EngineTypes.BACKWARD_PRE_CONTEXT: "BACKWARD",
E_EngineTypes.BACKWARD_INPUT_POSITION: None,
E_EngineTypes.INDENTATION_COUNTER: "FORWARD",
# There is never a reload on backward input position detection.
# The lexeme to parse must lie inside the borders!
}[EngineType]
assert direction is not None, \
"There is no reload during BACKWARD_INPUT_POSITION detection."
# 'DoorIndex == 0' is the entry into the state without any actions.
on_success = get_address("$entry",
entry_action.DoorID(StateIndex, DoorIndex=0),
U=True)
if InitStateIndexF and EngineType == E_EngineTypes.FORWARD:
on_fail = get_address("$terminal-EOF", U=True)
else:
on_fail = get_address("$drop-out", StateIndex, U=True, R=True)
get_label("$state-router", U=True) # Mark as 'referenced'
get_label("$reload-%s" % direction, U=True) # ...
return "QUEX_GOTO_RELOAD_%s(%s, %s);" % (direction, on_success,
on_fail)
def GOTO_RELOAD(self, StateIndex, InitStateIndexF, EngineType):
"""On reload a special section is entered that tries to reload data. Reload
has two possible results:
-- Data has been loaded: Now, a new input character can be determined
and the current transition map can be reentered. For convenience,
'RELOAD' expects to jump to right before the place where the input
pointer is adapted.
-- No data available to be loaded: Then the current state's drop-out
section must be entered. The forward init state immediate jumps
to 'end of stream'.
Thus: The reload behavior can be determined based on **one** state index.
The related drop-out label can be determined here.
"""
direction = {
E_EngineTypes.FORWARD: "FORWARD",
E_EngineTypes.BACKWARD_PRE_CONTEXT: "BACKWARD",
E_EngineTypes.BACKWARD_INPUT_POSITION: None,
E_EngineTypes.INDENTATION_COUNTER: "FORWARD",
# There is never a reload on backward input position detection.
# The lexeme to parse must lie inside the borders!
}[EngineType]
assert direction is not None, \
"There is no reload during BACKWARD_INPUT_POSITION detection."
# 'DoorIndex == 0' is the entry into the state without any actions.
on_success = get_address("$entry", entry_action.DoorID(StateIndex, DoorIndex=0), U=True)
if InitStateIndexF and EngineType == E_EngineTypes.FORWARD:
on_fail = get_address("$terminal-EOF", U=True)
else:
on_fail = get_address("$drop-out", StateIndex, U=True, R=True)
get_label("$state-router", U=True) # Mark as 'referenced'
get_label("$reload-%s" % direction, U=True) # ...
return "QUEX_GOTO_RELOAD_%s(%s, %s);" % (direction, on_success, on_fail)
def get_terminal_code(AcceptanceID, pattern_action_info, LanguageDB):
pattern = pattern_action_info.pattern()
#
action_code = pattern_action_info.action().get_code()
# (*) The 'normal' terminal state can also be reached by the terminal
# router and, thus, **must** restore the acceptance input position. This is so,
# because when the 'goto last_acceptance' is triggered the 'last_acceptance'
# may lay backwards and needs to be restored.
result = []
for letter in pattern_action_info.pattern_string():
if letter in ['\\', '"', '\n', '\t', '\r', '\a', '\v']:
result.append("\\")
result.append(letter)
safe_pattern = "".join(result)
input_position_search_backward_str = ""
if pattern.input_position_search_backward_sm is not None:
# Pseudo Ambiguous Post Contexts:
# (Retrieving the input position for the next run)
# -- Requires that the end of the core pattern is to be searched! One
# cannot simply restore some stored input position.
# -- The pseudo-ambiguous post condition is translated into a 'normal'
# pattern. However, after a match a backward detection of the end
# of the core pattern is done. Here, we first need to go to the point
# where the 'normal' pattern ended, then we can do a backward detection.
bipd_id = pattern.input_position_search_backward_sm.get_id()
bipd_str = " goto %s;\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_DETECTOR(
bipd_id)
# After having finished the analyzis, enter the terminal code, here.
bipd_str += "%s:\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_RETURN(
bipd_id)
input_position_search_backward_str = bipd_str
txt = [
"\nTERMINAL_%i:\n" % AcceptanceID, input_position_search_backward_str,
" __quex_debug(\"* terminal %i: %s\\n\");\n" %
(AcceptanceID, safe_pattern), action_code, "\n",
" goto %s;\n" % get_label("$re-start", U=True)
]
return txt
def get_terminal_code(AcceptanceID, pattern_action_info, LanguageDB):
pattern = pattern_action_info.pattern()
#
action_code = pattern_action_info.action().get_code()
# (*) The 'normal' terminal state can also be reached by the terminal
# router and, thus, **must** restore the acceptance input position. This is so,
# because when the 'goto last_acceptance' is triggered the 'last_acceptance'
# may lay backwards and needs to be restored.
result = []
for letter in pattern_action_info.pattern_string():
if letter in ['\\', '"', '\n', '\t', '\r', '\a', '\v']:
result.append("\\")
result.append(letter)
safe_pattern = "".join(result)
input_position_search_backward_str = ""
if pattern.input_position_search_backward_sm is not None:
# Pseudo Ambiguous Post Contexts:
# (Retrieving the input position for the next run)
# -- Requires that the end of the core pattern is to be searched! One
# cannot simply restore some stored input position.
# -- The pseudo-ambiguous post condition is translated into a 'normal'
# pattern. However, after a match a backward detection of the end
# of the core pattern is done. Here, we first need to go to the point
# where the 'normal' pattern ended, then we can do a backward detection.
bipd_id = pattern.input_position_search_backward_sm.get_id()
bipd_str = " goto %s;\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_DETECTOR(bipd_id)
# After having finished the analyzis, enter the terminal code, here.
bipd_str += "%s:\n" % LanguageDB.LABEL_NAME_BACKWARD_INPUT_POSITION_RETURN(bipd_id)
input_position_search_backward_str = bipd_str
txt = [
"\nTERMINAL_%i:\n" % AcceptanceID,
input_position_search_backward_str,
" __quex_debug(\"* terminal %i: %s\\n\");\n" % (AcceptanceID, safe_pattern),
action_code, "\n",
" goto %s;\n" % get_label("$re-start", U=True)
]
return txt
def get_info(StateIndexList):
# In some strange cases, a 'dummy' state router is required so that
# 'goto __STATE_ROUTER;' does not reference a non-existing label. Then,
# we return an empty text array.
if len(StateIndexList) == 0: return []
# Make sure, that for every state the 'drop-out' state is also mentioned
result = [None] * len(StateIndexList)
for i, index in enumerate(StateIndexList):
assert type(index) != str
if index >= 0:
# Transition to state entry
code = "goto %s; " % get_label_of_address(index)
result[i] = (index, code)
else:
# Transition to a templates 'drop-out'
code = "goto " + get_label("$drop-out", - index) + "; "
result[i] = (get_address("$drop-out", - index), code)
return result
def get_info(StateIndexList):
# In some strange cases, a 'dummy' state router is required so that
# 'goto __STATE_ROUTER;' does not reference a non-existing label. Then,
# we return an empty text array.
if len(StateIndexList) == 0: return []
# Make sure, that for every state the 'drop-out' state is also mentioned
result = [None] * len(StateIndexList)
for i, index in enumerate(StateIndexList):
assert type(index) != str
if index >= 0:
# Transition to state entry
code = "goto %s; " % get_label_of_address(index)
result[i] = (index, code)
else:
# Transition to a templates 'drop-out'
code = "goto " + get_label("$drop-out", -index) + "; "
result[i] = (get_address("$drop-out", -index), code)
return result
def __terminal_states(StateMachineName, action_db, OnFailureAction, EndOfStreamAction,
PreConditionIDList, LanguageDB, VariableDB, OnAfterMatchStr, LexemeNullObjectName):
"""NOTE: During backward-lexing, for a pre-condition, there is not need for terminal
states, since only the flag 'pre-condition fulfilled is raised.
"""
# (*) specific terminal states of patterns (entered from acceptance states)
specific_terminal_states = []
for pattern_id, pattern_action_info in action_db.items():
if pattern_id in PreConditionIDList: continue
code = get_terminal_code(pattern_id, pattern_action_info, LanguageDB)
specific_terminal_states.extend(code)
delete_pre_context_flags = []
for pre_context_sm_id in PreConditionIDList:
delete_pre_context_flags.append(" ")
delete_pre_context_flags.append(LanguageDB.ASSIGN("pre_context_%s_fulfilled_f" % __nice(pre_context_sm_id), 0))
# If there is at least a single terminal, the the 're-entry' preparation must be accomplished
if len(action_db) != 0: get_label("$re-start", U=True)
# -- execute 'on_failure' pattern action
# -- goto initial state
end_of_stream_code_action_str = EndOfStreamAction.action().get_code()
# -- FAILURE ACTION: Under 'normal' circumstances the on_failure action is simply to be executed
# since the 'get_forward()' incremented the 'current' pointer.
# HOWEVER, when end of file has been reached the 'current' pointer has to
# be reset so that the initial state can drop out on the buffer limit code
# and then transit to the end of file action.
# NOTE: It is possible that 'miss' happens after a chain of characters appeared. In any case the input
# pointer must be setup right after the lexeme start. This way, the lexer becomes a new chance as
# soon as possible.
on_failure = __terminal_on_failure_prolog(LanguageDB)
msg = OnFailureAction.action().get_code()
on_failure.append(msg)
prolog = blue_print(__terminal_state_prolog,
[
["$$LEXEME_LENGTH$$", LanguageDB.LEXEME_LENGTH()],
["$$INPUT_P$$", LanguageDB.INPUT_P()],
["$$LEXEME_NULL_OBJECT$$", LexemeNullObjectName],
]
)
router = Address("$terminal-router", None,
[
blue_print(__terminal_router_prolog_str,
[
["$$TERMINAL_FAILURE-REF$$", "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")],
["$$TERMINAL_FAILURE$$", get_label("$terminal-FAILURE")],
]),
# DO NOT 'U=True' for the state router. This is done automatically if
# 'goto reload' is used.
get_label("$state-router"), ";",
__terminal_router_epilog_str,
])
epilog = blue_print(__terminal_state_epilog,
[
["$$FAILURE_ACTION$$", "".join(on_failure)],
["$$END_OF_STREAM_ACTION$$", end_of_stream_code_action_str],
["$$TERMINAL_END_OF_STREAM-DEF$$", get_label("$terminal-EOF")],
["$$TERMINAL_FAILURE-DEF$$", get_label("$terminal-FAILURE")],
["$$STATE_MACHINE_NAME$$", StateMachineName],
["$$GOTO_START_PREPARATION$$", get_label("$re-start", U=True)],
])
reset_last_acceptance_str = ""
if VariableDB.has_key("last_acceptance"):
reset_last_acceptance_str = "last_acceptance = $$TERMINAL_FAILURE-REF$$; /* TERMINAL: FAILURE */"
return_preparation = ""
if OnAfterMatchStr != "":
return_preparation = blue_print(__return_preparation_str,
[["$$ON_AFTER_MATCH$$", OnAfterMatchStr]])
reentry_preparation = blue_print(__reentry_preparation_str,
[["$$REENTRY_PREPARATION$$", get_label("$re-start")],
["$$DELETE_PRE_CONDITION_FULLFILLED_FLAGS$$", "".join(delete_pre_context_flags)],
["$$GOTO_START$$", get_label("$start", U=True)],
["$$ON_AFTER_MATCH$$", OnAfterMatchStr],
["$$RESET_LAST_ACCEPTANCE$$", reset_last_acceptance_str],
["$$COMMENT_ON_POST_CONTEXT_INITIALIZATION$$", comment_on_post_context_position_init_str],
["$$TERMINAL_FAILURE-REF$$", "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")],
])
txt = []
txt.append(router)
txt.append(prolog)
txt.extend(specific_terminal_states)
txt.append(epilog)
txt.append(return_preparation)
txt.append(reentry_preparation)
return txt
def __terminal_states(StateMachineName, action_db, OnFailureAction,
EndOfStreamAction, PreConditionIDList, LanguageDB,
VariableDB, OnAfterMatchStr, LexemeNullObjectName):
"""NOTE: During backward-lexing, for a pre-condition, there is not need for terminal
states, since only the flag 'pre-condition fulfilled is raised.
"""
# (*) specific terminal states of patterns (entered from acceptance states)
specific_terminal_states = []
for pattern_id, pattern_action_info in action_db.items():
if pattern_id in PreConditionIDList: continue
code = get_terminal_code(pattern_id, pattern_action_info, LanguageDB)
specific_terminal_states.extend(code)
delete_pre_context_flags = []
for pre_context_sm_id in PreConditionIDList:
delete_pre_context_flags.append(" ")
delete_pre_context_flags.append(
LanguageDB.ASSIGN(
"pre_context_%s_fulfilled_f" % __nice(pre_context_sm_id), 0))
# If there is at least a single terminal, the the 're-entry' preparation must be accomplished
if len(action_db) != 0: get_label("$re-start", U=True)
# -- execute 'on_failure' pattern action
# -- goto initial state
end_of_stream_code_action_str = EndOfStreamAction.action().get_code()
# -- FAILURE ACTION: Under 'normal' circumstances the on_failure action is simply to be executed
# since the 'get_forward()' incremented the 'current' pointer.
# HOWEVER, when end of file has been reached the 'current' pointer has to
# be reset so that the initial state can drop out on the buffer limit code
# and then transit to the end of file action.
# NOTE: It is possible that 'miss' happens after a chain of characters appeared. In any case the input
# pointer must be setup right after the lexeme start. This way, the lexer becomes a new chance as
# soon as possible.
on_failure = __terminal_on_failure_prolog(LanguageDB)
msg = OnFailureAction.action().get_code()
on_failure.append(msg)
prolog = blue_print(__terminal_state_prolog, [
["$$LEXEME_LENGTH$$", LanguageDB.LEXEME_LENGTH()],
["$$INPUT_P$$", LanguageDB.INPUT_P()],
["$$LEXEME_NULL_OBJECT$$", LexemeNullObjectName],
])
router = Address(
"$terminal-router",
None,
[
blue_print(__terminal_router_prolog_str, [
[
"$$TERMINAL_FAILURE-REF$$",
"QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")
],
["$$TERMINAL_FAILURE$$",
get_label("$terminal-FAILURE")],
]),
# DO NOT 'U=True' for the state router. This is done automatically if
# 'goto reload' is used.
get_label("$state-router"),
";",
__terminal_router_epilog_str,
])
epilog = blue_print(__terminal_state_epilog, [
["$$FAILURE_ACTION$$", "".join(on_failure)],
["$$END_OF_STREAM_ACTION$$", end_of_stream_code_action_str],
["$$TERMINAL_END_OF_STREAM-DEF$$",
get_label("$terminal-EOF")],
["$$TERMINAL_FAILURE-DEF$$",
get_label("$terminal-FAILURE")],
["$$STATE_MACHINE_NAME$$", StateMachineName],
["$$GOTO_START_PREPARATION$$",
get_label("$re-start", U=True)],
])
reset_last_acceptance_str = ""
if VariableDB.has_key("last_acceptance"):
reset_last_acceptance_str = "last_acceptance = $$TERMINAL_FAILURE-REF$$; /* TERMINAL: FAILURE */"
return_preparation = ""
if OnAfterMatchStr != "":
return_preparation = blue_print(
__return_preparation_str,
[["$$ON_AFTER_MATCH$$", OnAfterMatchStr]])
reentry_preparation = blue_print(__reentry_preparation_str, [
["$$REENTRY_PREPARATION$$",
get_label("$re-start")],
[
"$$DELETE_PRE_CONDITION_FULLFILLED_FLAGS$$",
"".join(delete_pre_context_flags)
],
["$$GOTO_START$$", get_label("$start", U=True)],
["$$ON_AFTER_MATCH$$", OnAfterMatchStr],
["$$RESET_LAST_ACCEPTANCE$$", reset_last_acceptance_str],
[
"$$COMMENT_ON_POST_CONTEXT_INITIALIZATION$$",
comment_on_post_context_position_init_str
],
[
"$$TERMINAL_FAILURE-REF$$",
"QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")
],
])
txt = []
txt.append(router)
txt.append(prolog)
txt.extend(specific_terminal_states)
txt.append(epilog)
txt.append(return_preparation)
txt.append(reentry_preparation)
return txt
def GOTO_DROP_OUT(self, StateIndex):
return "goto %s;" % get_label("$drop-out", StateIndex, U=True, R=True)
def __analyzer_function(StateMachineName,
Setup,
variable_definitions,
function_body,
ModeNameList=[]):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
if a stand-alone lexical engine is required (without the
complete mode-handling framework of quex).
"""
LanguageDB = Setup.language_db
SingleModeAnalyzerF = Setup.single_mode_analyzer_f
txt = [
"#include <quex/code_base/temporary_macros_on>\n",
__function_signature.replace("$$STATE_MACHINE_NAME$$",
StateMachineName),
]
txt.extend(variable_definitions)
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
for name in ModeNameList:
txt.append("# define %s%s (QUEX_NAME(%s))\n" %
(name, " " * (L - len(name)), name))
txt.extend([
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
" || defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
])
txt.append(get_label("$start") + ":\n")
# -- entry to the actual function body
txt.append(" %s\n" % LanguageDB.LEXEME_START_SET())
txt.append(" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n")
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.append(
"\n" \
" /* Prevent compiler warning 'unused variable': use variables once in a part of the code*/\n" \
" /* that is never reached (and deleted by the compiler anyway).*/\n")
# Mode Names are defined as macros, so the following is not necessary.
# for mode_name in ModeNameList:
# txt.append(" (void)%s;\n" % mode_name)
txt.append( \
" (void)QUEX_LEXEME_NULL;\n" \
" (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n" \
" QUEX_ERROR_EXIT(\"Unreachable code has been reached.\\n\");\n")
## This was once we did not know ... if there was a goto to the initial state or not.
## txt += " goto %s;\n" % label.get(StateMachineName, InitialStateIndex)
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
for name in ModeNameList:
txt.append("# undef %s\n" % name)
txt.append("# undef self\n")
txt.append("}\n")
txt.append("#include <quex/code_base/temporary_macros_off>\n")
return txt
def get_skipper(OpenerSequence,
CloserSequence,
Mode=None,
IndentationCounterTerminalID=None,
OnSkipRangeOpenStr=""):
assert OpenerSequence.__class__ == list
assert len(OpenerSequence) >= 1
assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
assert CloserSequence.__class__ == list
assert len(CloserSequence) >= 1
assert map(type, CloserSequence) == [int] * len(CloserSequence)
assert OpenerSequence != CloserSequence
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
opener_length = len(OpenerSequence)
closer_str, closer_comment_str = get_character_sequence(CloserSequence)
closer_length = len(CloserSequence)
if not end_delimiter_is_subset_of_indentation_counter_newline(
Mode, CloserSequence):
goto_after_end_of_skipping_str = LanguageDB.GOTO(
E_StateIndices.ANALYZER_REENTRY)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID is not None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(
IndentationCounterTerminalID)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)
local_variable_db = {}
variable_db.enter(local_variable_db,
"reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
# variable_db.enter(local_variable_db, "text_end")
variable_db.enter(local_variable_db, "counter")
variable_db.enter(local_variable_db,
"Skipper%i_Opener",
"{ %s }" % opener_str,
ElementN=opener_length,
Index=skipper_index)
variable_db.enter(local_variable_db,
"Skipper%i_OpenerEnd",
"Skipper%i_Opener + (ptrdiff_t)%i" %
(skipper_index, opener_length),
Index=skipper_index)
variable_db.enter(local_variable_db,
"Skipper%i_Opener_it",
"0x0",
Index=skipper_index)
variable_db.enter(local_variable_db,
"Skipper%i_Closer",
"{ %s }" % closer_str,
ElementN=closer_length,
Index=skipper_index)
variable_db.enter(local_variable_db,
"Skipper%i_CloserEnd",
"Skipper%i_Closer + (ptrdiff_t)%i" %
(skipper_index, closer_length),
Index=skipper_index)
variable_db.enter(local_variable_db,
"Skipper%i_Closer_it",
"0x0",
Index=skipper_index)
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
code_str = blue_print(
template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER_LENGTH$$", "%i" % opener_length],
["$$INPUT_P_INCREMENT$$",
LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
LanguageDB.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$GOTO_RELOAD$$",
get_label("$reload", skipper_index)],
[
"$$INPUT_P_TO_LEXEME_START$$",
LanguageDB.INPUT_P_TO_LEXEME_START()
],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$",
LanguageDB.GOTO(skipper_index)],
["$$MARK_LEXEME_START$$",
LanguageDB.LEXEME_START_SET()],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$",
line_column_counter_in_loop()],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return code_str, local_variable_db
def get_skipper(OpenerSequence, CloserSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert OpenerSequence.__class__ == list
assert len(OpenerSequence) >= 1
assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
assert CloserSequence.__class__ == list
assert len(CloserSequence) >= 1
assert map(type, CloserSequence) == [int] * len(CloserSequence)
assert OpenerSequence != CloserSequence
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
opener_length = len(OpenerSequence)
closer_str, closer_comment_str = get_character_sequence(CloserSequence)
closer_length = len(CloserSequence)
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, CloserSequence):
goto_after_end_of_skipping_str = LanguageDB.GOTO(E_StateIndices.ANALYZER_REENTRY)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID is not None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(IndentationCounterTerminalID)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)
local_variable_db = {}
variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
# variable_db.enter(local_variable_db, "text_end")
variable_db.enter(local_variable_db, "counter")
variable_db.enter(local_variable_db, "Skipper%i_Opener", "{ %s }" % opener_str, ElementN=opener_length,
Index = skipper_index)
variable_db.enter(local_variable_db, "Skipper%i_OpenerEnd",
"Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length),
Index = skipper_index)
variable_db.enter(local_variable_db, "Skipper%i_Opener_it", "0x0",
Index = skipper_index)
variable_db.enter(local_variable_db, "Skipper%i_Closer", "{ %s }" % closer_str, ElementN=closer_length,
Index = skipper_index)
variable_db.enter(local_variable_db, "Skipper%i_CloserEnd",
"Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length),
Index = skipper_index)
variable_db.enter(local_variable_db, "Skipper%i_Closer_it", "0x0",
Index = skipper_index)
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
code_str = blue_print(template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER_LENGTH$$", "%i" % opener_length],
["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)],
["$$INPUT_P_TO_LEXEME_START$$", LanguageDB.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", LanguageDB.GOTO(skipper_index)],
["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop()],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return code_str, local_variable_db
def __parse_option(fh, new_mode):
def get_pattern_object(SM):
if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
else: result = SM
result = hopcroft.do(result, CreateNewStateMachineF=False)
return Pattern(result, AllowStateMachineTrafoF=True)
identifier = read_option_start(fh)
if identifier is None: return False
verify_word_in_list(identifier, mode_option_info_db.keys(), "mode option",
fh.name, get_current_line_info_number(fh))
if identifier == "skip":
# A skipper 'eats' characters at the beginning of a pattern that belong
# to a specified set of characters. A useful application is most probably
# the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
# implement a very effective way to skip these regions.
pattern_str, trigger_set = regular_expression.parse_character_set(
fh, PatternStringF=True)
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'." % identifier,
fh)
if trigger_set.is_empty():
error_msg("Empty trigger set for skipper." % identifier, fh)
# TriggerSet skipping is implemented the following way: As soon as one element of the
# trigger set appears, the state machine enters the 'trigger set skipper section'.
# Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
# NOTE: The correspondent CodeFragment for skipping is created in 'implement_skippers(...)'
pattern_sm = StateMachine()
pattern_sm.add_transition(pattern_sm.init_state_index,
trigger_set,
AcceptanceF=True)
# Skipper code is to be generated later
action = GeneratedCode(skip_character_set.do,
FileName=fh.name,
LineN=get_current_line_info_number(fh))
action.data["character_set"] = trigger_set
new_mode.add_match(pattern_str,
action,
get_pattern_object(pattern_sm),
Comment=E_SpecialPatterns.SKIP)
return True
elif identifier in ["skip_range", "skip_nested_range"]:
# A non-nesting skipper can contain a full fledged regular expression as opener,
# since it only effects the trigger. Not so the nested range skipper-see below.
# -- opener
skip_whitespace(fh)
if identifier == "skip_nested_range":
# Nested range state machines only accept 'strings' not state machines
opener_str, opener_sequence = __parse_string(
fh, "Opener pattern for 'skip_nested_range'")
opener_sm = StateMachine.from_sequence(opener_sequence)
else:
opener_str, opener_pattern = regular_expression.parse(fh)
opener_sm = opener_pattern.sm
# For 'range skipping' the opener sequence is not needed, only the opener state
# machine is webbed into the pattern matching state machine.
opener_sequence = None
skip_whitespace(fh)
# -- closer
closer_str, closer_sequence = __parse_string(
fh, "Closing pattern for 'skip_range' or 'skip_nested_range'")
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'" % identifier,
fh)
# Skipper code is to be generated later
generator_function, comment = {
"skip_range": (skip_range.do, E_SpecialPatterns.SKIP_RANGE),
"skip_nested_range":
(skip_nested_range.do, E_SpecialPatterns.SKIP_NESTED_RANGE),
}[identifier]
action = GeneratedCode(generator_function,
FileName=fh.name,
LineN=get_current_line_info_number(fh))
action.data["opener_sequence"] = opener_sequence
action.data["closer_sequence"] = closer_sequence
action.data["mode_name"] = new_mode.name
new_mode.add_match(opener_str,
action,
get_pattern_object(opener_sm),
Comment=comment)
return True
elif identifier == "indentation":
value = indentation_setup.do(fh)
# Enter 'Newline' and 'Suppressed Newline' as matches into the engine.
# Similar to skippers, the indentation count is then triggered by the newline.
# -- Suppressed Newline = Suppressor followed by Newline,
# then newline does not trigger indentation counting.
suppressed_newline_pattern_str = ""
if value.newline_suppressor_state_machine.get() is not None:
suppressed_newline_pattern_str = \
"(" + value.newline_suppressor_state_machine.pattern_string() + ")" \
+ "(" + value.newline_state_machine.pattern_string() + ")"
suppressed_newline_sm = \
sequentialize.do([value.newline_suppressor_state_machine.get(),
value.newline_state_machine.get()])
FileName = value.newline_suppressor_state_machine.file_name
LineN = value.newline_suppressor_state_machine.line_n
# Go back to start.
code = UserCodeFragment("goto %s;" % get_label("$start", U=True),
FileName, LineN)
new_mode.add_match(
suppressed_newline_pattern_str,
code,
get_pattern_object(suppressed_newline_sm),
Comment=E_SpecialPatterns.SUPPRESSED_INDENTATION_NEWLINE)
# When there is an empty line, then there shall be no indentation count on it.
# Here comes the trick:
#
# Let newline
# be defined as: newline ([space]* newline])*
#
# This way empty lines are eating away before the indentation count is activated.
# -- 'space'
x0 = StateMachine()
x0.add_transition(x0.init_state_index,
value.indentation_count_character_set(),
AcceptanceF=True)
# -- '[space]*'
x1 = repeat.do(x0)
# -- '[space]* newline'
x2 = sequentialize.do([x1, value.newline_state_machine.get()])
# -- '([space]* newline)*'
x3 = repeat.do(x2)
# -- 'newline ([space]* newline)*'
x4 = sequentialize.do([value.newline_state_machine.get(), x3])
# -- nfa to dfa; hopcroft optimization
sm = beautifier.do(x4)
FileName = value.newline_state_machine.file_name
LineN = value.newline_state_machine.line_n
action = GeneratedCode(indentation_counter.do, FileName, LineN)
action.data["indentation_setup"] = value
new_mode.add_match(value.newline_state_machine.pattern_string(),
action,
get_pattern_object(sm),
Comment=E_SpecialPatterns.INDENTATION_NEWLINE)
# Announce the mode to which the setup belongs
value.set_containing_mode_name(new_mode.name)
else:
value = read_option_value(fh)
# The 'verify_word_in_list()' call must have ensured that the following holds
assert mode_option_info_db.has_key(identifier)
# Is the option of the appropriate value?
option_info = mode_option_info_db[identifier]
if option_info.domain is not None and value not in option_info.domain:
error_msg("Tried to set value '%s' for option '%s'. " % (value, identifier) + \
"Though, possible for this option are only: %s." % repr(option_info.domain)[1:-1], fh)
# Finally, set the option
new_mode.add_option(identifier, value)
return True
def get_skipper(EndSequence,
Mode=None,
IndentationCounterTerminalID=None,
OnSkipRangeOpenStr=""):
assert type(EndSequence) == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
local_variable_db = {}
global template_str
LanguageDB = Setup.language_db
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
delimiter_length = len(EndSequence)
tmp = []
LanguageDB.COMMENT(
tmp, " Delimiter: %s" % delimiter_comment_str)
delimiter_comment_str = "".join(tmp)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " %s\n" % LanguageDB.ASSIGN(
"input", LanguageDB.INPUT_P_DEREFERENCE(i - 1))
txt += " %s" % LanguageDB.IF_INPUT(
"!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " %s" % LanguageDB.GOTO(skipper_index)
txt += " %s" % LanguageDB.END_IF()
delimiter_remainder_test_str = txt
if not end_delimiter_is_subset_of_indentation_counter_newline(
Mode, EndSequence):
goto_after_end_of_skipping_str = LanguageDB.GOTO(
E_StateIndices.ANALYZER_REENTRY)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID is not None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(
IndentationCounterTerminalID)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)
# The main part
code_str = blue_print(
template_str,
[
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$INPUT_P_INCREMENT$$",
LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
LanguageDB.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_ENTRY$$",
LanguageDB.GOTO(skipper_index)],
[
"$$INPUT_P_TO_LEXEME_START$$",
LanguageDB.INPUT_P_TO_LEXEME_START()
],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$MARK_LEXEME_START$$",
LanguageDB.LEXEME_START_SET()],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(
code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$",
get_label("$reload", skipper_index)]])
if reference_p_f:
variable_db.enter(local_variable_db,
"reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.enter(local_variable_db,
"Skipper%i",
"{ %s }" % delimiter_str,
delimiter_length,
Index=skipper_index)
variable_db.enter(local_variable_db,
"Skipper%iL",
"%i" % delimiter_length,
Index=skipper_index)
variable_db.enter(local_variable_db, "text_end")
return code_str, local_variable_db
def GOTO_DROP_OUT(self, StateIndex):
return "goto %s;" % get_label("$drop-out", StateIndex, U=True, R=True)
def do(Data):
"""The generated code is very similar to the 'skipper' code. It is to be executed
as soon as a 'real' newline arrived. Then it skips whitespace until the next
non-whitepace (also newline may trigger a 'stop').
Dependent on the setup the indentation is determined.
"""
IndentationSetup = Data["indentation_setup"]
assert IndentationSetup.__class__.__name__ == "IndentationSetup"
LanguageDB = Setup.language_db
Mode = None
if IndentationSetup.containing_mode_name() != "":
Mode = blackboard.mode_db[IndentationSetup.containing_mode_name()]
counter_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
trigger_map = []
# If the indentation consists only of spaces, than it is 'uniform' ...
if IndentationSetup.has_only_single_spaces():
# Count indentation/column at end of run;
# simply: current position - reference_p
character_set = IndentationSetup.space_db.values()[0]
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, counter_index])
# Reference Pointer: Define Variable, Initialize, determine how to subtact.
end_procedure = \
" me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n"
else:
# Count the indentation/column during the 'run'
# Add the space counters
for count, character_set in IndentationSetup.space_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("space", count, counter_index)])
# Add the grid counters
for count, character_set in IndentationSetup.grid_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])
# Reference Pointer: Not required.
# No subtraction 'current_position - reference_p'.
# (however, we pass 'reference_p' to indentation handler)
end_procedure = ""
# Bad character detection
if IndentationSetup.bad_character_set.get().is_empty() == False:
for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])
# Since we do not use a 'TransitionMap', there are some things we need
# to do by hand.
arrange_trigger_map(trigger_map)
local_variable_db = { "reference_p" :
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0")
}
init_reference_p = " reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n" + \
" me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"
iteration_code = []
transition_block.do(iteration_code, trigger_map,
StateIndex = counter_index,
EngineType = E_EngineTypes.INDENTATION_COUNTER,
GotoReload_Str = "goto %s;" % get_label("$reload", counter_index))
tmp = []
LanguageDB.COMMENT(tmp, "Skip whitespace at line begin; count indentation.")
comment_str = "".join(tmp)
# NOTE: Line and column number counting is off
# -- No newline can occur
# -- column number = indentation at the end of the process
end_procedure += " __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
if Mode is None or Mode.default_indentation_handler_sufficient():
end_procedure += " QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
else:
# Definition of '%s_on_indentation' in mode_classes.py.
end_procedure += " QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" \
% Mode.name
# The finishing touch
prolog = blue_print(prolog_txt,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$INIT_REFERENCE_POINTER$$", init_reference_p],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
])
# The finishing touch
epilog = blue_print(epilog_txt,
[
["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "SkipDelimiter$$COUNTER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$LOOP_REENTRANCE$$", LanguageDB.LABEL(counter_index)],
["$$IF_INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB.IF_INPUT("==", LanguageDB.BUFFER_LIMIT_CODE)],
["$$RELOAD$$", get_label("$reload", counter_index)],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
["$$LEXEME_START_SET_TO_REF$$", LanguageDB.LEXEME_START_SET("reference_p")],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$END_PROCEDURE$$", end_procedure],
["$$BAD_CHARACTER_HANDLING$$", get_bad_character_handler(Mode, IndentationSetup, counter_index)],
])
txt = [prolog]
txt.extend(iteration_code)
# txt.append(Address("$drop-out", counter_index))
txt.append("\n")
txt.append(epilog)
return txt, local_variable_db
def do(Data):
"""The generated code is very similar to the 'skipper' code. It is to be executed
as soon as a 'real' newline arrived. Then it skips whitespace until the next
non-whitepace (also newline may trigger a 'stop').
Dependent on the setup the indentation is determined.
"""
IndentationSetup = Data["indentation_setup"]
assert IndentationSetup.__class__.__name__ == "IndentationSetup"
LanguageDB = Setup.language_db
Mode = None
if IndentationSetup.containing_mode_name() != "":
Mode = blackboard.mode_db[IndentationSetup.containing_mode_name()]
counter_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
trigger_map = []
# If the indentation consists only of spaces, than it is 'uniform' ...
if IndentationSetup.has_only_single_spaces():
# Count indentation/column at end of run;
# simply: current position - reference_p
character_set = IndentationSetup.space_db.values()[0]
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, counter_index])
# Reference Pointer: Define Variable, Initialize, determine how to subtact.
end_procedure = (
" me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n"
)
else:
# Count the indentation/column during the 'run'
# Add the space counters
for count, character_set in IndentationSetup.space_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("space", count, counter_index)])
# Add the grid counters
for count, character_set in IndentationSetup.grid_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])
# Reference Pointer: Not required.
# No subtraction 'current_position - reference_p'.
# (however, we pass 'reference_p' to indentation handler)
end_procedure = ""
# Bad character detection
if IndentationSetup.bad_character_set.get().is_empty() == False:
for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])
# Since we do not use a 'TransitionMap', there are some things we need
# to do by hand.
arrange_trigger_map(trigger_map)
local_variable_db = {
"reference_p": Variable(
"reference_p", "QUEX_TYPE_CHARACTER_POSITION", None, "(QUEX_TYPE_CHARACTER_POSITION)0x0"
)
}
init_reference_p = (
" reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n"
+ " me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"
)
iteration_code = []
transition_block.do(
iteration_code,
trigger_map,
StateIndex=counter_index,
EngineType=E_EngineTypes.INDENTATION_COUNTER,
GotoReload_Str="goto %s;" % get_label("$reload", counter_index),
)
tmp = []
LanguageDB.COMMENT(tmp, "Skip whitespace at line begin; count indentation.")
comment_str = "".join(tmp)
# NOTE: Line and column number counting is off
# -- No newline can occur
# -- column number = indentation at the end of the process
end_procedure += " __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
if Mode is None or Mode.default_indentation_handler_sufficient():
end_procedure += " QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
else:
# Definition of '%s_on_indentation' in mode_classes.py.
end_procedure += " QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" % Mode.name
# The finishing touch
prolog = blue_print(
prolog_txt,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$INIT_REFERENCE_POINTER$$", init_reference_p],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
],
)
# The finishing touch
epilog = blue_print(
epilog_txt,
[
["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "SkipDelimiter$$COUNTER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$LOOP_REENTRANCE$$", LanguageDB.LABEL(counter_index)],
["$$IF_INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB.IF_INPUT("==", LanguageDB.BUFFER_LIMIT_CODE)],
["$$RELOAD$$", get_label("$reload", counter_index)],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
["$$LEXEME_START_SET_TO_REF$$", LanguageDB.LEXEME_START_SET("reference_p")],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$END_PROCEDURE$$", end_procedure],
["$$BAD_CHARACTER_HANDLING$$", get_bad_character_handler(Mode, IndentationSetup, counter_index)],
],
)
txt = [prolog]
txt.extend(iteration_code)
# txt.append(Address("$drop-out", counter_index))
txt.append("\n")
txt.append(epilog)
return txt, local_variable_db
def get_skipper(TriggerSet):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
global template_str
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap(
) # (don't worry about 'drop-out-ranges' etc.)
transition_map.add_transition(TriggerSet, skipper_index)
# On buffer limit code, the skipper must transit to a dedicated reloader
goto_loop_str = [
LanguageDB.INPUT_P_INCREMENT(),
" goto _%s_LOOP;\n" % skipper_index
]
trigger_map = transition_map.get_trigger_map()
for i, info in enumerate(trigger_map):
interval, target = info
if target == E_StateIndices.DROP_OUT: continue
trigger_map[i] = (interval, TextTransitionCode(goto_loop_str))
iteration_code = []
transition_block.do(iteration_code,
trigger_map,
skipper_index,
E_EngineTypes.ELSE,
GotoReload_Str="goto %s;" %
get_label("$reload", skipper_index))
tmp = []
LanguageDB.COMMENT(tmp,
"Skip any character in " + TriggerSet.get_utf8_string())
comment_str = "".join(tmp)
# Line and column number counting
prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
epilog = __lc_counting_replacements(epilog_txt, TriggerSet)
prolog = blue_print(prolog, [
["$$DELIMITER_COMMENT$$", comment_str],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
])
epilog = blue_print(
epilog,
[
["$$INPUT_P_INCREMENT$$",
LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
LanguageDB.INPUT_P_DECREMENT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB.IF_INPUT("==", "SkipDelimiter$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$LOOP_REENTRANCE$$",
LanguageDB.LABEL(skipper_index)],
["$$BUFFER_LIMIT_CODE$$", LanguageDB.BUFFER_LIMIT_CODE],
["$$RELOAD$$", get_label("$reload", skipper_index)],
[
"$$DROP_OUT_DIRECT$$",
get_label("$drop-out", skipper_index, U=True)
],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
[
"$$LABEL_TERMINAL_EOF$$",
"QUEX_LABEL(%i)" % get_address("$terminal-EOF", U=True)
],
[
"$$LABEL_REF_AFTER_RELOAD$$",
"QUEX_LABEL(%i)" %
get_address("$skipper-reload", skipper_index, U=True)
],
["$$GOTO_TERMINAL_EOF$$",
get_label("$terminal-EOF", U=True)],
[
"$$LABEL_AFTER_RELOAD$$",
get_label("$skipper-reload", skipper_index)
],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$MARK_LEXEME_START$$",
LanguageDB.LEXEME_START_SET()],
])
code = [prolog]
code.extend(iteration_code)
code.append(epilog)
local_variable_db = {}
variable_db.enter(local_variable_db,
"reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code, local_variable_db
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert type(EndSequence) == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
local_variable_db = {}
global template_str
LanguageDB = Setup.language_db
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
delimiter_length = len(EndSequence)
tmp = []
LanguageDB.COMMENT(tmp, " Delimiter: %s" % delimiter_comment_str)
delimiter_comment_str = "".join(tmp)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " %s\n" % LanguageDB.ASSIGN("input", LanguageDB.INPUT_P_DEREFERENCE(i - 1))
txt += " %s" % LanguageDB.IF_INPUT("!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " %s" % LanguageDB.GOTO(skipper_index)
txt += " %s" % LanguageDB.END_IF()
delimiter_remainder_test_str = txt
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence):
goto_after_end_of_skipping_str = LanguageDB.GOTO(E_StateIndices.ANALYZER_REENTRY)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID is not None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(IndentationCounterTerminalID)
if OnSkipRangeOpenStr != "":
on_skip_range_open_str = OnSkipRangeOpenStr
else:
on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)
# The main part
code_str = blue_print(
template_str,
[
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$ENTRY$$", LanguageDB.LABEL(skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_ENTRY$$", LanguageDB.GOTO(skipper_index)],
["$$INPUT_P_TO_LEXEME_START$$", LanguageDB.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
],
)
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(
code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]],
)
if reference_p_f:
variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.enter(local_variable_db, "Skipper%i", "{ %s }" % delimiter_str, delimiter_length, Index=skipper_index)
variable_db.enter(local_variable_db, "Skipper%iL", "%i" % delimiter_length, Index=skipper_index)
variable_db.enter(local_variable_db, "text_end")
return code_str, local_variable_db
def __analyzer_function(StateMachineName, Setup,
variable_definitions, function_body, ModeNameList=[]):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
if a stand-alone lexical engine is required (without the
complete mode-handling framework of quex).
"""
LanguageDB = Setup.language_db
SingleModeAnalyzerF = Setup.single_mode_analyzer_f
txt = [
"#include <quex/code_base/temporary_macros_on>\n",
__function_signature.replace("$$STATE_MACHINE_NAME$$", StateMachineName),
]
txt.extend(variable_definitions)
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
for name in ModeNameList:
txt.append("# define %s%s (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name))
txt.extend([
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
" || defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
])
txt.append(get_label("$start") + ":\n")
# -- entry to the actual function body
txt.append(" %s\n" % LanguageDB.LEXEME_START_SET())
txt.append(" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n")
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.append(
"\n" \
" /* Prevent compiler warning 'unused variable': use variables once in a part of the code*/\n" \
" /* that is never reached (and deleted by the compiler anyway).*/\n")
# Mode Names are defined as macros, so the following is not necessary.
# for mode_name in ModeNameList:
# txt.append(" (void)%s;\n" % mode_name)
txt.append( \
" (void)QUEX_LEXEME_NULL;\n" \
" (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n" \
" QUEX_ERROR_EXIT(\"Unreachable code has been reached.\\n\");\n")
## This was once we did not know ... if there was a goto to the initial state or not.
## txt += " goto %s;\n" % label.get(StateMachineName, InitialStateIndex)
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
for name in ModeNameList:
txt.append("# undef %s\n" % name)
txt.append("# undef self\n")
txt.append("}\n")
txt.append("#include <quex/code_base/temporary_macros_off>\n")
return txt
def __parse_option(fh, new_mode):
def get_pattern_object(SM):
if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
else: result = SM
result = hopcroft.do(result, CreateNewStateMachineF=False)
return Pattern(result, AllowStateMachineTrafoF=True)
identifier = read_option_start(fh)
if identifier is None: return False
verify_word_in_list(identifier, mode_option_info_db.keys(),
"mode option", fh.name, get_current_line_info_number(fh))
if identifier == "skip":
# A skipper 'eats' characters at the beginning of a pattern that belong
# to a specified set of characters. A useful application is most probably
# the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
# implement a very effective way to skip these regions.
pattern_str, trigger_set = regular_expression.parse_character_set(fh, PatternStringF=True)
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'." % identifier, fh)
if trigger_set.is_empty():
error_msg("Empty trigger set for skipper." % identifier, fh)
# TriggerSet skipping is implemented the following way: As soon as one element of the
# trigger set appears, the state machine enters the 'trigger set skipper section'.
# Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
# NOTE: The correspondent CodeFragment for skipping is created in 'implement_skippers(...)'
pattern_sm = StateMachine()
pattern_sm.add_transition(pattern_sm.init_state_index, trigger_set, AcceptanceF=True)
# Skipper code is to be generated later
action = GeneratedCode(skip_character_set.do,
FileName = fh.name,
LineN = get_current_line_info_number(fh))
action.data["character_set"] = trigger_set
new_mode.add_match(pattern_str, action, get_pattern_object(pattern_sm),
Comment=E_SpecialPatterns.SKIP)
return True
elif identifier in ["skip_range", "skip_nested_range"]:
# A non-nesting skipper can contain a full fledged regular expression as opener,
# since it only effects the trigger. Not so the nested range skipper-see below.
# -- opener
skip_whitespace(fh)
if identifier == "skip_nested_range":
# Nested range state machines only accept 'strings' not state machines
opener_str, opener_sequence = __parse_string(fh, "Opener pattern for 'skip_nested_range'")
opener_sm = StateMachine.from_sequence(opener_sequence)
else:
opener_str, opener_pattern = regular_expression.parse(fh)
opener_sm = opener_pattern.sm
# For 'range skipping' the opener sequence is not needed, only the opener state
# machine is webbed into the pattern matching state machine.
opener_sequence = None
skip_whitespace(fh)
# -- closer
closer_str, closer_sequence = __parse_string(fh, "Closing pattern for 'skip_range' or 'skip_nested_range'")
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'" % identifier, fh)
# Skipper code is to be generated later
generator_function, comment = {
"skip_range": (skip_range.do, E_SpecialPatterns.SKIP_RANGE),
"skip_nested_range": (skip_nested_range.do, E_SpecialPatterns.SKIP_NESTED_RANGE),
}[identifier]
action = GeneratedCode(generator_function,
FileName = fh.name,
LineN = get_current_line_info_number(fh))
action.data["opener_sequence"] = opener_sequence
action.data["closer_sequence"] = closer_sequence
action.data["mode_name"] = new_mode.name
new_mode.add_match(opener_str, action, get_pattern_object(opener_sm), Comment=comment)
return True
elif identifier == "indentation":
value = indentation_setup.do(fh)
# Enter 'Newline' and 'Suppressed Newline' as matches into the engine.
# Similar to skippers, the indentation count is then triggered by the newline.
# -- Suppressed Newline = Suppressor followed by Newline,
# then newline does not trigger indentation counting.
suppressed_newline_pattern_str = ""
if value.newline_suppressor_state_machine.get() is not None:
suppressed_newline_pattern_str = \
"(" + value.newline_suppressor_state_machine.pattern_string() + ")" \
+ "(" + value.newline_state_machine.pattern_string() + ")"
suppressed_newline_sm = \
sequentialize.do([value.newline_suppressor_state_machine.get(),
value.newline_state_machine.get()])
FileName = value.newline_suppressor_state_machine.file_name
LineN = value.newline_suppressor_state_machine.line_n
# Go back to start.
code = UserCodeFragment("goto %s;" % get_label("$start", U=True), FileName, LineN)
new_mode.add_match(suppressed_newline_pattern_str, code,
get_pattern_object(suppressed_newline_sm),
Comment=E_SpecialPatterns.SUPPRESSED_INDENTATION_NEWLINE)
# When there is an empty line, then there shall be no indentation count on it.
# Here comes the trick:
#
# Let newline
# be defined as: newline ([space]* newline])*
#
# This way empty lines are eating away before the indentation count is activated.
# -- 'space'
x0 = StateMachine()
x0.add_transition(x0.init_state_index, value.indentation_count_character_set(),
AcceptanceF=True)
# -- '[space]*'
x1 = repeat.do(x0)
# -- '[space]* newline'
x2 = sequentialize.do([x1, value.newline_state_machine.get()])
# -- '([space]* newline)*'
x3 = repeat.do(x2)
# -- 'newline ([space]* newline)*'
x4 = sequentialize.do([value.newline_state_machine.get(), x3])
# -- nfa to dfa; hopcroft optimization
sm = beautifier.do(x4)
FileName = value.newline_state_machine.file_name
LineN = value.newline_state_machine.line_n
action = GeneratedCode(indentation_counter.do, FileName, LineN)
action.data["indentation_setup"] = value
new_mode.add_match(value.newline_state_machine.pattern_string(), action,
get_pattern_object(sm),
Comment=E_SpecialPatterns.INDENTATION_NEWLINE)
# Announce the mode to which the setup belongs
value.set_containing_mode_name(new_mode.name)
else:
value = read_option_value(fh)
# The 'verify_word_in_list()' call must have ensured that the following holds
assert mode_option_info_db.has_key(identifier)
# Is the option of the appropriate value?
option_info = mode_option_info_db[identifier]
if option_info.domain is not None and value not in option_info.domain:
error_msg("Tried to set value '%s' for option '%s'. " % (value, identifier) + \
"Though, possible for this option are only: %s." % repr(option_info.domain)[1:-1], fh)
# Finally, set the option
new_mode.add_option(identifier, value)
return True
def get_skipper(TriggerSet):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
global template_str
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap() # (don't worry about 'drop-out-ranges' etc.)
transition_map.add_transition(TriggerSet, skipper_index)
# On buffer limit code, the skipper must transit to a dedicated reloader
goto_loop_str = [LanguageDB.INPUT_P_INCREMENT(), " goto _%s_LOOP;\n" % skipper_index]
trigger_map = transition_map.get_trigger_map()
for i, info in enumerate(trigger_map):
interval, target = info
if target == E_StateIndices.DROP_OUT: continue
trigger_map[i] = (interval, TextTransitionCode(goto_loop_str))
iteration_code = []
transition_block.do(iteration_code,
trigger_map,
skipper_index,
E_EngineTypes.ELSE,
GotoReload_Str="goto %s;" % get_label("$reload", skipper_index))
tmp = []
LanguageDB.COMMENT(tmp, "Skip any character in " + TriggerSet.get_utf8_string())
comment_str = "".join(tmp)
# Line and column number counting
prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
epilog = __lc_counting_replacements(epilog_txt, TriggerSet)
prolog = blue_print(prolog,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()],
])
epilog = blue_print(epilog,
[
["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "SkipDelimiter$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB.END_IF()],
["$$LOOP_REENTRANCE$$", LanguageDB.LABEL(skipper_index)],
["$$BUFFER_LIMIT_CODE$$", LanguageDB.BUFFER_LIMIT_CODE],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$DROP_OUT_DIRECT$$", get_label("$drop-out", skipper_index, U=True)],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$LABEL_TERMINAL_EOF$$", "QUEX_LABEL(%i)" % get_address("$terminal-EOF", U=True)],
["$$LABEL_REF_AFTER_RELOAD$$", "QUEX_LABEL(%i)" % get_address("$skipper-reload", skipper_index, U=True)],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
["$$LABEL_AFTER_RELOAD$$", get_label("$skipper-reload", skipper_index)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()],
])
code = [ prolog ]
code.extend(iteration_code)
code.append(epilog)
local_variable_db = {}
variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code, local_variable_db
