def __frame(FunctionName, IteratorName, CodeTxt, DoorIdReturn, DoorIdBeyond):
txt = [ \
"#ifdef __QUEX_OPTION_COUNTER\n" \
+ "static void\n" \
+ "%s(QUEX_TYPE_ANALYZER* me, QUEX_TYPE_LEXATOM* LexemeBegin, QUEX_TYPE_LEXATOM* LexemeEnd)\n" \
% FunctionName \
+ "{\n" \
+ "# define self (*me)\n" \
+ "/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n"
+ "# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router())
]
# Following function refers to the global 'variable_db'
txt.append(Lng.VARIABLE_DEFINITIONS(variable_db))
txt.append(
" (void)me;\n"
" __QUEX_IF_COUNT_SHIFT_VALUES();\n"
" /* Allow LexemeBegin == LexemeEnd (e.g. END_OF_STREAM)\n"
" * => Caller does not need to check\n"
" * BUT, if so quit immediately after 'shift values'. */\n"
" __quex_assert(LexemeBegin <= LexemeEnd);\n"
" if(LexemeBegin == LexemeEnd) return;\n"
" %s = LexemeBegin;\n" % IteratorName
)
txt.extend(CodeTxt)
door_id_failure = DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE)
door_id_bad_lexatom = DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM)
txt.append(
"%s /* TERMINAL: BAD_LEXATOM */\n;\n" % Lng.LABEL(door_id_bad_lexatom)
+ "%s /* TERMINAL: FAILURE */\n%s\n" % (Lng.LABEL(door_id_failure), Lng.GOTO(DoorIdBeyond))
)
txt.append(
"%s:\n" % dial_db.get_label_by_door_id(DoorIdReturn) \
+ " /* Assert: lexeme in codec's character boundaries. */\n" \
+ " __quex_assert(%s == LexemeEnd);\n" % IteratorName \
+ " return;\n" \
+ "".join(generator.do_state_router()) \
+ "# undef self\n" \
+ "# undef QUEX_LABEL_STATE_ROUTER\n"
# If there is no MATCH_FAILURE, then DoorIdBeyond is still referenced as 'gotoed',
# but MATCH_FAILURE is never implemented, later on, because its DoorId is not
# referenced.
+ "# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n"
+ " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router())
+ " %s /* to BAD_LEXATOM */\n" % Lng.GOTO(DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM))
+ "# endif\n"
+ " /* Avoid compiler warning: Unused label for 'TERMINAL <BEYOND>' */\n" \
+ " %s\n" % Lng.GOTO(DoorIdBeyond) \
+ " %s\n" % Lng.GOTO(door_id_failure) \
+ " (void)target_state_index;\n"
+ " (void)target_state_else_index;\n"
+ "}\n" \
+ "#endif /* __QUEX_OPTION_COUNTER */\n"
)
return "".join(Lng.GET_PLAIN_STRINGS(txt))
def HEADER_DEFINITIONS(self):
return blue_print(cpp_header_definition_str, [
("$$CONTINUE_WITH_ON_AFTER_MATCH$$",
dial_db.get_label_by_door_id(
DoorID.continue_with_on_after_match())),
("$$RETURN_WITH_ON_AFTER_MATCH$$",
dial_db.get_label_by_door_id(
DoorID.return_with_on_after_match())),
])
def __frame(FunctionName, IteratorName, CodeTxt, DoorIdReturn, DoorIdBeyond):
txt = [ \
"#ifdef __QUEX_OPTION_COUNTER\n" \
+ "static void\n" \
+ "%s(QUEX_TYPE_ANALYZER* me, QUEX_TYPE_CHARACTER* LexemeBegin, QUEX_TYPE_CHARACTER* LexemeEnd)\n" \
% FunctionName \
+ "{\n" \
+ "# define self (*me)\n" \
+ "/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n"
+ "# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router())
]
# Following function refers to the global 'variable_db'
txt.append(Lng.VARIABLE_DEFINITIONS(variable_db))
txt.append(
" (void)me;\n"
" __QUEX_IF_COUNT_SHIFT_VALUES();\n"
" /* Allow LexemeBegin == LexemeEnd (e.g. END_OF_STREAM)\n"
" * => Caller does not need to check\n"
" * BUT, if so quit immediately after 'shift values'. */\n"
" __quex_assert(LexemeBegin <= LexemeEnd);\n"
" if(LexemeBegin == LexemeEnd) return;\n"
" %s = LexemeBegin;\n" % IteratorName
)
txt.extend(CodeTxt)
door_id_failure = DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE)
txt.append(
"%s /* TERMINAL: FAILURE */\n%s\n" % (Lng.LABEL(door_id_failure), Lng.GOTO(DoorIdBeyond))
)
txt.append(
"%s:\n" % dial_db.get_label_by_door_id(DoorIdReturn) \
+ " __quex_assert(%s == LexemeEnd); /* Otherwise, lexeme violates codec character boundaries. */\n" \
% IteratorName \
+ " return;\n" \
+ "".join(generator.do_state_router()) \
+ "# undef self\n" \
+ "# undef QUEX_LABEL_STATE_ROUTER\n"
# If there is no MATCH_FAILURE, then DoorIdBeyond is still referenced as 'gotoed',
# but MATCH_FAILURE is never implemented, later on, because its DoorId is not
# referenced.
+ "# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n"
+ " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router())
+ "# endif\n"
+ " /* Avoid compiler warning: Unused label for 'TERMINAL <BEYOND>' */\n" \
+ " %s\n" % Lng.GOTO(DoorIdBeyond) \
+ " %s\n" % Lng.GOTO(door_id_failure) \
+ " (void)target_state_index;\n"
+ " (void)target_state_else_index;\n"
+ "}\n" \
+ "#endif /* __QUEX_OPTION_COUNTER */\n"
)
return "".join(Lng.GET_PLAIN_STRINGS(txt))
def do_pre_context(SM, PreContextSmIdList):
"""Pre-context detecting state machine (backward).
---------------------------------------------------------------------------
Micro actions are: pre-context fullfilled_f
DropOut --> Begin of 'main' state machine.
BLC --> ReloadStateBackward
EndOfStream --> 'error'
Variables (potentially) required:
pre_context_fulfilled_f[N] --> Array of flags for pre-context
indication.
RETURNS: [0] generated code text
[1] reload state BACKWARD, to be generated later.
"""
if SM is None:
return [], None
txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT)
txt.append(
"\n%s:" %
dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check()))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START())
for sm_id in PreContextSmIdList:
variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id)
variable_db.require("input")
return txt, analyzer
def get_info(StateIndexList):
"""
NOTE: At least a 'dummy' state router is always equired so that 'goto
QUEX_STATE_ROUTER;' does not reference a non-existing label. Then, we
return an empty text array.
<fschaef9/13y10m15d: "Still true with current dial_db implementation?">
"""
if len(StateIndexList) == 0: return []
# Make sure, that for every state the 'drop-out' state is also mentioned
result = [None] * len(StateIndexList)
for i, index in enumerate(StateIndexList):
assert type(index) != str
if index >= 0:
# Transition to state entry
label = dial_db.get_label_by_address(index)
else:
assert False, "Is this still an issue?"
# Transition to a templates 'drop-out'
door_id = DoorID.drop_out(- index)
label = dial_db.get_label_by_door_id(door_id)
result[i] = (index, "goto %s; " % label)
dial_db.mark_label_as_gotoed(label)
return result
def get_info(StateIndexList):
"""
NOTE: At least a 'dummy' state router is always equired so that 'goto
QUEX_STATE_ROUTER;' does not reference a non-existing label. Then, we
return an empty text array.
<fschaef9/13y10m15d: "Still true with current dial_db implementation?">
"""
if len(StateIndexList) == 0: return []
# Make sure, that for every state the 'drop-out' state is also mentioned
result = [None] * len(StateIndexList)
for i, index in enumerate(StateIndexList):
assert type(index) != str
if index >= 0:
# Transition to state entry
label = dial_db.get_label_by_address(index)
else:
assert False, "Is this still an issue?"
# Transition to a templates 'drop-out'
door_id = DoorID.drop_out(-index)
label = dial_db.get_label_by_door_id(door_id)
result[i] = (index, "goto %s; " % label)
dial_db.mark_label_as_gotoed(label)
return result
def get_main_function(tm0, TranstionTxt, Codec):
def indent(Txt, N):
return (" " * N) + (Txt.replace("\n", "\n" + (" " * N)))
if Codec == "UTF8": qtc_str = "uint8_t"
else: qtc_str = "uint32_t"
input_preperation = get_read_preparation(codec)
entry_list = [(0 if interval.begin < 0 else interval.begin, target)
for interval, target in tm0]
entry_list.append((tm0[-1][0].begin, -1))
entry_list.append((0x1FFFF, -1))
expected_array = [
" { 0x%06X, %s },\n" % (begin, target)
for begin, target in entry_list
]
txt = main_template.replace("$$ENTRY_LIST$$", "".join(expected_array))
txt = txt.replace("$$QUEX_TYPE_LEXATOM$$", qtc_str)
txt = txt.replace("$$TRANSITION$$", indent(TranstionTxt, 4))
txt = txt.replace("$$PREPARE_INPUT$$", input_preperation)
door_id = DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM)
txt = txt.replace("$$ON_BAD_LEXATOM$$",
dial_db.get_label_by_door_id(door_id))
txt = txt.replace("MATCH_FAILURE", "((int)-1)")
return txt
def do_pre_context(SM, PreContextSmIdList):
"""Pre-context detecting state machine (backward).
---------------------------------------------------------------------------
Micro actions are: pre-context fullfilled_f
DropOut --> Begin of 'main' state machine.
BLC --> ReloadStateBackward
EndOfStream --> 'error'
Variables (potentially) required:
pre_context_fulfilled_f[N] --> Array of flags for pre-context
indication.
RETURNS: [0] generated code text
[1] reload state BACKWARD, to be generated later.
"""
if SM is None:
return [], None
txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT)
txt.append("\n%s:" % dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check()))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START())
for sm_id in PreContextSmIdList:
variable_db.require("pre_context_%i_fulfilled_f", Index = sm_id)
variable_db.require("input")
return txt, analyzer
def get_transition_function(iid_map, Codec):
if Codec == "UTF8": Setup.buffer_codec_prepare("utf8", Module=utf8_state_split)
else: Setup.buffer_codec_prepare("unicode")
cssm = CharacterSetStateMachine(iid_map, MaintainLexemeF=False)
analyzer = analyzer_generator.do(cssm.sm, engine.CHARACTER_COUNTER)
tm_txt = do_analyzer(analyzer)
tm_txt = Lng.GET_PLAIN_STRINGS(tm_txt)
tm_txt.append("\n")
label = dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE))
for character_set, iid in iid_map:
tm_txt.append("%s return (int)%s;\n" % (Lng.LABEL(DoorID.incidence(iid)), iid))
tm_txt.append("%s return (int)-1;\n" % Lng.LABEL(DoorID.drop_out(-1)))
return "".join(tm_txt)
def get_transition_function(iid_map, Codec):
if Codec == "UTF8":
Setup.buffer_codec_prepare("utf8", Module=utf8_state_split)
else:
Setup.buffer_codec_prepare("unicode")
cssm = CharacterSetStateMachine(iid_map, MaintainLexemeF=False)
analyzer = analyzer_generator.do(cssm.sm, engine.CHARACTER_COUNTER)
tm_txt = do_analyzer(analyzer)
tm_txt = Lng.GET_PLAIN_STRINGS(tm_txt)
tm_txt.append("\n")
label = dial_db.get_label_by_door_id(
DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE))
for character_set, iid in iid_map:
tm_txt.append("%s return (int)%s;\n" %
(Lng.LABEL(DoorID.incidence(iid)), iid))
tm_txt.append("%s return (int)-1;\n" % Lng.LABEL(DoorID.drop_out(-1)))
return "".join(tm_txt)
def get_main_function(tm0, TranstionTxt, Codec):
def indent(Txt, N):
return (" " * N) + (Txt.replace("\n", "\n" + (" " * N)))
if Codec == "UTF8": qtc_str = "uint8_t"
else: qtc_str = "uint32_t"
input_preperation = get_read_preparation(codec)
entry_list = [ (0 if interval.begin < 0 else interval.begin, target) for interval, target in tm0 ]
entry_list.append((tm0[-1][0].begin, -1))
entry_list.append((0x1FFFF, -1))
expected_array = [ " { 0x%06X, %s },\n" % (begin, target) for begin, target in entry_list ]
txt = main_template.replace("$$ENTRY_LIST$$", "".join(expected_array))
txt = txt.replace("$$QUEX_TYPE_LEXATOM$$", qtc_str)
txt = txt.replace("$$TRANSITION$$", indent(TranstionTxt, 4))
txt = txt.replace("$$PREPARE_INPUT$$", input_preperation)
door_id = DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM)
txt = txt.replace("$$ON_BAD_LEXATOM$$", dial_db.get_label_by_door_id(door_id))
txt = txt.replace("MATCH_FAILURE", "((int)-1)")
return txt
def RELOAD_PROCEDURE(self, ForwardF):
assert self.__code_generation_reload_label is None
if ForwardF:
txt = cpp_reload_forward_str
txt = txt.replace("$$ON_BAD_LEXATOM$$",
dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM)))
txt = txt.replace("$$ON_LOAD_FAILURE$$",
dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.LOAD_FAILURE)))
txt = txt.replace("$$ON_NO_SPACE_FOR_LOAD$$",
dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.OVERFLOW)))
else:
txt = cpp_reload_backward_str
txt = txt.replace("$$ON_BAD_LEXATOM$$",
dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM)))
txt = txt.replace("$$ON_LOAD_FAILURE$$",
dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.LOAD_FAILURE)))
txt = txt.replace("$$ON_NO_SPACE_FOR_LOAD$$",
dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.OVERFLOW)))
return txt
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName,
OnSkipRangeOpen, DoorIdAfter):
assert len(OpenerSequence) >= 1
assert len(CloserSequence) >= 1
assert OpenerSequence != CloserSequence
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
opener_length = len(OpenerSequence)
closer_str, closer_comment_str = get_character_sequence(CloserSequence)
closer_length = len(CloserSequence)
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require("counter")
variable_db.require_array("Skipper%i_Opener",
Initial="{ %s }" % opener_str,
ElementN=opener_length,
Index=skipper_index)
variable_db.require("Skipper%i_OpenerEnd",
"Skipper%i_Opener + (ptrdiff_t)%i" %
(skipper_index, opener_length),
Index=skipper_index)
variable_db.require("Skipper%i_Opener_it", "0x0", Index=skipper_index)
variable_db.require_array("Skipper%i_Closer",
Initial="{ %s }" % closer_str,
ElementN=closer_length,
Index=skipper_index)
variable_db.require("Skipper%i_CloserEnd",
"Skipper%i_Closer + (ptrdiff_t)%i" %
(skipper_index, closer_length),
Index=skipper_index)
variable_db.require("Skipper%i_Closer_it", "0x0", Index=skipper_index)
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
reload_door_id = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen,
CloserPattern,
NestedF=True)
code_str = blue_print(
template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER_LENGTH$$", "%i" % opener_length],
["$$INPUT_P_INCREMENT$$",
Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$", Lng.LABEL(skipper_door_id)],
["$$RELOAD$$",
dial_db.get_label_by_door_id(reload_door_id)],
["$$GOTO_AFTER_END_OF_SKIPPING$$",
Lng.GOTO(DoorIdAfter)],
["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)],
["$$INPUT_P_TO_LEXEME_START$$",
Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$MARK_LEXEME_START$$",
Lng.LEXEME_START_SET()],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$",
line_column_counter_in_loop()],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return [code_str]
def GOTO(self, DoorId):
if DoorId.last_acceptance_f():
return "QUEX_GOTO_TERMINAL(last_acceptance);"
return "goto %s;" % dial_db.get_label_by_door_id(DoorId, GotoedF=True)
def LABEL(self, DoorId):
return "%s:" % dial_db.get_label_by_door_id(DoorId)
def HEADER_DEFINITIONS(self):
return blue_print(cpp_header_definition_str, [
("$$CONTINUE_WITH_ON_AFTER_MATCH$$", dial_db.get_label_by_door_id(DoorID.continue_with_on_after_match())),
("$$RETURN_WITH_ON_AFTER_MATCH$$", dial_db.get_label_by_door_id(DoorID.return_with_on_after_match())),
])
def LABEL(self, DoorId):
return "%s:" % dial_db.get_label_by_door_id(DoorId)
def GOTO(self, DoorId):
if DoorId.last_acceptance_f():
return "QUEX_GOTO_TERMINAL(last_acceptance);"
return "goto %s;" % dial_db.get_label_by_door_id(DoorId, GotoedF=True)
def _analyzer_function(StateMachineName, Setup, variable_definitions,
function_body, ModeNameList=[]):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
if a stand-alone lexical engine is required (without the
complete mode-handling framework of quex).
"""
Lng = Setup.language_db
SingleModeAnalyzerF = Setup.single_mode_analyzer_f
mode_definition_str = ""
mode_undefinition_str = ""
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
mode_definition_str = "".join(
"# define %s%s (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name)
for name in ModeNameList
)
mode_undefinition_str = "".join(
"# undef %s\n" % name
for name in ModeNameList
)
function_signature_str = __function_signature.replace("$$STATE_MACHINE_NAME$$",
StateMachineName)
txt = [
"#include <quex/code_base/temporary_macros_on>\n",
function_signature_str,
#
# Macro definitions
#
"# ifdef self\n",
"# undef self\n",
"# endif\n",
"# define self (*((QUEX_TYPE_ANALYZER*)me))\n",
"/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n",
"# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router()),
mode_definition_str,
Lng.LEXEME_MACRO_SETUP(),
#
variable_definitions,
#
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
" || defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
#
# Entry to the actual function body
#
"%s\n" % Lng.LABEL(DoorID.global_reentry()),
" %s\n" % Lng.LEXEME_START_SET(),
" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n",
]
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.extend([
"\n",
" __quex_assert_no_passage();\n",
"\n",
" /* Following labels are referenced in macros. It cannot be detected\n"
" * whether the macros are applied in user code or not. To avoid compiler.\n"
" * warnings of unused labels, they are referenced in unreachable code. */\n"
" %s /* in RETURN */\n" % Lng.GOTO(DoorID.return_with_on_after_match()),
" %s /* in CONTINUE */\n" % Lng.GOTO(DoorID.continue_with_on_after_match()),
" %s /* in CONTINUE and skippers */\n" % Lng.GOTO(DoorID.continue_without_on_after_match()),
"# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n",
" %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router()),
"# endif\n",
"\n",
" /* Prevent compiler warning 'unused variable'. */\n",
" (void)QUEX_LEXEME_NULL;\n",
" (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n",
" /* target_state_index and target_state_else_index appear when \n",
" * QUEX_GOTO_STATE is used without computed goto-s. */\n",
" (void)target_state_index;\n",
" (void)target_state_else_index;\n",
#
# Macro undefinitions
#
lexeme_macro_clean_up,
mode_undefinition_str,
"# undef self\n",
"# undef QUEX_LABEL_STATE_ROUTER\n",
"}\n",
"#include <quex/code_base/temporary_macros_off>\n",
])
return txt
def get_skipper(EndSequence, CloserPattern, ModeName, OnSkipRangeOpen,
DoorIdAfter):
assert len(EndSequence) >= 1
global template_str
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
end_sequence_transformed = transformation.do_sequence(EndSequence)
# Determine the $$DELIMITER$$
delimiter_length = len(end_sequence_transformed)
delimiter_comment_str = Lng.COMMENT(
" Delimiter: %s" % delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = "".join(" %s" %
Lng.IF_GOTO(Lng.INPUT_P_DEREFERENCE(
i - 1), "!=", "Skipper$$SKIPPER_INDEX$$[%i]" %
i, skipper_door_id, i == 1)
for i, letter in enumerate(EndSequence[1:], start=1))
delimiter_remainder_test_str = txt
door_id_reload = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern)
# The main part
code_str = blue_print(
template_str,
[
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$INPUT_P_INCREMENT$$",
Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$",
dial_db.get_label_by_door_id(skipper_door_id)],
["$$RELOAD$$",
dial_db.get_label_by_door_id(door_id_reload)],
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$INPUT_P_TO_LEXEME_START$$",
Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$",
Lng.GOTO(DoorIdAfter)],
["$$MARK_LEXEME_START$$",
Lng.LEXEME_START_SET()],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(
code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$", Lng.GOTO(door_id_reload)]])
if reference_p_f:
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require_array("Skipper%i",
Initial="{ %s }" % delimiter_str,
ElementN=delimiter_length,
Index=skipper_index)
variable_db.require("Skipper%iL",
"%i" % delimiter_length,
Index=skipper_index)
variable_db.require("text_end")
variable_db.require("input")
return [code_str]
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter):
assert len(OpenerSequence) >= 1
assert len(CloserSequence) >= 1
assert OpenerSequence != CloserSequence
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
opener_length = len(OpenerSequence)
closer_str, closer_comment_str = get_character_sequence(CloserSequence)
closer_length = len(CloserSequence)
variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require("counter")
variable_db.require_array("Skipper%i_Opener", Initial="{ %s }" % opener_str, ElementN=opener_length, Index = skipper_index)
variable_db.require("Skipper%i_OpenerEnd", "Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length), Index = skipper_index)
variable_db.require("Skipper%i_Opener_it", "0x0", Index = skipper_index)
variable_db.require_array("Skipper%i_Closer", Initial="{ %s }" % closer_str, ElementN=closer_length, Index = skipper_index)
variable_db.require("Skipper%i_CloserEnd", "Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length), Index = skipper_index)
variable_db.require("Skipper%i_Closer_it", "0x0", Index = skipper_index)
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
reload_door_id = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern, NestedF=True)
code_str = blue_print(template_str, [
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER_LENGTH$$", "%i" % opener_length],
["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$", Lng.LABEL(skipper_door_id)],
["$$RELOAD$$", dial_db.get_label_by_door_id(reload_door_id)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)],
["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)],
["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop()],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return [ code_str ]
def _analyzer_function(StateMachineName, Setup, variable_definitions,
function_body, ModeNameList=[]):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
if a stand-alone lexical engine is required (without the
complete mode-handling framework of quex).
"""
Lng = Setup.language_db
SingleModeAnalyzerF = Setup.single_mode_analyzer_f
mode_definition_str = ""
mode_undefinition_str = ""
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
mode_definition_str = "".join(
"# define %s%s (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name)
for name in ModeNameList
)
mode_undefinition_str = "".join(
"# undef %s\n" % name
for name in ModeNameList
)
function_signature_str = __function_signature.replace("$$STATE_MACHINE_NAME$$",
StateMachineName)
txt = [
"#include <quex/code_base/temporary_macros_on>\n",
function_signature_str,
#
# Macro definitions
#
"# ifdef self\n",
"# undef self\n",
"# endif\n",
"# define self (*((QUEX_TYPE_ANALYZER*)me))\n",
"/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n",
"# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router()),
mode_definition_str,
Lng.LEXEME_MACRO_SETUP(),
#
variable_definitions,
#
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
" || defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
#
# Entry to the actual function body
#
"%s\n" % Lng.LABEL(DoorID.global_reentry()),
" %s\n" % Lng.LEXEME_START_SET(),
" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n",
]
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.extend([
"\n",
" __quex_assert_no_passage();\n",
"\n",
" /* Following labels are referenced in macros. It cannot be detected\n"
" * whether the macros are applied in user code or not. To avoid compiler.\n"
" * warnings of unused labels, they are referenced in unreachable code. */\n"
" %s /* in RETURN */\n" % Lng.GOTO(DoorID.return_with_on_after_match()),
" %s /* in CONTINUE */\n" % Lng.GOTO(DoorID.continue_with_on_after_match()),
" %s /* in CONTINUE and skippers */\n" % Lng.GOTO(DoorID.continue_without_on_after_match()),
"# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n",
" %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router()),
"# endif\n",
"\n",
" /* Prevent compiler warning 'unused variable'. */\n",
" (void)QUEX_LEXEME_NULL;\n",
" (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n",
" /* target_state_index and target_state_else_index appear when \n",
" * QUEX_GOTO_STATE is used without computed goto-s. */\n",
" (void)target_state_index;\n",
" (void)target_state_else_index;\n",
#
# Macro undefinitions
#
lexeme_macro_clean_up,
mode_undefinition_str,
"# undef self\n",
"# undef QUEX_LABEL_STATE_ROUTER\n",
"}\n",
"#include <quex/code_base/temporary_macros_off>\n",
])
return txt
def get_skipper(EndSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter):
assert len(EndSequence) >= 1
global template_str
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
end_sequence_transformed = transformation.do_sequence(EndSequence)
# Determine the $$DELIMITER$$
delimiter_length = len(end_sequence_transformed)
delimiter_comment_str = Lng.COMMENT(" Delimiter: %s" % delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = "".join(
" %s" % Lng.IF_GOTO(Lng.INPUT_P_DEREFERENCE(i-1), "!=",
"Skipper$$SKIPPER_INDEX$$[%i]" % i,
skipper_door_id, i == 1)
for i, letter in enumerate(EndSequence[1:], start=1)
)
delimiter_remainder_test_str = txt
door_id_reload = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern)
# The main part
code_str = blue_print(template_str,
[
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$", dial_db.get_label_by_door_id(skipper_door_id)],
["$$RELOAD$$", dial_db.get_label_by_door_id(door_id_reload)],
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)],
["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$", Lng.GOTO(door_id_reload)]])
if reference_p_f:
variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require_array("Skipper%i", Initial="{ %s }" % delimiter_str, ElementN=delimiter_length, Index=skipper_index)
variable_db.require("Skipper%iL", "%i" % delimiter_length, Index=skipper_index)
variable_db.require("text_end")
variable_db.require("input")
return [ code_str ]
