def __frame(FunctionName, IteratorName, CodeTxt, DoorIdReturn, DoorIdBeyond):
txt = [ \
"#ifdef __QUEX_OPTION_COUNTER\n" \
+ "static void\n" \
+ "%s(QUEX_TYPE_ANALYZER* me, QUEX_TYPE_CHARACTER* LexemeBegin, QUEX_TYPE_CHARACTER* LexemeEnd)\n" \
% FunctionName \
+ "{\n" \
+ "# define self (*me)\n" \
+ "/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n"
+ "# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router())
]
# Following function refers to the global 'variable_db'
txt.append(Lng.VARIABLE_DEFINITIONS(variable_db))
txt.append(
" (void)me;\n"
" __QUEX_IF_COUNT_SHIFT_VALUES();\n"
" /* Allow LexemeBegin == LexemeEnd (e.g. END_OF_STREAM)\n"
" * => Caller does not need to check\n"
" * BUT, if so quit immediately after 'shift values'. */\n"
" __quex_assert(LexemeBegin <= LexemeEnd);\n"
" if(LexemeBegin == LexemeEnd) return;\n"
" %s = LexemeBegin;\n" % IteratorName
)
txt.extend(CodeTxt)
door_id_failure = DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE)
txt.append(
"%s /* TERMINAL: FAILURE */\n%s\n" % (Lng.LABEL(door_id_failure), Lng.GOTO(DoorIdBeyond))
)
txt.append(
"%s:\n" % dial_db.get_label_by_door_id(DoorIdReturn) \
+ " __quex_assert(%s == LexemeEnd); /* Otherwise, lexeme violates codec character boundaries. */\n" \
% IteratorName \
+ " return;\n" \
+ "".join(generator.do_state_router()) \
+ "# undef self\n" \
+ "# undef QUEX_LABEL_STATE_ROUTER\n"
# If there is no MATCH_FAILURE, then DoorIdBeyond is still referenced as 'gotoed',
# but MATCH_FAILURE is never implemented, later on, because its DoorId is not
# referenced.
+ "# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n"
+ " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router())
+ "# endif\n"
+ " /* Avoid compiler warning: Unused label for 'TERMINAL <BEYOND>' */\n" \
+ " %s\n" % Lng.GOTO(DoorIdBeyond) \
+ " %s\n" % Lng.GOTO(door_id_failure) \
+ " (void)target_state_index;\n"
+ " (void)target_state_else_index;\n"
+ "}\n" \
+ "#endif /* __QUEX_OPTION_COUNTER */\n"
)
return "".join(Lng.GET_PLAIN_STRINGS(txt))
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def action(ThePattern, PatternName):
txt = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName: txt.append("return false;\n")
else: txt.append("return true;\n")
txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match()))
## print "#", txt
return CodeTerminal(txt)
def action(ThePattern, PatternName):
txt = []
if ThePattern.sm_bipd_to_be_reversed is not None:
terminal_factory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M2_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName:
txt.append(
"QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n"
)
else:
txt.append("return;\n")
txt.append(
"%s\n" %
Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db))
## print "#", txt
return CodeTerminal(txt)
def do_match_pattern(self, Code, ThePattern):
"""A pattern has matched."""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# IMPORTANT: Terminals can be entered by any kind of 'GOTO'. In order to
# be on the safe side, BIPD should be started from within the
# terminal itself. Otherwise, it may be missed due to some
# coding negligence.
text = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(text, ThePattern)
text.extend([
self.get_counter_text(ThePattern),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_with_on_after_match())
])
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code(),
LexemeRelevanceF=True,
LexemeBeginF=lexeme_begin_f,
LexemeTerminatingZeroF=terminating_zero_f)
name = TerminalFactory.name_pattern_match_terminal(
ThePattern.pattern_string())
return Terminal(code, name)
def do_match_failure(self, Code, ThePattern):
"""No pattern in the mode has matched. Line and column numbers are
still counted. But, no 'on_match' or 'on_after_match' action is
executed.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=False)
text = [
#Lng.IF_END_OF_FILE(),
# self.get_counter_text(None),
# Lng.GOTO(DoorID.continue_without_on_after_match()),
#Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False),
# Lng.INPUT_P_INCREMENT(),
#Lng.END_IF(),
self.get_counter_text(None),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_without_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "FAILURE")
def do_end_of_stream(self, Code, ThePattern):
"""End of Stream: The terminating zero has been reached and no further
content can be loaded.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# No indentation handler => Empty string.
text = [
Lng.DEFAULT_COUNTER_CALL(),
self.txt_indentation_handler_call,
#
adorned_code,
#
Lng.ML_COMMENT(
"End of Stream FORCES a return from the lexical analyzer, so that no\n"
"tokens can be filled after the termination token."),
Lng.GOTO(DoorID.return_with_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "END_OF_STREAM")
def do(txt, StateIndex, DropOut, TheAnalyzer, DefineLabelF=True, MentionStateIndexF=True):
if MentionStateIndexF:
txt.append(1)
txt.append("__quex_debug_drop_out(%i);\n" % StateIndex)
txt.extend(
Lng.GOTO(TheAnalyzer.drop_out_DoorID(StateIndex))
)
return
def _code_terminal_on_bad_indentation_character(code, ISetup, ModeName,
incidence_db,
BadIndentationIid):
if ISetup.bad_character_set.get() is None:
return
on_bad_indentation_txt = Lng.SOURCE_REFERENCED(
incidence_db[E_IncidenceIDs.INDENTATION_BAD])
code.extend([
"%s\n" % Lng.LABEL(DoorID.incidence(BadIndentationIid)),
"#define BadCharacter (me->buffer._input_p[-1])\n",
"%s\n" % on_bad_indentation_txt, "#undef BadCharacter\n",
"%s\n" % Lng.GOTO(DoorID.global_reentry())
])
def _get_state_machine_vs_terminal_list(CloserPattern, dial_db, DoorIdExit):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = CloserPattern.get_cloned_sm(StateMachineId=dial.new_incidence_id())
code = [Lng.GOTO(DoorIdExit, dial_db)]
mini_terminal = loop.MiniTerminal(code, "<SKIP RANGE TERMINATED>",
sm.get_id())
return [(sm, mini_terminal)]
def do(StateRouterInfoList, dial_db):
"""Create code that allows to jump to a state based on an integer value.
"""
# NOTE: Do not use 'IfDoorIdReferencedCode' because the state router may
# possibly not be tagged as 'gotoed', event if it is used.
result = [
" __quex_assert_no_passage();\n",
" %s /* prevent unused label */\n" %
Lng.GOTO(DoorID.global_state_router(dial_db), dial_db),
"%s\n" % Lng.LABEL(DoorID.global_state_router(dial_db), dial_db)
]
if not Setup.computed_gotos_f and StateRouterInfoList:
result.extend(__get_code(StateRouterInfoList))
return result
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = StateMachine.from_sequence(CloserSequence)
sm.set_id(dial_db.new_incidence_id())
code = [Lng.GOTO(DoorID.continue_without_on_after_match())]
terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>",
sm.get_id())
return [(sm, terminal)]
def __terminate_analysis_step(self, Code, Name, Comment):
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# No indentation handler => Empty string.
text = [
Lng.DEFAULT_COUNTER_CALL(),
self.txt_indentation_handler_call,
#
adorned_code,
#
Lng.ML_COMMENT(Comment),
Lng.GOTO(DoorID.return_with_on_after_match()),
]
return self.__terminal(text, Code, Name)
def PPT_indentation_handler_suppressed_newline(MHI, SmSuppressedNewline):
"""Generate a PPT for suppressed newline, that is:
-- its PatternPriority.
-- the Pattern object.
-- the Terminal object.
The terminal simply jumpts to the re-entry of the lexical analyzer.
"""
assert SmSuppressedNewline is not None
pattern = Pattern(SmSuppressedNewline,
PatternString="<indentation suppressed newline>")
code = CodeTerminal([Lng.GOTO(DoorID.global_reentry())])
# terminal = terminal_factory.do(E_TerminalType.PLAIN, code)
# terminal.set_name("INDENTATION COUNTER: SUPPRESSED_NEWLINE")
return PPT(PatternPriority(MHI, 1), pattern, code)
def _add_pair(psml, SmOriginal, Name):
"""Add a state machine-terminal pair to 'psml'. A terminal is generated
which transits to 'INDENTATION_HANDLER'. The state machine is cloned
for safety.
"""
if SmOriginal is None: return
incidence_id = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
sm = SmOriginal.clone(StateMachineId=incidence_id)
code = [Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))]
terminal = Terminal(CodeTerminal(code), Name, incidence_id)
# TRY: terminal.set_requires_goto_loop_entry_f()
# INSTEAD: GOTO 'INDENTATION_HANDLER'
psml.append((sm, terminal))
def __code(Node, TheState, done_set, GlobalEntryF):
"""Code a node of the command tree in a sequence of nodes from leaf to
root, parent by parent. As long as the parent is in the list, no goto is
required. If the parent is implemented already, an explicit goto is
implemented.
RETURNS: list of strings = code for node.
"""
done_set.add(Node.door_id)
txt = []
__label_node(txt, Node)
__comment(txt, Node, TheState, GlobalEntryF)
# (*) The code of commands of the node
txt.extend(
Lng.COMMAND_LIST(Node.command_list)
)
# (*) AFTER: COMMAND_LIST -- The state debug info.
txt.append(Lng.STATE_DEBUG_INFO(TheState, GlobalEntryF))
# (*) The 'goto parent'.
if Node.parent is None:
# This is the root.
return txt
elif Node.parent.door_id not in done_set:
# As long as the parent is not done, it will be implemented immediately
# after this node--no goto is required.
return txt
elif Node.command_list and Node.command_list[-1].is_conditionless_goto():
# Goto is futile if the last command is an unconditional goto.
return txt
# Append the 'goto parent'
txt.append(" %s\n" % Lng.GOTO(Node.parent.door_id))
return txt
def do_bipd_entry_and_return(txt, ThePattern):
"""(This is a very seldom case) After the pattern has matched, one needs
to determine the end of the lexeme by 'backward input position detection'
(bipd). Thus,
TERMINAL
'----------.
(goto) '---------> BIPD State Machine
...
(determine _input_p)
|
(label) .----------------'
.----------'
|
The actions on
pattern match.
"""
door_id_entry = DoorID.state_machine_entry(ThePattern.bipd_sm.get_id())
door_id_return = DoorID.bipd_return(ThePattern.incidence_id())
txt.append(" %s\n%s\n" % (
Lng.GOTO(door_id_entry), # Enter BIPD
Lng.LABEL(door_id_return)) # Return from BIPD
)
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName,
OnSkipRangeOpen, DoorIdAfter):
assert len(OpenerSequence) >= 1
assert len(CloserSequence) >= 1
assert OpenerSequence != CloserSequence
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
opener_length = len(OpenerSequence)
closer_str, closer_comment_str = get_character_sequence(CloserSequence)
closer_length = len(CloserSequence)
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require("counter")
variable_db.require_array("Skipper%i_Opener",
Initial="{ %s }" % opener_str,
ElementN=opener_length,
Index=skipper_index)
variable_db.require("Skipper%i_OpenerEnd",
"Skipper%i_Opener + (ptrdiff_t)%i" %
(skipper_index, opener_length),
Index=skipper_index)
variable_db.require("Skipper%i_Opener_it", "0x0", Index=skipper_index)
variable_db.require_array("Skipper%i_Closer",
Initial="{ %s }" % closer_str,
ElementN=closer_length,
Index=skipper_index)
variable_db.require("Skipper%i_CloserEnd",
"Skipper%i_Closer + (ptrdiff_t)%i" %
(skipper_index, closer_length),
Index=skipper_index)
variable_db.require("Skipper%i_Closer_it", "0x0", Index=skipper_index)
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
reload_door_id = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen,
CloserPattern,
NestedF=True)
code_str = blue_print(
template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER_LENGTH$$", "%i" % opener_length],
["$$INPUT_P_INCREMENT$$",
Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$", Lng.LABEL(skipper_door_id)],
["$$RELOAD$$",
dial_db.get_label_by_door_id(reload_door_id)],
["$$GOTO_AFTER_END_OF_SKIPPING$$",
Lng.GOTO(DoorIdAfter)],
["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)],
["$$INPUT_P_TO_LEXEME_START$$",
Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$MARK_LEXEME_START$$",
Lng.LEXEME_START_SET()],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$",
line_column_counter_in_loop()],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return [code_str]
def __frame(FunctionName, CodeTxt, IteratorName, DoorIdReturn, dial_db):
txt = [ \
"static void\n" \
+ "%s(QUEX_TYPE_ANALYZER* me, QUEX_TYPE_LEXATOM* LexemeBegin, QUEX_TYPE_LEXATOM* LexemeEnd)\n" % FunctionName \
+ "{\n" \
]
if IteratorName:
state_router_adr = DoorID.global_state_router(dial_db).related_address
state_router_label = Lng.LABEL_STR_BY_ADR(state_router_adr)
txt.extend([
"# define self (*me)\n",
"/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n",
"# define QUEX_LABEL_STATE_ROUTER %s\n" % state_router_label
])
# Following function refers to the global 'variable_db'
txt.append(Lng.VARIABLE_DEFINITIONS(variable_db))
txt.extend([
" (void)me;\n",
Lng.COUNTER_SHIFT_VALUES(),
"%s" % Lng.ML_COMMENT("Allow LexemeBegin == LexemeEnd (e.g. END_OF_STREAM)\n"
"=> Caller does not need to check\n"
"BUT, if so quit immediately after 'shift values'."),
" __quex_assert(LexemeBegin <= LexemeEnd);\n",
" %s" % Lng.IF("LexemeBegin", "==", "LexemeEnd"),
" %s\n" % Lng.PURE_RETURN,
" %s\n" % Lng.END_IF,
" %s = LexemeBegin;\n" % IteratorName
])
txt.extend(CodeTxt)
if IteratorName:
door_id_failure = DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE, dial_db)
door_id_bad_lexatom = DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM, dial_db)
txt.append(
"%s /* TERMINAL: BAD_LEXATOM */\n;\n" % Lng.LABEL(door_id_bad_lexatom)
# BETTER: A lexeme that is 'counted' has already matched!
# => FAILURE is impossible!
# "%s /* TERMINAL: FAILURE */\n%s\n" % Lng.UNREACHABLE
+ "%s /* TERMINAL: FAILURE */\n%s\n" % (Lng.LABEL(door_id_failure),
Lng.GOTO(DoorIdReturn, dial_db))
)
txt.append(
"%s\n" % Lng.LABEL(DoorIdReturn)
+ "%s\n" % Lng.COMMENT("Assert: lexeme in codec's character boundaries.") \
+ " __quex_assert(%s == LexemeEnd);\n" % IteratorName \
+ " return;\n" \
+ "".join(generator.do_state_router(dial_db)) \
+ "%s\n" % Lng.UNDEFINE("self")
+ "%s\n" % Lng.UNDEFINE("QUEX_LABEL_STATE_ROUTER")
# If there is no MATCH_FAILURE, then DoorIdBeyond is still referenced as 'gotoed',
# but MATCH_FAILURE is never implemented, later on, because its DoorId is not
# referenced.
+ "$$<not-computed-gotos>----------------------------------------------\n"
+ " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router(dial_db), dial_db)
+ " %s /* to BAD_LEXATOM */\n" % Lng.GOTO(DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM, dial_db), dial_db)
+ "$$------------------------------------------------------------------\n"
+ " %s\n" % Lng.COMMENT("Avoid compiler warning: 'Unused labels'") \
+ " %s\n" % Lng.GOTO(door_id_failure, dial_db) \
+ " (void)target_state_index;\n"
+ " (void)target_state_else_index;\n"
)
txt.append("}\n")
return "".join(Lng.GET_PLAIN_STRINGS(txt, dial_db))
def code(self):
return Lng.GOTO(self.__door_id, self.dial_db)
def get_code(self, LoopStateMachineId):
return [
Lng.GOTO(
DoorID.state_machine_entry(LoopStateMachineId, dial_db),
dial_db)
]
def get_skipper(EndSequence, CloserPattern, ModeName, OnSkipRangeOpen,
DoorIdAfter):
assert len(EndSequence) >= 1
global template_str
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
end_sequence_transformed = transformation.do_sequence(EndSequence)
# Determine the $$DELIMITER$$
delimiter_length = len(end_sequence_transformed)
delimiter_comment_str = Lng.COMMENT(
" Delimiter: %s" % delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = "".join(" %s" %
Lng.IF_GOTO(Lng.INPUT_P_DEREFERENCE(
i - 1), "!=", "Skipper$$SKIPPER_INDEX$$[%i]" %
i, skipper_door_id, i == 1)
for i, letter in enumerate(EndSequence[1:], start=1))
delimiter_remainder_test_str = txt
door_id_reload = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern)
# The main part
code_str = blue_print(
template_str,
[
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$INPUT_P_INCREMENT$$",
Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$",
dial_db.get_label_by_door_id(skipper_door_id)],
["$$RELOAD$$",
dial_db.get_label_by_door_id(door_id_reload)],
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$INPUT_P_TO_LEXEME_START$$",
Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$",
Lng.GOTO(DoorIdAfter)],
["$$MARK_LEXEME_START$$",
Lng.LEXEME_START_SET()],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(
code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$", Lng.GOTO(door_id_reload)]])
if reference_p_f:
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require_array("Skipper%i",
Initial="{ %s }" % delimiter_str,
ElementN=delimiter_length,
Index=skipper_index)
variable_db.require("Skipper%iL",
"%i" % delimiter_length,
Index=skipper_index)
variable_db.require("text_end")
variable_db.require("input")
return [code_str]
def _prepare_skip_character_set(ModeName, OptionsDb, CounterDb, IncidenceDb,
MHI):
"""MHI = Mode hierarchie index."""
SkipSetupList = OptionsDb.value_sequence("skip")
if SkipSetupList is None or len(SkipSetupList) == 0:
return [], []
iterable = SkipSetupList.__iter__()
pattern, total_set = iterable.next()
pattern_str = pattern.pattern_string()
source_reference = pattern.sr
# Multiple skippers from different modes are combined into one pattern.
# This means, that we cannot say exactly where a 'skip' was defined
# if it intersects with another pattern.
for ipattern, icharacter_set in iterable:
total_set.unite_with(icharacter_set)
pattern_str += "|" + ipattern.pattern_string()
# The column/line number count actions for the characters in the
# total_set may differ. Thus, derive a separate set of characters
# for each same count action, i.e.
#
# map: count action --> subset of total_set
#
# When the first character is matched, then its terminal 'TERMINAL_x*'
# is entered, i.e the count action for the first character is performed
# before the skipping starts. This will look like this:
#
# TERMINAL_x0:
# count action '0';
# goto __SKIP;
# TERMINAL_x1:
# count action '1';
# goto __SKIP;
# ...
# An optional codec transformation is done later. The state machines
# are entered as pure Unicode state machines.
# It is not necessary to store the count action along with the state
# machine. This is done in "action_preparation.do()" for each
# terminal.
data = {
"counter_db": CounterDb,
"character_set": total_set,
}
# The terminal is not related to a pattern, because it is entered
# from the sub_terminals. Each sub_terminal relates to a sub character
# set.
terminal_iid = E_IncidenceIDs.SKIP
goto_terminal_str = Lng.GOTO(DoorID.incidence(terminal_iid))
code = CodeGeneratedBlock(terminal_iid, skip_character_set.do, data,
"character set skipper")
# Counting actions are added to the terminal automatically by the
# terminal_factory. The only thing that remains for each sub-terminal:
# 'goto skipper'.
ccfactory = CountInfoMap.from_LineColumnCount(CounterDb, total_set,
Lng.INPUT_P())
new_ppt_list = [
PPT_character_set_skipper(MHI, character_set, incidence_id, CounterDb,
goto_terminal_str, source_reference)
for character_set, incidence_id in ccfactory.get_incidence_id_map()
]
return [code], new_ppt_list
def code(self):
return Lng.GOTO(self.__door_id)
def _prepare_indentation_counter(self, MHI, Loopers, CaMap, ReloadState):
"""Prepare indentation counter. An indentation counter is implemented by
the following:
'newline' pattern --> triggers as soon as an UNSUPPRESSED newline occurs.
--> entry to the INDENTATION COUNTER.
'suppressed newline' --> INDENTATION COUNTER is NOT triggered.
The supressed newline pattern is longer (and has precedence) over the
newline pattern. With the suppressed newline it is possible to write
lines which overstep the newline (see backslahs in Python, for example).
RETURNS: List of:
[0] newline PPT and
[1] optionally the PPT of the newline suppressor.
The primary pattern action pair list is to be the head of all pattern
action pairs.
MHI = Mode hierarchie index defining the priority of the current mode.
"""
ISetup = Loopers.indentation_handler
if ISetup is None: return [], [], []
check_indentation_setup(ISetup)
# Isolate the pattern objects, so alternatively,
# they may be treated in 'indentation_counter'.
pattern_newline = ISetup.pattern_newline.clone()
if ISetup.pattern_suppressed_newline:
pattern_suppressed_newline = ISetup.pattern_suppressed_newline.clone(
)
else:
pattern_suppressed_newline = None
new_analyzer_list, \
new_terminal_list, \
required_register_set, \
run_time_counter_f = indentation_counter.do(self.terminal_factory.mode_name,
CaMap,
ISetup,
self.terminal_factory.incidence_db,
ReloadState,
self.terminal_factory.dial_db)
self.terminal_factory.run_time_counter_required_f |= run_time_counter_f
self.required_register_set.update(required_register_set)
# 'newline' triggers --> indentation counter
pattern = pattern_newline.finalize(CaMap)
pattern = pattern.clone_with_new_incidence_id(
E_IncidenceIDs.INDENTATION_HANDLER)
terminal = self._terminal_goto_to_looper(new_analyzer_list,
None,
"<indentation handler>",
required_register_set,
Pattern=pattern)
new_ppt_list = [PPT(PatternPriority(MHI, 0), pattern, terminal)]
if pattern_suppressed_newline is not None:
# 'newline-suppressor' causes following 'newline' to be ignored.
# => next line not subject to new indentation counting.
new_incidence_id = dial.new_incidence_id()
pattern = pattern_suppressed_newline.finalize(CaMap)
pattern = pattern.clone_with_new_incidence_id(new_incidence_id)
door_id = DoorID.global_reentry(self.terminal_factory.dial_db)
code = CodeTerminal(
[Lng.GOTO(door_id, self.terminal_factory.dial_db)])
new_ppt_list.append(
PPT(
PatternPriority(MHI, 1), pattern,
self.terminal_factory.do_plain(
code, pattern,
"INDENTATION COUNTER SUPPRESSED_NEWLINE: ")))
return new_ppt_list, new_analyzer_list, new_terminal_list
def framework(txt, PWState, TheAnalyzer):
"""Implement the Pathwalker's framework. The scheme for a path-walker
is the following:
Pathwalker Head:
Compares the current 'input' character if it is still
on the path or not. If it is on the path we increment
the 'path_iterator' and re-enter the path walker. If
not, then the thread of control enters the transition
map.
Pathwalker Transition Map:
The transition map is the common transition map that all
implemented states had in common. Now, transitions to
states outside the path may happen.
"""
dial_db = PWState.entry.dial_db
# Three Versions of PathWalker Heads:
if PWState.uniform_door_id is not None:
# UNIFORM PATHS: Along the path, always the same (or no) commands are executed.
#
# PathWalker Head Implementation:
#
# if input == *path_iterator:
# path_iterator += 1
# if *path_iterator != TerminationCode: goto CommonPathWalkerDoor
# else: goto TerminalDoor
#
# -- "goto CommonPathWalkerDoor"
goto_next_door = " %s\n" % Lng.GOTO(PWState.uniform_door_id,
dial_db)
# -- "goto TerminalDoor"
if PWState.uniform_terminal_door_id is not None:
# All path have same terminal state and enter it at the same door
goto_terminal_door = " %s\n" % Lng.GOTO(
PWState.uniform_terminal_door_id, dial_db)
else:
# The terminals of the paths are different
#
# The "goto TerminalDoor" is implemented for each path. The single
# goto is split into a sequence:
#
# if path_iterator == path_0_end: goto TerminalDoorOfPath0
# else if path_iterator == path_1_end: goto TerminalDoorOfPath1
# else if path_iterator == path_2_end: goto TerminalDoorOfPath2
# ...
tmp = ""
offset = 0
for path_id, door_id_sequence in enumerate(
PWState.door_id_sequence_list):
offset += len(door_id_sequence) + 1
tmp += " %s" % Lng.IF("path_iterator", "==", "&path_walker_%i_path_base[%s]" % \
(PWState.index, offset - 1), FirstF=(path_id == 0)) \
+ " %s\n" % Lng.GOTO(door_id_sequence[-1], dial_db)
tmp += " %s" % Lng.ELSE_FOLLOWS
tmp += " %s\n" % Lng.UNREACHABLE
tmp += " %s\n" % Lng.END_IF
goto_terminal_door = tmp
path_walker_head = \
[" %s" % Lng.IF_INPUT("==", "*path_iterator"),
" %s\n" % Lng.PATH_ITERATOR_INCREMENT,
" %s" % Lng.IF("*path_iterator", "!=", "QUEX_SETTING_BUFFER_LEXATOM_PATH_TERMINATION"),
goto_next_door,
" %s\n" % Lng.ELSE_FOLLOWS,
goto_terminal_door,
" %s\n" % Lng.END_IF,
" %s\n" % Lng.END_IF]
else:
# NON UNIFORM PATHS
#
# PathWalker Head Implementation:
#
# if input == *path_iterator:
# path_iterator += 1
# goto NextDoor(path_iterator)
#
# Here, the "goto TerminalDoor" results from NextDoor(path_iterator)
# automatically, when the path_iterator stands on the last element.
#
label = "path_walker_%i_state_base[path_iterator - path_walker_%i_reference]" \
% (PWState.index, PWState.index)
goto_next_door = "%s" % (Lng.GOTO_BY_VARIABLE(label))
path_walker_head = [
" %s" % Lng.IF_INPUT("==", "*path_iterator"),
" %s\n" % Lng.PATH_ITERATOR_INCREMENT,
" %s\n" % goto_next_door,
" %s\n" % Lng.END_IF
]
txt.extend(path_walker_head)
return