def get_main_function(tm0, TranstionTxt, Codec):
def indent(Txt, N):
return (" " * N) + (Txt.replace("\n", "\n" + (" " * N)))
input_preperation = get_read_preparation(codec)
entry_list = [(0 if interval.begin < 0 else interval.begin, target)
for interval, target in tm0]
entry_list.append((tm0[-1][0].begin, -1))
entry_list.append((0x1FFFF, -1))
expected_array = [
" { 0x%06X, %s },\n" % (begin, target)
for begin, target in entry_list
]
txt = main_template.replace("$$ENTRY_LIST$$", "".join(expected_array))
txt = txt.replace("$$TRANSITION$$", indent(TranstionTxt, 4))
txt = txt.replace("$$PREPARE_INPUT$$", input_preperation)
door_id = DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM, dial_db)
txt = txt.replace("$$ON_BAD_LEXATOM$$", Lng.LABEL_STR(door_id, dial_db))
txt = txt.replace("$$DROP_OUT_MINUS_1$$",
Lng.LABEL_STR(DoorID.drop_out(-1, dial_db)))
txt = txt.replace("MATCH_FAILURE", "((int)-1)")
return txt
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb,
IncidenceDb):
"""Generate a PPT for a range skipper.
"""
# -- door_id_after: Where to go after the closing character sequence matched:
# + Normally: To the begin of the analyzer. Start again.
# + End(Sequence) == newline of indentation counter.
# => goto indentation counter.
if _match_indentation_counter_newline_pattern(
OptionsDb.value("indentation"), data["closer_sequence"]):
door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)
else:
door_id_after = DoorID.continue_without_on_after_match()
if NestedF:
name = "SKIP NESTED RANGE"
code_generator_func = skip_nested_range.do
else:
name = "SKIP RANGE"
code_generator_func = skip_range.do
# -- data for code generation
my_data = deepcopy(data)
my_data["mode_name"] = ModeName
my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN]
my_data["door_id_after"] = door_id_after
my_data["counter_db"] = CounterDb
# -- terminal and code generator
priority = PatternPriority(MHI, i)
pattern = deepcopy(my_data["opener_pattern"])
pattern.set_incidence_id(dial_db.new_incidence_id())
pattern.set_pattern_string("<skip_range>")
code = CodeGenerated(code_generator_func, my_data, name)
return PPT(priority, pattern, code)
def reentry_preparation(Lng, PreConditionIDList, OnAfterMatchCode, dial_db):
"""Reentry preperation (without returning from the function."""
# (*) Unset all pre-context flags which may have possibly been set
unset_pre_context_flags_str = Lng.PRE_CONTEXT_RESET(PreConditionIDList)
on_after_match_str = Lng.SOURCE_REFERENCED(OnAfterMatchCode)
return [
"\n%s\n" % Lng.LABEL(DoorID.return_with_on_after_match(dial_db)),
Lng.COMMENT("RETURN -- after executing 'on_after_match' code."),
on_after_match_str,
" %s\n\n" % Lng.PURE_RETURN,
#
"\n%s\n" % Lng.LABEL(DoorID.continue_with_on_after_match(dial_db)),
Lng.COMMENT("CONTINUE -- after executing 'on_after_match' code."),
on_after_match_str,
#
"\n%s\n" % Lng.LABEL(DoorID.continue_without_on_after_match(dial_db)),
Lng.COMMENT(
"CONTINUE -- without executing 'on_after_match' (e.g. on FAILURE)."
),
"\n",
#
__assert_no_mode_change,
"\n",
__return_if_queue_full_or_simple_analyzer,
"\n",
unset_pre_context_flags_str,
"\n%s\n" % Lng.GOTO(DoorID.global_reentry(dial_db), dial_db),
]
def _range_skipper_door_id_exit(self, IndentationHandler, CloserPattern,
dial_db):
if self._match_indentation_counter_newline_pattern(
IndentationHandler, CloserPattern):
return DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER,
dial_db)
else:
return DoorID.continue_without_on_after_match(dial_db)
def HEADER_DEFINITIONS(self):
return blue_print(cpp_header_definition_str, [
("$$CONTINUE_WITH_ON_AFTER_MATCH$$",
dial_db.get_label_by_door_id(
DoorID.continue_with_on_after_match())),
("$$RETURN_WITH_ON_AFTER_MATCH$$",
dial_db.get_label_by_door_id(
DoorID.return_with_on_after_match())),
])
def __frame(FunctionName, IteratorName, CodeTxt, DoorIdReturn, DoorIdBeyond):
txt = [ \
"#ifdef __QUEX_OPTION_COUNTER\n" \
+ "static void\n" \
+ "%s(QUEX_TYPE_ANALYZER* me, QUEX_TYPE_CHARACTER* LexemeBegin, QUEX_TYPE_CHARACTER* LexemeEnd)\n" \
% FunctionName \
+ "{\n" \
+ "# define self (*me)\n" \
+ "/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n"
+ "# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router())
]
# Following function refers to the global 'variable_db'
txt.append(Lng.VARIABLE_DEFINITIONS(variable_db))
txt.append(
" (void)me;\n"
" __QUEX_IF_COUNT_SHIFT_VALUES();\n"
" /* Allow LexemeBegin == LexemeEnd (e.g. END_OF_STREAM)\n"
" * => Caller does not need to check\n"
" * BUT, if so quit immediately after 'shift values'. */\n"
" __quex_assert(LexemeBegin <= LexemeEnd);\n"
" if(LexemeBegin == LexemeEnd) return;\n"
" %s = LexemeBegin;\n" % IteratorName
)
txt.extend(CodeTxt)
door_id_failure = DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE)
txt.append(
"%s /* TERMINAL: FAILURE */\n%s\n" % (Lng.LABEL(door_id_failure), Lng.GOTO(DoorIdBeyond))
)
txt.append(
"%s:\n" % dial_db.get_label_by_door_id(DoorIdReturn) \
+ " __quex_assert(%s == LexemeEnd); /* Otherwise, lexeme violates codec character boundaries. */\n" \
% IteratorName \
+ " return;\n" \
+ "".join(generator.do_state_router()) \
+ "# undef self\n" \
+ "# undef QUEX_LABEL_STATE_ROUTER\n"
# If there is no MATCH_FAILURE, then DoorIdBeyond is still referenced as 'gotoed',
# but MATCH_FAILURE is never implemented, later on, because its DoorId is not
# referenced.
+ "# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n"
+ " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router())
+ "# endif\n"
+ " /* Avoid compiler warning: Unused label for 'TERMINAL <BEYOND>' */\n" \
+ " %s\n" % Lng.GOTO(DoorIdBeyond) \
+ " %s\n" % Lng.GOTO(door_id_failure) \
+ " (void)target_state_index;\n"
+ " (void)target_state_else_index;\n"
+ "}\n" \
+ "#endif /* __QUEX_OPTION_COUNTER */\n"
)
return "".join(Lng.GET_PLAIN_STRINGS(txt))
def _get_indentation_handler_terminal(ModeName, dial_db):
code = Lng.COMMAND_LIST([
Op.IndentationHandlerCall(ModeName),
Op.GotoDoorId(DoorID.continue_without_on_after_match(dial_db))
], dial_db)
incidence_id = dial.new_incidence_id()
terminal = Terminal(CodeTerminal(code),
"<CALL INDENTATION HANDLER>",
incidence_id,
dial_db=dial_db)
return DoorID.incidence(incidence_id, dial_db), terminal
def _code_terminal_on_bad_indentation_character(code, ISetup, ModeName,
incidence_db,
BadIndentationIid):
if ISetup.bad_character_set.get() is None:
return
on_bad_indentation_txt = Lng.SOURCE_REFERENCED(
incidence_db[E_IncidenceIDs.INDENTATION_BAD])
code.extend([
"%s\n" % Lng.LABEL(DoorID.incidence(BadIndentationIid)),
"#define BadCharacter (me->buffer._input_p[-1])\n",
"%s\n" % on_bad_indentation_txt, "#undef BadCharacter\n",
"%s\n" % Lng.GOTO(DoorID.global_reentry())
])
def get_loop_terminal_code(self, TheLoopMapEntry, DoorIdLoop,
DoorIdLoopExit):
"""RETURNS: A loop terminal.
A terminal: (i) Counts,
(ii) checks possibly for the lexeme end, and
(iii)a either re-enters the loop, or
(iii)b transits to an appendix state machine (couple terminal).
"""
IncidenceId = TheLoopMapEntry.incidence_id
AppendixSmId = TheLoopMapEntry.appendix_sm_id
TheCountAction = TheLoopMapEntry.count_action
code = []
if TheCountAction is not None:
code.extend(
TheCountAction.get_OpList(self.column_number_per_code_unit))
if AppendixSmId is not None:
if not lei.appendix_sm_has_transitions_f:
# If there is no appendix, directly goto to the terminal.
code.extend([Op.GotoDoorId(DoorID.incidence_id(AppendixSmId))])
else:
assert not self.lexeme_end_check_f
# Couple Terminal: transit to appendix state machine.
code.extend([
Op.Assign(E_R.ReferenceP, E_R.InputP),
Op.GotoDoorId(DoorID.state_machine_entry(AppendixSmId))
])
elif not self.lexeme_end_check_f:
# Loop Terminal: directly re-enter loop.
code.append(Op.GotoDoorId(DoorIdLoop))
else:
# Check Terminal: check against lexeme end before re-entering loop.
code.append(
Op.GotoDoorIdIfInputPNotEqualPointer(DoorIdLoop,
E_R.LexemeEnd))
if self.column_number_per_code_unit is not None \
and TheCountAction is not None \
and TheCountAction.cc_type == E_CharacterCountType.COLUMN:
# With reference counting, no column counting while looping.
# => Do it now, before leaving.
code.append(
Op.ColumnCountReferencePDeltaAdd(
E_R.InputP, self.column_number_per_code_unit, False))
code.append(Op.GotoDoorId(DoorIdLoopExit))
return Terminal(CodeTerminal(Lng.COMMAND_LIST(code)),
"<LOOP TERMINAL %i>" % IncidenceId, IncidenceId)
def do(StateRouterInfoList, dial_db):
"""Create code that allows to jump to a state based on an integer value.
"""
# NOTE: Do not use 'IfDoorIdReferencedCode' because the state router may
# possibly not be tagged as 'gotoed', event if it is used.
result = [
" __quex_assert_no_passage();\n",
" %s /* prevent unused label */\n" %
Lng.GOTO(DoorID.global_state_router(dial_db), dial_db),
"%s\n" % Lng.LABEL(DoorID.global_state_router(dial_db), dial_db)
]
if not Setup.computed_gotos_f and StateRouterInfoList:
result.extend(__get_code(StateRouterInfoList))
return result
def get_info(StateIndexList, dial_db):
"""
NOTE: At least a 'dummy' state router is always equired so that 'goto
QUEX_STATE_ROUTER;' does not reference a non-existing label. Then, we
return an empty text array.
<fschaef9/13y10m15d: "Still true with current dial_db implementation?">
"""
if len(StateIndexList) == 0: return []
# Make sure, that for every state the 'drop-out' state is also mentioned
result = [None] * len(StateIndexList)
for i, index in enumerate(StateIndexList):
assert type(index) != str
if index >= 0:
# Transition to state entry
adr = index
else:
assert False, "Is this still an issue?"
# Transition to a templates 'drop-out'
adr = DoorID.drop_out(-index, dial_db).related_address
result[i] = (index, Lng.GOTO_ADDRESS(adr, dial_db))
return result
def _get_pre_context_epilog_definition(dial_db):
backup_position = Lng.REGISTER_NAME(E_R.BackupStreamPositionOfLexemeStartP)
txt = [
Lng.LABEL(DoorID.global_end_of_pre_context_check(dial_db)),
#-------------------
Lng.IF(backup_position, "!=", "((QUEX_TYPE_STREAM_POSITION)-1)"),
# "QUEX_NAME(Buffer_print_content)(&me->buffer);\n",
# "std::cout << std::endl;\n",
Lng.IF("false", "==", Lng.BUFFER_SEEK(backup_position)),
Lng.RAISE_ERROR_FLAG("E_Error_File_SeekFailed"),
Lng.RETURN,
Lng.END_IF,
Lng.LEXEME_START_SET(PositionStorage=None), # use '_read_p'
# "std::cout << \"lexst \" << me->buffer._lexeme_start_p[0] << std::endl;",
# "std::cout << \"readp \" << me->buffer._read_p[0] << std::endl;",
# "QUEX_NAME(Buffer_print_content)(&me->buffer);\n",
# "std::cout << std::endl;\n",
Lng.ASSIGN(backup_position, "((QUEX_TYPE_STREAM_POSITION)-1)"),
Lng.ELSE_FOLLOWS,
#-----------------------
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
Lng.INPUT_P_TO_LEXEME_START(),
Lng.END_IF,
]
return [ "%s\n" % line for line in txt ]
def _terminal_goto_to_looper(self,
analyzer_list,
IncidenceId,
Name,
required_register_set,
Pattern=None):
"""Generate terminal that jumps to a 'looper'.
"""
door_id = DoorID.state_machine_entry(analyzer_list[0].state_machine_id,
self.terminal_factory.dial_db)
goto_op = Op.GotoDoorId(door_id)
goto_code = [Lng.COMMAND(goto_op, self.terminal_factory.dial_db)]
if Pattern is None:
return Terminal(CodeTerminal(goto_code),
Name,
IncidenceId,
RequiredRegisterSet=required_register_set,
dial_db=self.terminal_factory.dial_db)
else:
return self.terminal_factory.do_plain(
CodeTerminal(goto_code),
Pattern,
Name,
RequiredRegisterSet=required_register_set)
def _get_loop_terminal_list(LoopMap, EventHandler, IidLoopAfterAppendixDropOut,
DoorIdLoop, IidLoopExit):
"""RETURNS: List of terminals of the loop state:
(i) Counting terminals: Count and return to loop entry.
(ii) Couple terminals: Count and goto appendix state machine.
(iii) Exit terminal: Exit loop.
The '<LOOP>' terminal serves as an address for the appendix state machines.
If they fail, they can accept its incidence id and re-enter the loop from
there.
"""
door_id_loop_exit = DoorID.incidence(IidLoopExit)
# Terminal: Normal Loop Characters
# (LOOP EXIT terminal is generated later, see below).
result = [
EventHandler.get_loop_terminal_code(lei, DoorIdLoop, door_id_loop_exit)
for lei in LoopMap if lei.incidence_id != IidLoopExit
]
# Terminal: Re-enter Loop
if IidLoopAfterAppendixDropOut is not None:
result.append(
Terminal(
CodeTerminal(
EventHandler.on_loop_after_appendix_drop_out(DoorIdLoop)),
"<LOOP>", IidLoopAfterAppendixDropOut))
# Terminal: Exit Loop
result.append(
Terminal(CodeTerminal(EventHandler.on_loop_exit_text()), "<LOOP EXIT>",
IidLoopExit))
return result
def do_pre_context(SM, PreContextSmIdList):
"""Pre-context detecting state machine (backward).
---------------------------------------------------------------------------
Micro actions are: pre-context fullfilled_f
DropOut --> Begin of 'main' state machine.
BLC --> ReloadStateBackward
EndOfStream --> 'error'
Variables (potentially) required:
pre_context_fulfilled_f[N] --> Array of flags for pre-context
indication.
RETURNS: [0] generated code text
[1] reload state BACKWARD, to be generated later.
"""
if SM is None:
return [], None
txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT)
txt.append(
"\n%s:" %
dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check()))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START())
for sm_id in PreContextSmIdList:
variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id)
variable_db.require("input")
return txt, analyzer
def get_main_function(tm0, TranstionTxt, Codec):
def indent(Txt, N):
return (" " * N) + (Txt.replace("\n", "\n" + (" " * N)))
if Codec == "UTF8": qtc_str = "uint8_t"
else: qtc_str = "uint32_t"
input_preperation = get_read_preparation(codec)
entry_list = [(0 if interval.begin < 0 else interval.begin, target)
for interval, target in tm0]
entry_list.append((tm0[-1][0].begin, -1))
entry_list.append((0x1FFFF, -1))
expected_array = [
" { 0x%06X, %s },\n" % (begin, target)
for begin, target in entry_list
]
txt = main_template.replace("$$ENTRY_LIST$$", "".join(expected_array))
txt = txt.replace("$$QUEX_TYPE_LEXATOM$$", qtc_str)
txt = txt.replace("$$TRANSITION$$", indent(TranstionTxt, 4))
txt = txt.replace("$$PREPARE_INPUT$$", input_preperation)
door_id = DoorID.incidence(E_IncidenceIDs.BAD_LEXATOM)
txt = txt.replace("$$ON_BAD_LEXATOM$$",
dial_db.get_label_by_door_id(door_id))
txt = txt.replace("MATCH_FAILURE", "((int)-1)")
return txt
def action(ThePattern, PatternName):
txt = []
if ThePattern.sm_bipd_to_be_reversed is not None:
terminal_factory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M2_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName:
txt.append(
"QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n"
)
else:
txt.append("return;\n")
txt.append(
"%s\n" %
Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db))
## print "#", txt
return CodeTerminal(txt)
def action(ThePattern, PatternName):
txt = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName: txt.append("return false;\n")
else: txt.append("return true;\n")
txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match()))
## print "#", txt
return CodeTerminal(txt)
def get_info(StateIndexList):
"""
NOTE: At least a 'dummy' state router is always equired so that 'goto
QUEX_STATE_ROUTER;' does not reference a non-existing label. Then, we
return an empty text array.
<fschaef9/13y10m15d: "Still true with current dial_db implementation?">
"""
if len(StateIndexList) == 0: return []
# Make sure, that for every state the 'drop-out' state is also mentioned
result = [None] * len(StateIndexList)
for i, index in enumerate(StateIndexList):
assert type(index) != str
if index >= 0:
# Transition to state entry
label = dial_db.get_label_by_address(index)
else:
assert False, "Is this still an issue?"
# Transition to a templates 'drop-out'
door_id = DoorID.drop_out(-index)
label = dial_db.get_label_by_door_id(door_id)
result[i] = (index, "goto %s; " % label)
dial_db.mark_label_as_gotoed(label)
return result
def do_end_of_stream(self, Code, ThePattern):
"""End of Stream: The terminating zero has been reached and no further
content can be loaded.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# No indentation handler => Empty string.
text = [
Lng.DEFAULT_COUNTER_CALL(),
self.txt_indentation_handler_call,
#
adorned_code,
#
Lng.ML_COMMENT(
"End of Stream FORCES a return from the lexical analyzer, so that no\n"
"tokens can be filled after the termination token."),
Lng.GOTO(DoorID.return_with_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "END_OF_STREAM")
def do_match_pattern(self, Code, ThePattern):
"""A pattern has matched."""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# IMPORTANT: Terminals can be entered by any kind of 'GOTO'. In order to
# be on the safe side, BIPD should be started from within the
# terminal itself. Otherwise, it may be missed due to some
# coding negligence.
text = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(text, ThePattern)
text.extend([
self.get_counter_text(ThePattern),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_with_on_after_match())
])
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code(),
LexemeRelevanceF=True,
LexemeBeginF=lexeme_begin_f,
LexemeTerminatingZeroF=terminating_zero_f)
name = TerminalFactory.name_pattern_match_terminal(
ThePattern.pattern_string())
return Terminal(code, name)
def do_match_failure(self, Code, ThePattern):
"""No pattern in the mode has matched. Line and column numbers are
still counted. But, no 'on_match' or 'on_after_match' action is
executed.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=False)
text = [
#Lng.IF_END_OF_FILE(),
# self.get_counter_text(None),
# Lng.GOTO(DoorID.continue_without_on_after_match()),
#Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False),
# Lng.INPUT_P_INCREMENT(),
#Lng.END_IF(),
self.get_counter_text(None),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_without_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "FAILURE")
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def TERMINAL_CODE(self, TerminalStateList, TheAnalyzer):
text = [
cpp._terminal_state_prolog
]
terminal_door_id_list = []
for terminal in sorted(TerminalStateList, key=lambda x: x.incidence_id()):
door_id = DoorID.incidence(terminal.incidence_id())
terminal_door_id_list.append(door_id)
t_txt = ["%s\n __quex_debug(\"* TERMINAL %s\\n\");\n" % \
(self.LABEL(door_id), terminal.name())]
code = terminal.code(TheAnalyzer)
assert none_isinstance(code, list)
t_txt.extend(code)
t_txt.append("\n")
text.extend(t_txt)
text.append(
"if(0) {\n"
" /* Avoid unreferenced labels. */\n"
)
text.extend(
" %s\n" % self.GOTO(door_id)
for door_id in terminal_door_id_list
)
text.append("}\n")
return text
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def prepare(AppendixSm, OriginalIidDb):
lcci = SmLineColumnCountInfo.from_DFA(CaMap, AppendixSm, False,
Setup.buffer_encoding)
cmd_list = self.get_count_code(lcci)
target_door_id = DoorID.incidence(
OriginalIidDb[AppendixSm.get_id()], self.dial_db)
cmd_list.append(Op.GotoDoorId(target_door_id))
return (AppendixSm.get_id(), cmd_list)
def get_transition_function(iid_map, Codec):
if Codec == "UTF8": Setup.buffer_codec_set(bc_factory.do("utf8"), 1)
else: Setup.buffer_codec_set(bc_factory.do("unicode"), -1)
sm = StateMachine.from_IncidenceIdMap(iid_map)
dummy, sm = Setup.buffer_codec.do_state_machine(sm, beautifier)
analyzer = analyzer_generator.do(sm, engine.CHARACTER_COUNTER)
tm_txt = do_analyzer(analyzer)
tm_txt = Lng.GET_PLAIN_STRINGS(tm_txt)
tm_txt.append("\n")
#label = dial_db.get_label_by_door_id(DoorID.incidence(E_IncidenceIDs.MATCH_FAILURE))
for character_set, iid in iid_map:
tm_txt.append("%s return (int)%s;\n" %
(Lng.LABEL(DoorID.incidence(iid)), iid))
tm_txt.append("%s return (int)-1;\n" % Lng.LABEL(DoorID.drop_out(-1)))
return "".join(tm_txt)
def create_DropOut(self, SM_State):
if SM_State.is_acceptance():
incidence_id = self.__incidence_id_of_bipd
return CommandList(
QuexDebug('pattern %i: backward input position detected\\n' %
incidence_id), InputPIncrement(),
GotoDoorId(DoorID.bipd_return(incidence_id)))
else:
return CommandList(QuexAssertNoPassage())
def create_DropOut(self, SM_State, dial_db):
if SM_State.is_acceptance():
incidence_id = self.__incidence_id_of_bipd
return OpList(
Op.QuexDebug(
'pattern %i: backward input position detected\\n' %
incidence_id), Op.Increment(E_R.InputP),
Op.GotoDoorId(DoorID.bipd_return(incidence_id, dial_db)))
else:
return OpList(Op.QuexAssertNoPassage())
