def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(SmComment,
CounterDb,
CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Op.Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Op.Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
Op.LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _get_terminal(X, get_appendix):
cl = self._command(X.cc_type, X.parameter)
appendix = get_appendix(self, X.cc_type)
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(chain(cl, appendix))),
Name="%s" % X.cc_type)
terminal.set_incidence_id(X.incidence_id)
return terminal
def _get_loop_terminal_list(LoopMap, EventHandler, IidLoopAfterAppendixDropOut,
DoorIdLoop, IidLoopExit):
"""RETURNS: List of terminals of the loop state:
(i) Counting terminals: Count and return to loop entry.
(ii) Couple terminals: Count and goto appendix state machine.
(iii) Exit terminal: Exit loop.
The '<LOOP>' terminal serves as an address for the appendix state machines.
If they fail, they can accept its incidence id and re-enter the loop from
there.
"""
door_id_loop_exit = DoorID.incidence(IidLoopExit)
# Terminal: Normal Loop Characters
# (LOOP EXIT terminal is generated later, see below).
result = [
EventHandler.get_loop_terminal_code(lei, DoorIdLoop, door_id_loop_exit)
for lei in LoopMap if lei.incidence_id != IidLoopExit
]
# Terminal: Re-enter Loop
if IidLoopAfterAppendixDropOut is not None:
result.append(
Terminal(
CodeTerminal(
EventHandler.on_loop_after_appendix_drop_out(DoorIdLoop)),
"<LOOP>", IidLoopAfterAppendixDropOut))
# Terminal: Exit Loop
result.append(
Terminal(CodeTerminal(EventHandler.on_loop_exit_text()), "<LOOP EXIT>",
IidLoopExit))
return result
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
Op.LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _get_terminal(X, get_appendix):
cl = self._command(X.cc_type, X.parameter)
appendix = get_appendix(self, X.cc_type)
terminal = Terminal(CodeTerminal(
Lng.COMMAND_LIST(chain(cl, appendix))),
Name="%s" % X.cc_type)
terminal.set_incidence_id(X.incidence_id)
return terminal
def __get_terminal_beyond(OnBeyond, BeyondIid):
"""Generate Terminal to be executed upon exit from the 'loop'.
BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of
the terminal to be generated.
"""
code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond))
result = Terminal(code_on_beyond, "<BEYOND>") # Put last considered character back
result.set_incidence_id(BeyondIid)
return result
def __get_terminal_beyond(OnBeyond, BeyondIid):
"""Generate Terminal to be executed upon exit from the 'loop'.
BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of
the terminal to be generated.
"""
code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond))
result = Terminal(code_on_beyond,
"<BEYOND>") # Put last considered character back
result.set_incidence_id(BeyondIid)
return result
def _get_state_machine_and_terminal(Sequence, Name, OpList):
"""Create state machine that detects the 'Sequence', names the terminal
with 'Name', and implements the 'CmdList' in the terminal.
RETURNS: (state machine, terminal)
"""
sm = StateMachine.from_sequence(Sequence)
sm.set_id(dial_db.new_incidence_id())
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(OpList)), Name, sm.get_id())
terminal.set_requires_goto_loop_entry_f() # --> Goto Loop Entry
return sm, terminal
def _get_state_machine_and_terminal(Sequence, Name, OpList):
"""Create state machine that detects the 'Sequence', names the terminal
with 'Name', and implements the 'CmdList' in the terminal.
RETURNS: (state machine, terminal)
"""
sm = StateMachine.from_sequence(Sequence)
sm.set_id(dial_db.new_incidence_id())
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(OpList)), Name,
sm.get_id())
terminal.set_requires_goto_loop_entry_f() # --> Goto Loop Entry
return sm, terminal
def _get_terminal_list_for_loop(loop_map, loop_config, DoorIdLoop):
"""RETURNS: List of terminals of the loop state:
(i) Counting terminals: Count and return to loop entry.
(ii) Couple terminals: Count and goto appendix state machine.
(iii) Exit terminal: Exit loop.
The '<LOOP>' terminal serves as an address for the appendix state machines.
If they fail, they can accept its incidence id and re-enter the loop from
there.
"""
# Terminal: Normal Loop Characters
# (LOOP EXIT terminal is generated later, see below).
result = []
done = set()
for lme in loop_map:
if lme.iid_couple_terminal in done: continue
elif lme.iid_couple_terminal == loop_config.iid_loop_exit: continue
elif lme.code is None: continue
done.add(lme.iid_couple_terminal)
result.append(
Terminal(loop_config.CodeTerminal_without_Lazy_DoorIdLoop(
lme.code, DoorIdLoop),
"<LOOP TERMINAL %s>" % lme.iid_couple_terminal,
IncidenceId=lme.iid_couple_terminal,
dial_db=loop_config.dial_db))
# Terminal: Re-enter Loop
if loop_config.iid_loop_after_appendix_drop_out is not None:
txt = Lng.COMMAND_LIST(
loop_config.events.on_loop_after_appendix_drop_out(
DoorIdLoop, loop_config.column_number_per_code_unit),
loop_config.dial_db)
result.append(
Terminal(CodeTerminal(txt),
"<LOOP>",
loop_config.iid_loop_after_appendix_drop_out,
dial_db=loop_config.dial_db))
# Terminal: Exit Loop
result.append(
Terminal(CodeTerminal(
loop_config.events.on_loop_exit_text(loop_config.dial_db)),
"<LOOP EXIT>",
loop_config.iid_loop_exit,
dial_db=loop_config.dial_db))
return result
def _terminal_goto_to_looper(self,
analyzer_list,
IncidenceId,
Name,
required_register_set,
Pattern=None):
"""Generate terminal that jumps to a 'looper'.
"""
door_id = DoorID.state_machine_entry(analyzer_list[0].state_machine_id,
self.terminal_factory.dial_db)
goto_op = Op.GotoDoorId(door_id)
goto_code = [Lng.COMMAND(goto_op, self.terminal_factory.dial_db)]
if Pattern is None:
return Terminal(CodeTerminal(goto_code),
Name,
IncidenceId,
RequiredRegisterSet=required_register_set,
dial_db=self.terminal_factory.dial_db)
else:
return self.terminal_factory.do_plain(
CodeTerminal(goto_code),
Pattern,
Name,
RequiredRegisterSet=required_register_set)
def do_match_failure(self, Code, ThePattern):
"""No pattern in the mode has matched. Line and column numbers are
still counted. But, no 'on_match' or 'on_after_match' action is
executed.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=False)
text = [
#Lng.IF_END_OF_FILE(),
# self.get_counter_text(None),
# Lng.GOTO(DoorID.continue_without_on_after_match()),
#Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False),
# Lng.INPUT_P_INCREMENT(),
#Lng.END_IF(),
self.get_counter_text(None),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_without_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "FAILURE")
def do_match_pattern(self, Code, ThePattern):
"""A pattern has matched."""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# IMPORTANT: Terminals can be entered by any kind of 'GOTO'. In order to
# be on the safe side, BIPD should be started from within the
# terminal itself. Otherwise, it may be missed due to some
# coding negligence.
text = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(text, ThePattern)
text.extend([
self.get_counter_text(ThePattern),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_with_on_after_match())
])
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code(),
LexemeRelevanceF=True,
LexemeBeginF=lexeme_begin_f,
LexemeTerminatingZeroF=terminating_zero_f)
name = TerminalFactory.name_pattern_match_terminal(
ThePattern.pattern_string())
return Terminal(code, name)
def do_end_of_stream(self, Code, ThePattern):
"""End of Stream: The terminating zero has been reached and no further
content can be loaded.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# No indentation handler => Empty string.
text = [
Lng.DEFAULT_COUNTER_CALL(),
self.txt_indentation_handler_call,
#
adorned_code,
#
Lng.ML_COMMENT(
"End of Stream FORCES a return from the lexical analyzer, so that no\n"
"tokens can be filled after the termination token."),
Lng.GOTO(DoorID.return_with_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "END_OF_STREAM")
def get_Terminal(self, PreCode, dial_db, LoopStateMachineId):
assert LoopStateMachineId is not None
return Terminal(CodeTerminal(PreCode +
self.get_code(LoopStateMachineId)),
self.name,
self.incidence_id,
dial_db=dial_db)
def _get_terminal_list_for_appendix_sms(loop_config, DoorIdLoop,
appendix_cmd_list_db):
return [
Terminal(loop_config.CodeTerminal_without_Lazy_DoorIdLoop(
cmd_list, DoorIdLoop),
"<LOOP APPENDIX TERMINAL %s>" % appendix_sm_id,
IncidenceId=appendix_sm_id,
dial_db=loop_config.dial_db)
for appendix_sm_id, cmd_list in appendix_cmd_list_db.iteritems()
]
def _get_indentation_handler_terminal(ModeName, dial_db):
code = Lng.COMMAND_LIST([
Op.IndentationHandlerCall(ModeName),
Op.GotoDoorId(DoorID.continue_without_on_after_match(dial_db))
], dial_db)
incidence_id = dial.new_incidence_id()
terminal = Terminal(CodeTerminal(code),
"<CALL INDENTATION HANDLER>",
incidence_id,
dial_db=dial_db)
return DoorID.incidence(incidence_id, dial_db), terminal
def _prepare_skip_character_set(self, MHI, Loopers, CaMap, ReloadState):
"""MHI = Mode hierarchie index."""
if Loopers.skip is None: return [], [], []
skipped_character_set, \
pattern_str, \
aux_source_reference = Loopers.combined_skip(CaMap)
new_analyzer_list, \
new_terminal_list, \
loop_map, \
required_register_set = skip_character_set.do(self.terminal_factory.mode_name,
CaMap, skipped_character_set,
ReloadState,
self.terminal_factory.dial_db)
self.required_register_set.update(required_register_set)
extra_terminal_list = [
self._terminal_goto_to_looper(new_analyzer_list,
E_IncidenceIDs.SKIP, "<skip>",
required_register_set)
]
extra_terminal_list.extend(t for t in new_terminal_list)
# Any skipped character must enter the skipper entry.
goto_code = [
Op.GotoDoorId(
DoorID.incidence(E_IncidenceIDs.SKIP,
self.terminal_factory.dial_db))
]
new_ppt_list = []
for lei in loop_map:
new_incidence_id = dial.new_incidence_id()
pattern = Pattern.from_character_set(lei.character_set,
new_incidence_id,
PatternString="<skip>",
Sr=aux_source_reference)
# There is no reference pointer => Add directly
count_code = lei.aux_count_action.get_OpList(None)
code = Lng.COMMAND_LIST(count_code + goto_code,
self.terminal_factory.dial_db)
terminal = Terminal(CodeTerminal(code),
"ENTER SKIP:",
new_incidence_id,
dial_db=self.terminal_factory.dial_db)
new_ppt_list.append(
PPT(PatternPriority(MHI, new_incidence_id), pattern, terminal))
return new_ppt_list, new_analyzer_list, extra_terminal_list
def do_plain(self, Code, ThePattern):
"""Plain source code text as generated by quex."""
text = [self.get_counter_text(ThePattern)]
text.extend(Code.get_code())
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_code())
if ThePattern is None: name = "<no name>"
else: name = ThePattern.pattern_string()
return Terminal(code, name)
def __terminal(self, Text, Code, Name,
IncidenceId=None,
LexemeTerminatingZeroF=False, LexemeBeginF=False,
RequiredRegisterSet=None):
code = CodeTerminal(Text, SourceReference = Code.sr,
PureCode = Code.get_pure_code())
return Terminal(code, Name,
IncidenceId = IncidenceId,
RequireLexemeTerminatingZeroF = LexemeTerminatingZeroF,
RequiresLexemeBeginF = LexemeBeginF,
RequiredRegisterSet = RequiredRegisterSet,
dial_db = self.dial_db)
def get_loop_terminal_code(self, TheLoopMapEntry, DoorIdLoop,
DoorIdLoopExit):
"""RETURNS: A loop terminal.
A terminal: (i) Counts,
(ii) checks possibly for the lexeme end, and
(iii)a either re-enters the loop, or
(iii)b transits to an appendix state machine (couple terminal).
"""
IncidenceId = TheLoopMapEntry.incidence_id
AppendixSmId = TheLoopMapEntry.appendix_sm_id
TheCountAction = TheLoopMapEntry.count_action
code = []
if TheCountAction is not None:
code.extend(
TheCountAction.get_OpList(self.column_number_per_code_unit))
if AppendixSmId is not None:
if not lei.appendix_sm_has_transitions_f:
# If there is no appendix, directly goto to the terminal.
code.extend([Op.GotoDoorId(DoorID.incidence_id(AppendixSmId))])
else:
assert not self.lexeme_end_check_f
# Couple Terminal: transit to appendix state machine.
code.extend([
Op.Assign(E_R.ReferenceP, E_R.InputP),
Op.GotoDoorId(DoorID.state_machine_entry(AppendixSmId))
])
elif not self.lexeme_end_check_f:
# Loop Terminal: directly re-enter loop.
code.append(Op.GotoDoorId(DoorIdLoop))
else:
# Check Terminal: check against lexeme end before re-entering loop.
code.append(
Op.GotoDoorIdIfInputPNotEqualPointer(DoorIdLoop,
E_R.LexemeEnd))
if self.column_number_per_code_unit is not None \
and TheCountAction is not None \
and TheCountAction.cc_type == E_CharacterCountType.COLUMN:
# With reference counting, no column counting while looping.
# => Do it now, before leaving.
code.append(
Op.ColumnCountReferencePDeltaAdd(
E_R.InputP, self.column_number_per_code_unit, False))
code.append(Op.GotoDoorId(DoorIdLoopExit))
return Terminal(CodeTerminal(Lng.COMMAND_LIST(code)),
"<LOOP TERMINAL %i>" % IncidenceId, IncidenceId)
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = StateMachine.from_sequence(CloserSequence)
sm.set_id(dial_db.new_incidence_id())
code = [Lng.GOTO(DoorID.continue_without_on_after_match())]
terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>",
sm.get_id())
return [(sm, terminal)]
def _add_pair(psml, SmOriginal, Name):
"""Add a state machine-terminal pair to 'psml'. A terminal is generated
which transits to 'INDENTATION_HANDLER'. The state machine is cloned
for safety.
"""
if SmOriginal is None: return
incidence_id = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
sm = SmOriginal.clone(StateMachineId=incidence_id)
code = [Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))]
terminal = Terminal(CodeTerminal(code), Name, incidence_id)
# TRY: terminal.set_requires_goto_loop_entry_f()
# INSTEAD: GOTO 'INDENTATION_HANDLER'
psml.append((sm, terminal))
def __terminal(self, Text, Code, Name,
LexemeRelevanceF=bool, LexemeTerminatingZeroF=bool, LexemeBeginF=bool):
code = CodeTerminal(Text, SourceReference = Code.sr,
PureCode = Code.get_pure_code())
return Terminal(code, Name)
def create_state_machine_function(PatternActionPairList,
PatternDictionary,
BufferLimitCode,
SecondModeF=False):
# (*) Initialize address handling
dial_db.clear() # BEFORE constructor of generator;
variable_db.variable_db.init(
) # because constructor creates some addresses.
blackboard.required_support_begin_of_line_set()
def action(ThePattern, PatternName):
txt = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName: txt.append("return false;\n")
else: txt.append("return true;\n")
txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match()))
## print "#", txt
return CodeTerminal(txt)
# -- Display Setup: Patterns and the related Actions
print "(*) Lexical Analyser Patterns:"
for pair in PatternActionPairList:
print "%20s --> %s" % (pair[0], pair[1])
if not SecondModeF: sm_name = "Mr"
else: sm_name = "Mrs"
Setup.analyzer_class_name = sm_name
pattern_action_list = [
(regex.do(pattern_str, PatternDictionary), action_str)
for pattern_str, action_str in PatternActionPairList
]
support_begin_of_line_f = False
for pattern, action_str in pattern_action_list:
support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f
for pattern, action_str in pattern_action_list:
pattern.prepare_count_info(LineColumnCount_Default(),
CodecTrafoInfo=None)
pattern.mount_post_context_sm()
pattern.mount_pre_context_sm()
pattern.cut_character_list(signal_character_list(Setup))
# -- PatternList/TerminalDb
# (Terminals can only be generated after the 'mount procedure', because,
# the bipd_sm is generated through mounting.)
on_failure = CodeTerminal(["return false;\n"])
support_begin_of_line_f = False
terminal_db = {
E_IncidenceIDs.MATCH_FAILURE:
Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE),
E_IncidenceIDs.END_OF_STREAM:
Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM),
E_IncidenceIDs.BAD_LEXATOM:
Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM),
E_IncidenceIDs.OVERFLOW:
Terminal(on_failure, "NO_SPACE_TO_LOAD", E_IncidenceIDs.OVERFLOW),
E_IncidenceIDs.LOAD_FAILURE:
Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE),
}
for pattern, action_str in pattern_action_list:
name = safe_string(pattern.pattern_string())
terminal = Terminal(action(pattern, action_str), name)
terminal.set_incidence_id(pattern.incidence_id())
terminal_db[pattern.incidence_id()] = terminal
# -- create default action that prints the name and the content of the token
# store_last_character_str = ""
# if support_begin_of_line_f:
# store_last_character_str = " %s = %s;\n" % \
# ("me->buffer._lexatom_before_lexeme_start",
# "*(me->buffer._read_p - 1)")
# set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n"
# prefix = store_last_character_str + set_terminating_zero_str
print "## (1) code generation"
pattern_list = [pattern for pattern, action_str in pattern_action_list]
function_body, variable_definitions = cpp_generator.do_core(
pattern_list, terminal_db)
function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n"
function_txt = cpp_generator.wrap_up(sm_name,
function_body,
variable_definitions,
ModeNameList=[])
assert all_isinstance(function_txt, str)
return "#define __QUEX_OPTION_UNIT_TEST\n" \
+ nonsense_default_counter(not SecondModeF) \
+ "".join(function_txt)
def create_state_machine_function(PatternActionPairList,
PatternDictionary,
BufferLimitCode,
SecondModeF=False):
global dial_db
incidence_db = IncidenceDB()
if not SecondModeF: sm_name = "M"
else: sm_name = "M2"
Setup.analyzer_class_name = "TestAnalyzer"
Setup.analyzer_name_safe = "TestAnalyzer"
# (*) Initialize address handling
dial_db = DialDB() # BEFORE constructor of generator;
variable_db.variable_db.init(
) # because constructor creates some addresses.
blackboard.required_support_begin_of_line_set()
terminal_factory = TerminalFactory(sm_name, incidence_db, dial_db)
# -- Display Setup: Patterns and the related Actions
print "(*) Lexical Analyser Patterns:"
for pair in PatternActionPairList:
print "%20s --> %s" % (pair[0], pair[1])
def action(ThePattern, PatternName):
txt = []
if ThePattern.sm_bipd_to_be_reversed is not None:
terminal_factory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M2_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName:
txt.append(
"QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n"
)
else:
txt.append("return;\n")
txt.append(
"%s\n" %
Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db))
## print "#", txt
return CodeTerminal(txt)
pattern_action_list = [
(regex.do(pattern_str, PatternDictionary), action_str)
for pattern_str, action_str in PatternActionPairList
]
ca_map = LineColumnCount_Default()
pattern_list = []
terminal_db = {}
for pattern, action_str in pattern_action_list:
pattern = pattern.finalize(ca_map)
name = Lng.SAFE_STRING(pattern.pattern_string())
terminal = Terminal(action(pattern, action_str), name, dial_db=dial_db)
terminal.set_incidence_id(pattern.incidence_id)
pattern_list.append(pattern)
terminal_db[pattern.incidence_id] = terminal
# -- PatternList/TerminalDb
# (Terminals can only be generated after the 'mount procedure', because,
# the bipd_sm is generated through mounting.)
on_failure = CodeTerminal([
"QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n"
])
terminal_db.update({
E_IncidenceIDs.MATCH_FAILURE:
Terminal(on_failure,
"FAILURE",
E_IncidenceIDs.MATCH_FAILURE,
dial_db=dial_db),
E_IncidenceIDs.END_OF_STREAM:
Terminal(on_failure,
"END_OF_STREAM",
E_IncidenceIDs.END_OF_STREAM,
dial_db=dial_db),
E_IncidenceIDs.BAD_LEXATOM:
Terminal(on_failure,
"BAD_LEXATOM",
E_IncidenceIDs.BAD_LEXATOM,
dial_db=dial_db),
E_IncidenceIDs.LOAD_FAILURE:
Terminal(on_failure,
"LOAD_FAILURE",
E_IncidenceIDs.LOAD_FAILURE,
dial_db=dial_db),
})
mode = Mode(sm_name,
SourceRef_VOID,
pattern_list,
terminal_db, [],
incidence_db,
RunTimeCounterDb=None,
ReloadStateForward=None,
RequiredRegisterSet=set(),
dial_db=dial_db,
Documentation=ModeDocumentation([], [], [], [], []))
print "## (1) code generation"
txt = engine_generator.do_with_counter(mode, ["M", "M2"])
assert all_isinstance(txt, str)
return "#define QUEX_OPTION_UNIT_TEST_EXT\n" + "".join(txt)
def create_state_machine_function(PatternActionPairList, PatternDictionary,
BufferLimitCode, SecondModeF=False):
# (*) Initialize address handling
dial_db.clear() # BEFORE constructor of generator;
variable_db.variable_db.init() # because constructor creates some addresses.
blackboard.required_support_begin_of_line_set()
def action(ThePattern, PatternName):
txt = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName)
if "->1" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n")
elif "->2" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n")
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName: txt.append("return false;\n")
else: txt.append("return true;\n")
txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match()))
## print "#", txt
return CodeTerminal(txt)
# -- Display Setup: Patterns and the related Actions
print "(*) Lexical Analyser Patterns:"
for pair in PatternActionPairList:
print "%20s --> %s" % (pair[0], pair[1])
if not SecondModeF: sm_name = "Mr"
else: sm_name = "Mrs"
Setup.analyzer_class_name = sm_name
pattern_action_list = [
(regex.do(pattern_str, PatternDictionary), action_str)
for pattern_str, action_str in PatternActionPairList
]
support_begin_of_line_f = False
for pattern, action_str in pattern_action_list:
support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f
for pattern, action_str in pattern_action_list:
pattern.prepare_count_info(LineColumnCount_Default(), CodecTrafoInfo=None)
pattern.mount_post_context_sm()
pattern.mount_pre_context_sm()
pattern.cut_character_list(signal_character_list(Setup))
# -- PatternList/TerminalDb
# (Terminals can only be generated after the 'mount procedure', because,
# the bipd_sm is generated through mounting.)
on_failure = CodeTerminal(["return false;\n"])
support_begin_of_line_f = False
terminal_db = {
E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE",
E_IncidenceIDs.MATCH_FAILURE),
E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM",
E_IncidenceIDs.END_OF_STREAM),
E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM",
E_IncidenceIDs.BAD_LEXATOM),
E_IncidenceIDs.OVERFLOW: Terminal(on_failure, "NO_SPACE_TO_LOAD",
E_IncidenceIDs.OVERFLOW),
E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE",
E_IncidenceIDs.LOAD_FAILURE),
}
for pattern, action_str in pattern_action_list:
name = safe_string(pattern.pattern_string())
terminal = Terminal(action(pattern, action_str), name)
terminal.set_incidence_id(pattern.incidence_id())
terminal_db[pattern.incidence_id()] = terminal
# -- create default action that prints the name and the content of the token
# store_last_character_str = ""
# if support_begin_of_line_f:
# store_last_character_str = " %s = %s;\n" % \
# ("me->buffer._lexatom_before_lexeme_start",
# "*(me->buffer._read_p - 1)")
# set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n"
# prefix = store_last_character_str + set_terminating_zero_str
print "## (1) code generation"
pattern_list = [ pattern for pattern, action_str in pattern_action_list ]
function_body, variable_definitions = cpp_generator.do_core(pattern_list, terminal_db)
function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n"
function_txt = cpp_generator.wrap_up(sm_name, function_body,
variable_definitions,
ModeNameList=[])
assert all_isinstance(function_txt, str)
return "#define __QUEX_OPTION_UNIT_TEST\n" \
+ nonsense_default_counter(not SecondModeF) \
+ "".join(function_txt)
def do(ModeName, CaMap, OpenerPattern, CloserPattern, DoorIdExit, ReloadState,
dial_db):
"""
.---<---+----------<------+------------------.
| | | |
| | not | open_n += 1 |
.------. | Closer[0] | |
-------------------------->| Loop +--' | |
| | | yes |
| | | |
| | .-------------. |
| +----->----| Opener[1-N] | |
| | | ? | |
| | '-------------' |
| | | open_n > 0
| | .-------------. |
| +----->----| Closer[1-N] |--------------+------> RESTART
| | | ? | open_n -= 1 else
| | '-------------'
| |
| BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .------------------.
.--| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'---------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| SkipRangeOpen |
'---------------'
"""
psml, \
iid_aux_reentry = _get_state_machine_vs_terminal_list(CloserPattern, OpenerPattern,
DoorIdExit, dial_db)
if ReloadState: engine_type = ReloadState.engine_type
else: engine_type = None
# The first opening pattern must have matched --> counter = 1
entry_op_list = OpList(Op.AssignConstant(E_R.Counter, 1))
analyzer_list, \
terminal_list, \
loop_map, \
door_id_loop, \
required_register_set, \
run_time_counter_f = loop.do(CaMap,
BeforeEntryOpList = entry_op_list,
OnLoopExitDoorId = DoorIdExit,
EngineType = engine_type,
ReloadStateExtern = ReloadState,
ParallelSmTerminalPairList = psml,
dial_db = dial_db,
ModeName = ModeName)
reentry_op_list = [Op.GotoDoorId(door_id_loop)]
terminal_list.append(
Terminal(CodeTerminal(Lng.COMMAND_LIST(reentry_op_list, dial_db)),
Name="<SKIP NESTED RANGE REENTRY>",
IncidenceId=iid_aux_reentry,
dial_db=dial_db))
required_register_set.add(E_R.Counter)
return analyzer_list, \
terminal_list, \
required_register_set, \
run_time_counter_f