def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
Op.LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(SmComment,
CounterDb,
CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Op.Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Op.Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
Op.LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _get_terminal(X, get_appendix):
cl = self._command(X.cc_type, X.parameter)
appendix = get_appendix(self, X.cc_type)
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(chain(cl, appendix))),
Name="%s" % X.cc_type)
terminal.set_incidence_id(X.incidence_id)
return terminal
def _get_terminal(X, get_appendix):
cl = self._command(X.cc_type, X.parameter)
appendix = get_appendix(self, X.cc_type)
terminal = Terminal(CodeTerminal(
Lng.COMMAND_LIST(chain(cl, appendix))),
Name="%s" % X.cc_type)
terminal.set_incidence_id(X.incidence_id)
return terminal
def __get_terminal_beyond(OnBeyond, BeyondIid):
"""Generate Terminal to be executed upon exit from the 'loop'.
BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of
the terminal to be generated.
"""
code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond))
result = Terminal(code_on_beyond, "<BEYOND>") # Put last considered character back
result.set_incidence_id(BeyondIid)
return result
def __get_terminal_beyond(OnBeyond, BeyondIid):
"""Generate Terminal to be executed upon exit from the 'loop'.
BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of
the terminal to be generated.
"""
code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond))
result = Terminal(code_on_beyond,
"<BEYOND>") # Put last considered character back
result.set_incidence_id(BeyondIid)
return result
def create_state_machine_function(PatternActionPairList, PatternDictionary,
BufferLimitCode, SecondModeF=False):
# (*) Initialize address handling
dial_db.clear() # BEFORE constructor of generator;
variable_db.variable_db.init() # because constructor creates some addresses.
blackboard.required_support_begin_of_line_set()
def action(ThePattern, PatternName):
txt = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName)
if "->1" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n")
elif "->2" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n")
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName: txt.append("return false;\n")
else: txt.append("return true;\n")
txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match()))
## print "#", txt
return CodeTerminal(txt)
# -- Display Setup: Patterns and the related Actions
print "(*) Lexical Analyser Patterns:"
for pair in PatternActionPairList:
print "%20s --> %s" % (pair[0], pair[1])
if not SecondModeF: sm_name = "Mr"
else: sm_name = "Mrs"
Setup.analyzer_class_name = sm_name
pattern_action_list = [
(regex.do(pattern_str, PatternDictionary), action_str)
for pattern_str, action_str in PatternActionPairList
]
support_begin_of_line_f = False
for pattern, action_str in pattern_action_list:
support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f
for pattern, action_str in pattern_action_list:
pattern.prepare_count_info(LineColumnCount_Default(), CodecTrafoInfo=None)
pattern.mount_post_context_sm()
pattern.mount_pre_context_sm()
pattern.cut_character_list(signal_character_list(Setup))
# -- PatternList/TerminalDb
# (Terminals can only be generated after the 'mount procedure', because,
# the bipd_sm is generated through mounting.)
on_failure = CodeTerminal(["return false;\n"])
support_begin_of_line_f = False
terminal_db = {
E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE",
E_IncidenceIDs.MATCH_FAILURE),
E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM",
E_IncidenceIDs.END_OF_STREAM),
E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM",
E_IncidenceIDs.BAD_LEXATOM),
E_IncidenceIDs.OVERFLOW: Terminal(on_failure, "NO_SPACE_TO_LOAD",
E_IncidenceIDs.OVERFLOW),
E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE",
E_IncidenceIDs.LOAD_FAILURE),
}
for pattern, action_str in pattern_action_list:
name = safe_string(pattern.pattern_string())
terminal = Terminal(action(pattern, action_str), name)
terminal.set_incidence_id(pattern.incidence_id())
terminal_db[pattern.incidence_id()] = terminal
# -- create default action that prints the name and the content of the token
# store_last_character_str = ""
# if support_begin_of_line_f:
# store_last_character_str = " %s = %s;\n" % \
# ("me->buffer._lexatom_before_lexeme_start",
# "*(me->buffer._read_p - 1)")
# set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n"
# prefix = store_last_character_str + set_terminating_zero_str
print "## (1) code generation"
pattern_list = [ pattern for pattern, action_str in pattern_action_list ]
function_body, variable_definitions = cpp_generator.do_core(pattern_list, terminal_db)
function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n"
function_txt = cpp_generator.wrap_up(sm_name, function_body,
variable_definitions,
ModeNameList=[])
assert all_isinstance(function_txt, str)
return "#define __QUEX_OPTION_UNIT_TEST\n" \
+ nonsense_default_counter(not SecondModeF) \
+ "".join(function_txt)
def create_state_machine_function(PatternActionPairList,
PatternDictionary,
BufferLimitCode,
SecondModeF=False):
global dial_db
incidence_db = IncidenceDB()
if not SecondModeF: sm_name = "M"
else: sm_name = "M2"
Setup.analyzer_class_name = "TestAnalyzer"
Setup.analyzer_name_safe = "TestAnalyzer"
# (*) Initialize address handling
dial_db = DialDB() # BEFORE constructor of generator;
variable_db.variable_db.init(
) # because constructor creates some addresses.
blackboard.required_support_begin_of_line_set()
terminal_factory = TerminalFactory(sm_name, incidence_db, dial_db)
# -- Display Setup: Patterns and the related Actions
print "(*) Lexical Analyser Patterns:"
for pair in PatternActionPairList:
print "%20s --> %s" % (pair[0], pair[1])
def action(ThePattern, PatternName):
txt = []
if ThePattern.sm_bipd_to_be_reversed is not None:
terminal_factory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(M2_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName:
txt.append(
"QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n"
)
else:
txt.append("return;\n")
txt.append(
"%s\n" %
Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db))
## print "#", txt
return CodeTerminal(txt)
pattern_action_list = [
(regex.do(pattern_str, PatternDictionary), action_str)
for pattern_str, action_str in PatternActionPairList
]
ca_map = LineColumnCount_Default()
pattern_list = []
terminal_db = {}
for pattern, action_str in pattern_action_list:
pattern = pattern.finalize(ca_map)
name = Lng.SAFE_STRING(pattern.pattern_string())
terminal = Terminal(action(pattern, action_str), name, dial_db=dial_db)
terminal.set_incidence_id(pattern.incidence_id)
pattern_list.append(pattern)
terminal_db[pattern.incidence_id] = terminal
# -- PatternList/TerminalDb
# (Terminals can only be generated after the 'mount procedure', because,
# the bipd_sm is generated through mounting.)
on_failure = CodeTerminal([
"QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n"
])
terminal_db.update({
E_IncidenceIDs.MATCH_FAILURE:
Terminal(on_failure,
"FAILURE",
E_IncidenceIDs.MATCH_FAILURE,
dial_db=dial_db),
E_IncidenceIDs.END_OF_STREAM:
Terminal(on_failure,
"END_OF_STREAM",
E_IncidenceIDs.END_OF_STREAM,
dial_db=dial_db),
E_IncidenceIDs.BAD_LEXATOM:
Terminal(on_failure,
"BAD_LEXATOM",
E_IncidenceIDs.BAD_LEXATOM,
dial_db=dial_db),
E_IncidenceIDs.LOAD_FAILURE:
Terminal(on_failure,
"LOAD_FAILURE",
E_IncidenceIDs.LOAD_FAILURE,
dial_db=dial_db),
})
mode = Mode(sm_name,
SourceRef_VOID,
pattern_list,
terminal_db, [],
incidence_db,
RunTimeCounterDb=None,
ReloadStateForward=None,
RequiredRegisterSet=set(),
dial_db=dial_db,
Documentation=ModeDocumentation([], [], [], [], []))
print "## (1) code generation"
txt = engine_generator.do_with_counter(mode, ["M", "M2"])
assert all_isinstance(txt, str)
return "#define QUEX_OPTION_UNIT_TEST_EXT\n" + "".join(txt)
def create_state_machine_function(PatternActionPairList,
PatternDictionary,
BufferLimitCode,
SecondModeF=False):
# (*) Initialize address handling
dial_db.clear() # BEFORE constructor of generator;
variable_db.variable_db.init(
) # because constructor creates some addresses.
blackboard.required_support_begin_of_line_set()
def action(ThePattern, PatternName):
txt = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(txt, pattern)
txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(
blackboard.required_support_begin_of_line()))
txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True))
txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' %
PatternName)
if "->1" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n"
)
elif "->2" in PatternName:
txt.append(
"me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n"
)
if "CONTINUE" in PatternName: txt.append("")
elif "STOP" in PatternName: txt.append("return false;\n")
else: txt.append("return true;\n")
txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match()))
## print "#", txt
return CodeTerminal(txt)
# -- Display Setup: Patterns and the related Actions
print "(*) Lexical Analyser Patterns:"
for pair in PatternActionPairList:
print "%20s --> %s" % (pair[0], pair[1])
if not SecondModeF: sm_name = "Mr"
else: sm_name = "Mrs"
Setup.analyzer_class_name = sm_name
pattern_action_list = [
(regex.do(pattern_str, PatternDictionary), action_str)
for pattern_str, action_str in PatternActionPairList
]
support_begin_of_line_f = False
for pattern, action_str in pattern_action_list:
support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f
for pattern, action_str in pattern_action_list:
pattern.prepare_count_info(LineColumnCount_Default(),
CodecTrafoInfo=None)
pattern.mount_post_context_sm()
pattern.mount_pre_context_sm()
pattern.cut_character_list(signal_character_list(Setup))
# -- PatternList/TerminalDb
# (Terminals can only be generated after the 'mount procedure', because,
# the bipd_sm is generated through mounting.)
on_failure = CodeTerminal(["return false;\n"])
support_begin_of_line_f = False
terminal_db = {
E_IncidenceIDs.MATCH_FAILURE:
Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE),
E_IncidenceIDs.END_OF_STREAM:
Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM),
E_IncidenceIDs.BAD_LEXATOM:
Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM),
E_IncidenceIDs.OVERFLOW:
Terminal(on_failure, "NO_SPACE_TO_LOAD", E_IncidenceIDs.OVERFLOW),
E_IncidenceIDs.LOAD_FAILURE:
Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE),
}
for pattern, action_str in pattern_action_list:
name = safe_string(pattern.pattern_string())
terminal = Terminal(action(pattern, action_str), name)
terminal.set_incidence_id(pattern.incidence_id())
terminal_db[pattern.incidence_id()] = terminal
# -- create default action that prints the name and the content of the token
# store_last_character_str = ""
# if support_begin_of_line_f:
# store_last_character_str = " %s = %s;\n" % \
# ("me->buffer._lexatom_before_lexeme_start",
# "*(me->buffer._read_p - 1)")
# set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n"
# prefix = store_last_character_str + set_terminating_zero_str
print "## (1) code generation"
pattern_list = [pattern for pattern, action_str in pattern_action_list]
function_body, variable_definitions = cpp_generator.do_core(
pattern_list, terminal_db)
function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n"
function_txt = cpp_generator.wrap_up(sm_name,
function_body,
variable_definitions,
ModeNameList=[])
assert all_isinstance(function_txt, str)
return "#define __QUEX_OPTION_UNIT_TEST\n" \
+ nonsense_default_counter(not SecondModeF) \
+ "".join(function_txt)