def _add_newline(psml, SmNewlineOriginal): """Add a pair (newline state machine, terminal on newline) to 'psml'. When a newline occurs, the column count can be set to 1 and the line number is incremented. Then the indentation counting restarts. """ assert SmNewlineOriginal is not None # Disconnect from machines being used elsewhere. SmNewline = SmNewlineOriginal.clone() SmNewline.set_id(dial_db.new_incidence_id()) # The SmNewline has been used before in the main state machine with a # different incidence id. It is essential to clone! cl = [ Op.LineCountAdd(1), Op.AssignConstant(E_R.Column, 1), Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)) ] terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)), "<INDENTATION NEWLINE>") terminal.set_incidence_id(SmNewline.get_id()) psml.append((SmNewline, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb): """On matching the comment state machine goto a terminal that does the following: """ if SmCommentOriginal is None: return comment_skip_iid = dial_db.new_incidence_id() # Disconnect from machines being used elsewhere. SmComment = SmCommentOriginal.clone() SmComment.set_id(comment_skip_iid) if SmComment.last_character_set().contains_only(ord('\n')): code = Lng.COMMAND_LIST([ LineCountAdd(1), Op.AssignConstant(E_R.Column, 1), ]) else: count_info = CountInfo.from_StateMachine(SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec) code = [ Lng.COMMAND(Op.Assign(E_R.ReferenceP, E_R.LexemeStartP)), CounterDb.do_CountInfo(count_info), Lng.COMMAND(Op.Assign(E_R.LexemeStartP, E_R.ReferenceP)) ] code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))) terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT") terminal.set_incidence_id(comment_skip_iid) psml.append((SmComment, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb): """On matching the comment state machine goto a terminal that does the following: """ if SmCommentOriginal is None: return comment_skip_iid = dial_db.new_incidence_id() # Disconnect from machines being used elsewhere. SmComment = SmCommentOriginal.clone() SmComment.set_id(comment_skip_iid) if SmComment.last_character_set().contains_only(ord('\n')): code = Lng.COMMAND_LIST([ LineCountAdd(1), AssignConstant(E_R.Column, 1), ]) else: count_info = CountInfo.from_StateMachine( SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec) code = [ Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)), CounterDb.do_CountInfo(count_info), Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP)) ] code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))) terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT") terminal.set_incidence_id(comment_skip_iid) psml.append((SmComment, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal): """Add a pair (suppressed newline, terminal on suppressed newline to 'psml'. A suppresed newline is not like a newline--the next line is considered as being appended to the current line. Nevertheless the line number needs to incremented, just the column number is not reset to 1. Then, it continues with indentation counting. """ if SmSuppressedNewlineOriginal is None: return # Disconnect from machines being used elsewhere. SmSuppressedNewline = SmSuppressedNewlineOriginal.clone() SmSuppressedNewline.set_id(dial_db.new_incidence_id()) # The parser MUST ensure that if there is a newline suppressor, there MUST # be a newline being defined. cl = [ Op.LineCountAdd(1), Op.AssignConstant(E_R.Column, 1), Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)), ] terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)), "<INDENTATION SUPPRESSED NEWLINE>") terminal.set_incidence_id(SmSuppressedNewline.get_id()) psml.append((SmSuppressedNewline, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal): """Add a pair (suppressed newline, terminal on suppressed newline to 'psml'. A suppresed newline is not like a newline--the next line is considered as being appended to the current line. Nevertheless the line number needs to incremented, just the column number is not reset to 1. Then, it continues with indentation counting. """ if SmSuppressedNewlineOriginal is None: return # Disconnect from machines being used elsewhere. SmSuppressedNewline = SmSuppressedNewlineOriginal.clone() SmSuppressedNewline.set_id(dial_db.new_incidence_id()) # The parser MUST ensure that if there is a newline suppressor, there MUST # be a newline being defined. cl = [ LineCountAdd(1), AssignConstant(E_R.Column, 1), GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)), ] terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)), "<INDENTATION SUPPRESSED NEWLINE>") terminal.set_incidence_id(SmSuppressedNewline.get_id()) psml.append((SmSuppressedNewline, terminal))
def _add_newline(psml, SmNewlineOriginal): """Add a pair (newline state machine, terminal on newline) to 'psml'. When a newline occurs, the column count can be set to 1 and the line number is incremented. Then the indentation counting restarts. """ assert SmNewlineOriginal is not None # Disconnect from machines being used elsewhere. SmNewline = SmNewlineOriginal.clone() SmNewline.set_id(dial_db.new_incidence_id()) # The SmNewline has been used before in the main state machine with a # different incidence id. It is essential to clone! cl = [ LineCountAdd(1), AssignConstant(E_R.Column, 1), GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)) ] terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)), "<INDENTATION NEWLINE>") terminal.set_incidence_id(SmNewline.get_id()) psml.append((SmNewline, terminal))
def _get_terminal(X, get_appendix): cl = self._command(X.cc_type, X.parameter) appendix = get_appendix(self, X.cc_type) terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(chain(cl, appendix))), Name="%s" % X.cc_type) terminal.set_incidence_id(X.incidence_id) return terminal
def _get_terminal(X, get_appendix): cl = self._command(X.cc_type, X.parameter) appendix = get_appendix(self, X.cc_type) terminal = Terminal(CodeTerminal( Lng.COMMAND_LIST(chain(cl, appendix))), Name="%s" % X.cc_type) terminal.set_incidence_id(X.incidence_id) return terminal
def __get_terminal_beyond(OnBeyond, BeyondIid): """Generate Terminal to be executed upon exit from the 'loop'. BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of the terminal to be generated. """ code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond)) result = Terminal(code_on_beyond, "<BEYOND>") # Put last considered character back result.set_incidence_id(BeyondIid) return result
def __get_terminal_beyond(OnBeyond, BeyondIid): """Generate Terminal to be executed upon exit from the 'loop'. BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of the terminal to be generated. """ code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond)) result = Terminal(code_on_beyond, "<BEYOND>") # Put last considered character back result.set_incidence_id(BeyondIid) return result
def create_state_machine_function(PatternActionPairList, PatternDictionary, BufferLimitCode, SecondModeF=False): # (*) Initialize address handling dial_db.clear() # BEFORE constructor of generator; variable_db.variable_db.init() # because constructor creates some addresses. blackboard.required_support_begin_of_line_set() def action(ThePattern, PatternName): txt = [] if ThePattern.bipd_sm is not None: TerminalFactory.do_bipd_entry_and_return(txt, pattern) txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(blackboard.required_support_begin_of_line())) txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True)) txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName) if "->1" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n") elif "->2" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n") if "CONTINUE" in PatternName: txt.append("") elif "STOP" in PatternName: txt.append("return false;\n") else: txt.append("return true;\n") txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match())) ## print "#", txt return CodeTerminal(txt) # -- Display Setup: Patterns and the related Actions print "(*) Lexical Analyser Patterns:" for pair in PatternActionPairList: print "%20s --> %s" % (pair[0], pair[1]) if not SecondModeF: sm_name = "Mr" else: sm_name = "Mrs" Setup.analyzer_class_name = sm_name pattern_action_list = [ (regex.do(pattern_str, PatternDictionary), action_str) for pattern_str, action_str in PatternActionPairList ] support_begin_of_line_f = False for pattern, action_str in pattern_action_list: support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f for pattern, action_str in pattern_action_list: pattern.prepare_count_info(LineColumnCount_Default(), CodecTrafoInfo=None) pattern.mount_post_context_sm() pattern.mount_pre_context_sm() pattern.cut_character_list(signal_character_list(Setup)) # -- PatternList/TerminalDb # (Terminals can only be generated after the 'mount procedure', because, # the bipd_sm is generated through mounting.) on_failure = CodeTerminal(["return false;\n"]) support_begin_of_line_f = False terminal_db = { E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE), E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM), E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM), E_IncidenceIDs.OVERFLOW: Terminal(on_failure, "NO_SPACE_TO_LOAD", E_IncidenceIDs.OVERFLOW), E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE), } for pattern, action_str in pattern_action_list: name = safe_string(pattern.pattern_string()) terminal = Terminal(action(pattern, action_str), name) terminal.set_incidence_id(pattern.incidence_id()) terminal_db[pattern.incidence_id()] = terminal # -- create default action that prints the name and the content of the token # store_last_character_str = "" # if support_begin_of_line_f: # store_last_character_str = " %s = %s;\n" % \ # ("me->buffer._lexatom_before_lexeme_start", # "*(me->buffer._read_p - 1)") # set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n" # prefix = store_last_character_str + set_terminating_zero_str print "## (1) code generation" pattern_list = [ pattern for pattern, action_str in pattern_action_list ] function_body, variable_definitions = cpp_generator.do_core(pattern_list, terminal_db) function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n" function_txt = cpp_generator.wrap_up(sm_name, function_body, variable_definitions, ModeNameList=[]) assert all_isinstance(function_txt, str) return "#define __QUEX_OPTION_UNIT_TEST\n" \ + nonsense_default_counter(not SecondModeF) \ + "".join(function_txt)
def create_state_machine_function(PatternActionPairList, PatternDictionary, BufferLimitCode, SecondModeF=False): global dial_db incidence_db = IncidenceDB() if not SecondModeF: sm_name = "M" else: sm_name = "M2" Setup.analyzer_class_name = "TestAnalyzer" Setup.analyzer_name_safe = "TestAnalyzer" # (*) Initialize address handling dial_db = DialDB() # BEFORE constructor of generator; variable_db.variable_db.init( ) # because constructor creates some addresses. blackboard.required_support_begin_of_line_set() terminal_factory = TerminalFactory(sm_name, incidence_db, dial_db) # -- Display Setup: Patterns and the related Actions print "(*) Lexical Analyser Patterns:" for pair in PatternActionPairList: print "%20s --> %s" % (pair[0], pair[1]) def action(ThePattern, PatternName): txt = [] if ThePattern.sm_bipd_to_be_reversed is not None: terminal_factory.do_bipd_entry_and_return(txt, pattern) txt.append("%s\n" % Lng.STORE_LAST_CHARACTER( blackboard.required_support_begin_of_line())) txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True)) txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName) if "->1" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(M_analyzer_function);\n" ) elif "->2" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(M2_analyzer_function);\n" ) if "CONTINUE" in PatternName: txt.append("") elif "STOP" in PatternName: txt.append( "QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n" ) else: txt.append("return;\n") txt.append( "%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db)) ## print "#", txt return CodeTerminal(txt) pattern_action_list = [ (regex.do(pattern_str, PatternDictionary), action_str) for pattern_str, action_str in PatternActionPairList ] ca_map = LineColumnCount_Default() pattern_list = [] terminal_db = {} for pattern, action_str in pattern_action_list: pattern = pattern.finalize(ca_map) name = Lng.SAFE_STRING(pattern.pattern_string()) terminal = Terminal(action(pattern, action_str), name, dial_db=dial_db) terminal.set_incidence_id(pattern.incidence_id) pattern_list.append(pattern) terminal_db[pattern.incidence_id] = terminal # -- PatternList/TerminalDb # (Terminals can only be generated after the 'mount procedure', because, # the bipd_sm is generated through mounting.) on_failure = CodeTerminal([ "QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n" ]) terminal_db.update({ E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE, dial_db=dial_db), E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM, dial_db=dial_db), E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM, dial_db=dial_db), E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE, dial_db=dial_db), }) mode = Mode(sm_name, SourceRef_VOID, pattern_list, terminal_db, [], incidence_db, RunTimeCounterDb=None, ReloadStateForward=None, RequiredRegisterSet=set(), dial_db=dial_db, Documentation=ModeDocumentation([], [], [], [], [])) print "## (1) code generation" txt = engine_generator.do_with_counter(mode, ["M", "M2"]) assert all_isinstance(txt, str) return "#define QUEX_OPTION_UNIT_TEST_EXT\n" + "".join(txt)
def create_state_machine_function(PatternActionPairList, PatternDictionary, BufferLimitCode, SecondModeF=False): # (*) Initialize address handling dial_db.clear() # BEFORE constructor of generator; variable_db.variable_db.init( ) # because constructor creates some addresses. blackboard.required_support_begin_of_line_set() def action(ThePattern, PatternName): txt = [] if ThePattern.bipd_sm is not None: TerminalFactory.do_bipd_entry_and_return(txt, pattern) txt.append("%s\n" % Lng.STORE_LAST_CHARACTER( blackboard.required_support_begin_of_line())) txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True)) txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName) if "->1" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n" ) elif "->2" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n" ) if "CONTINUE" in PatternName: txt.append("") elif "STOP" in PatternName: txt.append("return false;\n") else: txt.append("return true;\n") txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match())) ## print "#", txt return CodeTerminal(txt) # -- Display Setup: Patterns and the related Actions print "(*) Lexical Analyser Patterns:" for pair in PatternActionPairList: print "%20s --> %s" % (pair[0], pair[1]) if not SecondModeF: sm_name = "Mr" else: sm_name = "Mrs" Setup.analyzer_class_name = sm_name pattern_action_list = [ (regex.do(pattern_str, PatternDictionary), action_str) for pattern_str, action_str in PatternActionPairList ] support_begin_of_line_f = False for pattern, action_str in pattern_action_list: support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f for pattern, action_str in pattern_action_list: pattern.prepare_count_info(LineColumnCount_Default(), CodecTrafoInfo=None) pattern.mount_post_context_sm() pattern.mount_pre_context_sm() pattern.cut_character_list(signal_character_list(Setup)) # -- PatternList/TerminalDb # (Terminals can only be generated after the 'mount procedure', because, # the bipd_sm is generated through mounting.) on_failure = CodeTerminal(["return false;\n"]) support_begin_of_line_f = False terminal_db = { E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE), E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM), E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM), E_IncidenceIDs.OVERFLOW: Terminal(on_failure, "NO_SPACE_TO_LOAD", E_IncidenceIDs.OVERFLOW), E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE), } for pattern, action_str in pattern_action_list: name = safe_string(pattern.pattern_string()) terminal = Terminal(action(pattern, action_str), name) terminal.set_incidence_id(pattern.incidence_id()) terminal_db[pattern.incidence_id()] = terminal # -- create default action that prints the name and the content of the token # store_last_character_str = "" # if support_begin_of_line_f: # store_last_character_str = " %s = %s;\n" % \ # ("me->buffer._lexatom_before_lexeme_start", # "*(me->buffer._read_p - 1)") # set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n" # prefix = store_last_character_str + set_terminating_zero_str print "## (1) code generation" pattern_list = [pattern for pattern, action_str in pattern_action_list] function_body, variable_definitions = cpp_generator.do_core( pattern_list, terminal_db) function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n" function_txt = cpp_generator.wrap_up(sm_name, function_body, variable_definitions, ModeNameList=[]) assert all_isinstance(function_txt, str) return "#define __QUEX_OPTION_UNIT_TEST\n" \ + nonsense_default_counter(not SecondModeF) \ + "".join(function_txt)