def get_pure_text(self): """RETURNS: Text The returned text is free from any annotations. """ assert all_isinstance(self.__code, (str, unicode)) return "".join(self.__code)
def get_text(self): """RETURNS: Text May contain annotations to the code may by the derived class. """ code = self.get_code() assert all_isinstance(code, (str, unicode)) return "".join(code)
def wrap_up(ModeName, FunctionBody, VariableDefs, ModeNameList, dial_db): txt_function = Lng.ANALYZER_FUNCTION(ModeName, Setup, VariableDefs, FunctionBody, dial_db, ModeNameList) txt_header = Lng.HEADER_DEFINITIONS(dial_db) assert isinstance(txt_header, (str, unicode)) txt_analyzer = get_plain_strings(txt_function, dial_db) assert all_isinstance(txt_analyzer, (str, unicode)) return [ txt_header ] + txt_analyzer
def wrap_up(ModeName, FunctionBody, VariableDefs, ModeNameList): txt_function = Lng.ANALYZER_FUNCTION(ModeName, Setup, VariableDefs, FunctionBody, ModeNameList) txt_header = Lng.HEADER_DEFINITIONS() assert isinstance(txt_header, (str, unicode)) txt_analyzer = get_plain_strings(txt_function) assert all_isinstance(txt_analyzer, (str, unicode)) return [ txt_header ] + txt_analyzer
def __get_setting_list(self, Name): """RETURNS: [ OptionSetting ] for a given option's Name. This function does additional checks for consistency. """ setting_list = dict.get(self, Name) if setting_list is None: return None assert isinstance(setting_list, list) assert all_isinstance(setting_list, OptionSetting) assert (not mode_option_info_db[Name].single_setting_f()) or len(setting_list) == 1 return setting_list
def __init__(self, PatternList): assert isinstance(PatternList, list) assert len(PatternList) > 0 assert all_isinstance(PatternList, Pattern) assert all_true(PatternList, lambda p: p.incidence_id() is not None) # (*) Core SM, Pre-Context SM, ... # ... and sometimes backward input position SMs. self.sm, \ self.pre_context_sm, \ self.bipd_sm_db, \ self.pre_context_sm_id_list = self.__prepare(PatternList)
def __get_setting_list(self, Name): """RETURNS: [ OptionSetting ] for a given option's Name. This function does additional checks for consistency. """ setting_list = dict.get(self, Name) if setting_list is None: return None assert isinstance(setting_list, list) assert all_isinstance(setting_list, OptionSetting) assert (not mode_option_info_db[Name].single_setting_f() ) or len(setting_list) == 1 return setting_list
def get_documentation(self): L = max(map(lambda mode: len(mode.name), self.__base_mode_sequence)) txt = "\nMODE: %s\n" % self.name txt += "\n" if len(self.__base_mode_sequence) != 1: txt += " BASE MODE SEQUENCE:\n" base_mode_name_list = map(lambda mode: mode.name, self.__base_mode_sequence[:-1]) base_mode_name_list.reverse() for name in base_mode_name_list: txt += " %s\n" % name txt += "\n" if len(self.__doc_history_deletion) != 0: txt += " DELETION ACTIONS:\n" for entry in self.__doc_history_deletion: txt += " %s: %s%s (from mode %s)\n" % \ (entry[0], " " * (L - len(self.name)), entry[1], entry[2]) txt += "\n" if len(self.__doc_history_reprioritization) != 0: txt += " PRIORITY-MARK ACTIONS:\n" self.__doc_history_reprioritization.sort( lambda x, y: cmp(x[4], y[4])) for entry in self.__doc_history_reprioritization: txt += " %s: %s%s (from mode %s) (%i) --> (%i)\n" % \ (entry[0], " " * (L - len(self.name)), entry[1], entry[2], entry[3], entry[4]) txt += "\n" assert all_isinstance(self.__pattern_list, Pattern) if len(self.__pattern_list) != 0: txt += " PATTERN LIST:\n" for x in self.__pattern_list: space = " " * (L - len(x.sr.mode_name)) txt += " (%3i) %s: %s%s\n" % \ (x.incidence_id(), x.sr.mode_name, space, x.pattern_string()) txt += "\n" return txt
def get_documentation(self): L = max(map(lambda mode: len(mode.name), self.__base_mode_sequence)) txt = "\nMODE: %s\n" % self.name txt += "\n" if len(self.__base_mode_sequence) != 1: txt += " BASE MODE SEQUENCE:\n" base_mode_name_list = map(lambda mode: mode.name, self.__base_mode_sequence[:-1]) base_mode_name_list.reverse() for name in base_mode_name_list: txt += " %s\n" % name txt += "\n" if len(self.__doc_history_deletion) != 0: txt += " DELETION ACTIONS:\n" for entry in self.__doc_history_deletion: txt += " %s: %s%s (from mode %s)\n" % \ (entry[0], " " * (L - len(self.name)), entry[1], entry[2]) txt += "\n" if len(self.__doc_history_reprioritization) != 0: txt += " PRIORITY-MARK ACTIONS:\n" self.__doc_history_reprioritization.sort(lambda x, y: cmp(x[4], y[4])) for entry in self.__doc_history_reprioritization: txt += " %s: %s%s (from mode %s) (%i) --> (%i)\n" % \ (entry[0], " " * (L - len(self.name)), entry[1], entry[2], entry[3], entry[4]) txt += "\n" assert all_isinstance(self.__pattern_list, Pattern) if len(self.__pattern_list) != 0: txt += " PATTERN LIST:\n" for x in self.__pattern_list: space = " " * (L - len(x.sr.mode_name)) txt += " (%3i) %s: %s%s\n" % \ (x.incidence_id(), x.sr.mode_name, space, x.pattern_string()) txt += "\n" return txt
def create_state_machine_function(PatternActionPairList, PatternDictionary, BufferLimitCode, SecondModeF=False): # (*) Initialize address handling dial_db.clear() # BEFORE constructor of generator; variable_db.variable_db.init( ) # because constructor creates some addresses. blackboard.required_support_begin_of_line_set() def action(ThePattern, PatternName): txt = [] if ThePattern.bipd_sm is not None: TerminalFactory.do_bipd_entry_and_return(txt, pattern) txt.append("%s\n" % Lng.STORE_LAST_CHARACTER( blackboard.required_support_begin_of_line())) txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True)) txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName) if "->1" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n" ) elif "->2" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n" ) if "CONTINUE" in PatternName: txt.append("") elif "STOP" in PatternName: txt.append("return false;\n") else: txt.append("return true;\n") txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match())) ## print "#", txt return CodeTerminal(txt) # -- Display Setup: Patterns and the related Actions print "(*) Lexical Analyser Patterns:" for pair in PatternActionPairList: print "%20s --> %s" % (pair[0], pair[1]) if not SecondModeF: sm_name = "Mr" else: sm_name = "Mrs" Setup.analyzer_class_name = sm_name pattern_action_list = [ (regex.do(pattern_str, PatternDictionary), action_str) for pattern_str, action_str in PatternActionPairList ] support_begin_of_line_f = False for pattern, action_str in pattern_action_list: support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f for pattern, action_str in pattern_action_list: pattern.prepare_count_info(LineColumnCount_Default(), CodecTrafoInfo=None) pattern.mount_post_context_sm() pattern.mount_pre_context_sm() pattern.cut_character_list(signal_character_list(Setup)) # -- PatternList/TerminalDb # (Terminals can only be generated after the 'mount procedure', because, # the bipd_sm is generated through mounting.) on_failure = CodeTerminal(["return false;\n"]) support_begin_of_line_f = False terminal_db = { E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE), E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM), E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM), E_IncidenceIDs.OVERFLOW: Terminal(on_failure, "NO_SPACE_TO_LOAD", E_IncidenceIDs.OVERFLOW), E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE), } for pattern, action_str in pattern_action_list: name = safe_string(pattern.pattern_string()) terminal = Terminal(action(pattern, action_str), name) terminal.set_incidence_id(pattern.incidence_id()) terminal_db[pattern.incidence_id()] = terminal # -- create default action that prints the name and the content of the token # store_last_character_str = "" # if support_begin_of_line_f: # store_last_character_str = " %s = %s;\n" % \ # ("me->buffer._lexatom_before_lexeme_start", # "*(me->buffer._read_p - 1)") # set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n" # prefix = store_last_character_str + set_terminating_zero_str print "## (1) code generation" pattern_list = [pattern for pattern, action_str in pattern_action_list] function_body, variable_definitions = cpp_generator.do_core( pattern_list, terminal_db) function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n" function_txt = cpp_generator.wrap_up(sm_name, function_body, variable_definitions, ModeNameList=[]) assert all_isinstance(function_txt, str) return "#define __QUEX_OPTION_UNIT_TEST\n" \ + nonsense_default_counter(not SecondModeF) \ + "".join(function_txt)
def __init__(self, Code, DropOutF=False): assert type(DropOutF) == bool self.__drop_out_f = DropOutF if isinstance(Code, list): self.__code = Code else: self.__code = [Code] assert all_isinstance(self.__code, (int, str, unicode))
def __init__(self, Code, DropOutF=False): assert type(DropOutF) == bool self.__drop_out_f = DropOutF if isinstance(Code, list): self.__code = Code else: self.__code = [ Code ] assert all_isinstance(self.__code, (int, str, unicode))
def create_state_machine_function(PatternActionPairList, PatternDictionary, BufferLimitCode, SecondModeF=False): global dial_db incidence_db = IncidenceDB() if not SecondModeF: sm_name = "M" else: sm_name = "M2" Setup.analyzer_class_name = "TestAnalyzer" Setup.analyzer_name_safe = "TestAnalyzer" # (*) Initialize address handling dial_db = DialDB() # BEFORE constructor of generator; variable_db.variable_db.init( ) # because constructor creates some addresses. blackboard.required_support_begin_of_line_set() terminal_factory = TerminalFactory(sm_name, incidence_db, dial_db) # -- Display Setup: Patterns and the related Actions print "(*) Lexical Analyser Patterns:" for pair in PatternActionPairList: print "%20s --> %s" % (pair[0], pair[1]) def action(ThePattern, PatternName): txt = [] if ThePattern.sm_bipd_to_be_reversed is not None: terminal_factory.do_bipd_entry_and_return(txt, pattern) txt.append("%s\n" % Lng.STORE_LAST_CHARACTER( blackboard.required_support_begin_of_line())) txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True)) txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName) if "->1" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(M_analyzer_function);\n" ) elif "->2" in PatternName: txt.append( "me->current_analyzer_function = QUEX_NAME(M2_analyzer_function);\n" ) if "CONTINUE" in PatternName: txt.append("") elif "STOP" in PatternName: txt.append( "QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n" ) else: txt.append("return;\n") txt.append( "%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db)) ## print "#", txt return CodeTerminal(txt) pattern_action_list = [ (regex.do(pattern_str, PatternDictionary), action_str) for pattern_str, action_str in PatternActionPairList ] ca_map = LineColumnCount_Default() pattern_list = [] terminal_db = {} for pattern, action_str in pattern_action_list: pattern = pattern.finalize(ca_map) name = Lng.SAFE_STRING(pattern.pattern_string()) terminal = Terminal(action(pattern, action_str), name, dial_db=dial_db) terminal.set_incidence_id(pattern.incidence_id) pattern_list.append(pattern) terminal_db[pattern.incidence_id] = terminal # -- PatternList/TerminalDb # (Terminals can only be generated after the 'mount procedure', because, # the bipd_sm is generated through mounting.) on_failure = CodeTerminal([ "QUEX_NAME(MF_error_code_set_if_first)(me, E_Error_UnitTest_Termination); return;\n" ]) terminal_db.update({ E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE, dial_db=dial_db), E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM, dial_db=dial_db), E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM, dial_db=dial_db), E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE, dial_db=dial_db), }) mode = Mode(sm_name, SourceRef_VOID, pattern_list, terminal_db, [], incidence_db, RunTimeCounterDb=None, ReloadStateForward=None, RequiredRegisterSet=set(), dial_db=dial_db, Documentation=ModeDocumentation([], [], [], [], [])) print "## (1) code generation" txt = engine_generator.do_with_counter(mode, ["M", "M2"]) assert all_isinstance(txt, str) return "#define QUEX_OPTION_UNIT_TEST_EXT\n" + "".join(txt)
def create_state_machine_function(PatternActionPairList, PatternDictionary, BufferLimitCode, SecondModeF=False): # (*) Initialize address handling dial_db.clear() # BEFORE constructor of generator; variable_db.variable_db.init() # because constructor creates some addresses. blackboard.required_support_begin_of_line_set() def action(ThePattern, PatternName): txt = [] if ThePattern.bipd_sm is not None: TerminalFactory.do_bipd_entry_and_return(txt, pattern) txt.append("%s\n" % Lng.STORE_LAST_CHARACTER(blackboard.required_support_begin_of_line())) txt.append("%s\n" % Lng.LEXEME_TERMINATING_ZERO_SET(True)) txt.append('printf("%19s \'%%s\'\\n", Lexeme); fflush(stdout);\n' % PatternName) if "->1" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mr_analyzer_function);\n") elif "->2" in PatternName: txt.append("me->current_analyzer_function = QUEX_NAME(Mrs_analyzer_function);\n") if "CONTINUE" in PatternName: txt.append("") elif "STOP" in PatternName: txt.append("return false;\n") else: txt.append("return true;\n") txt.append("%s\n" % Lng.GOTO(DoorID.continue_with_on_after_match())) ## print "#", txt return CodeTerminal(txt) # -- Display Setup: Patterns and the related Actions print "(*) Lexical Analyser Patterns:" for pair in PatternActionPairList: print "%20s --> %s" % (pair[0], pair[1]) if not SecondModeF: sm_name = "Mr" else: sm_name = "Mrs" Setup.analyzer_class_name = sm_name pattern_action_list = [ (regex.do(pattern_str, PatternDictionary), action_str) for pattern_str, action_str in PatternActionPairList ] support_begin_of_line_f = False for pattern, action_str in pattern_action_list: support_begin_of_line_f |= pattern.pre_context_trivial_begin_of_line_f for pattern, action_str in pattern_action_list: pattern.prepare_count_info(LineColumnCount_Default(), CodecTrafoInfo=None) pattern.mount_post_context_sm() pattern.mount_pre_context_sm() pattern.cut_character_list(signal_character_list(Setup)) # -- PatternList/TerminalDb # (Terminals can only be generated after the 'mount procedure', because, # the bipd_sm is generated through mounting.) on_failure = CodeTerminal(["return false;\n"]) support_begin_of_line_f = False terminal_db = { E_IncidenceIDs.MATCH_FAILURE: Terminal(on_failure, "FAILURE", E_IncidenceIDs.MATCH_FAILURE), E_IncidenceIDs.END_OF_STREAM: Terminal(on_failure, "END_OF_STREAM", E_IncidenceIDs.END_OF_STREAM), E_IncidenceIDs.BAD_LEXATOM: Terminal(on_failure, "BAD_LEXATOM", E_IncidenceIDs.BAD_LEXATOM), E_IncidenceIDs.OVERFLOW: Terminal(on_failure, "NO_SPACE_TO_LOAD", E_IncidenceIDs.OVERFLOW), E_IncidenceIDs.LOAD_FAILURE: Terminal(on_failure, "LOAD_FAILURE", E_IncidenceIDs.LOAD_FAILURE), } for pattern, action_str in pattern_action_list: name = safe_string(pattern.pattern_string()) terminal = Terminal(action(pattern, action_str), name) terminal.set_incidence_id(pattern.incidence_id()) terminal_db[pattern.incidence_id()] = terminal # -- create default action that prints the name and the content of the token # store_last_character_str = "" # if support_begin_of_line_f: # store_last_character_str = " %s = %s;\n" % \ # ("me->buffer._lexatom_before_lexeme_start", # "*(me->buffer._read_p - 1)") # set_terminating_zero_str = " QUEX_LEXEME_TERMINATING_ZERO_SET(&me->buffer);\n" # prefix = store_last_character_str + set_terminating_zero_str print "## (1) code generation" pattern_list = [ pattern for pattern, action_str in pattern_action_list ] function_body, variable_definitions = cpp_generator.do_core(pattern_list, terminal_db) function_body += "if(0) { __QUEX_COUNT_VOID((QUEX_TYPE_ANALYZER*)0, (QUEX_TYPE_LEXATOM*)0, (QUEX_TYPE_LEXATOM*)0); }\n" function_txt = cpp_generator.wrap_up(sm_name, function_body, variable_definitions, ModeNameList=[]) assert all_isinstance(function_txt, str) return "#define __QUEX_OPTION_UNIT_TEST\n" \ + nonsense_default_counter(not SecondModeF) \ + "".join(function_txt)