def __get_mode_init_call(mode): analyzer_function = "QUEX_NAME(%s_analyzer_function)" % mode.name on_indentation = "QUEX_NAME(%s_on_indentation)" % mode.name on_entry = "QUEX_NAME(%s_on_entry)" % mode.name on_exit = "QUEX_NAME(%s_on_exit)" % mode.name has_base = "QUEX_NAME(%s_has_base)" % mode.name has_entry_from = "QUEX_NAME(%s_has_entry_from)" % mode.name has_exit_to = "QUEX_NAME(%s_has_exit_to)" % mode.name if mode.abstract_f(): analyzer_function = "QUEX_NAME(Mode_uncallable_analyzer_function)" if not mode.incidence_db.has_key(E_IncidenceIDs.MODE_ENTRY): on_entry = "QUEX_NAME(Mode_on_entry_exit_null_function)" if not mode.incidence_db.has_key(E_IncidenceIDs.MODE_EXIT): on_exit = "QUEX_NAME(Mode_on_entry_exit_null_function)" if not mode.incidence_db.has_key(E_IncidenceIDs.INDENTATION_HANDLER): on_indentation = "QUEX_NAME(Mode_on_indentation_null_function)" txt = blue_print(quex_mode_init_call_str, [["$$MN$$", mode.name], ["$analyzer_function", analyzer_function], ["$on_indentation", on_indentation], ["$on_entry", on_entry], ["$on_exit", on_exit], ["$has_base", has_base], ["$has_entry_from", has_entry_from], ["$has_exit_to", has_exit_to]]) return txt
def __lc_counting_replacements(code_str, CharacterSet): """Line and Column Number Counting(Range Skipper): -- in loop if there appears a newline, then do: increment line_n set position from where to count column_n -- at end of skipping do one of the following: if end delimiter contains newline: column_n = number of letters since last new line in end delimiter increment line_n by number of newlines in end delimiter. (NOTE: in this case the setting of the position from where to count the column_n can be omitted.) else: column_n = current_position - position from where to count column number. NOTE: On reload we do count the column numbers and reset the column_p. """ set_reference = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" in_loop = "" # Does the end delimiter contain a newline? if CharacterSet.contains(ord("\n")): in_loop = line_column_counter_in_loop() end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(me->buffer._input_p - reference_p));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(me->buffer._input_p - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = me->buffer._input_p);\n" return blue_print(code_str, [["$$LC_COUNT_COLUMN_N_POINTER_REFERENCE$$", set_reference], ["$$LC_COUNT_IN_LOOP$$", in_loop], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ])
def do_implementation(ModeDB): FileTemplate = os.path.normpath(QUEX_PATH + Lng["$code_base"] + "/analyzer/TXT-Cpp.i") func_txt = get_file_content_or_die(FileTemplate) func_txt = blue_print(func_txt, [ [ "$$CONSTRUCTOR_EXTENSTION$$", Lng.SOURCE_REFERENCED(blackboard.class_constructor_extension) ], [ "$$CONVERTER_HELPER_I$$", Setup.get_file_reference(Setup.output_buffer_codec_header_i) ], [ "$$CONSTRUCTOR_MODE_DB_INITIALIZATION_CODE$$", get_constructor_code(ModeDB.values()) ], [ "$$MEMENTO_EXTENSIONS_PACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_pack_extension) ], [ "$$MEMENTO_EXTENSIONS_UNPACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_unpack_extension) ], ]) return func_txt
def __get_mode_init_call(mode): analyzer_function = "QUEX_NAME(%s_analyzer_function)" % mode.name on_indentation = "QUEX_NAME(%s_on_indentation)" % mode.name on_entry = "QUEX_NAME(%s_on_entry)" % mode.name on_exit = "QUEX_NAME(%s_on_exit)" % mode.name has_base = "QUEX_NAME(%s_has_base)" % mode.name has_entry_from = "QUEX_NAME(%s_has_entry_from)" % mode.name has_exit_to = "QUEX_NAME(%s_has_exit_to)" % mode.name if mode.options["inheritable"] == "only": analyzer_function = "QUEX_NAME(Mode_uncallable_analyzer_function)" if len(mode.get_code_fragment_list("on_entry")) == 0: on_entry = "QUEX_NAME(Mode_on_entry_exit_null_function)" if len(mode.get_code_fragment_list("on_exit")) == 0: on_exit = "QUEX_NAME(Mode_on_entry_exit_null_function)" if len(mode.get_code_fragment_list("on_indentation")) == 0: on_indentation = "QUEX_NAME(Mode_on_indentation_null_function)" txt = blue_print(quex_mode_init_call_str, [["$$MN$$", mode.name], ["$analyzer_function", analyzer_function], ["$on_indentation", on_indentation], ["$on_entry", on_entry], ["$on_exit", on_exit], ["$has_base", has_base], ["$has_entry_from", has_entry_from], ["$has_exit_to", has_exit_to]]) return txt
def do_map_id_to_name_function(): """Generate function which maps from token-id to string with the name of the token id. """ L = max(map(lambda name: len(name), token_id_db.keys())) # -- define the function for token names switch_cases = [] token_names = [] for token_name in sorted(token_id_db.keys()): if token_name in standard_token_id_list: continue # UCS codepoints are coded directly as pure numbers if len(token_name) > 2 and token_name[:2] == "--": token = token_id_db[token_name] switch_cases.append(" case 0x%06X: return token_id_str_%s;\n" % \ (token.number, token.name)) token_names.append(" static const char token_id_str_%s[]%s = \"%s\";\n" % \ (token.name, space(L, token.name), token.name)) else: switch_cases.append(" case %s%s:%s return token_id_str_%s;\n" % \ (Setup.token_id_prefix, token_name, space(L, token_name), token_name)) token_names.append(" static const char token_id_str_%s[]%s = \"%s\";\n" % \ (token_name, space(L, token_name), token_name)) return blue_print(func_str, [["$$TOKEN_ID_CASES$$", "".join(switch_cases)], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$TOKEN_NAMES$$", "".join(token_names)], ])
def do_map_id_to_name_cases(): """Generate function which maps from token-id to string with the name of the token id. """ if not token_db.token_id_db: return "" L = max(len(name) for name in token_db.token_id_db.keys()) def get_case(token_name, token): if token_name.startswith("--"): # UCS codepoints are coded directly as pure numbers return " case 0x%06X: return \"%s\";\n" % \ (token.number, token.name) else: return " case %s%s:%s return \"%s\";\n" % \ (Setup.token_id_prefix, token_name, space(L, token_name), token_name) # -- define the function for token names switch_cases = [ get_case(token_name, token) for token_name, token in sorted(token_db.token_id_db.iteritems()) if token_name not in standard_token_id_list ] txt = blue_print(map_id_to_name_cases, [["$$TOKEN_ID_CASES$$", "".join(switch_cases)], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix]]) return txt
def do(Modes): LexerClassName = Setup.analyzer_class_name DerivedClassName = Setup.analyzer_derived_class_name DerivedClassHeaderFileName = Setup.analyzer_derived_class_file if DerivedClassHeaderFileName != "": txt = "#include <" + Setup.get_file_reference(DerivedClassHeaderFileName) +">\n" else: txt = "#include \"" + Setup.get_file_reference(Setup.output_header_file) +"\"\n" txt += "#include <quex/code_base/analyzer/C-adaptions.h>\n" # -- mode class member function definitions (on_entry, on_exit, has_base, ...) mode_class_member_functions_txt = write_member_functions(Modes.values()) mode_objects_txt = "" for mode_name, mode in Modes.items(): if mode.options["inheritable"] == "only": continue mode_objects_txt += "/* Global */QUEX_NAME(Mode) QUEX_NAME(%s);\n" % mode_name txt += "QUEX_NAMESPACE_MAIN_OPEN\n" txt += mode_objects_txt txt += mode_class_member_functions_txt txt += "QUEX_NAMESPACE_MAIN_CLOSE\n" txt = blue_print(txt, [["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_DERIVED_CLASS_NAME$$", DerivedClassName]]) return txt
def get_graphviz_string(self, NormalizeF=False, Option="utf8"): assert Option in ["hex", "utf8"] # (*) normalize the state indices index_map, inverse_index_map, index_sequence = self.get_state_index_normalization(NormalizeF) # (*) Border of plot block frame_txt = "digraph state_machine_%i {\n" % self.get_id() frame_txt += "rankdir=LR;\n" frame_txt += "size=\"8,5\"\n" frame_txt += "node [shape = doublecircle]; $$ACCEPTANCE_STATES$$\n" frame_txt += "node [shape = circle];\n" frame_txt += "$$TRANSITIONS$$" frame_txt += "}\n" transition_str = "" acceptance_state_str = "" for printed_state_i, state in sorted(imap(lambda i: (index_map[i], self.states[i]), index_sequence)): if state.is_acceptance(): acceptance_state_str += "%i; " % int(printed_state_i) transition_str += state.get_graphviz_string(printed_state_i, index_map, Option) if acceptance_state_str != "": acceptance_state_str = acceptance_state_str[:-2] + ";" return blue_print(frame_txt, [["$$ACCEPTANCE_STATES$$", acceptance_state_str ], ["$$TRANSITIONS$$", transition_str]])
def get_graphviz_string(self, NormalizeF=False, Option="utf8"): assert Option in ["hex", "utf8"] # (*) normalize the state indices index_map, inverse_index_map, index_sequence = self.get_state_index_normalization( NormalizeF) # (*) Border of plot block frame_txt = "digraph state_machine_%i {\n" % self.get_id() frame_txt += "rankdir=LR;\n" frame_txt += "size=\"8,5\"\n" frame_txt += "node [shape = doublecircle]; $$ACCEPTANCE_STATES$$\n" frame_txt += "node [shape = circle];\n" frame_txt += "$$TRANSITIONS$$" frame_txt += "}\n" transition_str = "" acceptance_state_str = "" for printed_state_i, state in sorted( imap(lambda i: (index_map[i], self.states[i]), index_sequence)): if state.is_acceptance(): acceptance_state_str += "%i; " % int(printed_state_i) transition_str += state.get_graphviz_string( printed_state_i, index_map, Option) if acceptance_state_str != "": acceptance_state_str = acceptance_state_str[:-2] + ";" return blue_print(frame_txt, [["$$ACCEPTANCE_STATES$$", acceptance_state_str], ["$$TRANSITIONS$$", transition_str]])
def get_on_indentation_handler(Mode): # 'on_dedent' and 'on_n_dedent cannot be defined at the same time. assert not ( E_IncidenceIDs.INDENTATION_DEDENT in Mode.incidence_db \ and E_IncidenceIDs.INDENTATION_N_DEDENT in Mode.incidence_db) # A mode that deals only with the default indentation handler relies # on what is defined in '$QUEX_PATH/analyzer/member/on_indentation.i' if Mode.incidence_db.default_indentation_handler_f(): return " return;" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_INDENT) if code_fragment is not None: on_indent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_indent_str = "self_send(__QUEX_SETTING_TOKEN_ID_INDENT);" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_NODENT) if code_fragment is not None: on_nodent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_nodent_str = "self_send(__QUEX_SETTING_TOKEN_ID_NODENT);" on_dedent_str = "" on_n_dedent_str = "" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_DEDENT) if code_fragment is not None: on_dedent_str = Lng.SOURCE_REFERENCED(code_fragment) code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_N_DEDENT) if code_fragment is not None: on_n_dedent_str = Lng.SOURCE_REFERENCED(code_fragment) if (not on_dedent_str) and (not on_n_dedent_str): # If no 'on_dedent' and no 'on_n_dedent' is defined ... on_dedent_str = "" on_n_dedent_str = "#if defined(QUEX_OPTION_TOKEN_REPETITION_SUPPORT)\n" on_n_dedent_str += " self_send_n(ClosedN, __QUEX_SETTING_TOKEN_ID_DEDENT);\n" on_n_dedent_str += "#else\n" on_n_dedent_str += " while( start-- != stack->back ) self_send(__QUEX_SETTING_TOKEN_ID_DEDENT);\n" on_n_dedent_str += "#endif\n" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_ERROR) if code_fragment is not None: on_indentation_error = Lng.SOURCE_REFERENCED(code_fragment) else: # Default: Blow the program if there is an indentation error. on_indentation_error = 'QUEX_ERROR_EXIT("Lexical analyzer mode \'%s\': indentation error detected!\\n"' \ % Mode.name + \ ' "No \'on_indentation_error\' handler has been specified.\\n");' # Note: 'on_indentation_bad' is applied in code generation for # indentation counter in 'indentation_counter.py'. txt = blue_print( on_indentation_str, [["$$INDENT-PROCEDURE$$", on_indent_str], ["$$NODENT-PROCEDURE$$", on_nodent_str], ["$$DEDENT-PROCEDURE$$", on_dedent_str], ["$$N-DEDENT-PROCEDURE$$", on_n_dedent_str], ["$$INDENTATION-ERROR-PROCEDURE$$", on_indentation_error]]) return txt
def __lc_counting_replacements(code_str, CharacterSet): """Line and Column Number Counting(Range Skipper): -- in loop if there appears a newline, then do: increment line_n set position from where to count column_n -- at end of skipping do one of the following: if end delimiter contains newline: column_n = number of letters since last new line in end delimiter increment line_n by number of newlines in end delimiter. (NOTE: in this case the setting of the position from where to count the column_n can be omitted.) else: column_n = current_position - position from where to count column number. NOTE: On reload we do count the column numbers and reset the column_p. """ set_reference = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" in_loop = "" # Does the end delimiter contain a newline? if CharacterSet.contains(ord("\n")): in_loop = line_column_counter_in_loop() end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(me->buffer._input_p - reference_p));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(me->buffer._input_p - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = me->buffer._input_p);\n" return blue_print(code_str, [ ["$$LC_COUNT_COLUMN_N_POINTER_REFERENCE$$", set_reference], ["$$LC_COUNT_IN_LOOP$$", in_loop], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ])
def __cpp_terminal_states(StateMachineName, sm, action_db, DefaultAction): # -- specific terminal states of patterns (entered from acceptance states) txt = "" for pattern_id in action_db.keys(): txt += " %s:\n" % get_label("", None, pattern_id) action_code = " " + action_db[pattern_id].replace("\n", "\n ") txt += " QUEX_STREAM_SEEK(last_acceptance_input_position);" txt += action_code + "\n" txt += " // if action code returns from the function, then the following is meaningless\n" if sm.states[sm.init_state_index].transitions().is_empty() == False: txt += " QUEX_STREAM_GET(input);" txt += " goto QUEX_LABEL_%s_ENTRY_INITIAL_STATE;\n" % StateMachineName specific_terminal_states_str = txt # -- general terminal state (entered from non-acceptance state) txt = "" for pattern_id in action_db.keys(): txt += " case %s: goto %s;\n" % \ (repr(pattern_id), get_label("", None, pattern_id)) jumps_to_acceptance_states_str = txt # -- execute default pattern action # -- reset character stream to last success # -- goto initial state txt = blue_print(__cpp_terminal_state_str, [["%%JUMPS_TO_ACCEPTANCE_STATE%%", jumps_to_acceptance_states_str], ["%%SPECIFIC_TERMINAL_STATES%%", specific_terminal_states_str], ["%%DEFAULT_ACTION%%", DefaultAction.replace("\n", " \n")], ["%%STATE_MACHINE_NAME%%", StateMachineName], ["%%INITIAL_STATE_INDEX_LABEL%%", get_label(StateMachineName, sm.init_state_index)]]) return txt
def __get_mode_init_call(mode): analyzer_function = "QUEX_NAME(%s_analyzer_function)" % mode.name on_indentation = "QUEX_NAME(%s_on_indentation)" % mode.name on_entry = "QUEX_NAME(%s_on_entry)" % mode.name on_exit = "QUEX_NAME(%s_on_exit)" % mode.name has_base = "QUEX_NAME(%s_has_base)" % mode.name has_entry_from = "QUEX_NAME(%s_has_entry_from)" % mode.name has_exit_to = "QUEX_NAME(%s_has_exit_to)" % mode.name if mode.options["inheritable"] == "only": analyzer_function = "QUEX_NAME(Mode_uncallable_analyzer_function)" if len(mode.get_code_fragment_list("on_entry")) == 0: on_entry = "QUEX_NAME(Mode_on_entry_exit_null_function)" if len(mode.get_code_fragment_list("on_exit")) == 0: on_exit = "QUEX_NAME(Mode_on_entry_exit_null_function)" if len(mode.get_code_fragment_list("on_indentation")) == 0: on_indentation = "QUEX_NAME(Mode_on_indentation_null_function)" txt = blue_print( quex_mode_init_call_str, [["$$MN$$", mode.name], ["$analyzer_function", analyzer_function], ["$on_indentation", on_indentation], ["$on_entry", on_entry], ["$on_exit", on_exit], ["$has_base", has_base], ["$has_entry_from", has_entry_from], ["$has_exit_to", has_exit_to]]) return txt
def get_on_indentation_handler(Mode): # 'on_dedent' and 'on_n_dedent cannot be defined at the same time. assert not ( E_IncidenceIDs.INDENTATION_DEDENT in Mode.incidence_db \ and E_IncidenceIDs.INDENTATION_N_DEDENT in Mode.incidence_db) # A mode that deals only with the default indentation handler relies # on what is defined in '$QUEX_PATH/analyzer/member/on_indentation.i' if Mode.incidence_db.default_indentation_handler_f(): return " return;" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_INDENT) if code_fragment is not None: on_indent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_indent_str = "self_send(__QUEX_SETTING_TOKEN_ID_INDENT);" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_NODENT) if code_fragment is not None: on_nodent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_nodent_str = "self_send(__QUEX_SETTING_TOKEN_ID_NODENT);" on_dedent_str = "" on_n_dedent_str = "" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_DEDENT) if code_fragment is not None: on_dedent_str = Lng.SOURCE_REFERENCED(code_fragment) code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_N_DEDENT) if code_fragment is not None: on_n_dedent_str = Lng.SOURCE_REFERENCED(code_fragment) if (not on_dedent_str) and (not on_n_dedent_str): # If no 'on_dedent' and no 'on_n_dedent' is defined ... on_dedent_str = "" on_n_dedent_str = "#if defined(QUEX_OPTION_TOKEN_REPETITION_SUPPORT)\n" on_n_dedent_str += " self_send_n(ClosedN, __QUEX_SETTING_TOKEN_ID_DEDENT);\n" on_n_dedent_str += "#else\n" on_n_dedent_str += " while( start-- != stack->back ) self_send(__QUEX_SETTING_TOKEN_ID_DEDENT);\n" on_n_dedent_str += "#endif\n" code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_ERROR) if code_fragment is not None: on_indentation_error = Lng.SOURCE_REFERENCED(code_fragment) else: # Default: Blow the program if there is an indentation error. on_indentation_error = 'QUEX_ERROR_EXIT("Lexical analyzer mode \'%s\': indentation error detected!\\n"' \ % Mode.name + \ ' "No \'on_indentation_error\' handler has been specified.\\n");' # Note: 'on_indentation_bad' is applied in code generation for # indentation counter in 'indentation_counter.py'. txt = blue_print(on_indentation_str, [["$$INDENT-PROCEDURE$$", on_indent_str], ["$$NODENT-PROCEDURE$$", on_nodent_str], ["$$DEDENT-PROCEDURE$$", on_dedent_str], ["$$N-DEDENT-PROCEDURE$$", on_n_dedent_str], ["$$INDENTATION-ERROR-PROCEDURE$$", on_indentation_error]]) return txt
def get_on_indentation_handler(Mode): # 'on_dedent' and 'on_n_dedent cannot be defined at the same time. assert not ( Mode.has_code_fragment_list("on_dedent") \ and Mode.has_code_fragment_list("on_n_dedent")) # A mode that deals only with the default indentation handler relies # on what is defined in '$QUEX_PATH/analayzer/member/on_indentation.i' if Mode.default_indentation_handler_sufficient(): return " return;" if Mode.has_code_fragment_list("on_indent"): on_indent_str, eol_f = action_preparation.get_code(Mode.get_code_fragment_list("on_indent")) else: on_indent_str = "self_send(__QUEX_SETTING_TOKEN_ID_INDENT);" if Mode.has_code_fragment_list("on_nodent"): on_nodent_str, eol_f = action_preparation.get_code(Mode.get_code_fragment_list("on_nodent")) else: on_nodent_str = "self_send(__QUEX_SETTING_TOKEN_ID_NODENT);" if Mode.has_code_fragment_list("on_dedent"): assert not Mode.has_code_fragment_list("on_n_dedent") on_dedent_str, eol_f = action_preparation.get_code(Mode.get_code_fragment_list("on_dedent")) on_n_dedent_str = "" elif Mode.has_code_fragment_list("on_n_dedent"): assert not Mode.has_code_fragment_list("on_dedent") on_n_dedent_str, eol_f = action_preparation.get_code(Mode.get_code_fragment_list("on_n_dedent")) on_dedent_str = "" else: # If no 'on_dedent' and no 'on_n_dedent' is defined ... on_dedent_str = "" on_n_dedent_str = "#if defined(QUEX_OPTION_TOKEN_REPETITION_SUPPORT)\n" on_n_dedent_str += " self_send_n(ClosedN, __QUEX_SETTING_TOKEN_ID_DEDENT);\n" on_n_dedent_str += "#else\n" on_n_dedent_str += " while( start-- != stack->back ) self_send(__QUEX_SETTING_TOKEN_ID_DEDENT);\n" on_n_dedent_str += "#endif\n" if not Mode.has_code_fragment_list("on_indentation_error"): # Default: Blow the program if there is an indentation error. on_indentation_error = 'QUEX_ERROR_EXIT("Lexical analyzer mode \'%s\': indentation error detected!\\n"' \ % Mode.name + \ ' "No \'on_indentation_error\' handler has been specified.\\n");' else: on_indentation_error, eol_f = action_preparation.get_code(Mode.get_code_fragment_list("on_indentation_error")) # Note: 'on_indentation_bad' is applied in code generation for # indentation counter in 'indentation_counter.py'. txt = blue_print(on_indentation_str, [["$$INDENT-PROCEDURE$$", on_indent_str], ["$$NODENT-PROCEDURE$$", on_nodent_str], ["$$DEDENT-PROCEDURE$$", on_dedent_str], ["$$N-DEDENT-PROCEDURE$$", on_n_dedent_str], ["$$INDENTATION-ERROR-PROCEDURE$$", on_indentation_error]]) return txt
def do(setup): """________________________________________________________________________ (1) Error Check (2) Generates a file containing: -- token id definitions (if they are not done in '--foreign-token-id-file'). -- const string& TokenClass::map_id_to_name(), i.e. a function which can convert token ids into strings. ________________________________________________________________________ """ global file_str # At this point, assume that the token type has been generated. assert blackboard.token_type_definition is not None # (1) Error Check # __warn_implicit_token_definitions() if len(Setup.token_id_foreign_definition_file) == 0: __autogenerate_token_id_numbers() __warn_on_double_definition() # If a mandatory token id is missing, this means that Quex did not # properly do implicit token definitions. Program error-abort. __error_on_mandatory_token_id_missing(AssertF=True) else: __error_on_mandatory_token_id_missing() __error_on_no_specific_token_ids() # (2) Generate token id file (if not specified outside) # if len(Setup.token_id_foreign_definition_file) != 0: # Content of file = inclusion of 'Setup.token_id_foreign_definition_file'. token_id_txt = [ "#include \"%s\"\n" % Setup.get_file_reference(Setup.token_id_foreign_definition_file) ] else: token_id_txt = __get_token_id_definition_txt() include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper()) content = blue_print(file_str, [ ["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], [ "$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference( blackboard.token_type_definition.get_file_name()) ], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", include_guard_ext], ]) return content
def HEADER_DEFINITIONS(self): return blue_print(cpp_header_definition_str, [ ("$$CONTINUE_WITH_ON_AFTER_MATCH$$", dial_db.get_label_by_door_id( DoorID.continue_with_on_after_match())), ("$$RETURN_WITH_ON_AFTER_MATCH$$", dial_db.get_label_by_door_id( DoorID.return_with_on_after_match())), ])
def prepare(template, name, code_unit_type, max_code_unit_n): result = blue_print(string_template, [ ["$$DRAIN_ENCODING$$", name], ["$$DRAIN_CODE_UNIT_TYPE$$", code_unit_type], ["$$DRAIN_ENCODING_MAX_CODE_UNIT_N$$", str(max_code_unit_n)], ]) if name == "wchar_t": result = "\n".join([ "#if ! defined(QUEX_OPTION_WCHAR_T_DISABLED_EXT)", result, "#endif" ]) return result
def do(setup): """________________________________________________________________________ (1) Error Check (2) Generates a file containing: -- token id definitions (if they are not done in '--foreign-token-id-file'). -- const string& TokenClass::map_id_to_name(), i.e. a function which can convert token ids into strings. ________________________________________________________________________ """ global file_str # At this point, assume that the token type has been generated. assert blackboard.token_type_definition is not None # (1) Error Check # __warn_implicit_token_definitions() if len(Setup.token_id_foreign_definition_file) == 0: __autogenerate_token_id_numbers() __warn_on_double_definition() # If a mandatory token id is missing, this means that Quex did not # properly do implicit token definitions. Program error-abort. __error_on_mandatory_token_id_missing(AssertF=True) else: __error_on_mandatory_token_id_missing() __error_on_no_specific_token_ids() # (2) Generate token id file (if not specified outside) # if len(Setup.token_id_foreign_definition_file) != 0: # Content of file = inclusion of 'Setup.token_id_foreign_definition_file'. token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(Setup.token_id_foreign_definition_file)] else: token_id_txt = __get_token_id_definition_txt() include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper()) content = blue_print(file_str, [["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", include_guard_ext], ]) return content
def replace_keywords(program_txt, Lng, NoIndentF): """Replaces pseudo-code keywords with keywords of the given language.""" txt = blue_print(program_txt, Lng.items()) if NoIndentF == False: # delete the last newline, to prevent additional indentation if txt[-1] == "\n": txt = txt[:-1] # indent by four spaces # (if this happens in recursively called functions nested indented blocks # are correctly indented, see NumberSet::get_condition_code() for example) txt = txt.replace("\n", "\n ") + "\n" return txt
def replace_keywords(program_txt, LanguageDB, NoIndentF): """Replaces pseudo-code keywords with keywords of the given language.""" txt = blue_print(program_txt, LanguageDB.items()) if NoIndentF == False: # delete the last newline, to prevent additional indentation if txt[-1] == "\n": txt = txt[:-1] # indent by four spaces # (if this happens in recursively called functions nested indented blocks # are correctly indented, see NumberSet::get_condition_code() for example) txt = txt.replace("\n", "\n ") + "\n" return txt
def do_implementation(ModeDB): FileTemplate = os.path.normpath(QUEX_PATH + Lng["$code_base"] + "/analyzer/TXT-Cpp.i") func_txt = get_file_content_or_die(FileTemplate) func_txt = blue_print(func_txt, [ ["$$CONSTRUCTOR_EXTENSTION$$", Lng.SOURCE_REFERENCED(blackboard.class_constructor_extension)], ["$$CONVERTER_HELPER_I$$", Setup.get_file_reference(Setup.output_buffer_codec_header_i)], ["$$CONSTRUCTOR_MODE_DB_INITIALIZATION_CODE$$", get_constructor_code(ModeDB.values())], ["$$MEMENTO_EXTENSIONS_PACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_pack_extension)], ["$$MEMENTO_EXTENSIONS_UNPACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_unpack_extension)], ]) return func_txt
def do(Mode, ModeNameList): assert required_support_indentation_count() # 'on_dedent' and 'on_n_dedent cannot be defined at the same time. assert not ( E_IncidenceIDs.INDENTATION_DEDENT in Mode.incidence_db \ and E_IncidenceIDs.INDENTATION_N_DEDENT in Mode.incidence_db) code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_INDENT) if code_fragment is not None: on_indent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_indent_str = Lng.TOKEN_SEND("QUEX_SETTING_TOKEN_ID_INDENT") code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_NODENT) if code_fragment is not None: on_nodent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_nodent_str = Lng.TOKEN_SEND("QUEX_SETTING_TOKEN_ID_NODENT") code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_DEDENT) if code_fragment is not None: on_dedent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_dedent_str = Lng.TOKEN_SEND("QUEX_SETTING_TOKEN_ID_DEDENT") code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_N_DEDENT) if code_fragment is not None: on_n_dedent_str = Lng.SOURCE_REFERENCED(code_fragment) else: on_n_dedent_str = Lng.TOKEN_SEND_N("ClosedN", "QUEX_SETTING_TOKEN_ID_DEDENT") code_fragment = Mode.incidence_db.get(E_IncidenceIDs.INDENTATION_ERROR) on_indentation_error = "" if code_fragment is not None: on_indentation_error = Lng.SOURCE_REFERENCED(code_fragment) # Note: 'on_indentation_bad' is applied in code generation for # indentation counter in 'indentation_counter.py'. return blue_print(on_indentation_str, [ ["$$DEFINE_SELF$$", Lng.DEFINE_SELF("me")], ["$$MODE_DEFINITION$$", Lng.MODE_DEFINITION(ModeNameList)], ["$$MODE_UNDEFINITION$$", Lng.MODE_UNDEFINITION(ModeNameList)], ["$$INDENT-PROCEDURE$$", on_indent_str], ["$$NODENT-PROCEDURE$$", on_nodent_str], ["$$DEDENT-PROCEDURE$$", on_dedent_str], ["$$N-DEDENT-PROCEDURE$$", on_n_dedent_str], ["$$INDENTATION-ERROR-PROCEDURE$$", on_indentation_error] ])
def do(ModeDb): LexerClassName = Setup.analyzer_class_name DerivedClassName = Setup.analyzer_derived_class_name mode_setup_txt = __setup(ModeDb) # -- mode class member function definitions (on_entry, on_exit, has_base, ...) mode_class_member_functions_txt = write_member_functions(ModeDb.values()) txt = Lng.FRAME_IN_NAMESPACE_MAIN("".join([ mode_setup_txt, mode_class_member_functions_txt, ])) return blue_print(txt, [["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_DERIVED_CLASS_NAME$$", DerivedClassName]])
def do_implementation(ModeDB, MemberFunctionSignatureList): func_txt = Lng.open_template(Lng.analyzer_template_i_file()) if blackboard.mode_db: map_token_ids_to_names_str = token_id_maker.do_map_id_to_name_cases() else: map_token_ids_to_names_str = "" func_txt = blue_print(func_txt, [ [ "$$MEMBER_FUNCTION_ASSIGNMENT$$", Lng.MEMBER_FUNCTION_ASSIGNMENT(MemberFunctionSignatureList) ], [ "$$CONSTRUCTOR_EXTENSTION$$", Lng.SOURCE_REFERENCED(blackboard.class_constructor_extension) ], [ "$$DESTRUCTOR_EXTENSTION$$", Lng.SOURCE_REFERENCED(blackboard.class_destructor_extension) ], [ "$$USER_DEFINED_PRINT$$", Lng.SOURCE_REFERENCED(blackboard.class_print_extension) ], [ "$$RESET_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.reset_extension) ], [ "$$MEMENTO_EXTENSIONS_PACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_pack_extension) ], [ "$$MEMENTO_EXTENSIONS_UNPACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_unpack_extension) ], [ "$$INCLUDE_TOKEN_ID_HEADER$$", Lng.INCLUDE(Setup.output_token_id_file_ref) ], ["$$MAP_ID_TO_NAME_CASES$$", map_token_ids_to_names_str], ]) return "\n%s\n" % func_txt
def do(setup): """________________________________________________________________________ (1) Error Check (2) Generates a file containing: -- token id definitions (if they are not done in '--foreign-token-id-file'). -- const string& TokenClass::map_token_id_to_name(), i.e. a function which can convert token ids into strings. ________________________________________________________________________ """ global file_str # At this point, assume that the token type has been generated. assert token_db.token_type_definition is not None # (1) Error Check # __warn_implicit_token_definitions() if not Setup.token_class_only_f: __error_on_no_specific_token_ids() if Setup.extern_token_id_file: __error_on_mandatory_token_id_missing() return None __autogenerate_token_id_numbers() __warn_on_double_definition() # If a mandatory token id is missing, this means that Quex did not # properly do implicit token definitions. Program error-abort. __error_on_mandatory_token_id_missing(AssertF=True) # (2) Generate token id file (if not specified outside) # if not Setup.extern_token_id_file: token_id_txt = __get_token_id_definition_txt() else: # Content of file = inclusion of 'Setup.extern_token_id_file'. token_id_txt = ["%s\n" % Lng.INCLUDE(Setup.extern_token_id_file)] return blue_print(file_str, [ ["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ])
def __replace_function_names(txt, mode, NullFunctionsF=True): def name(Name, ModeName): return Lng.NAME_IN_NAMESPACE_MAIN("%s_%s" % (ModeName, Name)) if NullFunctionsF and not mode.incidence_db.has_key( E_IncidenceIDs.MODE_ENTRY): on_entry = Lng.NAME_IN_NAMESPACE_MAIN( "Mode_on_entry_exit_null_function") else: on_entry = name("on_entry", mode.name) if NullFunctionsF and not mode.incidence_db.has_key( E_IncidenceIDs.MODE_EXIT): on_exit = Lng.NAME_IN_NAMESPACE_MAIN( "Mode_on_entry_exit_null_function") else: on_exit = name("on_exit", mode.name) if NullFunctionsF and not mode.incidence_db.has_key( E_IncidenceIDs.INDENTATION_HANDLER): on_indentation = Lng.NAME_IN_NAMESPACE_MAIN( "Mode_on_indentation_null_function") else: on_indentation = name("on_indentation", mode.name) return blue_print(txt, [ ["$$MN$$", mode.name], ["$analyzer_function", name("analyzer_function", mode.name)], ["$on_indentation", on_indentation], ["$on_entry", on_entry], ["$on_exit", on_exit], ["$has_base", name("has_base", mode.name)], ["$has_entry_from", name("has_entry_from", mode.name)], ["$has_exit_to", name("has_exit_to", mode.name)], [ "$on_buffer_before_change", name("on_buffer_before_change", mode.name) ], ["$on_buffer_overflow", name("on_buffer_overflow", mode.name)], ])
def _do(UnicodeTrafoInfo): """ PURPOSE: Writes converters for conversion towards UTF8/UTF16/UCS2/UCS4. UnicodeTrafoInfo: Provides the information about the relation of character codes in a particular coding to unicode character codes. It is provided in the following form: # Codec Values Unicode Values [ (Source0_Begin, Source0_End, TargetInterval0_Begin), (Source1_Begin, Source1_End, TargetInterval1_Begin), (Source2_Begin, Source2_End, TargetInterval2_Begin), ... ] """ codec_name = make_safe_identifier(UnicodeTrafoInfo.name).lower() utf8_epilog, utf8_function_body = ConverterWriterUTF8().do(UnicodeTrafoInfo) utf16_prolog, utf16_function_body = ConverterWriterUTF16().do(UnicodeTrafoInfo) dummy, utf32_function_body = ConverterWriterUTF32().do(UnicodeTrafoInfo) # Provide only the constant which are necessary FileName = os.path.normpath( QUEX_PATH + Lng["$code_base"] + "/converter_helper/TXT-from-codec-buffer.i") codec_header = Setup.get_file_reference(Setup.output_buffer_codec_header) txt_i = blue_print(get_file_content_or_die(FileName), [["$$CODEC$$", codec_name], ["$$EPILOG$$", utf8_epilog], ["$$CODEC_HEADER$$", codec_header], ["$$BODY_UTF8$$", utf8_function_body], ["$$BODY_UTF16$$", utf16_function_body], ["$$BODY_UTF32$$", utf32_function_body]]) # A separate declaration header is required FileName = os.path.normpath( QUEX_PATH + Lng["$code_base"] + "/converter_helper/TXT-from-codec-buffer") template_h_txt = get_file_content_or_die(FileName) txt_h = template_h_txt.replace("$$CODEC$$", codec_name) return txt_h, txt_i
def do(ModeDb): LexerClassName = Setup.analyzer_class_name DerivedClassName = Setup.analyzer_derived_class_name mode_db_setup_txt = __setup(ModeDb) # -- mode class member function definitions (on_entry, on_exit, has_base, ...) mode_class_member_functions_txt = write_member_functions(ModeDb.values()) txt = "".join([ "QUEX_NAMESPACE_MAIN_OPEN\n", mode_db_setup_txt, mode_class_member_functions_txt, "QUEX_NAMESPACE_MAIN_CLOSE\n" ]) txt = blue_print(txt, [["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_DERIVED_CLASS_NAME$$", DerivedClassName]]) return txt
def do(Modes): LexerClassName = Setup.analyzer_class_name DerivedClassName = Setup.analyzer_derived_class_name # -- mode class member function definitions (on_entry, on_exit, has_base, ...) mode_class_member_functions_txt = write_member_functions(Modes.values()) mode_objects_txt = "" for mode_name, mode in Modes.items(): if mode.abstract_f(): continue mode_objects_txt += "/* Global */QUEX_NAME(Mode) QUEX_NAME(%s);\n" % mode_name txt = "%s%s%s%s" % ("QUEX_NAMESPACE_MAIN_OPEN\n", mode_objects_txt, mode_class_member_functions_txt, "QUEX_NAMESPACE_MAIN_CLOSE\n") txt = blue_print(txt, [["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_DERIVED_CLASS_NAME$$", DerivedClassName]]) return txt
def _do(UnicodeTrafoInfo): """ PURPOSE: Writes converters for conversion towards UTF8/UTF16/UCS2/UCS4. UnicodeTrafoInfo: Provides the information about the relation of character codes in a particular coding to unicode character codes. It is provided in the following form: # Codec Values Unicode Values [ (Source0_Begin, Source0_End, TargetInterval0_Begin), (Source1_Begin, Source1_End, TargetInterval1_Begin), (Source2_Begin, Source2_End, TargetInterval2_Begin), ... ] """ codec_name = make_safe_identifier(UnicodeTrafoInfo.name).lower() utf8_epilog, utf8_function_body = ConverterWriterUTF8().do( UnicodeTrafoInfo) utf16_prolog, utf16_function_body = ConverterWriterUTF16().do( UnicodeTrafoInfo) dummy, utf32_function_body = ConverterWriterUTF32().do(UnicodeTrafoInfo) # Provide only the constant which are necessary FileName = os.path.normpath(QUEX_PATH + Lng["$code_base"] + "/converter_helper/TXT-from-codec-buffer.i") codec_header = Setup.get_file_reference(Setup.output_buffer_codec_header) txt_i = blue_print(get_file_content_or_die(FileName), [["$$CODEC$$", codec_name], ["$$EPILOG$$", utf8_epilog], ["$$CODEC_HEADER$$", codec_header], ["$$BODY_UTF8$$", utf8_function_body], ["$$BODY_UTF16$$", utf16_function_body], ["$$BODY_UTF32$$", utf32_function_body]]) # A separate declaration header is required FileName = os.path.normpath(QUEX_PATH + Lng["$code_base"] + "/converter_helper/TXT-from-codec-buffer") template_h_txt = get_file_content_or_die(FileName) txt_h = template_h_txt.replace("$$CODEC$$", codec_name) return txt_h, txt_i
def __cpp_terminal_states(StateMachineName, sm, action_db, DefaultAction): # -- specific terminal states of patterns (entered from acceptance states) txt = "" for acceptance_id in action_db.keys(): txt += " %s:\n" % get_label("", None, acceptance_id) action_code = " " + action_db[acceptance_id].replace("\n", "\n ") txt += " QUEX_STREAM_SEEK(last_acceptance_input_position);" txt += action_code + "\n" txt += " // if action code returns from the function, then the following is meaningless\n" if sm.states[sm.init_state_index].target_map.is_empty() == False: txt += " QUEX_STREAM_GET(input);" txt += " goto QUEX_LABEL_%s_ENTRY_INITIAL_STATE;\n" % StateMachineName specific_terminal_states_str = txt # -- general terminal state (entered from non-acceptance state) txt = "" for acceptance_id in action_db.keys(): txt += " case %s: goto %s;\n" % \ (repr(acceptance_id), get_label("", None, acceptance_id)) jumps_to_acceptance_states_str = txt # -- execute default pattern action # -- reset character stream to last success # -- goto initial state txt = blue_print( __cpp_terminal_state_str, [["%%JUMPS_TO_ACCEPTANCE_STATE%%", jumps_to_acceptance_states_str], ["%%SPECIFIC_TERMINAL_STATES%%", specific_terminal_states_str], ["%%DEFAULT_ACTION%%", DefaultAction.replace("\n", " \n")], ["%%STATE_MACHINE_NAME%%", StateMachineName], [ "%%INITIAL_STATE_INDEX_LABEL%%", get_label(StateMachineName, sm.init_state_index) ]]) return txt
def do(): """RETURNS: list of (content, file_name) where 'content' is the content to be written into 'file_name'. """ if not Setup.converter_only_f: source_name = "lexeme" else: if Setup.converter_source_name: source_name = Setup.converter_source_name else: source_name = Setup.buffer_encoding.name header_txt = Lng.template_converter_header() implementation_txt = blue_print( Lng.template_converter_implementation(), [("$$CONVERTER_HEADER$$", Lng.file_name_converter_header(source_name)), ("$$CHARACTER_CONVERTERS$$", _character_converters()), ("$$STRING_CONVERTERS$$", _string_converters())]) if Setup.converter_only_f: implementation_txt = implementation_txt.replace( "QUEX_TYPE_LEXATOM", Setup.lexatom.type) implementation_txt = implementation_txt.replace( "QUEX_INLINE", Lng.INLINE) implementation_txt = Lng.Match_QUEX_NAME_lexeme.sub( "QUEX_NAME(%s_" % source_name, implementation_txt) header_txt = header_txt.replace("QUEX_TYPE_LEXATOM", Setup.lexatom.type) header_txt = header_txt.replace("QUEX_INLINE", Lng.INLINE) header_txt = Lng.Match_QUEX_NAME_lexeme.sub( "QUEX_NAME(%s_" % source_name, header_txt) return [ (header_txt, Lng.file_name_converter_header(source_name)), (implementation_txt, Lng.file_name_converter_implementation(source_name)), ]
def do(Modes): LexerClassName = Setup.analyzer_class_name DerivedClassName = Setup.analyzer_derived_class_name # -- mode class member function definitions (on_entry, on_exit, has_base, ...) mode_class_member_functions_txt = write_member_functions(Modes.values()) mode_objects_txt = "" for mode_name, mode in Modes.items(): if mode.abstract_f(): continue mode_objects_txt += "/* Global */QUEX_NAME(Mode) QUEX_NAME(%s);\n" % mode_name txt = "%s%s%s%s" % ( "QUEX_NAMESPACE_MAIN_OPEN\n", mode_objects_txt, mode_class_member_functions_txt, "QUEX_NAMESPACE_MAIN_CLOSE\n" ) txt = blue_print(txt, [["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_DERIVED_CLASS_NAME$$", DerivedClassName]]) return txt
def _table_character_converters(unicode_trafo_info): """ PURPOSE: Writes converters for conversion towards UTF8/UTF16/UCS2/UCS4. UnicodeTrafoInfo: Provides the information about the relation of character codes in a particular coding to unicode character codes: # Codec Values Unicode Values [ (Source0_Begin, Source0_End, TargetInterval0_Begin), (Source1_Begin, Source1_End, TargetInterval1_Begin), (Source2_Begin, Source2_End, TargetInterval2_Begin), ... ] """ encoding_name = Lng.SAFE_IDENTIFIER(unicode_trafo_info.name) if encoding_name in ("utf32", "unicode"): source_interval_begin = 0 lexatom_size_in_byte = Setup.lexatom.size_in_byte if lexatom_size_in_byte == -1: lexatom_size_in_byte = 4 source_interval_end = min(256**lexatom_size_in_byte, 0x200000) target_interval_begin = 0 unicode_trafo_info = [(source_interval_begin, source_interval_end, target_interval_begin)] utf8_function_body = ConverterWriterUTF8().do(unicode_trafo_info) utf16_function_body = ConverterWriterUTF16().do(unicode_trafo_info) utf32_function_body = ConverterWriterUTF32().do(unicode_trafo_info) return blue_print(Lng.template_converter_character_functions(), [["$$BODY_UTF8$$", utf8_function_body], ["$$BODY_UTF16$$", utf16_function_body], ["$$BODY_UTF32$$", utf32_function_body]])
def LEXEME_MACRO_SETUP(self): return blue_print(cpp.lexeme_macro_setup, [ ["$$LEXEME_LENGTH$$", self.LEXEME_LENGTH()], ["$$INPUT_P$$", self.INPUT_P()], ])
def do(ModeDb): assert ModeDb txt = Lng.open_template(Lng.analyzer_configuration_file()) lexatom_loader_seek_buffer_size = 512 indentation_stack_size = 64 buffer_size = 131072 buffer_size_min = 32768 converter_buffer_size = 65536 mode_stack_size = 64 if buffer_size_min >= 1024: fallback_n = 256 elif buffer_size_min >= 16: fallback_n = buffer_size >> 4 else: fallback_n = 0 if Setup.fallback_optional_f: fallback_n = 0 else: pre_context_lengths = [m.longest_pre_context() for m in ModeDb.itervalues()] if None in pre_context_lengths or not pre_context_lengths: # Assert: any specification has taken care of constraint: assert not Setup.fallback_mandatory_f fallback_n = 0 else: fallback_n = "%s" % max(pre_context_lengths) adaptable_list = [ ("BUFFER_SIZE", "%s" % buffer_size), ("BUFFER_FALLBACK_N", "%s" % fallback_n), ("BUFFER_SIZE_MIN", "QUEX_SETTING_BUFFER_SIZE"), ("INDENTATION_STACK_SIZE", "%s" % indentation_stack_size), ("BUFFER_LEXATOM_LOADER_CONVERTER_BUFFER_SIZE", "(size_t)%s" % converter_buffer_size), ("BUFFER_LEXATOM_LOADER_SEEK_BUFFER_SIZE", lexatom_loader_seek_buffer_size), ("MODE_STACK_SIZE", "(size_t)%s" % mode_stack_size), ("TOKEN_QUEUE_SIZE", "(size_t)%s" % repr(Setup.token_queue_size)), ] immutable_list = [ ("VERSION", '"%s"' % QUEX_VERSION), ("ANALYZER_VERSION", '"%s"' % Setup.user_application_version_id), ("BUILD_DATE", '"%s"' % time.asctime()), ("MODE_INITIAL_P", '&%s' % Lng.NAME_IN_NAMESPACE_MAIN(blackboard.initial_mode.get_pure_text())), ("BUFFER_LEXATOM_BUFFER_BORDER", "0x%X" % Setup.buffer_limit_code), ("BUFFER_LEXATOM_NEWLINE", _lexatom_newline_in_engine_encoding()), ("BUFFER_LEXATOM_PATH_TERMINATION", "0x%X" % Setup.path_limit_code), ("BUFFER_FALLBACK_N", "%s" % fallback_n), ("FALLBACK_MANDATORY", {True: Lng.TRUE, False: Lng.FALSE}[Setup.fallback_mandatory_f]) ] adaptable_txt = [ Lng.QUEX_SETTING_DEF(name, value) for name, value in adaptable_list ] immutable_txt = [ Lng.QUEX_SETTING_DEF(name, value) for name, value in immutable_list ] setting_list = [ name for name, dummy in chain(adaptable_list, immutable_list) ] txt = blue_print(txt, [ ["$$ADAPTABLE$$", "\n".join(adaptable_txt)], ["$$IMMUTABLE$$", "\n".join(immutable_txt)], ["$$TYPE_DEFINITIONS$$", _type_definitions()], ["$$HELP_IF_CONFIGURATION_BY_CMAKE$$", Lng.HELP_IF_CONFIGURATION_BY_CMAKE(adaptable_list + immutable_list)], ["$$ERROR_IF_NO_CONFIGURATION_BY_MACRO$$", Lng.ERROR_IF_DEFINED_AND_NOT_CONFIGURATION_BY_MACRO(setting_list)], ]) return txt # adapt.do(txt, Setup.output_directory)
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""): assert type(EndSequence) == list assert len(EndSequence) >= 1 assert map(type, EndSequence) == [int] * len(EndSequence) local_variable_db = {} global template_str LanguageDB = Setup.language_db # Name the $$SKIPPER$$ skipper_index = sm_index.get() # Determine the $$DELIMITER$$ delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence) delimiter_length = len(EndSequence) tmp = [] LanguageDB.COMMENT(tmp, " Delimiter: %s" % delimiter_comment_str) delimiter_comment_str = "".join(tmp) # Determine the check for the tail of the delimiter delimiter_remainder_test_str = "" if len(EndSequence) != 1: txt = "" i = 0 for letter in EndSequence[1:]: i += 1 txt += " %s\n" % LanguageDB.ASSIGN("input", LanguageDB.INPUT_P_DEREFERENCE(i - 1)) txt += " %s" % LanguageDB.IF_INPUT("!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i) txt += " %s" % LanguageDB.GOTO(skipper_index) txt += " %s" % LanguageDB.END_IF() delimiter_remainder_test_str = txt if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence): goto_after_end_of_skipping_str = LanguageDB.GOTO(E_StateIndices.ANALYZER_REENTRY) else: # If there is indentation counting involved, then the counter's terminal id must # be determined at this place. assert IndentationCounterTerminalID is not None # If the ending delimiter is a subset of what the 'newline' pattern triggers # in indentation counting => move on to the indentation counter. goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(IndentationCounterTerminalID) if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr else: on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence) # The main part code_str = blue_print( template_str, [ ["$$DELIMITER_COMMENT$$", delimiter_comment_str], ["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()], ["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")], ["$$ENDIF$$", LanguageDB.END_IF()], ["$$ENTRY$$", LanguageDB.LABEL(skipper_index)], ["$$RELOAD$$", get_label("$reload", skipper_index)], ["$$GOTO_ENTRY$$", LanguageDB.GOTO(skipper_index)], ["$$INPUT_P_TO_LEXEME_START$$", LanguageDB.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str], ["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()], ["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str], ], ) # Line and column number counting code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence) # The finishing touch code_str = blue_print( code_str, [["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]], ) if reference_p_f: variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") variable_db.enter(local_variable_db, "Skipper%i", "{ %s }" % delimiter_str, delimiter_length, Index=skipper_index) variable_db.enter(local_variable_db, "Skipper%iL", "%i" % delimiter_length, Index=skipper_index) variable_db.enter(local_variable_db, "text_end") return code_str, local_variable_db
def do(ModeDB): assert blackboard.token_type_definition is not None QuexClassHeaderFileTemplate = os.path.normpath( QUEX_PATH + Lng["$code_base"] + Lng["$analyzer_template_file"]).replace("//","/") LexerClassName = Setup.analyzer_class_name quex_converter_coding_name_str = Setup.converter_ucs_coding_name mode_id_definition_str = "" # NOTE: First mode-id needs to be '1' for compatibility with flex generated engines for i, info in enumerate(ModeDB.items()): name = info[0] mode = info[1] if mode.abstract_f(): continue mode_id_definition_str += " QUEX_NAME(ModeID_%s) = %i,\n" % (name, i) if mode_id_definition_str != "": mode_id_definition_str = mode_id_definition_str[:-2] # -- instances of mode classes as members of the lexer mode_object_members_txt, \ mode_specific_functions_txt, \ friend_txt = get_mode_class_related_code_fragments(ModeDB.values()) # -- define a pointer that directly has the type of the derived class if Setup.analyzer_derived_class_name != "": analyzer_derived_class_name = Setup.analyzer_derived_class_name derived_class_type_declaration = "class %s;" % Setup.analyzer_derived_class_name else: analyzer_derived_class_name = Setup.analyzer_class_name derived_class_type_declaration = "" token_class_file_name = blackboard.token_type_definition.get_file_name() token_class_name = blackboard.token_type_definition.class_name token_class_name_safe = blackboard.token_type_definition.class_name_safe template_code_txt = get_file_content_or_die(QuexClassHeaderFileTemplate) include_guard_ext = get_include_guard_extension( Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name) if len(Setup.token_id_foreign_definition_file) != 0: token_id_definition_file = Setup.token_id_foreign_definition_file else: token_id_definition_file = Setup.output_token_id_file lexer_name_space_safe = get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space)) txt = blue_print(template_code_txt, [ ["$$___SPACE___$$", " " * (len(LexerClassName) + 1)], ["$$CLASS_BODY_EXTENSION$$", Lng.SOURCE_REFERENCED(blackboard.class_body_extension)], ["$$CONVERTER_HELPER$$", Setup.get_file_reference(Setup.output_buffer_codec_header)], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_ext], ["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_NAME_SPACE$$", lexer_name_space_safe], ["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe], ["$$LEXER_CONFIG_FILE$$", Setup.get_file_reference(Setup.output_configuration_file)], ["$$LEXER_DERIVED_CLASS_DECL$$", derived_class_type_declaration], ["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name], ["$$QUEX_MODE_ID_DEFINITIONS$$", mode_id_definition_str], ["$$MEMENTO_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.memento_class_extension)], ["$$MODE_CLASS_FRIENDS$$", friend_txt], ["$$MODE_OBJECTS$$", mode_object_members_txt], ["$$MODE_SPECIFIC_ANALYSER_FUNCTIONS$$", mode_specific_functions_txt], ["$$PRETTY_INDENTATION$$", " " + " " * (len(LexerClassName)*2 + 2)], ["$$QUEX_TEMPLATE_DIR$$", QUEX_PATH + Lng["$code_base"]], ["$$QUEX_VERSION$$", QUEX_VERSION], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(token_class_file_name)], ["$$TOKEN_CLASS$$", token_class_name], ["$$TOKEN_CLASS_NAME_SAFE$$", token_class_name_safe], ["$$TOKEN_ID_DEFINITION_FILE$$", Setup.get_file_reference(token_id_definition_file)], ["$$CORE_ENGINE_CHARACTER_CODING$$", quex_converter_coding_name_str], ["$$USER_DEFINED_HEADER$$", Lng.SOURCE_REFERENCED(blackboard.header) + "\n"], ]) return txt
def get_skipper(EndSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter): assert len(EndSequence) >= 1 global template_str # Name the $$SKIPPER$$ skipper_index = sm_index.get() skipper_door_id = dial_db.new_door_id(skipper_index) delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence) end_sequence_transformed = transformation.do_sequence(EndSequence) # Determine the $$DELIMITER$$ delimiter_length = len(end_sequence_transformed) delimiter_comment_str = Lng.COMMENT(" Delimiter: %s" % delimiter_comment_str) # Determine the check for the tail of the delimiter delimiter_remainder_test_str = "" if len(EndSequence) != 1: txt = "".join( " %s" % Lng.IF_GOTO(Lng.INPUT_P_DEREFERENCE(i-1), "!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i, skipper_door_id, i == 1) for i, letter in enumerate(EndSequence[1:], start=1) ) delimiter_remainder_test_str = txt door_id_reload = dial_db.new_door_id() on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern) # The main part code_str = blue_print(template_str, [ ["$$DELIMITER_COMMENT$$", delimiter_comment_str], ["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", Lng.ACCESS_INPUT()], ["$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")], ["$$ENDIF$$", Lng.END_IF()], ["$$ENTRY$$", dial_db.get_label_by_door_id(skipper_door_id)], ["$$RELOAD$$", dial_db.get_label_by_door_id(door_id_reload)], ["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)], ["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)], ["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()], ["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open], ]) # Line and column number counting code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence) # The finishing touch code_str = blue_print(code_str, [["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", Lng.GOTO(door_id_reload)]]) if reference_p_f: variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") variable_db.require_array("Skipper%i", Initial="{ %s }" % delimiter_str, ElementN=delimiter_length, Index=skipper_index) variable_db.require("Skipper%iL", "%i" % delimiter_length, Index=skipper_index) variable_db.require("text_end") variable_db.require("input") return [ code_str ]
def __terminal_states(StateMachineName, action_db, OnFailureAction, EndOfStreamAction, PreConditionIDList, LanguageDB, VariableDB, OnAfterMatchStr, LexemeNullObjectName): """NOTE: During backward-lexing, for a pre-condition, there is not need for terminal states, since only the flag 'pre-condition fulfilled is raised. """ # (*) specific terminal states of patterns (entered from acceptance states) specific_terminal_states = [] for pattern_id, pattern_action_info in action_db.items(): if pattern_id in PreConditionIDList: continue code = get_terminal_code(pattern_id, pattern_action_info, LanguageDB) specific_terminal_states.extend(code) delete_pre_context_flags = [] for pre_context_sm_id in PreConditionIDList: delete_pre_context_flags.append(" ") delete_pre_context_flags.append( LanguageDB.ASSIGN( "pre_context_%s_fulfilled_f" % __nice(pre_context_sm_id), 0)) # If there is at least a single terminal, the the 're-entry' preparation must be accomplished if len(action_db) != 0: get_label("$re-start", U=True) # -- execute 'on_failure' pattern action # -- goto initial state end_of_stream_code_action_str = EndOfStreamAction.action().get_code() # -- FAILURE ACTION: Under 'normal' circumstances the on_failure action is simply to be executed # since the 'get_forward()' incremented the 'current' pointer. # HOWEVER, when end of file has been reached the 'current' pointer has to # be reset so that the initial state can drop out on the buffer limit code # and then transit to the end of file action. # NOTE: It is possible that 'miss' happens after a chain of characters appeared. In any case the input # pointer must be setup right after the lexeme start. This way, the lexer becomes a new chance as # soon as possible. on_failure = __terminal_on_failure_prolog(LanguageDB) msg = OnFailureAction.action().get_code() on_failure.append(msg) prolog = blue_print(__terminal_state_prolog, [ ["$$LEXEME_LENGTH$$", LanguageDB.LEXEME_LENGTH()], ["$$INPUT_P$$", LanguageDB.INPUT_P()], ["$$LEXEME_NULL_OBJECT$$", LexemeNullObjectName], ]) router = Address( "$terminal-router", None, [ blue_print(__terminal_router_prolog_str, [ [ "$$TERMINAL_FAILURE-REF$$", "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE") ], ["$$TERMINAL_FAILURE$$", get_label("$terminal-FAILURE")], ]), # DO NOT 'U=True' for the state router. This is done automatically if # 'goto reload' is used. get_label("$state-router"), ";", __terminal_router_epilog_str, ]) epilog = blue_print(__terminal_state_epilog, [ ["$$FAILURE_ACTION$$", "".join(on_failure)], ["$$END_OF_STREAM_ACTION$$", end_of_stream_code_action_str], ["$$TERMINAL_END_OF_STREAM-DEF$$", get_label("$terminal-EOF")], ["$$TERMINAL_FAILURE-DEF$$", get_label("$terminal-FAILURE")], ["$$STATE_MACHINE_NAME$$", StateMachineName], ["$$GOTO_START_PREPARATION$$", get_label("$re-start", U=True)], ]) reset_last_acceptance_str = "" if VariableDB.has_key("last_acceptance"): reset_last_acceptance_str = "last_acceptance = $$TERMINAL_FAILURE-REF$$; /* TERMINAL: FAILURE */" return_preparation = "" if OnAfterMatchStr != "": return_preparation = blue_print( __return_preparation_str, [["$$ON_AFTER_MATCH$$", OnAfterMatchStr]]) reentry_preparation = blue_print(__reentry_preparation_str, [ ["$$REENTRY_PREPARATION$$", get_label("$re-start")], [ "$$DELETE_PRE_CONDITION_FULLFILLED_FLAGS$$", "".join(delete_pre_context_flags) ], ["$$GOTO_START$$", get_label("$start", U=True)], ["$$ON_AFTER_MATCH$$", OnAfterMatchStr], ["$$RESET_LAST_ACCEPTANCE$$", reset_last_acceptance_str], [ "$$COMMENT_ON_POST_CONTEXT_INITIALIZATION$$", comment_on_post_context_position_init_str ], [ "$$TERMINAL_FAILURE-REF$$", "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE") ], ]) txt = [] txt.append(router) txt.append(prolog) txt.extend(specific_terminal_states) txt.append(epilog) txt.append(return_preparation) txt.append(reentry_preparation) return txt
def __lc_counting_replacements(code_str, EndSequence): """Line and Column Number Counting(Range Skipper): -- in loop if there appears a newline, then do: increment line_n set position from where to count column_n -- at end of skipping do one of the following: if end delimiter contains newline: column_n = number of letters since last new line in end delimiter increment line_n by number of newlines in end delimiter. (NOTE: in this case the setting of the position from where to count the column_n can be omitted.) else: column_n = current_position - position from where to count column number. NOTE: On reload we do count the column numbers and reset the column_p. """ def get_character_n_after_last_newline(Sequence): tmp = copy(Sequence) tmp.reverse() try: return tmp.index(ord('\n')) except: return -1 char_n_after_last_newline = get_character_n_after_last_newline(EndSequence) reference_p_def = "" in_loop = "" end_procedure = "" before_reload = "" after_reload = "" on_first_delimiter = "" reference_p_required_f = False # Line/Column Counting: newline_number_in_delimiter = EndSequence.count(ord('\n')) if EndSequence == map(ord, "\n") or EndSequence == map(ord, "\r\n"): # (1) If the end-delimiter is a newline # => there cannot appear a newline inside the comment # => IN LOOP: no line number increment # no reference pointer required for column counting end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n" end_procedure += " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n" else: # (2) If end-delimiter is NOT newline # => there can appear a newline inside the comment if newline_number_in_delimiter == 0: # -- no newlines in delimiter => line and column number # must be counted. in_loop = line_column_counter_in_loop() end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" reference_p_required_f = True else: # -- newline inside delimiter => line number must be counted # column number is fixed. end_procedure = " __QUEX_IF_COUNT_COLUMNS_SET((size_t)%i);\n" \ % (char_n_after_last_newline + 1) if EndSequence[0] == ord('\n') \ or len(EndSequence) > 1 and EndSequence[0:2] == [ord('\r'), ord('\n')]: # If the first character in the sequence is newline, then the line counting # may is prevented by the loop exit. Now, we need to count. on_first_delimiter = "/* First delimiter char was a newline */\n" + \ " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n" end_procedure += " __QUEX_IF_COUNT_LINES_ADD((size_t)%i);\n" % (newline_number_in_delimiter - 1) else: in_loop = line_counter_in_loop() end_procedure += " __QUEX_IF_COUNT_LINES_ADD((size_t)%i);\n" % newline_number_in_delimiter if reference_p_required_f: reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" if len(EndSequence) > 1: end_procedure = "%s\n%s" % (Lng.INPUT_P_ADD(len(EndSequence)-1), end_procedure) return blue_print(code_str, [["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def], ["$$LC_COUNT_IN_LOOP$$", in_loop], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ["$$LC_ON_FIRST_DELIMITER$$", on_first_delimiter], ]), \ reference_p_required_f
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter): assert len(OpenerSequence) >= 1 assert len(CloserSequence) >= 1 assert OpenerSequence != CloserSequence skipper_index = sm_index.get() skipper_door_id = dial_db.new_door_id(skipper_index) opener_str, opener_comment_str = get_character_sequence(OpenerSequence) opener_length = len(OpenerSequence) closer_str, closer_comment_str = get_character_sequence(CloserSequence) closer_length = len(CloserSequence) variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") variable_db.require("counter") variable_db.require_array("Skipper%i_Opener", Initial="{ %s }" % opener_str, ElementN=opener_length, Index = skipper_index) variable_db.require("Skipper%i_OpenerEnd", "Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length), Index = skipper_index) variable_db.require("Skipper%i_Opener_it", "0x0", Index = skipper_index) variable_db.require_array("Skipper%i_Closer", Initial="{ %s }" % closer_str, ElementN=closer_length, Index = skipper_index) variable_db.require("Skipper%i_CloserEnd", "Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length), Index = skipper_index) variable_db.require("Skipper%i_Closer_it", "0x0", Index = skipper_index) reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" if CloserSequence[-1] == ord('\n'): end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n" end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n" else: end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" reload_door_id = dial_db.new_door_id() on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern, NestedF=True) code_str = blue_print(template_str, [ ["$$SKIPPER_INDEX$$", __nice(skipper_index)], # ["$$OPENER_LENGTH$$", "%i" % opener_length], ["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", Lng.ACCESS_INPUT()], ["$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")], ["$$ENDIF$$", Lng.END_IF()], ["$$ENTRY$$", Lng.LABEL(skipper_door_id)], ["$$RELOAD$$", dial_db.get_label_by_door_id(reload_door_id)], ["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)], ["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)], ["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)], ["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open], # ["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def], ["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop()], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ]) return [ code_str ]
def __terminal_states(StateMachineName, action_db, OnFailureAction, EndOfStreamAction, PreConditionIDList, LanguageDB, VariableDB, OnAfterMatchStr, LexemeNullObjectName): """NOTE: During backward-lexing, for a pre-condition, there is not need for terminal states, since only the flag 'pre-condition fulfilled is raised. """ # (*) specific terminal states of patterns (entered from acceptance states) specific_terminal_states = [] for pattern_id, pattern_action_info in action_db.items(): if pattern_id in PreConditionIDList: continue code = get_terminal_code(pattern_id, pattern_action_info, LanguageDB) specific_terminal_states.extend(code) delete_pre_context_flags = [] for pre_context_sm_id in PreConditionIDList: delete_pre_context_flags.append(" ") delete_pre_context_flags.append(LanguageDB.ASSIGN("pre_context_%s_fulfilled_f" % __nice(pre_context_sm_id), 0)) # If there is at least a single terminal, the the 're-entry' preparation must be accomplished if len(action_db) != 0: get_label("$re-start", U=True) # -- execute 'on_failure' pattern action # -- goto initial state end_of_stream_code_action_str = EndOfStreamAction.action().get_code() # -- FAILURE ACTION: Under 'normal' circumstances the on_failure action is simply to be executed # since the 'get_forward()' incremented the 'current' pointer. # HOWEVER, when end of file has been reached the 'current' pointer has to # be reset so that the initial state can drop out on the buffer limit code # and then transit to the end of file action. # NOTE: It is possible that 'miss' happens after a chain of characters appeared. In any case the input # pointer must be setup right after the lexeme start. This way, the lexer becomes a new chance as # soon as possible. on_failure = __terminal_on_failure_prolog(LanguageDB) msg = OnFailureAction.action().get_code() on_failure.append(msg) prolog = blue_print(__terminal_state_prolog, [ ["$$LEXEME_LENGTH$$", LanguageDB.LEXEME_LENGTH()], ["$$INPUT_P$$", LanguageDB.INPUT_P()], ["$$LEXEME_NULL_OBJECT$$", LexemeNullObjectName], ] ) router = Address("$terminal-router", None, [ blue_print(__terminal_router_prolog_str, [ ["$$TERMINAL_FAILURE-REF$$", "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")], ["$$TERMINAL_FAILURE$$", get_label("$terminal-FAILURE")], ]), # DO NOT 'U=True' for the state router. This is done automatically if # 'goto reload' is used. get_label("$state-router"), ";", __terminal_router_epilog_str, ]) epilog = blue_print(__terminal_state_epilog, [ ["$$FAILURE_ACTION$$", "".join(on_failure)], ["$$END_OF_STREAM_ACTION$$", end_of_stream_code_action_str], ["$$TERMINAL_END_OF_STREAM-DEF$$", get_label("$terminal-EOF")], ["$$TERMINAL_FAILURE-DEF$$", get_label("$terminal-FAILURE")], ["$$STATE_MACHINE_NAME$$", StateMachineName], ["$$GOTO_START_PREPARATION$$", get_label("$re-start", U=True)], ]) reset_last_acceptance_str = "" if VariableDB.has_key("last_acceptance"): reset_last_acceptance_str = "last_acceptance = $$TERMINAL_FAILURE-REF$$; /* TERMINAL: FAILURE */" return_preparation = "" if OnAfterMatchStr != "": return_preparation = blue_print(__return_preparation_str, [["$$ON_AFTER_MATCH$$", OnAfterMatchStr]]) reentry_preparation = blue_print(__reentry_preparation_str, [["$$REENTRY_PREPARATION$$", get_label("$re-start")], ["$$DELETE_PRE_CONDITION_FULLFILLED_FLAGS$$", "".join(delete_pre_context_flags)], ["$$GOTO_START$$", get_label("$start", U=True)], ["$$ON_AFTER_MATCH$$", OnAfterMatchStr], ["$$RESET_LAST_ACCEPTANCE$$", reset_last_acceptance_str], ["$$COMMENT_ON_POST_CONTEXT_INITIALIZATION$$", comment_on_post_context_position_init_str], ["$$TERMINAL_FAILURE-REF$$", "QUEX_LABEL(%i)" % get_address("$terminal-FAILURE")], ]) txt = [] txt.append(router) txt.append(prolog) txt.extend(specific_terminal_states) txt.append(epilog) txt.append(return_preparation) txt.append(reentry_preparation) return txt
def HEADER_DEFINITIONS(self): return blue_print(cpp_header_definition_str, [ ("$$CONTINUE_WITH_ON_AFTER_MATCH$$", dial_db.get_label_by_door_id(DoorID.continue_with_on_after_match())), ("$$RETURN_WITH_ON_AFTER_MATCH$$", dial_db.get_label_by_door_id(DoorID.return_with_on_after_match())), ])
#! /usr/bin/env python import sys import os sys.path.insert(0, os.environ["QUEX_PATH"]) from quex.engine.misc.string_handling import blue_print DistributionDir = sys.argv[1] Version = sys.argv[2] template_str = open(os.environ["QUEX_PATH"] + "/adm/packager/install-builder-template.xml").read() result = blue_print(template_str, [["$$VERSION$$", Version ], ["$$QUEX_PATH$$", os.environ["QUEX_PATH"]], ["$$DISTRIBUTION_ROOT_DIR$$", DistributionDir], ]) open("install-builder.xml", "w").write(result) print "DIR: %s CONTAINS: %s" % (os.getcwd(), "install-builder.xml")
def do(ModeDescriptionDB): IndentationSupportF = blackboard.required_support_indentation_count() BeginOfLineSupportF = blackboard.required_support_begin_of_line() LexerClassName = Setup.analyzer_class_name ConfigurationTemplateFile =( QUEX_PATH \ + Lng["$code_base"] \ + "/analyzer/configuration/TXT").replace("//","/") txt = get_file_content_or_die(ConfigurationTemplateFile) # -- check if exit/entry handlers have to be active entry_handler_active_f = False exit_handler_active_f = False for mode in ModeDescriptionDB.values(): entry_handler_active_f |= mode.incidence_db.has_key(E_IncidenceIDs.MODE_ENTRY) exit_handler_active_f |= mode.incidence_db.has_key(E_IncidenceIDs.MODE_EXIT) # Buffer filler converter (0x0 means: no buffer filler converter) converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW " if Setup.converter_user_new_func != "": converter_new_str += Setup.converter_user_new_func + "()" else: converter_new_str = "/* " + converter_new_str + " */" # Token repetition support token_repeat_test_txt = "" for token_id_str in blackboard.token_repetition_token_id_list: token_repeat_test_txt += "TokenID == %s || " % token_id_str if token_repeat_test_txt != "": token_repeat_test_txt = token_repeat_test_txt[:-3] else: token_repeat_test_txt = "false" if Setup.analyzer_derived_class_name != "": analyzer_derived_class_name = Setup.analyzer_derived_class_name else: analyzer_derived_class_name = Setup.analyzer_class_name txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING", Setup.count_column_number_f) txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False) txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f) txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f) txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK", Setup.include_stack_support_f) txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING", Setup.count_line_number_f) txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER", Setup.post_categorizer_f) txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK", Setup.mode_transition_check_f) txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR", Setup.string_accumulator_f) txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE", Setup.token_policy == "queue") txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE", Setup.token_policy == "single") txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT", token_repeat_test_txt != "false") txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY", Setup.token_memory_management_by_user_f) txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN", Setup.buffer_byte_order == "big") txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER", Setup.converter_helper_required_f) txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f) txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF) txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN", Setup.buffer_byte_order == "little") txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT", entry_handler_active_f) txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT", exit_handler_active_f) txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C") txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION", BeginOfLineSupportF) txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN", Setup.byte_order_is_that_of_current_system_f) txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS", Setup.buffer_based_analyzis_f) txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC", Setup.buffer_codec.name != "unicode") # -- token class related definitions token_descr = blackboard.token_type_definition # -- name of the character codec codec_name = make_safe_identifier(Setup.buffer_codec.name).lower() # Setup.buffer_element_size can be '-1'. This signals then that # sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value # is required here. character_size_str = "%i" % Setup.buffer_element_size def namespace(NameSpaceList): result = Lng.NAMESPACE_REFERENCE(NameSpaceList, TrailingDelimiterF=False) if len(result) == 0: return "" assert Setup.language.upper() != "C++" or len(result) > 2, \ "Error while generating namespace reference '%s'" % result return result txt = blue_print(txt, [ ["$$BUFFER_LIMIT_CODE$$", "0x%X" % Setup.buffer_limit_code], ["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name], ["$$INCLUDE_GUARD_EXTENSION$$", get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name)], ["$$INITIAL_LEXER_MODE_ID$$", "QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_text()], ["$$LEXER_BUILD_DATE$$", time.asctime()], ["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe], ["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name], ["$$MAX_MODE_CLASS_N$$", repr(len(ModeDescriptionDB))], ["$$NAMESPACE_MAIN$$", namespace(Setup.analyzer_name_space)], ["$$NAMESPACE_MAIN_CLOSE$$", Lng.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace("\n", "\\\n")], ["$$NAMESPACE_MAIN_OPEN$$", Lng.NAMESPACE_OPEN(Setup.analyzer_name_space).replace("\n", "\\\n")], ["$$NAMESPACE_TOKEN$$", namespace(token_descr.name_space)], ["$$NAMESPACE_TOKEN_CLOSE$$", Lng.NAMESPACE_CLOSE(token_descr.name_space).replace("\n", "\\\n")], ["$$NAMESPACE_TOKEN_OPEN$$", Lng.NAMESPACE_OPEN(token_descr.name_space).replace("\n", "\\\n")], ["$$PATH_TERMINATION_CODE$$", "0x%X" % Setup.path_limit_code], ["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str], ["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type], ["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str], ["$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$", Lng.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace("\n", "\\\n")], ["$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$", Lng.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace("\n", "\\\n")], ["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp], ["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe], ["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name], ["$$QUEX_VERSION$$", QUEX_VERSION], ["$$TOKEN_CLASS$$", token_descr.class_name], ["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe], ["$$TOKEN_COLUMN_N_TYPE$$", token_descr.column_number_type.get_pure_text()], ["$$TOKEN_ID_TYPE$$", token_descr.token_id_type.get_pure_text()], ["$$TOKEN_LINE_N_TYPE$$", token_descr.line_number_type.get_pure_text()], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$TOKEN_QUEUE_SAFETY_BORDER$$", repr(Setup.token_queue_safety_border)], ["$$TOKEN_QUEUE_SIZE$$", repr(Setup.token_queue_size)], ["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt], ["$$USER_LEXER_VERSION$$", Setup.user_application_version_id], ]) return txt
def get_implementation_of_mode_functions(mode, Modes): """Writes constructors and mode transition functions. void quex::lexer::enter_EXAMPLE_MODE() { ... } where EXAMPLE_MODE is a lexer mode from the given lexer-modes, and 'quex::lexer' is the lexical analysis class. """ def __filter_out_abstract_modes(ModeNameList): """Return only those names from ModeNameList where the mode is not abstract. That is, it can be implemented. """ result = [] for name in ModeNameList: for mode in Modes: if mode.name == name: if not mode.abstract_f(): result.append(name) break return result # (*) on enter on_entry_str = "# ifdef QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK\n" on_entry_str += " QUEX_NAME(%s).has_entry_from(FromMode);\n" % mode.name on_entry_str += "# endif\n" code_fragment = mode.incidence_db.get(E_IncidenceIDs.MODE_ENTRY) if code_fragment is not None: on_entry_str += Lng.SOURCE_REFERENCED(code_fragment) if on_entry_str[-1] == "\n": on_entry_str = on_entry_str[:-1] # (*) on exit on_exit_str = "# ifdef QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK\n" on_exit_str += " QUEX_NAME(%s).has_exit_to(ToMode);\n" % mode.name on_exit_str += "# endif\n" code_fragment = mode.incidence_db.get(E_IncidenceIDs.MODE_EXIT) if code_fragment is not None: on_exit_str += Lng.SOURCE_REFERENCED(code_fragment) # (*) on indentation on_indentation_str = get_on_indentation_handler(mode) # (*) has base mode if mode.has_base_mode(): base_mode_list = __filter_out_abstract_modes(mode.get_base_mode_name_list()) has_base_mode_str = get_IsOneOfThoseCode(base_mode_list, CheckBaseModeF=True) else: has_base_mode_str = " return false;" # (*) has entry from # check whether the source mode is a permissible 'entry' mode entry_list = __filter_out_abstract_modes(mode.entry_mode_name_list) has_entry_from_str = get_IsOneOfThoseCode(entry_list, ConsiderDerivedClassesF=True) # (*) has exit to # check whether the target mode is a permissible 'exit' mode exit_list = __filter_out_abstract_modes(mode.exit_mode_name_list) has_exit_to_str = get_IsOneOfThoseCode(exit_list, ConsiderDerivedClassesF=True) txt = blue_print(mode_function_implementation_str, [ ["$$ENTER-PROCEDURE$$", on_entry_str], ["$$EXIT-PROCEDURE$$", on_exit_str], # ["$$ON_INDENTATION-PROCEDURE$$", on_indentation_str], # ["$$HAS_BASE_MODE$$", has_base_mode_str], ["$$HAS_ENTRANCE_FROM$$", has_entry_from_str], ["$$HAS_EXIT_TO$$", has_exit_to_str], # ["$$MODE_NAME$$", mode.name], ]) return txt
def get_implementation_of_mode_functions(mode, Modes): """Writes constructors and mode transition functions. void quex::lexer::enter_EXAMPLE_MODE() { ... } where EXAMPLE_MODE is a lexer mode from the given lexer-modes, and 'quex::lexer' is the lexical analysis class. """ def __filter_out_inheritable_only(ModeNameList): result = [] for name in ModeNameList: for mode in Modes: if mode.name == name: if mode.options["inheritable"] != "only": result.append(name) break return result # (*) on enter on_entry_str = "# ifdef QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK\n" on_entry_str += " QUEX_NAME(%s).has_entry_from(FromMode);\n" % mode.name on_entry_str += "# endif\n" for code_info in mode.get_code_fragment_list("on_entry"): on_entry_str += code_info.get_code() if on_entry_str[-1] == "\n": on_entry_str = on_entry_str[:-1] # (*) on exit on_exit_str = "# ifdef QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK\n" on_exit_str += " QUEX_NAME(%s).has_exit_to(ToMode);\n" % mode.name on_exit_str += "# endif\n" for code_info in mode.get_code_fragment_list("on_exit"): on_exit_str += code_info.get_code() # (*) on indentation on_indentation_str = get_on_indentation_handler(mode) # (*) has base mode if mode.has_base_mode(): base_mode_list = __filter_out_inheritable_only(mode.get_base_mode_name_list()) has_base_mode_str = get_IsOneOfThoseCode(base_mode_list, CheckBaseModeF=True) else: has_base_mode_str = " return false;" # (*) has entry from try: entry_list = __filter_out_inheritable_only(mode.options["entry"]) has_entry_from_str = get_IsOneOfThoseCode(entry_list, __filter_out_inheritable_only(ConsiderDerivedClassesF=True)) # check whether the mode we come from is an allowed mode except: has_entry_from_str = " return true; /* default */" # (*) has exit to try: exit_list = __filter_out_inheritable_only(mode.options["exit"]) has_exit_to_str = get_IsOneOfThoseCode(exit_list, ConsiderDerivedClassesF=True) except: has_exit_to_str = " return true; /* default */" txt = blue_print(mode_function_implementation_str, [ ["$$ENTER-PROCEDURE$$", on_entry_str], ["$$EXIT-PROCEDURE$$", on_exit_str], # ["$$ON_INDENTATION-PROCEDURE$$", on_indentation_str], # ["$$HAS_BASE_MODE$$", has_base_mode_str], ["$$HAS_ENTRANCE_FROM$$", has_entry_from_str], ["$$HAS_EXIT_TO$$", has_exit_to_str], # ["$$MODE_NAME$$", mode.name], ]) return txt
def get_skipper(TriggerSet): """This function implements simple 'skipping' in the sense of passing by characters that belong to a given set of characters--the TriggerSet. """ global template_str assert TriggerSet.__class__.__name__ == "NumberSet" assert not TriggerSet.is_empty() LanguageDB = Setup.language_db skipper_index = sm_index.get() # Mini trigger map: [ trigger set ] --> loop start # That means: As long as characters of the trigger set appear, we go to the loop start. transition_map = TransitionMap( ) # (don't worry about 'drop-out-ranges' etc.) transition_map.add_transition(TriggerSet, skipper_index) # On buffer limit code, the skipper must transit to a dedicated reloader goto_loop_str = [ LanguageDB.INPUT_P_INCREMENT(), " goto _%s_LOOP;\n" % skipper_index ] trigger_map = transition_map.get_trigger_map() for i, info in enumerate(trigger_map): interval, target = info if target == E_StateIndices.DROP_OUT: continue trigger_map[i] = (interval, TextTransitionCode(goto_loop_str)) iteration_code = [] transition_block.do(iteration_code, trigger_map, skipper_index, E_EngineTypes.ELSE, GotoReload_Str="goto %s;" % get_label("$reload", skipper_index)) tmp = [] LanguageDB.COMMENT(tmp, "Skip any character in " + TriggerSet.get_utf8_string()) comment_str = "".join(tmp) # Line and column number counting prolog = __lc_counting_replacements(prolog_txt, TriggerSet) epilog = __lc_counting_replacements(epilog_txt, TriggerSet) prolog = blue_print(prolog, [ ["$$DELIMITER_COMMENT$$", comment_str], ["$$SKIPPER_INDEX$$", "%i" % skipper_index], ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()], ]) epilog = blue_print( epilog, [ ["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()], [ "$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "SkipDelimiter$$SKIPPER_INDEX$$[0]") ], ["$$ENDIF$$", LanguageDB.END_IF()], ["$$LOOP_REENTRANCE$$", LanguageDB.LABEL(skipper_index)], ["$$BUFFER_LIMIT_CODE$$", LanguageDB.BUFFER_LIMIT_CODE], ["$$RELOAD$$", get_label("$reload", skipper_index)], [ "$$DROP_OUT_DIRECT$$", get_label("$drop-out", skipper_index, U=True) ], ["$$SKIPPER_INDEX$$", "%i" % skipper_index], [ "$$LABEL_TERMINAL_EOF$$", "QUEX_LABEL(%i)" % get_address("$terminal-EOF", U=True) ], [ "$$LABEL_REF_AFTER_RELOAD$$", "QUEX_LABEL(%i)" % get_address("$skipper-reload", skipper_index, U=True) ], ["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)], [ "$$LABEL_AFTER_RELOAD$$", get_label("$skipper-reload", skipper_index) ], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_START$$", get_label("$start", U=True)], ["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()], ]) code = [prolog] code.extend(iteration_code) code.append(epilog) local_variable_db = {} variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") return code, local_variable_db
def get_skipper(OpenerSequence, CloserSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""): assert OpenerSequence.__class__ == list assert len(OpenerSequence) >= 1 assert map(type, OpenerSequence) == [int] * len(OpenerSequence) assert CloserSequence.__class__ == list assert len(CloserSequence) >= 1 assert map(type, CloserSequence) == [int] * len(CloserSequence) assert OpenerSequence != CloserSequence LanguageDB = Setup.language_db skipper_index = sm_index.get() opener_str, opener_comment_str = get_character_sequence(OpenerSequence) opener_length = len(OpenerSequence) closer_str, closer_comment_str = get_character_sequence(CloserSequence) closer_length = len(CloserSequence) if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, CloserSequence): goto_after_end_of_skipping_str = LanguageDB.GOTO(E_StateIndices.ANALYZER_REENTRY) else: # If there is indentation counting involved, then the counter's terminal id must # be determined at this place. assert IndentationCounterTerminalID is not None # If the ending delimiter is a subset of what the 'newline' pattern triggers # in indentation counting => move on to the indentation counter. goto_after_end_of_skipping_str = LanguageDB.GOTO_TERMINAL(IndentationCounterTerminalID) if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence) local_variable_db = {} variable_db.enter(local_variable_db, "reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") # variable_db.enter(local_variable_db, "text_end") variable_db.enter(local_variable_db, "counter") variable_db.enter(local_variable_db, "Skipper%i_Opener", "{ %s }" % opener_str, ElementN=opener_length, Index = skipper_index) variable_db.enter(local_variable_db, "Skipper%i_OpenerEnd", "Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length), Index = skipper_index) variable_db.enter(local_variable_db, "Skipper%i_Opener_it", "0x0", Index = skipper_index) variable_db.enter(local_variable_db, "Skipper%i_Closer", "{ %s }" % closer_str, ElementN=closer_length, Index = skipper_index) variable_db.enter(local_variable_db, "Skipper%i_CloserEnd", "Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length), Index = skipper_index) variable_db.enter(local_variable_db, "Skipper%i_Closer_it", "0x0", Index = skipper_index) reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" if CloserSequence[-1] == ord('\n'): end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n" end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n" else: end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" code_str = blue_print(template_str, [ ["$$SKIPPER_INDEX$$", __nice(skipper_index)], # ["$$OPENER_LENGTH$$", "%i" % opener_length], ["$$INPUT_P_INCREMENT$$", LanguageDB.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", LanguageDB.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", LanguageDB.ACCESS_INPUT()], ["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")], ["$$ENDIF$$", LanguageDB.END_IF()], ["$$ENTRY$$", LanguageDB.LABEL(skipper_index)], ["$$RELOAD$$", get_label("$reload", skipper_index)], ["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str], ["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)], ["$$INPUT_P_TO_LEXEME_START$$", LanguageDB.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_ENTRY$$", LanguageDB.GOTO(skipper_index)], ["$$MARK_LEXEME_START$$", LanguageDB.LEXEME_START_SET()], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str], # ["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def], ["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop()], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ]) return code_str, local_variable_db
def _do(Descr): # The following things must be ensured before the function is called assert Descr is not None assert Descr.__class__.__name__ == "TokenTypeDescriptor" ## ALLOW: Descr.get_member_db().keys() == empty TemplateFile = QUEX_PATH \ + Lng["$code_base"] \ + Lng["$token_template_file"] TemplateIFile = QUEX_PATH \ + Lng["$code_base"] \ + Lng["$token_template_i_file"] template_str = open_file_or_die(TemplateFile, Mode="rb").read() template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read() virtual_destructor_str = "" if Descr.open_for_derivation_f: virtual_destructor_str = "virtual " if Descr.copy is None: # Default copy operation: Plain Copy of token memory copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n" else: copy_str = Lng.SOURCE_REFERENCED(Descr.copy) if Descr.take_text is None: take_text_str = "return true;\n" else: take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text) include_guard_extension_str = get_include_guard_extension( Lng.NAMESPACE_REFERENCE(Descr.name_space) + "__" + Descr.class_name) # In case of plain 'C' the class name must incorporate the namespace (list) token_class_name = Descr.class_name if Setup.language == "C": token_class_name = Setup.token_class_name_safe converter_declaration_include, \ converter_implementation_include, \ converter_string, \ converter_wstring = __get_converter_configuration(include_guard_extension_str) extra_at_begin_str = lexeme_null_declaration() extra_at_end_str = "" if Setup.token_class_only_f: extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \ + extra_at_begin_str extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \ + extra_at_end_str namespace_open, namespace_close = __namespace_brackets() helper_variable_replacements = [ ["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include], ["$INCLUDE_CONVERTER_IMPLEMENTATION", converter_implementation_include], ["$CONVERTER_STRING", converter_string], ["$CONVERTER_WSTRING", converter_wstring], ["$NAMESPACE_CLOSE", namespace_close], ["$NAMESPACE_OPEN", namespace_open], ["$TOKEN_CLASS", token_class_name], ] txt = blue_print(template_str, [ ["$$EXTRA_AT_BEGIN$$", extra_at_begin_str], ["$$EXTRA_AT_END$$", extra_at_end_str], ]) txt = blue_print(txt, [ ["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)], ["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)], ["$$COPY$$", copy_str], ["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)], ["$$DISTINCT_MEMBERS$$", get_distinct_members(Descr)], ["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)], ["$$FUNC_TAKE_TEXT$$", take_text_str], ["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str], ["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)], ["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)], ["$$QUICK_SETTERS$$", get_quick_setters(Descr)], ["$$SETTERS_GETTERS$$", get_setter_getter(Descr)], ["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)], ["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)], ["$$UNION_MEMBERS$$", get_union_members(Descr)], ["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str], ["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe], ]) txt = blue_print(txt, helper_variable_replacements) if Setup.language.upper() != "C++" and Setup.token_class_only_f: extra_at_begin_str += local_strlen_str % (Descr.class_name_safe, Setup.buffer_element_type, Setup.buffer_element_type) txt_i = blue_print(template_i_str, [ ["$$EXTRA_AT_BEGIN$$", extra_at_begin_str], ["$$EXTRA_AT_END$$", extra_at_end_str], ]) txt_i = blue_print(txt_i, [ ["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)], ["$$COPY$$", copy_str], ["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)], ["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)], ["$$FUNC_TAKE_TEXT$$", take_text_str], ["$$TOKEN_CLASS_HEADER$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str], ["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)], ["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)], ["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)], ["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)], ["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe], ]) txt_i = blue_print(txt_i, helper_variable_replacements) return txt, txt_i
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter): assert len(OpenerSequence) >= 1 assert len(CloserSequence) >= 1 assert OpenerSequence != CloserSequence skipper_index = sm_index.get() skipper_door_id = dial_db.new_door_id(skipper_index) opener_str, opener_comment_str = get_character_sequence(OpenerSequence) opener_length = len(OpenerSequence) closer_str, closer_comment_str = get_character_sequence(CloserSequence) closer_length = len(CloserSequence) variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") variable_db.require("counter") variable_db.require_array("Skipper%i_Opener", Initial="{ %s }" % opener_str, ElementN=opener_length, Index=skipper_index) variable_db.require("Skipper%i_OpenerEnd", "Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length), Index=skipper_index) variable_db.require("Skipper%i_Opener_it", "0x0", Index=skipper_index) variable_db.require_array("Skipper%i_Closer", Initial="{ %s }" % closer_str, ElementN=closer_length, Index=skipper_index) variable_db.require("Skipper%i_CloserEnd", "Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length), Index=skipper_index) variable_db.require("Skipper%i_Closer_it", "0x0", Index=skipper_index) reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" if CloserSequence[-1] == ord('\n'): end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n" end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n" else: end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" reload_door_id = dial_db.new_door_id() on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern, NestedF=True) code_str = blue_print( template_str, [ ["$$SKIPPER_INDEX$$", __nice(skipper_index)], # ["$$OPENER_LENGTH$$", "%i" % opener_length], ["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", Lng.ACCESS_INPUT()], [ "$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]") ], ["$$ENDIF$$", Lng.END_IF()], ["$$ENTRY$$", Lng.LABEL(skipper_door_id)], ["$$RELOAD$$", dial_db.get_label_by_door_id(reload_door_id)], ["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)], ["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)], ["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)], ["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open], # ["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def], ["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop()], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ]) return [code_str]
def do(setup): """Creates a file of token-ids from a given set of names. Creates also a function: const string& $$token$$::map_id_to_name(). """ global file_str LanguageDB = Setup.language_db __propose_implicit_token_definitions() for standard_token_id in standard_token_id_list: assert token_id_db.has_key(standard_token_id) assert blackboard.token_type_definition is not None, \ "Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \ "handle this." # (*) Token ID File ________________________________________________________________ # # The token id file can either be specified as database of # token-id names, or as a file that directly assigns the token-ids # to variables. If the flag '--user-token-id-file' is defined, then # then the token-id file is provided by the user. Otherwise, the # token id file is created by the token-id maker. # # The token id maker considers the file passed by the option '-t' # as the database file and creates a C++ file with the output filestem # plus the suffix "--token-ids". Note, that the token id file is a # header file. # if len(token_id_db.keys()) == len(standard_token_id_list): token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \ (setup.token_id_prefix_plain, setup.token_id_prefix_plain) # TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done. error_msg("Only token ids %s are defined.\n" % token_id_str + \ "Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \ "specify at least one other token id.") #______________________________________________________________________________________ L = max(map(lambda name: len(name), token_id_db.keys())) def space(Name): return " " * (L - len(Name)) # -- define values for the token ids def define_this(txt, token): if setup.language == "C": txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \ % (setup.token_id_prefix_plain, token.name, space(token.name), token.number)) else: txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \ % (setup.token_id_prefix_plain, token.name, space(token.name), token.number)) if setup.token_id_foreign_definition_file != "": token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(setup.token_id_foreign_definition_file)] else: if setup.language == "C": prolog = "" epilog = "" else: prolog = LanguageDB.NAMESPACE_OPEN(setup.token_id_prefix_name_space) epilog = LanguageDB.NAMESPACE_CLOSE(setup.token_id_prefix_name_space) token_id_txt = [prolog] # Assign values to tokens with no numeric identifier # NOTE: This has not to happen if token's are defined by the user's provided file. i = setup.token_id_counter_offset # Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name' # may contain '--' to indicate a unicode value, so do not use it as name. for dummy_name, token in sorted(token_id_db.items()): if token.number is None: while __is_token_id_occupied(i): i += 1 token.number = i; define_this(token_id_txt, token) # Double check that no token id appears twice # Again, this can only happen, if quex itself produced the numeric values for the token token_list = token_id_db.values() for i, x in enumerate(token_list): for y in token_list[i+1:]: if x.number != y.number: continue error_msg("Token id '%s'" % x.name, x.file_name, x.line_n, DontExitF=True) error_msg("and token id '%s' have same numeric value '%s'." \ % (y.name, x.number), y.file_name, y.line_n, DontExitF=True) token_id_txt.append(epilog) content = blue_print(file_str, [["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$TOKEN_PREFIX$$", setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \ Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper())], ]) return content