def do(setup): """________________________________________________________________________ (1) Error Check (2) Generates a file containing: -- token id definitions (if they are not done in '--foreign-token-id-file'). -- const string& TokenClass::map_id_to_name(), i.e. a function which can convert token ids into strings. ________________________________________________________________________ """ global file_str # At this point, assume that the token type has been generated. assert blackboard.token_type_definition is not None # (1) Error Check # __warn_implicit_token_definitions() if len(Setup.token_id_foreign_definition_file) == 0: __autogenerate_token_id_numbers() __warn_on_double_definition() # If a mandatory token id is missing, this means that Quex did not # properly do implicit token definitions. Program error-abort. __error_on_mandatory_token_id_missing(AssertF=True) else: __error_on_mandatory_token_id_missing() __error_on_no_specific_token_ids() # (2) Generate token id file (if not specified outside) # if len(Setup.token_id_foreign_definition_file) != 0: # Content of file = inclusion of 'Setup.token_id_foreign_definition_file'. token_id_txt = [ "#include \"%s\"\n" % Setup.get_file_reference(Setup.token_id_foreign_definition_file) ] else: token_id_txt = __get_token_id_definition_txt() include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper()) content = blue_print(file_str, [ ["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], [ "$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference( blackboard.token_type_definition.get_file_name()) ], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", include_guard_ext], ]) return content
def do(setup): """________________________________________________________________________ (1) Error Check (2) Generates a file containing: -- token id definitions (if they are not done in '--foreign-token-id-file'). -- const string& TokenClass::map_id_to_name(), i.e. a function which can convert token ids into strings. ________________________________________________________________________ """ global file_str # At this point, assume that the token type has been generated. assert blackboard.token_type_definition is not None # (1) Error Check # __warn_implicit_token_definitions() if len(Setup.token_id_foreign_definition_file) == 0: __autogenerate_token_id_numbers() __warn_on_double_definition() # If a mandatory token id is missing, this means that Quex did not # properly do implicit token definitions. Program error-abort. __error_on_mandatory_token_id_missing(AssertF=True) else: __error_on_mandatory_token_id_missing() __error_on_no_specific_token_ids() # (2) Generate token id file (if not specified outside) # if len(Setup.token_id_foreign_definition_file) != 0: # Content of file = inclusion of 'Setup.token_id_foreign_definition_file'. token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(Setup.token_id_foreign_definition_file)] else: token_id_txt = __get_token_id_definition_txt() include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper()) content = blue_print(file_str, [["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", include_guard_ext], ]) return content
def do(ModeDB): assert blackboard.token_type_definition is not None QuexClassHeaderFileTemplate = os.path.normpath( QUEX_PATH + Lng["$code_base"] + Lng["$analyzer_template_file"]).replace("//","/") LexerClassName = Setup.analyzer_class_name quex_converter_coding_name_str = Setup.converter_ucs_coding_name mode_id_definition_str = "" # NOTE: First mode-id needs to be '1' for compatibility with flex generated engines for i, info in enumerate(ModeDB.items()): name = info[0] mode = info[1] if mode.abstract_f(): continue mode_id_definition_str += " QUEX_NAME(ModeID_%s) = %i,\n" % (name, i) if mode_id_definition_str != "": mode_id_definition_str = mode_id_definition_str[:-2] # -- instances of mode classes as members of the lexer mode_object_members_txt, \ mode_specific_functions_txt, \ friend_txt = get_mode_class_related_code_fragments(ModeDB.values()) # -- define a pointer that directly has the type of the derived class if Setup.analyzer_derived_class_name != "": analyzer_derived_class_name = Setup.analyzer_derived_class_name derived_class_type_declaration = "class %s;" % Setup.analyzer_derived_class_name else: analyzer_derived_class_name = Setup.analyzer_class_name derived_class_type_declaration = "" token_class_file_name = blackboard.token_type_definition.get_file_name() token_class_name = blackboard.token_type_definition.class_name token_class_name_safe = blackboard.token_type_definition.class_name_safe template_code_txt = get_file_content_or_die(QuexClassHeaderFileTemplate) include_guard_ext = get_include_guard_extension( Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name) if len(Setup.token_id_foreign_definition_file) != 0: token_id_definition_file = Setup.token_id_foreign_definition_file else: token_id_definition_file = Setup.output_token_id_file lexer_name_space_safe = get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space)) txt = blue_print(template_code_txt, [ ["$$___SPACE___$$", " " * (len(LexerClassName) + 1)], ["$$CLASS_BODY_EXTENSION$$", Lng.SOURCE_REFERENCED(blackboard.class_body_extension)], ["$$CONVERTER_HELPER$$", Setup.get_file_reference(Setup.output_buffer_codec_header)], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_ext], ["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_NAME_SPACE$$", lexer_name_space_safe], ["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe], ["$$LEXER_CONFIG_FILE$$", Setup.get_file_reference(Setup.output_configuration_file)], ["$$LEXER_DERIVED_CLASS_DECL$$", derived_class_type_declaration], ["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name], ["$$QUEX_MODE_ID_DEFINITIONS$$", mode_id_definition_str], ["$$MEMENTO_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.memento_class_extension)], ["$$MODE_CLASS_FRIENDS$$", friend_txt], ["$$MODE_OBJECTS$$", mode_object_members_txt], ["$$MODE_SPECIFIC_ANALYSER_FUNCTIONS$$", mode_specific_functions_txt], ["$$PRETTY_INDENTATION$$", " " + " " * (len(LexerClassName)*2 + 2)], ["$$QUEX_TEMPLATE_DIR$$", QUEX_PATH + Lng["$code_base"]], ["$$QUEX_VERSION$$", QUEX_VERSION], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(token_class_file_name)], ["$$TOKEN_CLASS$$", token_class_name], ["$$TOKEN_CLASS_NAME_SAFE$$", token_class_name_safe], ["$$TOKEN_ID_DEFINITION_FILE$$", Setup.get_file_reference(token_id_definition_file)], ["$$CORE_ENGINE_CHARACTER_CODING$$", quex_converter_coding_name_str], ["$$USER_DEFINED_HEADER$$", Lng.SOURCE_REFERENCED(blackboard.header) + "\n"], ]) return txt
def _do(Descr): # The following things must be ensured before the function is called assert Descr is not None assert Descr.__class__.__name__ == "TokenTypeDescriptor" ## ALLOW: Descr.get_member_db().keys() == empty TemplateFile = QUEX_PATH \ + Lng["$code_base"] \ + Lng["$token_template_file"] TemplateIFile = QUEX_PATH \ + Lng["$code_base"] \ + Lng["$token_template_i_file"] template_str = open_file_or_die(TemplateFile, Mode="rb").read() template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read() virtual_destructor_str = "" if Descr.open_for_derivation_f: virtual_destructor_str = "virtual " if Descr.copy is None: # Default copy operation: Plain Copy of token memory copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n" else: copy_str = Lng.SOURCE_REFERENCED(Descr.copy) if Descr.take_text is None: take_text_str = "return true;\n" else: take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text) include_guard_extension_str = get_include_guard_extension( Lng.NAMESPACE_REFERENCE(Descr.name_space) + "__" + Descr.class_name) # In case of plain 'C' the class name must incorporate the namespace (list) token_class_name = Descr.class_name if Setup.language == "C": token_class_name = Setup.token_class_name_safe converter_declaration_include, \ converter_implementation_include, \ converter_string, \ converter_wstring = __get_converter_configuration(include_guard_extension_str) extra_at_begin_str = lexeme_null_declaration() extra_at_end_str = "" if Setup.token_class_only_f: extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \ + extra_at_begin_str extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \ + extra_at_end_str namespace_open, namespace_close = __namespace_brackets() helper_variable_replacements = [ ["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include], [ "$INCLUDE_CONVERTER_IMPLEMENTATION", converter_implementation_include ], ["$CONVERTER_STRING", converter_string], ["$CONVERTER_WSTRING", converter_wstring], ["$NAMESPACE_CLOSE", namespace_close], ["$NAMESPACE_OPEN", namespace_open], ["$TOKEN_CLASS", token_class_name], ] txt = blue_print(template_str, [ ["$$EXTRA_AT_BEGIN$$", extra_at_begin_str], ["$$EXTRA_AT_END$$", extra_at_end_str], ]) txt = blue_print(txt, [ ["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)], ["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)], ["$$COPY$$", copy_str], ["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)], ["$$DISTINCT_MEMBERS$$", get_distinct_members(Descr)], ["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)], ["$$FUNC_TAKE_TEXT$$", take_text_str], ["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str], ["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)], ["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)], ["$$QUICK_SETTERS$$", get_quick_setters(Descr)], ["$$SETTERS_GETTERS$$", get_setter_getter(Descr)], [ "$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get) ], [ "$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set) ], ["$$UNION_MEMBERS$$", get_union_members(Descr)], ["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str], ["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe], ]) txt = blue_print(txt, helper_variable_replacements) if Setup.language.upper() != "C++" and Setup.token_class_only_f: extra_at_begin_str += local_strlen_str % (Descr.class_name_safe, Setup.buffer_element_type, Setup.buffer_element_type) txt_i = blue_print(template_i_str, [ ["$$EXTRA_AT_BEGIN$$", extra_at_begin_str], ["$$EXTRA_AT_END$$", extra_at_end_str], ]) txt_i = blue_print(txt_i, [ ["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)], ["$$COPY$$", copy_str], ["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)], ["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)], ["$$FUNC_TAKE_TEXT$$", take_text_str], [ "$$TOKEN_CLASS_HEADER$$", Setup.get_file_reference( blackboard.token_type_definition.get_file_name()) ], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str], ["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)], ["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)], [ "$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get) ], [ "$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set) ], ["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe], ]) txt_i = blue_print(txt_i, helper_variable_replacements) return txt, txt_i
def do(ModeDescriptionDB): IndentationSupportF = blackboard.required_support_indentation_count() BeginOfLineSupportF = blackboard.required_support_begin_of_line() LexerClassName = Setup.analyzer_class_name ConfigurationTemplateFile =( QUEX_PATH \ + Lng["$code_base"] \ + "/analyzer/configuration/TXT").replace("//","/") txt = get_file_content_or_die(ConfigurationTemplateFile) # -- check if exit/entry handlers have to be active entry_handler_active_f = False exit_handler_active_f = False for mode in ModeDescriptionDB.values(): entry_handler_active_f |= mode.incidence_db.has_key(E_IncidenceIDs.MODE_ENTRY) exit_handler_active_f |= mode.incidence_db.has_key(E_IncidenceIDs.MODE_EXIT) # Buffer filler converter (0x0 means: no buffer filler converter) converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW " if Setup.converter_user_new_func != "": converter_new_str += Setup.converter_user_new_func + "()" else: converter_new_str = "/* " + converter_new_str + " */" # Token repetition support token_repeat_test_txt = "" for token_id_str in blackboard.token_repetition_token_id_list: token_repeat_test_txt += "TokenID == %s || " % token_id_str if token_repeat_test_txt != "": token_repeat_test_txt = token_repeat_test_txt[:-3] else: token_repeat_test_txt = "false" if Setup.analyzer_derived_class_name != "": analyzer_derived_class_name = Setup.analyzer_derived_class_name else: analyzer_derived_class_name = Setup.analyzer_class_name txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING", Setup.count_column_number_f) txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False) txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f) txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f) txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK", Setup.include_stack_support_f) txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING", Setup.count_line_number_f) txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER", Setup.post_categorizer_f) txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK", Setup.mode_transition_check_f) txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR", Setup.string_accumulator_f) txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE", Setup.token_policy == "queue") txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE", Setup.token_policy == "single") txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT", token_repeat_test_txt != "false") txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY", Setup.token_memory_management_by_user_f) txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN", Setup.buffer_byte_order == "big") txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER", Setup.converter_helper_required_f) txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f) txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF) txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN", Setup.buffer_byte_order == "little") txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT", entry_handler_active_f) txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT", exit_handler_active_f) txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C") txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION", BeginOfLineSupportF) txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN", Setup.byte_order_is_that_of_current_system_f) txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS", Setup.buffer_based_analyzis_f) txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC", Setup.buffer_codec.name != "unicode") # -- token class related definitions token_descr = blackboard.token_type_definition # -- name of the character codec codec_name = make_safe_identifier(Setup.buffer_codec.name).lower() # Setup.buffer_element_size can be '-1'. This signals then that # sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value # is required here. character_size_str = "%i" % Setup.buffer_element_size def namespace(NameSpaceList): result = Lng.NAMESPACE_REFERENCE(NameSpaceList, TrailingDelimiterF=False) if len(result) == 0: return "" assert Setup.language.upper() != "C++" or len(result) > 2, \ "Error while generating namespace reference '%s'" % result return result txt = blue_print(txt, [ ["$$BUFFER_LIMIT_CODE$$", "0x%X" % Setup.buffer_limit_code], ["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name], ["$$INCLUDE_GUARD_EXTENSION$$", get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name)], ["$$INITIAL_LEXER_MODE_ID$$", "QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_text()], ["$$LEXER_BUILD_DATE$$", time.asctime()], ["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe], ["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name], ["$$MAX_MODE_CLASS_N$$", repr(len(ModeDescriptionDB))], ["$$NAMESPACE_MAIN$$", namespace(Setup.analyzer_name_space)], ["$$NAMESPACE_MAIN_CLOSE$$", Lng.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace("\n", "\\\n")], ["$$NAMESPACE_MAIN_OPEN$$", Lng.NAMESPACE_OPEN(Setup.analyzer_name_space).replace("\n", "\\\n")], ["$$NAMESPACE_TOKEN$$", namespace(token_descr.name_space)], ["$$NAMESPACE_TOKEN_CLOSE$$", Lng.NAMESPACE_CLOSE(token_descr.name_space).replace("\n", "\\\n")], ["$$NAMESPACE_TOKEN_OPEN$$", Lng.NAMESPACE_OPEN(token_descr.name_space).replace("\n", "\\\n")], ["$$PATH_TERMINATION_CODE$$", "0x%X" % Setup.path_limit_code], ["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str], ["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type], ["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str], ["$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$", Lng.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace("\n", "\\\n")], ["$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$", Lng.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace("\n", "\\\n")], ["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp], ["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe], ["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name], ["$$QUEX_VERSION$$", QUEX_VERSION], ["$$TOKEN_CLASS$$", token_descr.class_name], ["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe], ["$$TOKEN_COLUMN_N_TYPE$$", token_descr.column_number_type.get_pure_text()], ["$$TOKEN_ID_TYPE$$", token_descr.token_id_type.get_pure_text()], ["$$TOKEN_LINE_N_TYPE$$", token_descr.line_number_type.get_pure_text()], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], ["$$TOKEN_QUEUE_SAFETY_BORDER$$", repr(Setup.token_queue_safety_border)], ["$$TOKEN_QUEUE_SIZE$$", repr(Setup.token_queue_size)], ["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt], ["$$USER_LEXER_VERSION$$", Setup.user_application_version_id], ]) return txt
def do(ModeDB): IndentationSupportF = blackboard.requires_indentation_count(ModeDB) BeginOfLineSupportF = blackboard.requires_begin_of_line_condition_support( ModeDB) LanguageDB = Setup.language_db LexerClassName = Setup.analyzer_class_name ConfigurationTemplateFile =( QUEX_PATH \ + Setup.language_db["$code_base"] \ + "/analyzer/configuration/TXT").replace("//","/") txt = get_file_content_or_die(ConfigurationTemplateFile) # -- check if exit/entry handlers have to be active entry_handler_active_f = False exit_handler_active_f = False for mode in ModeDB.values(): if len(mode.get_code_fragment_list("on_entry")) != 0: entry_handler_active_f = True if len(mode.get_code_fragment_list("on_exit")) != 0: exit_handler_active_f = True # Buffer filler converter (0x0 means: no buffer filler converter) converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW " if Setup.converter_user_new_func != "": converter_new_str += Setup.converter_user_new_func + "()" else: converter_new_str = "/* " + converter_new_str + " */" # Token repetition support token_repeat_test_txt = "" for token_id_str in blackboard.token_repetition_token_id_list: token_repeat_test_txt += "TokenID == %s || " % token_id_str if token_repeat_test_txt != "": token_repeat_test_txt = token_repeat_test_txt[:-3] else: token_repeat_test_txt = "false" if Setup.analyzer_derived_class_name != "": analyzer_derived_class_name = Setup.analyzer_derived_class_name else: analyzer_derived_class_name = Setup.analyzer_class_name txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING", Setup.count_column_number_f) txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False) txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f) txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f) txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK", Setup.include_stack_support_f) txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING", Setup.count_line_number_f) txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER", Setup.post_categorizer_f) txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK", Setup.mode_transition_check_f) txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR", Setup.string_accumulator_f) txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE", Setup.token_policy == "queue") txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE", Setup.token_policy == "single") txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT", token_repeat_test_txt != "false") txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY", Setup.token_memory_management_by_user_f) txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN", Setup.buffer_byte_order == "big") txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER", Setup.converter_helper_required_f) txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f) txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF) txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN", Setup.buffer_byte_order == "little") txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT", entry_handler_active_f) txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT", exit_handler_active_f) txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C") txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION", BeginOfLineSupportF) txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN", Setup.byte_order_is_that_of_current_system_f) txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS", Setup.buffer_based_analyzis_f) txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC", Setup.buffer_codec != "unicode") # -- token class related definitions token_descr = blackboard.token_type_definition # -- name of the character codec codec_name = "unicode" if Setup.buffer_codec != "unicode": codec_name = make_safe_identifier(Setup.buffer_codec).lower() # Setup.buffer_element_size can be '-1'. This signals then that # sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value # is required here. character_size_str = "%i" % Setup.buffer_element_size def namespace(NameSpaceList): result = Setup.language_db.NAMESPACE_REFERENCE(NameSpaceList) if result == "::": return "" assert Setup.language.upper() != "C++" or len(result) > 2, \ "Error while generating namespace reference '%s'" % result return result[:-2] txt = blue_print(txt, [ ["$$BUFFER_LIMIT_CODE$$", "0x%X" % Setup.buffer_limit_code], ["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name], [ "$$INCLUDE_GUARD_EXTENSION$$", get_include_guard_extension( LanguageDB.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name) ], [ "$$INITIAL_LEXER_MODE_ID$$", "QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_code() ], ["$$LEXER_BUILD_DATE$$", time.asctime()], ["$$LEXER_CLASS_NAME$$", LexerClassName], ["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe], ["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name], ["$$MAX_MODE_CLASS_N$$", repr(len(ModeDB))], ["$$NAMESPACE_MAIN$$", namespace(Setup.analyzer_name_space)], [ "$$NAMESPACE_MAIN_CLOSE$$", LanguageDB.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace( "\n", "\\\n") ], [ "$$NAMESPACE_MAIN_OPEN$$", LanguageDB.NAMESPACE_OPEN(Setup.analyzer_name_space).replace( "\n", "\\\n") ], ["$$NAMESPACE_TOKEN$$", namespace(token_descr.name_space)], [ "$$NAMESPACE_TOKEN_CLOSE$$", LanguageDB.NAMESPACE_CLOSE(token_descr.name_space).replace( "\n", "\\\n") ], [ "$$NAMESPACE_TOKEN_OPEN$$", LanguageDB.NAMESPACE_OPEN(token_descr.name_space).replace( "\n", "\\\n") ], ["$$PATH_TERMINATION_CODE$$", "0x%X" % Setup.path_limit_code], ["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str], ["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type], ["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str], [ "$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$", LanguageDB.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace( "\n", "\\\n") ], [ "$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$", LanguageDB.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace( "\n", "\\\n") ], ["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp], ["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe], ["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name], ["$$QUEX_VERSION$$", QUEX_VERSION], ["$$TOKEN_CLASS$$", token_descr.class_name], ["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe], [ "$$TOKEN_COLUMN_N_TYPE$$", token_descr.column_number_type.get_pure_code() ], ["$$TOKEN_ID_TYPE$$", token_descr.token_id_type.get_pure_code()], [ "$$TOKEN_LINE_N_TYPE$$", token_descr.line_number_type.get_pure_code() ], ["$$TOKEN_PREFIX$$", Setup.token_id_prefix], [ "$$TOKEN_QUEUE_SAFETY_BORDER$$", repr(Setup.token_queue_safety_border) ], ["$$TOKEN_QUEUE_SIZE$$", repr(Setup.token_queue_size)], ["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt], ["$$USER_LEXER_VERSION$$", Setup.user_application_version_id], ]) return txt
def do(setup): """Creates a file of token-ids from a given set of names. Creates also a function: const string& $$token$$::map_id_to_name(). """ global file_str LanguageDB = Setup.language_db __propose_implicit_token_definitions() for standard_token_id in standard_token_id_list: assert token_id_db.has_key(standard_token_id) assert blackboard.token_type_definition is not None, \ "Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \ "handle this." # (*) Token ID File ________________________________________________________________ # # The token id file can either be specified as database of # token-id names, or as a file that directly assigns the token-ids # to variables. If the flag '--user-token-id-file' is defined, then # then the token-id file is provided by the user. Otherwise, the # token id file is created by the token-id maker. # # The token id maker considers the file passed by the option '-t' # as the database file and creates a C++ file with the output filestem # plus the suffix "--token-ids". Note, that the token id file is a # header file. # if len(token_id_db.keys()) == len(standard_token_id_list): token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \ (setup.token_id_prefix_plain, setup.token_id_prefix_plain) # TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done. error_msg("Only token ids %s are defined.\n" % token_id_str + \ "Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \ "specify at least one other token id.") #______________________________________________________________________________________ L = max(map(lambda name: len(name), token_id_db.keys())) def space(Name): return " " * (L - len(Name)) # -- define values for the token ids def define_this(txt, token): if setup.language == "C": txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \ % (setup.token_id_prefix_plain, token.name, space(token.name), token.number)) else: txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \ % (setup.token_id_prefix_plain, token.name, space(token.name), token.number)) if setup.token_id_foreign_definition_file != "": token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(setup.token_id_foreign_definition_file)] else: if setup.language == "C": prolog = "" epilog = "" else: prolog = LanguageDB.NAMESPACE_OPEN(setup.token_id_prefix_name_space) epilog = LanguageDB.NAMESPACE_CLOSE(setup.token_id_prefix_name_space) token_id_txt = [prolog] # Assign values to tokens with no numeric identifier # NOTE: This has not to happen if token's are defined by the user's provided file. i = setup.token_id_counter_offset # Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name' # may contain '--' to indicate a unicode value, so do not use it as name. for dummy_name, token in sorted(token_id_db.items()): if token.number is None: while __is_token_id_occupied(i): i += 1 token.number = i; define_this(token_id_txt, token) # Double check that no token id appears twice # Again, this can only happen, if quex itself produced the numeric values for the token token_list = token_id_db.values() for i, x in enumerate(token_list): for y in token_list[i+1:]: if x.number != y.number: continue error_msg("Token id '%s'" % x.name, x.file_name, x.line_n, DontExitF=True) error_msg("and token id '%s' have same numeric value '%s'." \ % (y.name, x.number), y.file_name, y.line_n, DontExitF=True) token_id_txt.append(epilog) content = blue_print(file_str, [["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$TOKEN_PREFIX$$", setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \ Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper())], ]) return content
def _do(Descr): # The following things must be ensured before the function is called assert Descr is not None assert Descr.__class__.__name__ == "TokenTypeDescriptor" ## ALLOW: Descr.get_member_db().keys() == empty TemplateFile = QUEX_PATH \ + Lng["$code_base"] \ + Lng["$token_template_file"] TemplateIFile = QUEX_PATH \ + Lng["$code_base"] \ + Lng["$token_template_i_file"] template_str = open_file_or_die(TemplateFile, Mode="rb").read() template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read() virtual_destructor_str = "" if Descr.open_for_derivation_f: virtual_destructor_str = "virtual " if Descr.copy is None: # Default copy operation: Plain Copy of token memory copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n" else: copy_str = Lng.SOURCE_REFERENCED(Descr.copy) if Descr.take_text is None: take_text_str = "return true;\n" else: take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text) include_guard_extension_str = get_include_guard_extension( Lng.NAMESPACE_REFERENCE(Descr.name_space) + "__" + Descr.class_name) # In case of plain 'C' the class name must incorporate the namespace (list) token_class_name = Descr.class_name if Setup.language == "C": token_class_name = Setup.token_class_name_safe converter_declaration_include, \ converter_implementation_include, \ converter_string, \ converter_wstring = __get_converter_configuration(include_guard_extension_str) extra_at_begin_str = lexeme_null_declaration() extra_at_end_str = "" if Setup.token_class_only_f: extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \ + extra_at_begin_str extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \ + extra_at_end_str namespace_open, namespace_close = __namespace_brackets() helper_variable_replacements = [ ["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include], ["$INCLUDE_CONVERTER_IMPLEMENTATION", converter_implementation_include], ["$CONVERTER_STRING", converter_string], ["$CONVERTER_WSTRING", converter_wstring], ["$NAMESPACE_CLOSE", namespace_close], ["$NAMESPACE_OPEN", namespace_open], ["$TOKEN_CLASS", token_class_name], ] txt = blue_print(template_str, [ ["$$EXTRA_AT_BEGIN$$", extra_at_begin_str], ["$$EXTRA_AT_END$$", extra_at_end_str], ]) txt = blue_print(txt, [ ["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)], ["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)], ["$$COPY$$", copy_str], ["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)], ["$$DISTINCT_MEMBERS$$", get_distinct_members(Descr)], ["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)], ["$$FUNC_TAKE_TEXT$$", take_text_str], ["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str], ["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)], ["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)], ["$$QUICK_SETTERS$$", get_quick_setters(Descr)], ["$$SETTERS_GETTERS$$", get_setter_getter(Descr)], ["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)], ["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)], ["$$UNION_MEMBERS$$", get_union_members(Descr)], ["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str], ["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe], ]) txt = blue_print(txt, helper_variable_replacements) if Setup.language.upper() != "C++" and Setup.token_class_only_f: extra_at_begin_str += local_strlen_str % (Descr.class_name_safe, Setup.buffer_element_type, Setup.buffer_element_type) txt_i = blue_print(template_i_str, [ ["$$EXTRA_AT_BEGIN$$", extra_at_begin_str], ["$$EXTRA_AT_END$$", extra_at_end_str], ]) txt_i = blue_print(txt_i, [ ["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)], ["$$COPY$$", copy_str], ["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)], ["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)], ["$$FUNC_TAKE_TEXT$$", take_text_str], ["$$TOKEN_CLASS_HEADER$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str], ["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)], ["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)], ["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)], ["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)], ["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe], ]) txt_i = blue_print(txt_i, helper_variable_replacements) return txt, txt_i
def do(setup): """Creates a file of token-ids from a given set of names. Creates also a function: const string& $$token$$::map_id_to_name(). """ global file_str LanguageDB = Setup.language_db __propose_implicit_token_definitions() for standard_token_id in standard_token_id_list: assert token_id_db.has_key(standard_token_id) assert blackboard.token_type_definition is not None, \ "Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \ "handle this." # (*) Token ID File ________________________________________________________________ # # The token id file can either be specified as database of # token-id names, or as a file that directly assigns the token-ids # to variables. If the flag '--user-token-id-file' is defined, then # then the token-id file is provided by the user. Otherwise, the # token id file is created by the token-id maker. # # The token id maker considers the file passed by the option '-t' # as the database file and creates a C++ file with the output filestem # plus the suffix "--token-ids". Note, that the token id file is a # header file. # if len(token_id_db.keys()) == len(standard_token_id_list): token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \ (setup.token_id_prefix_plain, setup.token_id_prefix_plain) # TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done. error_msg("Only token ids %s are defined.\n" % token_id_str + \ "Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \ "specify at least one other token id.") #______________________________________________________________________________________ L = max(map(lambda name: len(name), token_id_db.keys())) def space(Name): return " " * (L - len(Name)) # -- define values for the token ids def define_this(txt, token): if setup.language == "C": txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \ % (setup.token_id_prefix_plain, token.name, space(token.name), token.number)) else: txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \ % (setup.token_id_prefix_plain, token.name, space(token.name), token.number)) if setup.token_id_foreign_definition_file != "": token_id_txt = [ "#include \"%s\"\n" % Setup.get_file_reference(setup.token_id_foreign_definition_file) ] else: if setup.language == "C": prolog = "" epilog = "" else: prolog = LanguageDB.NAMESPACE_OPEN( setup.token_id_prefix_name_space) epilog = LanguageDB.NAMESPACE_CLOSE( setup.token_id_prefix_name_space) token_id_txt = [prolog] # Assign values to tokens with no numeric identifier # NOTE: This has not to happen if token's are defined by the user's provided file. i = setup.token_id_counter_offset # Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name' # may contain '--' to indicate a unicode value, so do not use it as name. for dummy_name, token in sorted(token_id_db.items()): if token.number is None: while __is_token_id_occupied(i): i += 1 token.number = i define_this(token_id_txt, token) # Double check that no token id appears twice # Again, this can only happen, if quex itself produced the numeric values for the token token_list = token_id_db.values() for i, x in enumerate(token_list): for y in token_list[i + 1:]: if x.number != y.number: continue error_msg("Token id '%s'" % x.name, x.file_name, x.line_n, DontExitF=True) error_msg("and token id '%s' have same numeric value '%s'." \ % (y.name, x.number), y.file_name, y.line_n, DontExitF=True) token_id_txt.append(epilog) content = blue_print(file_str, [["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)], ["$$DATE$$", time.asctime()], ["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())], ["$$TOKEN_PREFIX$$", setup.token_id_prefix], ["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \ Setup.analyzer_name_safe.upper() \ + "__" \ + Setup.token_class_name_safe.upper())], ]) return content