def do(setup):
"""________________________________________________________________________
(1) Error Check
(2) Generates a file containing:
-- token id definitions (if they are not done in '--foreign-token-id-file').
-- const string& TokenClass::map_id_to_name(), i.e. a function which can
convert token ids into strings.
________________________________________________________________________
"""
global file_str
# At this point, assume that the token type has been generated.
assert blackboard.token_type_definition is not None
# (1) Error Check
#
__warn_implicit_token_definitions()
if len(Setup.token_id_foreign_definition_file) == 0:
__autogenerate_token_id_numbers()
__warn_on_double_definition()
# If a mandatory token id is missing, this means that Quex did not
# properly do implicit token definitions. Program error-abort.
__error_on_mandatory_token_id_missing(AssertF=True)
else:
__error_on_mandatory_token_id_missing()
__error_on_no_specific_token_ids()
# (2) Generate token id file (if not specified outside)
#
if len(Setup.token_id_foreign_definition_file) != 0:
# Content of file = inclusion of 'Setup.token_id_foreign_definition_file'.
token_id_txt = [
"#include \"%s\"\n" %
Setup.get_file_reference(Setup.token_id_foreign_definition_file)
]
else:
token_id_txt = __get_token_id_definition_txt()
include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())
content = blue_print(file_str, [
["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
[
"$$TOKEN_CLASS_DEFINITION_FILE$$",
Setup.get_file_reference(
blackboard.token_type_definition.get_file_name())
],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", include_guard_ext],
])
return content
def do(setup):
"""________________________________________________________________________
(1) Error Check
(2) Generates a file containing:
-- token id definitions (if they are not done in '--foreign-token-id-file').
-- const string& TokenClass::map_id_to_name(), i.e. a function which can
convert token ids into strings.
________________________________________________________________________
"""
global file_str
# At this point, assume that the token type has been generated.
assert blackboard.token_type_definition is not None
# (1) Error Check
#
__warn_implicit_token_definitions()
if len(Setup.token_id_foreign_definition_file) == 0:
__autogenerate_token_id_numbers()
__warn_on_double_definition()
# If a mandatory token id is missing, this means that Quex did not
# properly do implicit token definitions. Program error-abort.
__error_on_mandatory_token_id_missing(AssertF=True)
else:
__error_on_mandatory_token_id_missing()
__error_on_no_specific_token_ids()
# (2) Generate token id file (if not specified outside)
#
if len(Setup.token_id_foreign_definition_file) != 0:
# Content of file = inclusion of 'Setup.token_id_foreign_definition_file'.
token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(Setup.token_id_foreign_definition_file)]
else:
token_id_txt = __get_token_id_definition_txt()
include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())
content = blue_print(file_str,
[["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", include_guard_ext],
])
return content
def do(ModeDB):
assert blackboard.token_type_definition is not None
QuexClassHeaderFileTemplate = os.path.normpath( QUEX_PATH
+ Lng["$code_base"]
+ Lng["$analyzer_template_file"]).replace("//","/")
LexerClassName = Setup.analyzer_class_name
quex_converter_coding_name_str = Setup.converter_ucs_coding_name
mode_id_definition_str = ""
# NOTE: First mode-id needs to be '1' for compatibility with flex generated engines
for i, info in enumerate(ModeDB.items()):
name = info[0]
mode = info[1]
if mode.abstract_f(): continue
mode_id_definition_str += " QUEX_NAME(ModeID_%s) = %i,\n" % (name, i)
if mode_id_definition_str != "":
mode_id_definition_str = mode_id_definition_str[:-2]
# -- instances of mode classes as members of the lexer
mode_object_members_txt, \
mode_specific_functions_txt, \
friend_txt = get_mode_class_related_code_fragments(ModeDB.values())
# -- define a pointer that directly has the type of the derived class
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
derived_class_type_declaration = "class %s;" % Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
derived_class_type_declaration = ""
token_class_file_name = blackboard.token_type_definition.get_file_name()
token_class_name = blackboard.token_type_definition.class_name
token_class_name_safe = blackboard.token_type_definition.class_name_safe
template_code_txt = get_file_content_or_die(QuexClassHeaderFileTemplate)
include_guard_ext = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space)
+ "__" + Setup.analyzer_class_name)
if len(Setup.token_id_foreign_definition_file) != 0:
token_id_definition_file = Setup.token_id_foreign_definition_file
else:
token_id_definition_file = Setup.output_token_id_file
lexer_name_space_safe = get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space))
txt = blue_print(template_code_txt,
[
["$$___SPACE___$$", " " * (len(LexerClassName) + 1)],
["$$CLASS_BODY_EXTENSION$$", Lng.SOURCE_REFERENCED(blackboard.class_body_extension)],
["$$CONVERTER_HELPER$$", Setup.get_file_reference(Setup.output_buffer_codec_header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_ext],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_NAME_SPACE$$", lexer_name_space_safe],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_CONFIG_FILE$$", Setup.get_file_reference(Setup.output_configuration_file)],
["$$LEXER_DERIVED_CLASS_DECL$$", derived_class_type_declaration],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$QUEX_MODE_ID_DEFINITIONS$$", mode_id_definition_str],
["$$MEMENTO_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.memento_class_extension)],
["$$MODE_CLASS_FRIENDS$$", friend_txt],
["$$MODE_OBJECTS$$", mode_object_members_txt],
["$$MODE_SPECIFIC_ANALYSER_FUNCTIONS$$", mode_specific_functions_txt],
["$$PRETTY_INDENTATION$$", " " + " " * (len(LexerClassName)*2 + 2)],
["$$QUEX_TEMPLATE_DIR$$", QUEX_PATH + Lng["$code_base"]],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(token_class_file_name)],
["$$TOKEN_CLASS$$", token_class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_class_name_safe],
["$$TOKEN_ID_DEFINITION_FILE$$", Setup.get_file_reference(token_id_definition_file)],
["$$CORE_ENGINE_CHARACTER_CODING$$", quex_converter_coding_name_str],
["$$USER_DEFINED_HEADER$$", Lng.SOURCE_REFERENCED(blackboard.header) + "\n"],
])
return txt
def do(ModeDB):
assert blackboard.token_type_definition is not None
QuexClassHeaderFileTemplate = os.path.normpath( QUEX_PATH
+ Lng["$code_base"]
+ Lng["$analyzer_template_file"]).replace("//","/")
LexerClassName = Setup.analyzer_class_name
quex_converter_coding_name_str = Setup.converter_ucs_coding_name
mode_id_definition_str = ""
# NOTE: First mode-id needs to be '1' for compatibility with flex generated engines
for i, info in enumerate(ModeDB.items()):
name = info[0]
mode = info[1]
if mode.abstract_f(): continue
mode_id_definition_str += " QUEX_NAME(ModeID_%s) = %i,\n" % (name, i)
if mode_id_definition_str != "":
mode_id_definition_str = mode_id_definition_str[:-2]
# -- instances of mode classes as members of the lexer
mode_object_members_txt, \
mode_specific_functions_txt, \
friend_txt = get_mode_class_related_code_fragments(ModeDB.values())
# -- define a pointer that directly has the type of the derived class
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
derived_class_type_declaration = "class %s;" % Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
derived_class_type_declaration = ""
token_class_file_name = blackboard.token_type_definition.get_file_name()
token_class_name = blackboard.token_type_definition.class_name
token_class_name_safe = blackboard.token_type_definition.class_name_safe
template_code_txt = get_file_content_or_die(QuexClassHeaderFileTemplate)
include_guard_ext = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space)
+ "__" + Setup.analyzer_class_name)
if len(Setup.token_id_foreign_definition_file) != 0:
token_id_definition_file = Setup.token_id_foreign_definition_file
else:
token_id_definition_file = Setup.output_token_id_file
lexer_name_space_safe = get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space))
txt = blue_print(template_code_txt,
[
["$$___SPACE___$$", " " * (len(LexerClassName) + 1)],
["$$CLASS_BODY_EXTENSION$$", Lng.SOURCE_REFERENCED(blackboard.class_body_extension)],
["$$CONVERTER_HELPER$$", Setup.get_file_reference(Setup.output_buffer_codec_header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_ext],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_NAME_SPACE$$", lexer_name_space_safe],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_CONFIG_FILE$$", Setup.get_file_reference(Setup.output_configuration_file)],
["$$LEXER_DERIVED_CLASS_DECL$$", derived_class_type_declaration],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$QUEX_MODE_ID_DEFINITIONS$$", mode_id_definition_str],
["$$MEMENTO_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.memento_class_extension)],
["$$MODE_CLASS_FRIENDS$$", friend_txt],
["$$MODE_OBJECTS$$", mode_object_members_txt],
["$$MODE_SPECIFIC_ANALYSER_FUNCTIONS$$", mode_specific_functions_txt],
["$$PRETTY_INDENTATION$$", " " + " " * (len(LexerClassName)*2 + 2)],
["$$QUEX_TEMPLATE_DIR$$", QUEX_PATH + Lng["$code_base"]],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(token_class_file_name)],
["$$TOKEN_CLASS$$", token_class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_class_name_safe],
["$$TOKEN_ID_DEFINITION_FILE$$", Setup.get_file_reference(token_id_definition_file)],
["$$CORE_ENGINE_CHARACTER_CODING$$", quex_converter_coding_name_str],
["$$USER_DEFINED_HEADER$$", Lng.SOURCE_REFERENCED(blackboard.header) + "\n"],
])
return txt
def _do(Descr):
# The following things must be ensured before the function is called
assert Descr is not None
assert Descr.__class__.__name__ == "TokenTypeDescriptor"
## ALLOW: Descr.get_member_db().keys() == empty
TemplateFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_file"]
TemplateIFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_i_file"]
template_str = open_file_or_die(TemplateFile, Mode="rb").read()
template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read()
virtual_destructor_str = ""
if Descr.open_for_derivation_f: virtual_destructor_str = "virtual "
if Descr.copy is None:
# Default copy operation: Plain Copy of token memory
copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n"
else:
copy_str = Lng.SOURCE_REFERENCED(Descr.copy)
if Descr.take_text is None:
take_text_str = "return true;\n"
else:
take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text)
include_guard_extension_str = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Descr.name_space) + "__" + Descr.class_name)
# In case of plain 'C' the class name must incorporate the namespace (list)
token_class_name = Descr.class_name
if Setup.language == "C":
token_class_name = Setup.token_class_name_safe
converter_declaration_include, \
converter_implementation_include, \
converter_string, \
converter_wstring = __get_converter_configuration(include_guard_extension_str)
extra_at_begin_str = lexeme_null_declaration()
extra_at_end_str = ""
if Setup.token_class_only_f:
extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \
+ extra_at_begin_str
extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \
+ extra_at_end_str
namespace_open, namespace_close = __namespace_brackets()
helper_variable_replacements = [
["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include],
[
"$INCLUDE_CONVERTER_IMPLEMENTATION",
converter_implementation_include
],
["$CONVERTER_STRING", converter_string],
["$CONVERTER_WSTRING", converter_wstring],
["$NAMESPACE_CLOSE", namespace_close],
["$NAMESPACE_OPEN", namespace_open],
["$TOKEN_CLASS", token_class_name],
]
txt = blue_print(template_str, [
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt = blue_print(txt, [
["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)],
["$$CONSTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$DISTINCT_MEMBERS$$",
get_distinct_members(Descr)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_CLOSE$$",
Lng.NAMESPACE_CLOSE(Descr.name_space)],
["$$NAMESPACE_OPEN$$",
Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$QUICK_SETTERS$$", get_quick_setters(Descr)],
["$$SETTERS_GETTERS$$",
get_setter_getter(Descr)],
[
"$$TOKEN_REPETITION_N_GET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_get)
],
[
"$$TOKEN_REPETITION_N_SET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_set)
],
["$$UNION_MEMBERS$$", get_union_members(Descr)],
["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt = blue_print(txt, helper_variable_replacements)
if Setup.language.upper() != "C++" and Setup.token_class_only_f:
extra_at_begin_str += local_strlen_str % (Descr.class_name_safe,
Setup.buffer_element_type,
Setup.buffer_element_type)
txt_i = blue_print(template_i_str, [
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt_i = blue_print(txt_i, [
["$$CONSTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
[
"$$TOKEN_CLASS_HEADER$$",
Setup.get_file_reference(
blackboard.token_type_definition.get_file_name())
],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_OPEN$$",
Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$NAMESPACE_CLOSE$$",
Lng.NAMESPACE_CLOSE(Descr.name_space)],
[
"$$TOKEN_REPETITION_N_GET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_get)
],
[
"$$TOKEN_REPETITION_N_SET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_set)
],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt_i = blue_print(txt_i, helper_variable_replacements)
return txt, txt_i
def do(ModeDescriptionDB):
IndentationSupportF = blackboard.required_support_indentation_count()
BeginOfLineSupportF = blackboard.required_support_begin_of_line()
LexerClassName = Setup.analyzer_class_name
ConfigurationTemplateFile =( QUEX_PATH \
+ Lng["$code_base"] \
+ "/analyzer/configuration/TXT").replace("//","/")
txt = get_file_content_or_die(ConfigurationTemplateFile)
# -- check if exit/entry handlers have to be active
entry_handler_active_f = False
exit_handler_active_f = False
for mode in ModeDescriptionDB.values():
entry_handler_active_f |= mode.incidence_db.has_key(E_IncidenceIDs.MODE_ENTRY)
exit_handler_active_f |= mode.incidence_db.has_key(E_IncidenceIDs.MODE_EXIT)
# Buffer filler converter (0x0 means: no buffer filler converter)
converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW "
if Setup.converter_user_new_func != "":
converter_new_str += Setup.converter_user_new_func + "()"
else:
converter_new_str = "/* " + converter_new_str + " */"
# Token repetition support
token_repeat_test_txt = ""
for token_id_str in blackboard.token_repetition_token_id_list:
token_repeat_test_txt += "TokenID == %s || " % token_id_str
if token_repeat_test_txt != "":
token_repeat_test_txt = token_repeat_test_txt[:-3]
else:
token_repeat_test_txt = "false"
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING", Setup.count_column_number_f)
txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f)
txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK", Setup.include_stack_support_f)
txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING", Setup.count_line_number_f)
txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER", Setup.post_categorizer_f)
txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK", Setup.mode_transition_check_f)
txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR", Setup.string_accumulator_f)
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE", Setup.token_policy == "queue")
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE", Setup.token_policy == "single")
txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT", token_repeat_test_txt != "false")
txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY", Setup.token_memory_management_by_user_f)
txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN", Setup.buffer_byte_order == "big")
txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER", Setup.converter_helper_required_f)
txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f)
txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF)
txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN", Setup.buffer_byte_order == "little")
txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT", entry_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT", exit_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C")
txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION", BeginOfLineSupportF)
txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN", Setup.byte_order_is_that_of_current_system_f)
txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS", Setup.buffer_based_analyzis_f)
txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC", Setup.buffer_codec.name != "unicode")
# -- token class related definitions
token_descr = blackboard.token_type_definition
# -- name of the character codec
codec_name = make_safe_identifier(Setup.buffer_codec.name).lower()
# Setup.buffer_element_size can be '-1'. This signals then that
# sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value
# is required here.
character_size_str = "%i" % Setup.buffer_element_size
def namespace(NameSpaceList):
result = Lng.NAMESPACE_REFERENCE(NameSpaceList, TrailingDelimiterF=False)
if len(result) == 0: return ""
assert Setup.language.upper() != "C++" or len(result) > 2, \
"Error while generating namespace reference '%s'" % result
return result
txt = blue_print(txt,
[
["$$BUFFER_LIMIT_CODE$$", "0x%X" % Setup.buffer_limit_code],
["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name],
["$$INCLUDE_GUARD_EXTENSION$$", get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name)],
["$$INITIAL_LEXER_MODE_ID$$", "QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_text()],
["$$LEXER_BUILD_DATE$$", time.asctime()],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$MAX_MODE_CLASS_N$$", repr(len(ModeDescriptionDB))],
["$$NAMESPACE_MAIN$$", namespace(Setup.analyzer_name_space)],
["$$NAMESPACE_MAIN_CLOSE$$", Lng.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace("\n", "\\\n")],
["$$NAMESPACE_MAIN_OPEN$$", Lng.NAMESPACE_OPEN(Setup.analyzer_name_space).replace("\n", "\\\n")],
["$$NAMESPACE_TOKEN$$", namespace(token_descr.name_space)],
["$$NAMESPACE_TOKEN_CLOSE$$", Lng.NAMESPACE_CLOSE(token_descr.name_space).replace("\n", "\\\n")],
["$$NAMESPACE_TOKEN_OPEN$$", Lng.NAMESPACE_OPEN(token_descr.name_space).replace("\n", "\\\n")],
["$$PATH_TERMINATION_CODE$$", "0x%X" % Setup.path_limit_code],
["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str],
["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type],
["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str],
["$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$", Lng.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace("\n", "\\\n")],
["$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$", Lng.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace("\n", "\\\n")],
["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp],
["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe],
["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS$$", token_descr.class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe],
["$$TOKEN_COLUMN_N_TYPE$$", token_descr.column_number_type.get_pure_text()],
["$$TOKEN_ID_TYPE$$", token_descr.token_id_type.get_pure_text()],
["$$TOKEN_LINE_N_TYPE$$", token_descr.line_number_type.get_pure_text()],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
["$$TOKEN_QUEUE_SAFETY_BORDER$$", repr(Setup.token_queue_safety_border)],
["$$TOKEN_QUEUE_SIZE$$", repr(Setup.token_queue_size)],
["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt],
["$$USER_LEXER_VERSION$$", Setup.user_application_version_id],
])
return txt
def do(ModeDB):
IndentationSupportF = blackboard.requires_indentation_count(ModeDB)
BeginOfLineSupportF = blackboard.requires_begin_of_line_condition_support(
ModeDB)
LanguageDB = Setup.language_db
LexerClassName = Setup.analyzer_class_name
ConfigurationTemplateFile =( QUEX_PATH \
+ Setup.language_db["$code_base"] \
+ "/analyzer/configuration/TXT").replace("//","/")
txt = get_file_content_or_die(ConfigurationTemplateFile)
# -- check if exit/entry handlers have to be active
entry_handler_active_f = False
exit_handler_active_f = False
for mode in ModeDB.values():
if len(mode.get_code_fragment_list("on_entry")) != 0:
entry_handler_active_f = True
if len(mode.get_code_fragment_list("on_exit")) != 0:
exit_handler_active_f = True
# Buffer filler converter (0x0 means: no buffer filler converter)
converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW "
if Setup.converter_user_new_func != "":
converter_new_str += Setup.converter_user_new_func + "()"
else:
converter_new_str = "/* " + converter_new_str + " */"
# Token repetition support
token_repeat_test_txt = ""
for token_id_str in blackboard.token_repetition_token_id_list:
token_repeat_test_txt += "TokenID == %s || " % token_id_str
if token_repeat_test_txt != "":
token_repeat_test_txt = token_repeat_test_txt[:-3]
else:
token_repeat_test_txt = "false"
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING",
Setup.count_column_number_f)
txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f)
txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK",
Setup.include_stack_support_f)
txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING",
Setup.count_line_number_f)
txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER",
Setup.post_categorizer_f)
txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK",
Setup.mode_transition_check_f)
txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR",
Setup.string_accumulator_f)
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE",
Setup.token_policy == "queue")
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE",
Setup.token_policy == "single")
txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT",
token_repeat_test_txt != "false")
txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY",
Setup.token_memory_management_by_user_f)
txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN",
Setup.buffer_byte_order == "big")
txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER",
Setup.converter_helper_required_f)
txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f)
txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF)
txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN",
Setup.buffer_byte_order == "little")
txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT",
entry_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT",
exit_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C")
txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION",
BeginOfLineSupportF)
txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN",
Setup.byte_order_is_that_of_current_system_f)
txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS",
Setup.buffer_based_analyzis_f)
txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC",
Setup.buffer_codec != "unicode")
# -- token class related definitions
token_descr = blackboard.token_type_definition
# -- name of the character codec
codec_name = "unicode"
if Setup.buffer_codec != "unicode":
codec_name = make_safe_identifier(Setup.buffer_codec).lower()
# Setup.buffer_element_size can be '-1'. This signals then that
# sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value
# is required here.
character_size_str = "%i" % Setup.buffer_element_size
def namespace(NameSpaceList):
result = Setup.language_db.NAMESPACE_REFERENCE(NameSpaceList)
if result == "::": return ""
assert Setup.language.upper() != "C++" or len(result) > 2, \
"Error while generating namespace reference '%s'" % result
return result[:-2]
txt = blue_print(txt, [
["$$BUFFER_LIMIT_CODE$$",
"0x%X" % Setup.buffer_limit_code],
["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name],
[
"$$INCLUDE_GUARD_EXTENSION$$",
get_include_guard_extension(
LanguageDB.NAMESPACE_REFERENCE(Setup.analyzer_name_space) +
"__" + Setup.analyzer_class_name)
],
[
"$$INITIAL_LEXER_MODE_ID$$",
"QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_code()
],
["$$LEXER_BUILD_DATE$$", time.asctime()],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$MAX_MODE_CLASS_N$$", repr(len(ModeDB))],
["$$NAMESPACE_MAIN$$",
namespace(Setup.analyzer_name_space)],
[
"$$NAMESPACE_MAIN_CLOSE$$",
LanguageDB.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace(
"\n", "\\\n")
],
[
"$$NAMESPACE_MAIN_OPEN$$",
LanguageDB.NAMESPACE_OPEN(Setup.analyzer_name_space).replace(
"\n", "\\\n")
],
["$$NAMESPACE_TOKEN$$",
namespace(token_descr.name_space)],
[
"$$NAMESPACE_TOKEN_CLOSE$$",
LanguageDB.NAMESPACE_CLOSE(token_descr.name_space).replace(
"\n", "\\\n")
],
[
"$$NAMESPACE_TOKEN_OPEN$$",
LanguageDB.NAMESPACE_OPEN(token_descr.name_space).replace(
"\n", "\\\n")
],
["$$PATH_TERMINATION_CODE$$",
"0x%X" % Setup.path_limit_code],
["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str],
["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type],
["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str],
[
"$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$",
LanguageDB.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace(
"\n", "\\\n")
],
[
"$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$",
LanguageDB.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace(
"\n", "\\\n")
],
["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp],
["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe],
["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS$$", token_descr.class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe],
[
"$$TOKEN_COLUMN_N_TYPE$$",
token_descr.column_number_type.get_pure_code()
],
["$$TOKEN_ID_TYPE$$",
token_descr.token_id_type.get_pure_code()],
[
"$$TOKEN_LINE_N_TYPE$$",
token_descr.line_number_type.get_pure_code()
],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
[
"$$TOKEN_QUEUE_SAFETY_BORDER$$",
repr(Setup.token_queue_safety_border)
],
["$$TOKEN_QUEUE_SIZE$$",
repr(Setup.token_queue_size)],
["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt],
["$$USER_LEXER_VERSION$$", Setup.user_application_version_id],
])
return txt
def do(setup):
"""Creates a file of token-ids from a given set of names.
Creates also a function:
const string& $$token$$::map_id_to_name().
"""
global file_str
LanguageDB = Setup.language_db
__propose_implicit_token_definitions()
for standard_token_id in standard_token_id_list:
assert token_id_db.has_key(standard_token_id)
assert blackboard.token_type_definition is not None, \
"Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \
"handle this."
# (*) Token ID File ________________________________________________________________
#
# The token id file can either be specified as database of
# token-id names, or as a file that directly assigns the token-ids
# to variables. If the flag '--user-token-id-file' is defined, then
# then the token-id file is provided by the user. Otherwise, the
# token id file is created by the token-id maker.
#
# The token id maker considers the file passed by the option '-t'
# as the database file and creates a C++ file with the output filestem
# plus the suffix "--token-ids". Note, that the token id file is a
# header file.
#
if len(token_id_db.keys()) == len(standard_token_id_list):
token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \
(setup.token_id_prefix_plain, setup.token_id_prefix_plain)
# TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done.
error_msg("Only token ids %s are defined.\n" % token_id_str + \
"Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \
"specify at least one other token id.")
#______________________________________________________________________________________
L = max(map(lambda name: len(name), token_id_db.keys()))
def space(Name):
return " " * (L - len(Name))
# -- define values for the token ids
def define_this(txt, token):
if setup.language == "C":
txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
else:
txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
if setup.token_id_foreign_definition_file != "":
token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(setup.token_id_foreign_definition_file)]
else:
if setup.language == "C":
prolog = ""
epilog = ""
else:
prolog = LanguageDB.NAMESPACE_OPEN(setup.token_id_prefix_name_space)
epilog = LanguageDB.NAMESPACE_CLOSE(setup.token_id_prefix_name_space)
token_id_txt = [prolog]
# Assign values to tokens with no numeric identifier
# NOTE: This has not to happen if token's are defined by the user's provided file.
i = setup.token_id_counter_offset
# Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name'
# may contain '--' to indicate a unicode value, so do not use it as name.
for dummy_name, token in sorted(token_id_db.items()):
if token.number is None:
while __is_token_id_occupied(i):
i += 1
token.number = i;
define_this(token_id_txt, token)
# Double check that no token id appears twice
# Again, this can only happen, if quex itself produced the numeric values for the token
token_list = token_id_db.values()
for i, x in enumerate(token_list):
for y in token_list[i+1:]:
if x.number != y.number: continue
error_msg("Token id '%s'" % x.name, x.file_name, x.line_n, DontExitF=True)
error_msg("and token id '%s' have same numeric value '%s'." \
% (y.name, x.number), y.file_name, y.line_n, DontExitF=True)
token_id_txt.append(epilog)
content = blue_print(file_str,
[["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$TOKEN_PREFIX$$", setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \
Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())],
])
return content
def _do(Descr):
# The following things must be ensured before the function is called
assert Descr is not None
assert Descr.__class__.__name__ == "TokenTypeDescriptor"
## ALLOW: Descr.get_member_db().keys() == empty
TemplateFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_file"]
TemplateIFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_i_file"]
template_str = open_file_or_die(TemplateFile, Mode="rb").read()
template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read()
virtual_destructor_str = ""
if Descr.open_for_derivation_f: virtual_destructor_str = "virtual "
if Descr.copy is None:
# Default copy operation: Plain Copy of token memory
copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n"
else:
copy_str = Lng.SOURCE_REFERENCED(Descr.copy)
if Descr.take_text is None:
take_text_str = "return true;\n"
else:
take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text)
include_guard_extension_str = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Descr.name_space)
+ "__" + Descr.class_name)
# In case of plain 'C' the class name must incorporate the namespace (list)
token_class_name = Descr.class_name
if Setup.language == "C":
token_class_name = Setup.token_class_name_safe
converter_declaration_include, \
converter_implementation_include, \
converter_string, \
converter_wstring = __get_converter_configuration(include_guard_extension_str)
extra_at_begin_str = lexeme_null_declaration()
extra_at_end_str = ""
if Setup.token_class_only_f:
extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \
+ extra_at_begin_str
extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \
+ extra_at_end_str
namespace_open, namespace_close = __namespace_brackets()
helper_variable_replacements = [
["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include],
["$INCLUDE_CONVERTER_IMPLEMENTATION", converter_implementation_include],
["$CONVERTER_STRING", converter_string],
["$CONVERTER_WSTRING", converter_wstring],
["$NAMESPACE_CLOSE", namespace_close],
["$NAMESPACE_OPEN", namespace_open],
["$TOKEN_CLASS", token_class_name],
]
txt = blue_print(template_str,
[
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt = blue_print(txt,
[
["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)],
["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$DISTINCT_MEMBERS$$", get_distinct_members(Descr)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)],
["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$QUICK_SETTERS$$", get_quick_setters(Descr)],
["$$SETTERS_GETTERS$$", get_setter_getter(Descr)],
["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)],
["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)],
["$$UNION_MEMBERS$$", get_union_members(Descr)],
["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt = blue_print(txt, helper_variable_replacements)
if Setup.language.upper() != "C++" and Setup.token_class_only_f:
extra_at_begin_str += local_strlen_str % (Descr.class_name_safe, Setup.buffer_element_type, Setup.buffer_element_type)
txt_i = blue_print(template_i_str,
[
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt_i = blue_print(txt_i,
[
["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
["$$TOKEN_CLASS_HEADER$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)],
["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)],
["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt_i = blue_print(txt_i, helper_variable_replacements)
return txt, txt_i
def do(setup):
"""Creates a file of token-ids from a given set of names.
Creates also a function:
const string& $$token$$::map_id_to_name().
"""
global file_str
LanguageDB = Setup.language_db
__propose_implicit_token_definitions()
for standard_token_id in standard_token_id_list:
assert token_id_db.has_key(standard_token_id)
assert blackboard.token_type_definition is not None, \
"Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \
"handle this."
# (*) Token ID File ________________________________________________________________
#
# The token id file can either be specified as database of
# token-id names, or as a file that directly assigns the token-ids
# to variables. If the flag '--user-token-id-file' is defined, then
# then the token-id file is provided by the user. Otherwise, the
# token id file is created by the token-id maker.
#
# The token id maker considers the file passed by the option '-t'
# as the database file and creates a C++ file with the output filestem
# plus the suffix "--token-ids". Note, that the token id file is a
# header file.
#
if len(token_id_db.keys()) == len(standard_token_id_list):
token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \
(setup.token_id_prefix_plain, setup.token_id_prefix_plain)
# TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done.
error_msg("Only token ids %s are defined.\n" % token_id_str + \
"Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \
"specify at least one other token id.")
#______________________________________________________________________________________
L = max(map(lambda name: len(name), token_id_db.keys()))
def space(Name):
return " " * (L - len(Name))
# -- define values for the token ids
def define_this(txt, token):
if setup.language == "C":
txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
else:
txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
if setup.token_id_foreign_definition_file != "":
token_id_txt = [
"#include \"%s\"\n" %
Setup.get_file_reference(setup.token_id_foreign_definition_file)
]
else:
if setup.language == "C":
prolog = ""
epilog = ""
else:
prolog = LanguageDB.NAMESPACE_OPEN(
setup.token_id_prefix_name_space)
epilog = LanguageDB.NAMESPACE_CLOSE(
setup.token_id_prefix_name_space)
token_id_txt = [prolog]
# Assign values to tokens with no numeric identifier
# NOTE: This has not to happen if token's are defined by the user's provided file.
i = setup.token_id_counter_offset
# Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name'
# may contain '--' to indicate a unicode value, so do not use it as name.
for dummy_name, token in sorted(token_id_db.items()):
if token.number is None:
while __is_token_id_occupied(i):
i += 1
token.number = i
define_this(token_id_txt, token)
# Double check that no token id appears twice
# Again, this can only happen, if quex itself produced the numeric values for the token
token_list = token_id_db.values()
for i, x in enumerate(token_list):
for y in token_list[i + 1:]:
if x.number != y.number: continue
error_msg("Token id '%s'" % x.name,
x.file_name,
x.line_n,
DontExitF=True)
error_msg("and token id '%s' have same numeric value '%s'." \
% (y.name, x.number), y.file_name, y.line_n, DontExitF=True)
token_id_txt.append(epilog)
content = blue_print(file_str,
[["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$TOKEN_PREFIX$$", setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \
Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())],
])
return content
