def analyzer_functions_get(ModeDB):
IndentationSupportF = blackboard.requires_indentation_count(ModeDB)
BeginOfLineSupportF = blackboard.requires_begin_of_line_condition_support(ModeDB)
inheritance_info_str = ""
analyzer_code = ""
# (*) Get list of modes that are actually implemented
# (abstract modes only serve as common base)
mode_list = [ mode for mode in ModeDB.itervalues()
if mode.options["inheritable"] != "only" ]
mode_name_list = [ mode.name for mode in mode_list ]
for mode in mode_list:
# -- some modes only define event handlers that are inherited
if len(mode.get_pattern_action_pair_list()) == 0: continue
# -- prepare the source code fragments for the generator
required_local_variables_db, \
pattern_action_pair_list, \
on_end_of_stream_action, \
on_failure_action, \
on_after_match_str = action_preparation.do(mode, IndentationSupportF, BeginOfLineSupportF)
# -- prepare code generation
generator = cpp_generator.Generator(StateMachineName = mode.name,
PatternActionPair_List = pattern_action_pair_list,
OnFailureAction = on_failure_action,
OnEndOfStreamAction = on_end_of_stream_action,
OnAfterMatch = on_after_match_str,
ModeNameList = mode_name_list)
# -- generate!
analyzer_code += "".join(generator.do(required_local_variables_db))
if Setup.comment_mode_patterns_f:
inheritance_info_str += mode.get_documentation()
# Bring the info about the patterns first
if Setup.comment_mode_patterns_f:
comment = []
Setup.language_db.ML_COMMENT(comment,
"BEGIN: MODE PATTERNS\n" + \
inheritance_info_str + \
"\nEND: MODE PATTERNS")
comment.append("\n") # For safety: New content may have to start in a newline, e.g. "#ifdef ..."
analyzer_code += "".join(comment)
# generate frame for analyser code
return cpp_generator.frame_this(analyzer_code)
def do(ModeDB):
IndentationSupportF = blackboard.requires_indentation_count(ModeDB)
BeginOfLineSupportF = blackboard.requires_begin_of_line_condition_support(
ModeDB)
LanguageDB = Setup.language_db
LexerClassName = Setup.analyzer_class_name
ConfigurationTemplateFile =( QUEX_PATH \
+ Setup.language_db["$code_base"] \
+ "/analyzer/configuration/TXT").replace("//","/")
txt = get_file_content_or_die(ConfigurationTemplateFile)
# -- check if exit/entry handlers have to be active
entry_handler_active_f = False
exit_handler_active_f = False
for mode in ModeDB.values():
if len(mode.get_code_fragment_list("on_entry")) != 0:
entry_handler_active_f = True
if len(mode.get_code_fragment_list("on_exit")) != 0:
exit_handler_active_f = True
# Buffer filler converter (0x0 means: no buffer filler converter)
converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW "
if Setup.converter_user_new_func != "":
converter_new_str += Setup.converter_user_new_func + "()"
else:
converter_new_str = "/* " + converter_new_str + " */"
# Token repetition support
token_repeat_test_txt = ""
for token_id_str in blackboard.token_repetition_token_id_list:
token_repeat_test_txt += "TokenID == %s || " % token_id_str
if token_repeat_test_txt != "":
token_repeat_test_txt = token_repeat_test_txt[:-3]
else:
token_repeat_test_txt = "false"
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING",
Setup.count_column_number_f)
txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f)
txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK",
Setup.include_stack_support_f)
txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING",
Setup.count_line_number_f)
txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER",
Setup.post_categorizer_f)
txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK",
Setup.mode_transition_check_f)
txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR",
Setup.string_accumulator_f)
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE",
Setup.token_policy == "queue")
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE",
Setup.token_policy == "single")
txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT",
token_repeat_test_txt != "false")
txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY",
Setup.token_memory_management_by_user_f)
txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN",
Setup.buffer_byte_order == "big")
txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER",
Setup.converter_helper_required_f)
txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f)
txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF)
txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN",
Setup.buffer_byte_order == "little")
txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT",
entry_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT",
exit_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C")
txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION",
BeginOfLineSupportF)
txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN",
Setup.byte_order_is_that_of_current_system_f)
txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS",
Setup.buffer_based_analyzis_f)
txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC",
Setup.buffer_codec != "unicode")
# -- token class related definitions
token_descr = blackboard.token_type_definition
# -- name of the character codec
codec_name = "unicode"
if Setup.buffer_codec != "unicode":
codec_name = make_safe_identifier(Setup.buffer_codec).lower()
# Setup.buffer_element_size can be '-1'. This signals then that
# sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value
# is required here.
character_size_str = "%i" % Setup.buffer_element_size
def namespace(NameSpaceList):
result = Setup.language_db.NAMESPACE_REFERENCE(NameSpaceList)
if result == "::": return ""
assert Setup.language.upper() != "C++" or len(result) > 2, \
"Error while generating namespace reference '%s'" % result
return result[:-2]
txt = blue_print(txt, [
["$$BUFFER_LIMIT_CODE$$",
"0x%X" % Setup.buffer_limit_code],
["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name],
[
"$$INCLUDE_GUARD_EXTENSION$$",
get_include_guard_extension(
LanguageDB.NAMESPACE_REFERENCE(Setup.analyzer_name_space) +
"__" + Setup.analyzer_class_name)
],
[
"$$INITIAL_LEXER_MODE_ID$$",
"QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_code()
],
["$$LEXER_BUILD_DATE$$", time.asctime()],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$MAX_MODE_CLASS_N$$", repr(len(ModeDB))],
["$$NAMESPACE_MAIN$$",
namespace(Setup.analyzer_name_space)],
[
"$$NAMESPACE_MAIN_CLOSE$$",
LanguageDB.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace(
"\n", "\\\n")
],
[
"$$NAMESPACE_MAIN_OPEN$$",
LanguageDB.NAMESPACE_OPEN(Setup.analyzer_name_space).replace(
"\n", "\\\n")
],
["$$NAMESPACE_TOKEN$$",
namespace(token_descr.name_space)],
[
"$$NAMESPACE_TOKEN_CLOSE$$",
LanguageDB.NAMESPACE_CLOSE(token_descr.name_space).replace(
"\n", "\\\n")
],
[
"$$NAMESPACE_TOKEN_OPEN$$",
LanguageDB.NAMESPACE_OPEN(token_descr.name_space).replace(
"\n", "\\\n")
],
["$$PATH_TERMINATION_CODE$$",
"0x%X" % Setup.path_limit_code],
["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str],
["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type],
["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str],
[
"$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$",
LanguageDB.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace(
"\n", "\\\n")
],
[
"$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$",
LanguageDB.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace(
"\n", "\\\n")
],
["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp],
["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe],
["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS$$", token_descr.class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe],
[
"$$TOKEN_COLUMN_N_TYPE$$",
token_descr.column_number_type.get_pure_code()
],
["$$TOKEN_ID_TYPE$$",
token_descr.token_id_type.get_pure_code()],
[
"$$TOKEN_LINE_N_TYPE$$",
token_descr.line_number_type.get_pure_code()
],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
[
"$$TOKEN_QUEUE_SAFETY_BORDER$$",
repr(Setup.token_queue_safety_border)
],
["$$TOKEN_QUEUE_SIZE$$",
repr(Setup.token_queue_size)],
["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt],
["$$USER_LEXER_VERSION$$", Setup.user_application_version_id],
])
return txt
def do(ModeDB):
IndentationSupportF = blackboard.requires_indentation_count(ModeDB)
BeginOfLineSupportF = blackboard.requires_begin_of_line_condition_support(ModeDB)
LanguageDB = Setup.language_db
LexerClassName = Setup.analyzer_class_name
ConfigurationTemplateFile =( QUEX_PATH \
+ Setup.language_db["$code_base"] \
+ "/analyzer/configuration/TXT").replace("//","/")
txt = get_file_content_or_die(ConfigurationTemplateFile)
# -- check if exit/entry handlers have to be active
entry_handler_active_f = False
exit_handler_active_f = False
for mode in ModeDB.values():
if len(mode.get_code_fragment_list("on_entry")) != 0: entry_handler_active_f = True
if len(mode.get_code_fragment_list("on_exit")) != 0: exit_handler_active_f = True
# Buffer filler converter (0x0 means: no buffer filler converter)
converter_new_str = "# define QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW "
if Setup.converter_user_new_func != "":
converter_new_str += Setup.converter_user_new_func + "()"
else:
converter_new_str = "/* " + converter_new_str + " */"
# Token repetition support
token_repeat_test_txt = ""
for token_id_str in blackboard.token_repetition_token_id_list:
token_repeat_test_txt += "TokenID == %s || " % token_id_str
if token_repeat_test_txt != "":
token_repeat_test_txt = token_repeat_test_txt[:-3]
else:
token_repeat_test_txt = "false"
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
txt = __switch(txt, "QUEX_OPTION_COLUMN_NUMBER_COUNTING", Setup.count_column_number_f)
txt = __switch(txt, "QUEX_OPTION_COMPUTED_GOTOS", False)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICONV", Setup.converter_iconv_f)
txt = __switch(txt, "QUEX_OPTION_CONVERTER_ICU", Setup.converter_icu_f)
txt = __switch(txt, "QUEX_OPTION_INCLUDE_STACK", Setup.include_stack_support_f)
txt = __switch(txt, "QUEX_OPTION_LINE_NUMBER_COUNTING", Setup.count_line_number_f)
txt = __switch(txt, "QUEX_OPTION_POST_CATEGORIZER", Setup.post_categorizer_f)
txt = __switch(txt, "QUEX_OPTION_RUNTIME_MODE_TRANSITION_CHECK", Setup.mode_transition_check_f)
txt = __switch(txt, "QUEX_OPTION_STRING_ACCUMULATOR", Setup.string_accumulator_f)
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_QUEUE", Setup.token_policy == "queue")
txt = __switch(txt, "QUEX_OPTION_TOKEN_POLICY_SINGLE", Setup.token_policy == "single")
txt = __switch(txt, "QUEX_OPTION_TOKEN_REPETITION_SUPPORT", token_repeat_test_txt != "false")
txt = __switch(txt, "QUEX_OPTION_USER_MANAGED_TOKEN_MEMORY", Setup.token_memory_management_by_user_f)
txt = __switch(txt, "__QUEX_OPTION_BIG_ENDIAN", Setup.buffer_byte_order == "big")
txt = __switch(txt, "__QUEX_OPTION_CONVERTER_HELPER", Setup.converter_helper_required_f)
txt = __switch(txt, "__QUEX_OPTION_CONVERTER", Setup.converter_f)
txt = __switch(txt, "QUEX_OPTION_INDENTATION_TRIGGER", IndentationSupportF)
txt = __switch(txt, "__QUEX_OPTION_LITTLE_ENDIAN", Setup.buffer_byte_order == "little")
txt = __switch(txt, "__QUEX_OPTION_ON_ENTRY_HANDLER_PRESENT", entry_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_ON_EXIT_HANDLER_PRESENT", exit_handler_active_f)
txt = __switch(txt, "__QUEX_OPTION_PLAIN_C", Setup.language.upper() == "C")
txt = __switch(txt, "__QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION", BeginOfLineSupportF)
txt = __switch(txt, "__QUEX_OPTION_SYSTEM_ENDIAN", Setup.byte_order_is_that_of_current_system_f)
txt = __switch(txt, "QUEX_OPTION_BUFFER_BASED_ANALYZIS", Setup.buffer_based_analyzis_f)
txt = __switch(txt, "__QUEX_OPTION_ENGINE_RUNNING_ON_CODEC", Setup.buffer_codec != "unicode")
# -- token class related definitions
token_descr = blackboard.token_type_definition
# -- name of the character codec
codec_name = "unicode"
if Setup.buffer_codec != "unicode":
codec_name = make_safe_identifier(Setup.buffer_codec).lower()
# Setup.buffer_element_size can be '-1'. This signals then that
# sizeof(QUEX_TYPE_CHARACTER) needs to be used. A numeric value
# is required here.
character_size_str = "%i" % Setup.buffer_element_size
def namespace(NameSpaceList):
result = Setup.language_db.NAMESPACE_REFERENCE(NameSpaceList)
if result == "::": return ""
assert Setup.language.upper() != "C++" or len(result) > 2, \
"Error while generating namespace reference '%s'" % result
return result[:-2]
txt = blue_print(txt,
[
["$$BUFFER_LIMIT_CODE$$", "0x%X" % Setup.buffer_limit_code],
["$$QUEX_SETTING_CHARACTER_CODEC$$", codec_name],
["$$INCLUDE_GUARD_EXTENSION$$", get_include_guard_extension(LanguageDB.NAMESPACE_REFERENCE(Setup.analyzer_name_space) + "__" + Setup.analyzer_class_name)],
["$$INITIAL_LEXER_MODE_ID$$", "QUEX_NAME(ModeID_%s)" % blackboard.initial_mode.get_pure_code()],
["$$LEXER_BUILD_DATE$$", time.asctime()],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$MAX_MODE_CLASS_N$$", repr(len(ModeDB))],
["$$NAMESPACE_MAIN$$", namespace(Setup.analyzer_name_space)],
["$$NAMESPACE_MAIN_CLOSE$$", LanguageDB.NAMESPACE_CLOSE(Setup.analyzer_name_space).replace("\n", "\\\n")],
["$$NAMESPACE_MAIN_OPEN$$", LanguageDB.NAMESPACE_OPEN(Setup.analyzer_name_space).replace("\n", "\\\n")],
["$$NAMESPACE_TOKEN$$", namespace(token_descr.name_space)],
["$$NAMESPACE_TOKEN_CLOSE$$", LanguageDB.NAMESPACE_CLOSE(token_descr.name_space).replace("\n", "\\\n")],
["$$NAMESPACE_TOKEN_OPEN$$", LanguageDB.NAMESPACE_OPEN(token_descr.name_space).replace("\n", "\\\n")],
["$$PATH_TERMINATION_CODE$$", "0x%X" % Setup.path_limit_code],
["$$QUEX_SETTING_BUFFER_FILLERS_CONVERTER_NEW$$", converter_new_str],
["$$QUEX_TYPE_CHARACTER$$", Setup.buffer_element_type],
["$$QUEX_SETTING_CHARACTER_SIZE$$", character_size_str],
["$$QUEX_NAMESPACE_LEXEME_NULL_OPEN$$", LanguageDB.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace("\n", "\\\n")],
["$$QUEX_NAMESPACE_LEXEME_NULL_CLOSE$$", LanguageDB.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace("\n", "\\\n")],
["$$QUEX_LEXEME_NULL$$", Setup.lexeme_null_full_name_cpp],
["$$QUEX_LEXEME_NULL_SAFE$$", Setup.lexeme_null_name_safe],
["$$QUEX_LEXEME_NULL_IN_ITS_NAMESPACE$$", Setup.lexeme_null_name],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS$$", token_descr.class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_descr.class_name_safe],
["$$TOKEN_COLUMN_N_TYPE$$", token_descr.column_number_type.get_pure_code()],
["$$TOKEN_ID_TYPE$$", token_descr.token_id_type.get_pure_code()],
["$$TOKEN_LINE_N_TYPE$$", token_descr.line_number_type.get_pure_code()],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
["$$TOKEN_QUEUE_SAFETY_BORDER$$", repr(Setup.token_queue_safety_border)],
["$$TOKEN_QUEUE_SIZE$$", repr(Setup.token_queue_size)],
["$$TOKEN_REPEAT_TEST$$", token_repeat_test_txt],
["$$USER_LEXER_VERSION$$", Setup.user_application_version_id],
])
return txt
