def do(PatternActionPairList, TestStr, PatternDictionary={}, Language="ANSI-C-PlainMemory", QuexBufferSize=15, # DO NOT CHANGE! SecondPatternActionPairList=[], QuexBufferFallbackN=-1, ShowBufferLoadsF=False, AssertsActionvation_str="-DQUEX_OPTION_ASSERTS"): BufferLimitCode = 0 Setup.buffer_limit_code = BufferLimitCode Setup.buffer_element_specification_prepare() Setup.buffer_codec_prepare("unicode", None) __Setup_init_language_database(Language) CompileOptionStr = "" computed_goto_f = False FullLanguage = Language if Language.find("StrangeStream") != -1: CompileOptionStr += " -DQUEX_OPTION_STRANGE_ISTREAM_IMPLEMENTATION " if Language.find("-CG") != -1: Language = Language.replace("-CG", "") CompileOptionStr += " -DQUEX_OPTION_COMPUTED_GOTOS " computed_goto_f = True if Language == "Cpp-Template": Language = "Cpp" # Shall template compression be used? Setup.compression_type_list = [ E_Compression.TEMPLATE ] Setup.compression_template_min_gain = 0 elif Language == "Cpp-Path": Language = "Cpp" Setup.compression_type_list = [ E_Compression.PATH ] elif Language == "Cpp-PathUniform": Language = "Cpp" Setup.compression_type_list = [ E_Compression.PATH_UNIFORM ] elif Language == "ANSI-C-PathTemplate": Language = "Cpp" Setup.compression_type_list = [ E_Compression.PATH, E_Compression.TEMPLATE ] Setup.compression_template_min_gain = 0 try: adapted_dict = {} for key, regular_expression in PatternDictionary.items(): string_stream = StringIO(regular_expression) pattern = regex.do(string_stream, adapted_dict) # It is ESSENTIAL that the state machines of defined patterns do not # have origins! Actually, there are not more than patterns waiting # to be applied in regular expressions. The regular expressions # can later be origins. assert pattern.sm.has_origins() == False adapted_dict[key] = PatternShorthand(key, pattern.sm) except RegularExpressionException, x: print "Dictionary Creation:\n" + repr(x)
def get_test_application(counter_db, ReferenceP, CT): Setup.buffer_element_specification_prepare() if codec == "utf_32_le" or codec == "ascii": Setup.buffer_codec_prepare("unicode") elif codec == "utf_8": Setup.buffer_codec_prepare("utf8", Module=utf8_state_split) elif codec == "utf_16_le": Setup.buffer_codec_prepare("utf16", Module=utf16_state_split) else: Setup.buffer_codec_prepare(codec) # (*) Generate Code ccfactory = CountOpFactory.from_ParserDataLineColumn(counter_db, Setup.buffer_codec.source_set, Lng.INPUT_P()) counter_function_name, \ counter_str = counter.get(ccfactory, "TEST_MODE") counter_str = counter_str.replace("static void", "void") # Make sure that the counter is implemented using reference pointer found_n = 0 for i, line in enumerate(counter_str.splitlines()): if line.find("reference_p") != -1: found_n += 1 if found_n == 3: break # [RP] Verify that a reference pointer has been used or not used according # to what was specified. # 1. place: definition, 2. place: reference pointer set, 3. place: add. if ReferenceP: assert found_n >= 3, "Counter has not been setup using a reference pointer." else: assert found_n == 0, "Counter has been setup using a reference pointer." open("./data/test.c", "wb").write("#include <data/check.h>\n\n" + counter_str) # (*) Compile os.system("rm -f test") compile_str = "gcc -Wall -Werror -I. -ggdb ./data/check.c ./data/test.c " \ + " -D__QUEX_OPTION_COUNTER" \ + " -DDEF_COUNTER_FUNCTION='%s' " % counter_function_name \ + " -DDEF_FILE_NAME='\"data/input.txt\"' " \ + " -DDEF_CHARACTER_TYPE=%s " % CT \ + " -o test" # + " -DDEF_DEBUG_TRACE " print "## %s" % compile_str os.system(compile_str)
def get_test_application(counter_db, ReferenceP, CT): Setup.buffer_element_specification_prepare() if codec == "utf_32_le" or codec == "ascii": Setup.buffer_codec_prepare("unicode") elif codec == "utf_8": Setup.buffer_codec_prepare("utf8", Module=utf8_state_split) elif codec == "utf_16_le": Setup.buffer_codec_prepare("utf16", Module=utf16_state_split) else: Setup.buffer_codec_prepare(codec) # (*) Generate Code ccfactory = CountOpFactory.from_ParserDataLineColumn( counter_db, Setup.buffer_codec.source_set, Lng.INPUT_P()) counter_function_name, \ counter_str = counter.get(ccfactory, "TEST_MODE") counter_str = counter_str.replace("static void", "void") # Make sure that the counter is implemented using reference pointer found_n = 0 for i, line in enumerate(counter_str.splitlines()): if line.find("reference_p") != -1: found_n += 1 if found_n == 3: break # [RP] Verify that a reference pointer has been used or not used according # to what was specified. # 1. place: definition, 2. place: reference pointer set, 3. place: add. if ReferenceP: assert found_n >= 3, "Counter has not been setup using a reference pointer." else: assert found_n == 0, "Counter has been setup using a reference pointer." open("./data/test.c", "wb").write("#include <data/check.h>\n\n" + counter_str) # (*) Compile os.system("rm -f test") compile_str = "gcc -Wall -Werror -I. -ggdb ./data/check.c ./data/test.c " \ + " -D__QUEX_OPTION_COUNTER" \ + " -DDEF_COUNTER_FUNCTION='%s' " % counter_function_name \ + " -DDEF_FILE_NAME='\"data/input.txt\"' " \ + " -DDEF_CHARACTER_TYPE=%s " % CT \ + " -o test" # + " -DDEF_DEBUG_TRACE " print "## %s" % compile_str os.system(compile_str)
def test(TestString): TestString = TestString.replace("\n", "\\n").replace("\t", "\\t") if "BeginOfLine" in sys.argv: TestString = "^%s" % TestString print ("expr. = " + TestString).replace("\n", "\\n").replace("\t", "\\t") pattern = core.do(TestString, {}) # Prepare transformation info according to choice. Setup.buffer_element_specification_prepare() if "UTF8" in sys.argv: Setup.buffer_codec_prepare("utf8", Module=utf8_state_split) else: Setup.buffer_codec_prepare("utf16", Module=utf16_state_split) # Count pattern.prepare_count_info(counter_db, Setup.buffer_codec) print ("info = {\n %s\n}\n" % str(pattern.count_info()).replace("\n", "\n "))
#! /usr/bin/env python import sys import os sys.path.append(os.environ["QUEX_PATH"]) from StringIO import StringIO import quex.input.files.counter as counter import quex.engine.misc.error as error from quex.engine.misc.file_in import EndOfStreamException from quex.blackboard import setup as Setup Setup.buffer_element_specification_prepare() Setup.buffer_codec_prepare("unicode") if "--hwut-info" in sys.argv: print "Parse Counter Setup;" print "CHOICES: basic, twice, intersection, intersection-2, non-numeric;" sys.exit() # choice = sys.argv[1] count_n = 0 def test(Text): global count_n count_n += 1 if Text.find("\n") == -1: print "(%i) |%s|\n" % (count_n, Text) else: print "(%i)\n::\n%s\n::\n" % (count_n, Text)
import quex.output.core.dictionary as languages from quex.output.core.base import do_analyzer from quex.engine.analyzer.door_id_address_label import DoorID import quex.engine.analyzer.core as analyzer_generator from quex.engine.analyzer.door_id_address_label import dial_db from quex.engine.analyzer.transition_map import TransitionMap from quex.engine.codec_db.core import CodecDynamicInfo, CodecInfo from quex.blackboard import setup as Setup, \ E_MapImplementationType, \ E_IncidenceIDs, \ Lng from collections import defaultdict Setup.language_db = languages.db["C++"] Setup.buffer_element_type = "uint32_t" Setup.buffer_element_specification_prepare() Setup.buffer_codec_prepare("unicode", None) dial_db.clear() if "--hwut-info" in sys.argv: print "Single State: Transition Code Generation;" print "CHOICES: A, B, C, A-UTF8, B-UTF8, C-UTF8;" sys.exit(0) choice, codec = { "A": ("A", ""), "B": ("B", ""), "C": ("C", ""), "A-UTF8": ("A", "UTF8"), "B-UTF8": ("B", "UTF8"),
def prepare(command_line, argv): """RETURN: True, if process needs to be started. False, if job is done. """ global Setup # (*) Classes and their namespace __setup_analyzer_class(Setup) __setup_token_class(Setup) __setup_token_id_prefix(Setup) __setup_lexeme_null(Setup) # Requires 'token_class_name_space' # (*) Output programming language Setup.language = Setup.language.upper() error.verify_word_in_list(Setup.language, output_language_db.keys(), "Programming language '%s' is not supported." % Setup.language) Setup.language_db = output_language_db[Setup.language] Setup.extension_db = global_extension_db[Setup.language] # Is the output file naming scheme provided by the extension database # (Validation must happen immediately) if Setup.extension_db.has_key(Setup.output_file_naming_scheme) == False: error.log("File extension scheme '%s' is not provided for language '%s'.\n" \ % (Setup.output_file_naming_scheme, Setup.language) + \ "Available schemes are: %s." % repr(Setup.extension_db.keys())[1:-1]) if Setup.buffer_byte_order == "<system>": Setup.buffer_byte_order = sys.byteorder Setup.byte_order_is_that_of_current_system_f = True else: Setup.byte_order_is_that_of_current_system_f = False Setup.buffer_element_specification_prepare() if Setup.buffer_codec_name == "utf8": module = utf8_state_split elif Setup.buffer_codec_name == "utf16": module = utf16_state_split else: module = None Setup.buffer_codec_prepare(Setup.buffer_codec_name, Setup.buffer_codec_file, module) # AFTER: Setup.buffer_codec_prepare() !!! if Setup.language not in ["DOT"]: prepare_file_names(Setup) type_info = global_character_type_db.get(Setup.buffer_element_type) if type_info is not None and len(type_info) >= 4 \ and type_info[3] != -1 and Setup.buffer_element_size != -1 \ and type_info[3] != Setup.buffer_element_size: error.log("\nBuffer element type ('--bet' or '--buffer-element-type') was set to '%s'.\n" \ % Setup.buffer_element_type \ + "It is well known to be of size %s[byte]. However, the buffer element size\n" \ % type_info[3] \ + "('-b' or '--buffer-element-type') was specified as '%s'.\n\n" \ % Setup.buffer_element_size \ + "Quex can continue, but the result is questionable.\n", \ DontExitF=True) Setup.converter_f = False if Setup.converter_iconv_f or Setup.converter_icu_f or len(Setup.converter_user_new_func) != 0: Setup.converter_f = True # The only case where no converter helper is required is where ASCII # (Unicode restricted to [0, FF] is used. Setup.converter_helper_required_f = True if Setup.converter_f == False and Setup.buffer_element_size == 1 and Setup.buffer_codec.name == "unicode": Setup.converter_helper_required_f = False validation.do(Setup, command_line, argv) if Setup.converter_ucs_coding_name == "": if global_character_type_db.has_key(Setup.buffer_element_type): if Setup.buffer_byte_order == "little": index = 1 else: index = 2 Setup.converter_ucs_coding_name = global_character_type_db[Setup.buffer_element_type][index] if len(Setup.token_id_foreign_definition) != 0: if len(Setup.token_id_foreign_definition) > 3: error.log("Option '--foreign-token-id-file' received > 3 followers.\n" "Found: %s" % str(Setup.token_id_foreign_definition)[1:-1]) if len(Setup.token_id_foreign_definition) > 1: Setup.token_id_foreign_definition_file_region_begin_re = \ __compile_regular_expression(Setup.token_id_foreign_definition[1], "token id region begin") if len(Setup.token_id_foreign_definition) > 2: Setup.token_id_foreign_definition_file_region_end_re = \ __compile_regular_expression(Setup.token_id_foreign_definition[2], "token id region end") Setup.token_id_foreign_definition_file = \ Setup.token_id_foreign_definition[0] CommentDelimiterList = [["//", "\n"], ["/*", "*/"]] token_id_file_parse(Setup.token_id_foreign_definition_file, CommentDelimiterList) # (*) Compression Types compression_type_list = [] for name, ctype in [("compression_template_f", E_Compression.TEMPLATE), ("compression_template_uniform_f", E_Compression.TEMPLATE_UNIFORM), ("compression_path_f", E_Compression.PATH), ("compression_path_uniform_f", E_Compression.PATH_UNIFORM)]: if command_line_args_defined(command_line, name): compression_type_list.append((command_line_arg_position(name), ctype)) compression_type_list.sort(key=itemgetter(0)) Setup.compression_type_list = map(lambda x: x[1], compression_type_list) # (*) return Setup ___________________________________________________________________ return True
def prepare(command_line, argv): """RETURN: True, if process needs to be started. False, if job is done. """ global Setup # (*) Classes and their namespace __setup_analyzer_class(Setup) __setup_token_class(Setup) __setup_token_id_prefix(Setup) __setup_lexeme_null(Setup) # Requires 'token_class_name_space' # (*) Output programming language Setup.language = Setup.language.upper() error.verify_word_in_list( Setup.language, output_language_db.keys(), "Programming language '%s' is not supported." % Setup.language) Setup.language_db = output_language_db[Setup.language] Setup.extension_db = global_extension_db[Setup.language] # Is the output file naming scheme provided by the extension database # (Validation must happen immediately) if Setup.extension_db.has_key(Setup.output_file_naming_scheme) == False: error.log("File extension scheme '%s' is not provided for language '%s'.\n" \ % (Setup.output_file_naming_scheme, Setup.language) + \ "Available schemes are: %s." % repr(Setup.extension_db.keys())[1:-1]) if Setup.buffer_byte_order == "<system>": Setup.buffer_byte_order = sys.byteorder Setup.byte_order_is_that_of_current_system_f = True else: Setup.byte_order_is_that_of_current_system_f = False Setup.buffer_element_specification_prepare() if Setup.buffer_codec_name == "utf8": module = utf8_state_split elif Setup.buffer_codec_name == "utf16": module = utf16_state_split else: module = None Setup.buffer_codec_prepare(Setup.buffer_codec_name, Setup.buffer_codec_file, module) # AFTER: Setup.buffer_codec_prepare() !!! if Setup.language not in ["DOT"]: prepare_file_names(Setup) type_info = global_character_type_db.get(Setup.buffer_element_type) if type_info is not None and len(type_info) >= 4 \ and type_info[3] != -1 and Setup.buffer_element_size != -1 \ and type_info[3] != Setup.buffer_element_size: error.log("\nBuffer element type ('--bet' or '--buffer-element-type') was set to '%s'.\n" \ % Setup.buffer_element_type \ + "It is well known to be of size %s[byte]. However, the buffer element size\n" \ % type_info[3] \ + "('-b' or '--buffer-element-type') was specified as '%s'.\n\n" \ % Setup.buffer_element_size \ + "Quex can continue, but the result is questionable.\n", \ DontExitF=True) Setup.converter_f = False if Setup.converter_iconv_f or Setup.converter_icu_f or len( Setup.converter_user_new_func) != 0: Setup.converter_f = True # The only case where no converter helper is required is where ASCII # (Unicode restricted to [0, FF] is used. Setup.converter_helper_required_f = True if Setup.converter_f == False and Setup.buffer_element_size == 1 and Setup.buffer_codec.name == "unicode": Setup.converter_helper_required_f = False validation.do(Setup, command_line, argv) if Setup.converter_ucs_coding_name == "": if global_character_type_db.has_key(Setup.buffer_element_type): if Setup.buffer_byte_order == "little": index = 1 else: index = 2 Setup.converter_ucs_coding_name = global_character_type_db[ Setup.buffer_element_type][index] if len(Setup.token_id_foreign_definition) != 0: if len(Setup.token_id_foreign_definition) > 3: error.log( "Option '--foreign-token-id-file' received > 3 followers.\n" "Found: %s" % str(Setup.token_id_foreign_definition)[1:-1]) if len(Setup.token_id_foreign_definition) > 1: Setup.token_id_foreign_definition_file_region_begin_re = \ __compile_regular_expression(Setup.token_id_foreign_definition[1], "token id region begin") if len(Setup.token_id_foreign_definition) > 2: Setup.token_id_foreign_definition_file_region_end_re = \ __compile_regular_expression(Setup.token_id_foreign_definition[2], "token id region end") Setup.token_id_foreign_definition_file = \ Setup.token_id_foreign_definition[0] CommentDelimiterList = [["//", "\n"], ["/*", "*/"]] token_id_file_parse(Setup.token_id_foreign_definition_file, CommentDelimiterList) # (*) Compression Types compression_type_list = [] for name, ctype in [ ("compression_template_f", E_Compression.TEMPLATE), ("compression_template_uniform_f", E_Compression.TEMPLATE_UNIFORM), ("compression_path_f", E_Compression.PATH), ("compression_path_uniform_f", E_Compression.PATH_UNIFORM) ]: if command_line_args_defined(command_line, name): compression_type_list.append( (command_line_arg_position(name), ctype)) compression_type_list.sort(key=itemgetter(0)) Setup.compression_type_list = map(lambda x: x[1], compression_type_list) # (*) return Setup ___________________________________________________________________ return True