def do(PatternActionPairList, TestStr, PatternDictionary={}, Language="ANSI-C-PlainMemory",
QuexBufferSize=15, # DO NOT CHANGE!
SecondPatternActionPairList=[], QuexBufferFallbackN=-1, ShowBufferLoadsF=False,
AssertsActionvation_str="-DQUEX_OPTION_ASSERTS"):
BufferLimitCode = 0
Setup.buffer_limit_code = BufferLimitCode
Setup.buffer_element_specification_prepare()
Setup.buffer_codec_prepare("unicode", None)
__Setup_init_language_database(Language)
CompileOptionStr = ""
computed_goto_f = False
FullLanguage = Language
if Language.find("StrangeStream") != -1:
CompileOptionStr += " -DQUEX_OPTION_STRANGE_ISTREAM_IMPLEMENTATION "
if Language.find("-CG") != -1:
Language = Language.replace("-CG", "")
CompileOptionStr += " -DQUEX_OPTION_COMPUTED_GOTOS "
computed_goto_f = True
if Language == "Cpp-Template":
Language = "Cpp"
# Shall template compression be used?
Setup.compression_type_list = [ E_Compression.TEMPLATE ]
Setup.compression_template_min_gain = 0
elif Language == "Cpp-Path":
Language = "Cpp"
Setup.compression_type_list = [ E_Compression.PATH ]
elif Language == "Cpp-PathUniform":
Language = "Cpp"
Setup.compression_type_list = [ E_Compression.PATH_UNIFORM ]
elif Language == "ANSI-C-PathTemplate":
Language = "Cpp"
Setup.compression_type_list = [ E_Compression.PATH, E_Compression.TEMPLATE ]
Setup.compression_template_min_gain = 0
try:
adapted_dict = {}
for key, regular_expression in PatternDictionary.items():
string_stream = StringIO(regular_expression)
pattern = regex.do(string_stream, adapted_dict)
# It is ESSENTIAL that the state machines of defined patterns do not
# have origins! Actually, there are not more than patterns waiting
# to be applied in regular expressions. The regular expressions
# can later be origins.
assert pattern.sm.has_origins() == False
adapted_dict[key] = PatternShorthand(key, pattern.sm)
except RegularExpressionException, x:
print "Dictionary Creation:\n" + repr(x)
def get_test_application(counter_db, ReferenceP, CT):
Setup.buffer_element_specification_prepare()
if codec == "utf_32_le" or codec == "ascii":
Setup.buffer_codec_prepare("unicode")
elif codec == "utf_8":
Setup.buffer_codec_prepare("utf8", Module=utf8_state_split)
elif codec == "utf_16_le":
Setup.buffer_codec_prepare("utf16", Module=utf16_state_split)
else:
Setup.buffer_codec_prepare(codec)
# (*) Generate Code
ccfactory = CountOpFactory.from_ParserDataLineColumn(counter_db,
Setup.buffer_codec.source_set,
Lng.INPUT_P())
counter_function_name, \
counter_str = counter.get(ccfactory, "TEST_MODE")
counter_str = counter_str.replace("static void", "void")
# Make sure that the counter is implemented using reference pointer
found_n = 0
for i, line in enumerate(counter_str.splitlines()):
if line.find("reference_p") != -1:
found_n += 1
if found_n == 3: break
# [RP] Verify that a reference pointer has been used or not used according
# to what was specified.
# 1. place: definition, 2. place: reference pointer set, 3. place: add.
if ReferenceP:
assert found_n >= 3, "Counter has not been setup using a reference pointer."
else:
assert found_n == 0, "Counter has been setup using a reference pointer."
open("./data/test.c", "wb").write("#include <data/check.h>\n\n" + counter_str)
# (*) Compile
os.system("rm -f test")
compile_str = "gcc -Wall -Werror -I. -ggdb ./data/check.c ./data/test.c " \
+ " -D__QUEX_OPTION_COUNTER" \
+ " -DDEF_COUNTER_FUNCTION='%s' " % counter_function_name \
+ " -DDEF_FILE_NAME='\"data/input.txt\"' " \
+ " -DDEF_CHARACTER_TYPE=%s " % CT \
+ " -o test"
# + " -DDEF_DEBUG_TRACE "
print "## %s" % compile_str
os.system(compile_str)
def get_test_application(counter_db, ReferenceP, CT):
Setup.buffer_element_specification_prepare()
if codec == "utf_32_le" or codec == "ascii":
Setup.buffer_codec_prepare("unicode")
elif codec == "utf_8":
Setup.buffer_codec_prepare("utf8", Module=utf8_state_split)
elif codec == "utf_16_le":
Setup.buffer_codec_prepare("utf16", Module=utf16_state_split)
else:
Setup.buffer_codec_prepare(codec)
# (*) Generate Code
ccfactory = CountOpFactory.from_ParserDataLineColumn(
counter_db, Setup.buffer_codec.source_set, Lng.INPUT_P())
counter_function_name, \
counter_str = counter.get(ccfactory, "TEST_MODE")
counter_str = counter_str.replace("static void", "void")
# Make sure that the counter is implemented using reference pointer
found_n = 0
for i, line in enumerate(counter_str.splitlines()):
if line.find("reference_p") != -1:
found_n += 1
if found_n == 3: break
# [RP] Verify that a reference pointer has been used or not used according
# to what was specified.
# 1. place: definition, 2. place: reference pointer set, 3. place: add.
if ReferenceP:
assert found_n >= 3, "Counter has not been setup using a reference pointer."
else:
assert found_n == 0, "Counter has been setup using a reference pointer."
open("./data/test.c",
"wb").write("#include <data/check.h>\n\n" + counter_str)
# (*) Compile
os.system("rm -f test")
compile_str = "gcc -Wall -Werror -I. -ggdb ./data/check.c ./data/test.c " \
+ " -D__QUEX_OPTION_COUNTER" \
+ " -DDEF_COUNTER_FUNCTION='%s' " % counter_function_name \
+ " -DDEF_FILE_NAME='\"data/input.txt\"' " \
+ " -DDEF_CHARACTER_TYPE=%s " % CT \
+ " -o test"
# + " -DDEF_DEBUG_TRACE "
print "## %s" % compile_str
os.system(compile_str)
def test(TestString):
TestString = TestString.replace("\n", "\\n").replace("\t", "\\t")
if "BeginOfLine" in sys.argv:
TestString = "^%s" % TestString
print ("expr. = " + TestString).replace("\n", "\\n").replace("\t", "\\t")
pattern = core.do(TestString, {})
# Prepare transformation info according to choice.
Setup.buffer_element_specification_prepare()
if "UTF8" in sys.argv:
Setup.buffer_codec_prepare("utf8", Module=utf8_state_split)
else:
Setup.buffer_codec_prepare("utf16", Module=utf16_state_split)
# Count
pattern.prepare_count_info(counter_db, Setup.buffer_codec)
print ("info = {\n %s\n}\n" % str(pattern.count_info()).replace("\n", "\n "))
#! /usr/bin/env python
import sys
import os
sys.path.append(os.environ["QUEX_PATH"])
from StringIO import StringIO
import quex.input.files.counter as counter
import quex.engine.misc.error as error
from quex.engine.misc.file_in import EndOfStreamException
from quex.blackboard import setup as Setup
Setup.buffer_element_specification_prepare()
Setup.buffer_codec_prepare("unicode")
if "--hwut-info" in sys.argv:
print "Parse Counter Setup;"
print "CHOICES: basic, twice, intersection, intersection-2, non-numeric;"
sys.exit()
# choice = sys.argv[1]
count_n = 0
def test(Text):
global count_n
count_n += 1
if Text.find("\n") == -1:
print "(%i) |%s|\n" % (count_n, Text)
else:
print "(%i)\n::\n%s\n::\n" % (count_n, Text)
import quex.output.core.dictionary as languages
from quex.output.core.base import do_analyzer
from quex.engine.analyzer.door_id_address_label import DoorID
import quex.engine.analyzer.core as analyzer_generator
from quex.engine.analyzer.door_id_address_label import dial_db
from quex.engine.analyzer.transition_map import TransitionMap
from quex.engine.codec_db.core import CodecDynamicInfo, CodecInfo
from quex.blackboard import setup as Setup, \
E_MapImplementationType, \
E_IncidenceIDs, \
Lng
from collections import defaultdict
Setup.language_db = languages.db["C++"]
Setup.buffer_element_type = "uint32_t"
Setup.buffer_element_specification_prepare()
Setup.buffer_codec_prepare("unicode", None)
dial_db.clear()
if "--hwut-info" in sys.argv:
print "Single State: Transition Code Generation;"
print "CHOICES: A, B, C, A-UTF8, B-UTF8, C-UTF8;"
sys.exit(0)
choice, codec = {
"A": ("A", ""),
"B": ("B", ""),
"C": ("C", ""),
"A-UTF8": ("A", "UTF8"),
"B-UTF8": ("B", "UTF8"),
def prepare(command_line, argv):
"""RETURN: True, if process needs to be started.
False, if job is done.
"""
global Setup
# (*) Classes and their namespace
__setup_analyzer_class(Setup)
__setup_token_class(Setup)
__setup_token_id_prefix(Setup)
__setup_lexeme_null(Setup) # Requires 'token_class_name_space'
# (*) Output programming language
Setup.language = Setup.language.upper()
error.verify_word_in_list(Setup.language, output_language_db.keys(),
"Programming language '%s' is not supported." % Setup.language)
Setup.language_db = output_language_db[Setup.language]
Setup.extension_db = global_extension_db[Setup.language]
# Is the output file naming scheme provided by the extension database
# (Validation must happen immediately)
if Setup.extension_db.has_key(Setup.output_file_naming_scheme) == False:
error.log("File extension scheme '%s' is not provided for language '%s'.\n" \
% (Setup.output_file_naming_scheme, Setup.language) + \
"Available schemes are: %s." % repr(Setup.extension_db.keys())[1:-1])
if Setup.buffer_byte_order == "<system>":
Setup.buffer_byte_order = sys.byteorder
Setup.byte_order_is_that_of_current_system_f = True
else:
Setup.byte_order_is_that_of_current_system_f = False
Setup.buffer_element_specification_prepare()
if Setup.buffer_codec_name == "utf8": module = utf8_state_split
elif Setup.buffer_codec_name == "utf16": module = utf16_state_split
else: module = None
Setup.buffer_codec_prepare(Setup.buffer_codec_name,
Setup.buffer_codec_file, module)
# AFTER: Setup.buffer_codec_prepare() !!!
if Setup.language not in ["DOT"]:
prepare_file_names(Setup)
type_info = global_character_type_db.get(Setup.buffer_element_type)
if type_info is not None and len(type_info) >= 4 \
and type_info[3] != -1 and Setup.buffer_element_size != -1 \
and type_info[3] != Setup.buffer_element_size:
error.log("\nBuffer element type ('--bet' or '--buffer-element-type') was set to '%s'.\n" \
% Setup.buffer_element_type \
+ "It is well known to be of size %s[byte]. However, the buffer element size\n" \
% type_info[3] \
+ "('-b' or '--buffer-element-type') was specified as '%s'.\n\n" \
% Setup.buffer_element_size \
+ "Quex can continue, but the result is questionable.\n", \
DontExitF=True)
Setup.converter_f = False
if Setup.converter_iconv_f or Setup.converter_icu_f or len(Setup.converter_user_new_func) != 0:
Setup.converter_f = True
# The only case where no converter helper is required is where ASCII
# (Unicode restricted to [0, FF] is used.
Setup.converter_helper_required_f = True
if Setup.converter_f == False and Setup.buffer_element_size == 1 and Setup.buffer_codec.name == "unicode":
Setup.converter_helper_required_f = False
validation.do(Setup, command_line, argv)
if Setup.converter_ucs_coding_name == "":
if global_character_type_db.has_key(Setup.buffer_element_type):
if Setup.buffer_byte_order == "little": index = 1
else: index = 2
Setup.converter_ucs_coding_name = global_character_type_db[Setup.buffer_element_type][index]
if len(Setup.token_id_foreign_definition) != 0:
if len(Setup.token_id_foreign_definition) > 3:
error.log("Option '--foreign-token-id-file' received > 3 followers.\n"
"Found: %s" % str(Setup.token_id_foreign_definition)[1:-1])
if len(Setup.token_id_foreign_definition) > 1:
Setup.token_id_foreign_definition_file_region_begin_re = \
__compile_regular_expression(Setup.token_id_foreign_definition[1], "token id region begin")
if len(Setup.token_id_foreign_definition) > 2:
Setup.token_id_foreign_definition_file_region_end_re = \
__compile_regular_expression(Setup.token_id_foreign_definition[2], "token id region end")
Setup.token_id_foreign_definition_file = \
Setup.token_id_foreign_definition[0]
CommentDelimiterList = [["//", "\n"], ["/*", "*/"]]
token_id_file_parse(Setup.token_id_foreign_definition_file,
CommentDelimiterList)
# (*) Compression Types
compression_type_list = []
for name, ctype in [("compression_template_f", E_Compression.TEMPLATE),
("compression_template_uniform_f", E_Compression.TEMPLATE_UNIFORM),
("compression_path_f", E_Compression.PATH),
("compression_path_uniform_f", E_Compression.PATH_UNIFORM)]:
if command_line_args_defined(command_line, name):
compression_type_list.append((command_line_arg_position(name), ctype))
compression_type_list.sort(key=itemgetter(0))
Setup.compression_type_list = map(lambda x: x[1], compression_type_list)
# (*) return Setup ___________________________________________________________________
return True
def prepare(command_line, argv):
"""RETURN: True, if process needs to be started.
False, if job is done.
"""
global Setup
# (*) Classes and their namespace
__setup_analyzer_class(Setup)
__setup_token_class(Setup)
__setup_token_id_prefix(Setup)
__setup_lexeme_null(Setup) # Requires 'token_class_name_space'
# (*) Output programming language
Setup.language = Setup.language.upper()
error.verify_word_in_list(
Setup.language, output_language_db.keys(),
"Programming language '%s' is not supported." % Setup.language)
Setup.language_db = output_language_db[Setup.language]
Setup.extension_db = global_extension_db[Setup.language]
# Is the output file naming scheme provided by the extension database
# (Validation must happen immediately)
if Setup.extension_db.has_key(Setup.output_file_naming_scheme) == False:
error.log("File extension scheme '%s' is not provided for language '%s'.\n" \
% (Setup.output_file_naming_scheme, Setup.language) + \
"Available schemes are: %s." % repr(Setup.extension_db.keys())[1:-1])
if Setup.buffer_byte_order == "<system>":
Setup.buffer_byte_order = sys.byteorder
Setup.byte_order_is_that_of_current_system_f = True
else:
Setup.byte_order_is_that_of_current_system_f = False
Setup.buffer_element_specification_prepare()
if Setup.buffer_codec_name == "utf8": module = utf8_state_split
elif Setup.buffer_codec_name == "utf16": module = utf16_state_split
else: module = None
Setup.buffer_codec_prepare(Setup.buffer_codec_name,
Setup.buffer_codec_file, module)
# AFTER: Setup.buffer_codec_prepare() !!!
if Setup.language not in ["DOT"]:
prepare_file_names(Setup)
type_info = global_character_type_db.get(Setup.buffer_element_type)
if type_info is not None and len(type_info) >= 4 \
and type_info[3] != -1 and Setup.buffer_element_size != -1 \
and type_info[3] != Setup.buffer_element_size:
error.log("\nBuffer element type ('--bet' or '--buffer-element-type') was set to '%s'.\n" \
% Setup.buffer_element_type \
+ "It is well known to be of size %s[byte]. However, the buffer element size\n" \
% type_info[3] \
+ "('-b' or '--buffer-element-type') was specified as '%s'.\n\n" \
% Setup.buffer_element_size \
+ "Quex can continue, but the result is questionable.\n", \
DontExitF=True)
Setup.converter_f = False
if Setup.converter_iconv_f or Setup.converter_icu_f or len(
Setup.converter_user_new_func) != 0:
Setup.converter_f = True
# The only case where no converter helper is required is where ASCII
# (Unicode restricted to [0, FF] is used.
Setup.converter_helper_required_f = True
if Setup.converter_f == False and Setup.buffer_element_size == 1 and Setup.buffer_codec.name == "unicode":
Setup.converter_helper_required_f = False
validation.do(Setup, command_line, argv)
if Setup.converter_ucs_coding_name == "":
if global_character_type_db.has_key(Setup.buffer_element_type):
if Setup.buffer_byte_order == "little": index = 1
else: index = 2
Setup.converter_ucs_coding_name = global_character_type_db[
Setup.buffer_element_type][index]
if len(Setup.token_id_foreign_definition) != 0:
if len(Setup.token_id_foreign_definition) > 3:
error.log(
"Option '--foreign-token-id-file' received > 3 followers.\n"
"Found: %s" % str(Setup.token_id_foreign_definition)[1:-1])
if len(Setup.token_id_foreign_definition) > 1:
Setup.token_id_foreign_definition_file_region_begin_re = \
__compile_regular_expression(Setup.token_id_foreign_definition[1], "token id region begin")
if len(Setup.token_id_foreign_definition) > 2:
Setup.token_id_foreign_definition_file_region_end_re = \
__compile_regular_expression(Setup.token_id_foreign_definition[2], "token id region end")
Setup.token_id_foreign_definition_file = \
Setup.token_id_foreign_definition[0]
CommentDelimiterList = [["//", "\n"], ["/*", "*/"]]
token_id_file_parse(Setup.token_id_foreign_definition_file,
CommentDelimiterList)
# (*) Compression Types
compression_type_list = []
for name, ctype in [
("compression_template_f", E_Compression.TEMPLATE),
("compression_template_uniform_f", E_Compression.TEMPLATE_UNIFORM),
("compression_path_f", E_Compression.PATH),
("compression_path_uniform_f", E_Compression.PATH_UNIFORM)
]:
if command_line_args_defined(command_line, name):
compression_type_list.append(
(command_line_arg_position(name), ctype))
compression_type_list.sort(key=itemgetter(0))
Setup.compression_type_list = map(lambda x: x[1], compression_type_list)
# (*) return Setup ___________________________________________________________________
return True