from quex.engine.generator.code_fragment_base import CodeFragment
from quex.engine.misc.enum import Enum
from quex.engine.interval_handling import NumberSet, Interval
from quex.input.setup import QuexSetup, SETUP_INFO
from copy import deepcopy
#-----------------------------------------------------------------------------------------
# setup: All information of the user's desired setup.
#-----------------------------------------------------------------------------------------
setup = QuexSetup(SETUP_INFO)
#-----------------------------------------------------------------------------------------
# StateIndices: Values to be used as target states for transitions
#-----------------------------------------------------------------------------------------
E_StateIndices = Enum("DROP_OUT", "RELOAD_PROCEDURE",
"INIT_STATE_TRANSITION_BLOCK",
"END_OF_PRE_CONTEXT_CHECK", "RECURSIVE", "ALL",
"ANALYZER_REENTRY", "NONE", "VOID")
E_PreContextIDs = Enum("NONE", "BEGIN_OF_LINE", "_DEBUG_NAME_PreContextIDs")
E_AcceptanceIDs = Enum("FAILURE", "PRE_CONTEXT_FULFILLED",
"TERMINAL_PRE_CONTEXT_CHECK",
"TERMINAL_BACKWARD_INPUT_POSITION", "VOID",
"_DEBUG_NAME_E_AcceptanceIDs")
E_PostContextIDs = Enum("NONE", "IRRELEVANT", "_DEBUG_NAME_E_PostContextIDs")
E_EngineTypes = Enum(
"FORWARD",
"BACKWARD_PRE_CONTEXT",
"BACKWARD_INPUT_POSITION",
from collections import namedtuple
from operator import attrgetter
from copy import deepcopy
import types
E_R = Enum(
"AcceptanceRegister",
"Buffer",
"Column",
"Input",
"Indentation",
"InputP",
"LexemeStartP",
"LexemeEnd",
"CharacterBeginP", # -> dynamic size codecs
"Line",
"PathIterator",
"PreContextFlags",
"ReferenceP",
"StandardOutput",
"PositionRegister",
"Pointer",
"TargetStateElseIndex",
"TargetStateIndex",
"TemplateStateKey",
"ThreadOfControl")
class Command(namedtuple("Command_tuple", ("id", "content", "my_hash"))):
"""_________________________________________________________________________
Information about an operation to be executed. It consists mainly of a
from quex.output.core.state.transition_map.bisection import Bisection
from quex.output.core.state.transition_map.branch_table import BranchTable
from quex.output.core.state.transition_map.comparison_sequence import ComparisonSequence
from quex.engine.analyzer.transition_map import TransitionMap
from quex.engine.misc.enum import Enum
E_Solution = Enum("COMPARISON_SEQUENCE", "BRANCH_TABLE", "BISECTIONING", "__DEBUG_Solution")
def do(TM):
return get_structure(TM)
def get_solution(TM):
"""RETURNS: [0] Solution from E_Solution
[1] Most often appearing target
"""
interval_n = len(TM)
assert interval_n > 0
most_often_appearing_target, target_n = TransitionMap.get_target_statistics(TM)
# If there's only one interval, there's no need to compare, just go!
# Otherwise, if there's a very low number of intervals, make a small
# comparison list that iterates linearly through the items.
if target_n < 4 and interval_n < 6:
return E_Solution.COMPARISON_SEQUENCE, None
# If the size of character ranges which do not target 'moat' is less
# than a certain number, implement the transition as branch table. The
# 'moat' is implemented as the 'default:' case.
sz_non_moat = TransitionMap.get_size_of_range_other_targets(TM, most_often_appearing_target)
#! /usr/bin/env python
import quex.engine.misc.file_in as file_in
from quex.engine.misc.file_operations import get_propperly_slash_based_file_name
from quex.engine.misc.enum import Enum
from quex.engine.misc.interval_handling import NumberSet, NumberSet_All
from quex.DEFINITIONS import QUEX_PATH
from quex.engine.state_machine.transformation.base import EncodingTrafoUnicode
import os
import sys
E_Files = Enum("HEADER", "HEADER_IMPLEMTATION", "SOURCE", "_DEBUG_Files")
class QuexSetup:
def __init__(self, SetupInfo, BcFactory):
self.init(SetupInfo)
range_max = NumberSet_All()
unit_test_bc = EncodingTrafoUnicode(range_max, range_max)
self.buffer_codec_set(unit_test_bc, -1)
def buffer_codec_set(self, BufferCodec, LexatomSizeInBytes):
self.buffer_codec = BufferCodec
self.buffer_lexatom_size_in_byte = LexatomSizeInBytes
self.buffer_codec.adapt_source_and_drain_range(LexatomSizeInBytes)
def init(self, SetupInfo):
for key, entry in SetupInfo.items():
if type(entry) != list:
default_value = entry
elif entry[1] in SetupParTypes:
# base_name = os.path.basename(x[0])
base_name = " " * len(os.path.basename(x[0]))
name = " " * (len(file_name) - len(base_name)) + base_name
prev_file_name = file_name
print "%s%s:%s:%s(...)" % (" " * (i * 4), name, x[1], x[2])
def pair_combinations(iterable):
other = tuple(iterable)
for i, x in enumerate(other):
for y in islice(other, i + 1, None):
yield x, y
E_Values = Enum("UNASSIGNED", "VOID", "DISABLED", "_DEBUG_E_Values")
def concatinate(one_list, other_list):
"""RETURNS: The 'one_list' extended with 'other_list' in case that
'other_list' is not None.
Both lists remain unchanged during the operation.
"""
if other_list is not None: return one_list + other_list
else: return one_list
def flatten_list_of_lists(ListOfListsIterable):
"""The very fastest way to flatten a list of lists of objects into a list
of objects.
i += 1
else:
# base_name = os.path.basename(x[0])
base_name = " " * len(os.path.basename(x[0]))
name = " " * (len(file_name) - len(base_name)) + base_name
prev_file_name = file_name
print "%s%s:%s:%s(...)" % (" " * (i*4), name, x[1], x[2])
def pair_combinations(iterable):
other = tuple(iterable)
for i, x in enumerate(other):
for y in islice(other, i+1, None):
yield x, y
E_Values = Enum("UNASSIGNED", "VOID", "DISABLED", "SIGMA", "RESTORE", "_DEBUG_E_Values")
# 'SIGMA' is something that is always different. It will cause always the
# content to become 'VOID', i.e. not uniform in UniformObject.
def concatinate(one_list, other_list):
"""RETURNS: The 'one_list' extended with 'other_list' in case that
'other_list' is not None.
Both lists remain unchanged during the operation.
"""
if other_list is not None: return one_list + other_list
else: return one_list
def flatten_list_of_lists(ListOfListsIterable):
"""The very fastest way to flatten a list of lists of objects into a list
of objects.
import os
sys.path.insert(0, os.environ["QUEX_PATH"])
from quex.engine.misc.enum import Enum
if "--hwut-info" in sys.argv:
print "Enums -- Functionality Test"
sys.exit(0)
# This test has been derived directly from the recipe.
# Extension were made to check important things such as 'safe usage in dictionaries'.
print "(*) Creating an Enum from tuples of names and values:"
print " ", ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']
Days = Enum('Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su')
Months = Enum('Ramadan', "Shawwal", "Dhu_alHidja", "Muharram")
print
print "(*) String Representations and hash values"
print "All:", Days
print "Mo: ", Days.Mo, hash(Days.Mo), Days.Mo.Value
print "Tu: ", Days.Tu, hash(Days.Tu), Days.Tu.Value
print "We: ", Days.We, hash(Days.We), Days.We.Value
print "Th: ", Days.Th, hash(Days.Th), Days.Th.Value
print "Fr: ", Days.Fr, hash(Days.Fr), Days.Fr.Value
print "Sa: ", Days.Sa, hash(Days.Sa), Days.Sa.Value
print "Su: ", Days.Su, hash(Days.Su), Days.Su.Value
print
print "(*) Length = %i" % len(Days)
from quex.engine.misc.enum import Enum
# import sys
INTEGER_MAX = 0x7fffffff
# E_Border's speed is critical for performance of NFA to DFA conversion.
# => implement as class
# E_Border = Enum("BEGIN", "END", "UNDEFINED", "_DEBUG_Border")
class E_Border:
BEGIN = 1 # BEGIN > END is essential for comparison!
END = 0
E_Files = Enum("HEADER", "HEADER_IMPLEMTATION", "SOURCE", "_DEBUG_Files")
#------------------------------------------------------------------------------
# StateIndices: Values to be used as target states for transitions
#------------------------------------------------------------------------------
E_StateIndices = Enum("DROP_OUT", "RELOAD_FORWARD", "RELOAD_BACKWARD",
"BEFORE_ENTRY", "AFTER_PATH_WALK", "VOID")
E_AcceptanceCondition = Enum("BEGIN_OF_LINE", "BEGIN_OF_STREAM",
"END_OF_STREAM",
"_DEBUG_NAME_E_AcceptanceCondition")
def E_AcceptanceConditionSet_string(ACSet):
if not ACSet: return ""
result = []
for acc_condition_id in sorted(ACSet):
import os
sys.path.insert(0, os.environ["QUEX_PATH"])
from quex.engine.misc.enum import Enum
if "--hwut-info" in sys.argv:
print "Enums -- Functionality Test"
sys.exit(0)
# This test has been derived directly from the recipe.
# Extension were made to check important things such as 'safe usage in dictionaries'.
print "(*) Creating an Enum from tuples of names and values:"
print " ", ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']
Days = Enum('Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su')
Months = Enum('Ramadan', "Shawwal", "Dhu_alHidja", "Muharram")
print
print "(*) String Representations and hash values"
print "All:", Days
print "Mo: ", Days.Mo, hash(Days.Mo), Days.Mo.Value
print "Tu: ", Days.Tu, hash(Days.Tu), Days.Tu.Value
print "We: ", Days.We, hash(Days.We), Days.We.Value
print "Th: ", Days.Th, hash(Days.Th), Days.Th.Value
print "Fr: ", Days.Fr, hash(Days.Fr), Days.Fr.Value
print "Sa: ", Days.Sa, hash(Days.Sa), Days.Sa.Value
print "Su: ", Days.Su, hash(Days.Su), Days.Su.Value
print
print "(*) Length = %i" % len(Days)
HEURISTIC: _____________________________________________________________
(1) If interval can be interpreted as set of 'exceptional' transitions
where the number of exceptions N <= N_linear, the implement trigger
map as Linear Comparison (iii).
(2)
"""
from quex.engine.misc.enum import Enum
from quex.blackboard import setup as Setup
from math import log
E_Type = Enum("SWITCH_CASE", "BISECTION", "COMPARISON_SEQUENCE", "TRANSITION")
#class Cost:
# def __init__(self, C, M):
# self.computation = C
# self.memory = M
#
class Region:
@classmethod
def init(cls):
del cls.candidate_list[:]
del cls.good_list[:]
@classmethod
def consider(cls, EntryIndex, TheInterval, TheTarget):
cls.generate_candidates(cls.candidate_list, EntryIndex, TheInterval, TheTarget)
(C) 2017 Frank-Rene Schaefer
"""
import os
import sys
sys.path.insert(0, os.path.abspath("../../../../"))
from quex.engine.misc.interval_handling import NumberSet
from quex.engine.misc.enum import Enum
from copy import copy
from collections import defaultdict, namedtuple
from itertools import tee, takewhile
E_SubLexemeId = Enum("SEQUENCE", "NUMBER_SET", "LOOP")
Step = namedtuple("Step", ("by_trigger_set", "target_si"))
def __unicode_char(X):
"""Avoid problems with python's narrow build--do not use 'UNICHR()'."""
return eval("u'\U%08X'" % N)
def get(Dfa):
"""RETURNS: 'set' of lexeme which are matched by 'Dfa'.
The lexeme representation is suited for quick comparison. All
elements are *immutables*, so they may also serve as hash keys
or set elements.
def __getattr__(self, Attr):
language_db = self.__setup.language_db
try:
return getattr(language_db, Attr)
except KeyError:
raise AttributeError
Lng = Lng_class(setup)
#------------------------------------------------------------------------------
# StateIndices: Values to be used as target states for transitions
#------------------------------------------------------------------------------
E_StateIndices = Enum("DROP_OUT", "RELOAD_FORWARD", "RELOAD_BACKWARD",
"END_OF_PRE_CONTEXT_CHECK", "RECURSIVE", "ALL",
"ANALYZER_REENTRY", "BEFORE_ENTRY", "VOID")
E_PreContextIDs = Enum("NONE", "BEGIN_OF_LINE", "_DEBUG_NAME_PreContextIDs")
E_PostContextIDs = Enum("NONE", "IRRELEVANT", "_DEBUG_NAME_E_PostContextIDs")
E_TransitionN = Enum("VOID", "LEXEME_START_PLUS_ONE", "IRRELEVANT",
"_DEBUG_NAME_TransitionNs")
E_TriggerIDs = Enum("NONE", "_DEBUG_NAME_TriggerIDs")
E_InputActions = Enum("DEREF", "INCREMENT", "INCREMENT_THEN_DEREF",
"DECREMENT", "DECREMENT_THEN_DEREF",
"_DEBUG_InputActions")
# import array
from quex.engine.interval_handling import NumberSet, Interval
from quex.engine.misc.file_in import error_msg
from quex.engine.misc.enum import Enum
from quex.blackboard import E_StateIndices
import sys
from operator import attrgetter
# definitions for 'history items':
E_Border = Enum("BEGIN", "END", "UNDEFINED")
class TransitionMap:
"""Members:
__db: map [target index] --> [trigger set that triggers to target]
__epsilon_target_index_list: list of target states that are entered via epsilon
transition.
"""
def __init__(self, DB=None, ETIL=None):
if DB is None: self.__db = {}
else: self.__db = DB
if ETIL is None: self.__epsilon_target_index_list = []
else: self.__epsilon_target_index_list = ETIL
## OPTIMIZATION OPTION: Store the trigger map in a 'cache' variable. This, however,
## requires that all possible changes to the database need to annulate the cache value.
## self.__DEBUG_trigger_map = None
header_file = self.extern_token_class_file
implementation_file = None # Must be linked outside!
else:
if self.token_class_only_f == False:
implementation_type = E_Files.HEADER_IMPLEMTATION
else:
implementation_type = E_Files.SOURCE
header_file = self.prepare_file_name("-token", E_Files.HEADER)
implementation_file = self.prepare_file_name(
"-token", implementation_type)
self.output_token_class_file = header_file
self.output_token_class_file_implementation = implementation_file
SetupParTypes = Enum("LIST", "INT_LIST", "FLAG", "NEGATED_FLAG", "STRING",
"OPTIONAL_STRING")
SETUP_INFO = {
# [Name in Setup] [ Flags ] [Default / Type]
"_debug_exception_f": [["--debug-exception"], SetupParTypes.FLAG],
"_debug_limit_recursion": [["--debug-limit-recursion"], 0],
"_debug_reference_original_paths_f": [["--debug-original-paths"],
SetupParTypes.FLAG],
"_debug_QUEX_TYPE_LEXATOM_EXT": [["--debug-QUEX_TYPE_LEXATOM_EXT"],
SetupParTypes.FLAG],
"fallback_mandatory_f": [["--fallback-mandatory", "--fbm"],
SetupParTypes.FLAG],
"fallback_optional_f": [["--fallback-optional", "--fbo"],
SetupParTypes.FLAG],
"quex_lib": [["--ql", "--quex-lib"], ""],
"configuration_by_macros_f": [["--cbm", "--config-by-macros"],