def split_interval_according_to_utf8_byte_sequence_length(X):
"""Split Unicode interval into intervals where all values
have the same utf8-byte sequence length.
"""
if X.begin < 0: X.begin = 0
if X.end > UTF8_MAX: X.end = UTF8_MAX + 1
if X.size() == 0: return None
db = {}
current_begin = X.begin
last_L = len(unicode_to_utf8(X.end -
1)) # Length of utf8 sequence corresponding
# # the last value inside the interval.
while 1 + 1 == 2:
L = len(unicode_to_utf8(
current_begin)) # Length of the first unicode in utf8
# Store the interval together with the required byte sequence length (as key)
current_end = UTF8_BORDERS[L - 1]
if L == last_L:
db[L] = Interval(current_begin, X.end)
break
db[L] = Interval(current_begin, current_end)
current_begin = current_end
return db
def split_contigous_intervals_for_surrogates(Begin, End):
"""Splits the interval X into sub interval so that no interval runs over a 'surrogate'
border of the last word. For that, it is simply checked if the End falls into the
same 'surrogate' domain of 'front' (start value of front = Begin). If it does not
an interval [front, end_of_domain) is split up and front is set to end of domain.
This procedure repeats until front and End lie in the same domain.
"""
assert Begin >= 0x10000
assert End <= 0x110000
front_seq = unicode_to_utf16(Begin)
back_seq = unicode_to_utf16(End - 1)
if front_seq[0] == back_seq[0]:
return [Interval(Begin, End)]
# Separate into three domains:
#
# (1) interval from Begin until second surrogate hits border 0xE000
# (2) interval where the first surrogate inreases while second
# surrogate iterates over [0xDC00, 0xDFFF]
# (3) interval from begin of last surrogate border to End
result = []
end = utf16_to_unicode([front_seq[0], 0xDFFF]) + 1
# The following **must** hold according to entry condition about front and back sequence
assert End > end
result.append(Interval(Begin, end))
if front_seq[0] + 1 != back_seq[0]:
mid_end = utf16_to_unicode([back_seq[0] - 1, 0xDFFF]) + 1
result.append(Interval(end, mid_end))
end = mid_end
result.append(Interval(end, End))
return result
def add_transition(self, Trigger, TargetStateIdx):
"""Adds a transition according to trigger and target index.
RETURNS: The target state index (may be created newly).
"""
assert type(TargetStateIdx) == long or TargetStateIdx is None
assert Trigger.__class__ in [int, long, list, Interval, NumberSet
] or Trigger is None
if Trigger is None: # This is a shorthand to trigger via the remaining triggers
Trigger = self.get_trigger_set_union().inverse()
elif type(Trigger) == long:
Trigger = Interval(int(Trigger), int(Trigger + 1))
elif type(Trigger) == int:
Trigger = Interval(Trigger, Trigger + 1)
elif type(Trigger) == list:
Trigger = NumberSet(Trigger, ArgumentIsYoursF=True)
if Trigger.__class__ == Interval:
if self.__db.has_key(TargetStateIdx):
self.__db[TargetStateIdx].add_interval(Trigger)
else:
self.__db[TargetStateIdx] = NumberSet(Trigger,
ArgumentIsYoursF=True)
else:
if self.__db.has_key(TargetStateIdx):
self.__db[TargetStateIdx].unite_with(Trigger)
else:
self.__db[TargetStateIdx] = Trigger
return TargetStateIdx
def split_contigous_intervals_for_surrogates(Begin, End):
"""Splits the interval X into sub interval so that no interval runs over a 'surrogate'
border of the last word. For that, it is simply checked if the End falls into the
same 'surrogate' domain of 'front' (start value of front = Begin). If it does not
an interval [front, end_of_domain) is split up and front is set to end of domain.
This procedure repeats until front and End lie in the same domain.
"""
global ForbiddenRange
assert Begin >= 0x10000
assert End <= 0x110000
assert End > Begin
front_seq = unicode_to_utf16(Begin)
back_seq = unicode_to_utf16(End - 1)
# (*) First word is the same.
# Then,
# -- it is either a one word character.
# -- it is a range of two word characters, but the range
# extends in one contigous range in the second surrogate.
# In both cases, the interval is contigous.
if front_seq[0] == back_seq[0]:
return [Interval(Begin, End)]
# (*) First word is NOT the same
# Separate into three domains:
#
# (1) Interval from Begin until second surrogate hits border 0xE000
# (2) Interval where the first surrogate inreases while second
# surrogate iterates over [0xDC00, 0xDFFF]
# (3) Interval from begin of last surrogate border to End
result = []
end = utf16_to_unicode([front_seq[0], ForbiddenRange.end - 1]) + 1
# (1) 'Begin' until second surrogate hits border 0xE000
# (The following **must** hold according to entry condition about
# front and back sequence.)
assert End > end
result.append(Interval(Begin, end))
if front_seq[0] + 1 != back_seq[0]:
# (2) Second surrogate iterates over [0xDC00, 0xDFFF]
mid_end = utf16_to_unicode([back_seq[0] - 1, ForbiddenRange.end - 1]) + 1
# (The following **must** hold according to entry condition about
# front and back sequence.)
assert mid_end > end
result.append(Interval(end, mid_end))
end = mid_end
# (3) Last surrogate border to End
if End > end:
result.append(Interval(end, End))
return result
def __separate_buffer_limit_code_transition(TransitionMap, EngineType):
"""This function ensures, that the buffer limit code is separated
into a single value interval. Thus the transition map can
transit immediate to the reload procedure.
"""
BLC = Setup.buffer_limit_code
for i, entry in enumerate(TransitionMap):
interval, target_index = entry
if target_index == E_StateIndices.RELOAD_PROCEDURE:
assert interval.contains_only(Setup.buffer_limit_code)
assert EngineType != E_EngineTypes.BACKWARD_INPUT_POSITION
# Transition 'buffer limit code --> E_StateIndices.RELOAD_PROCEDURE'
# has been setup already.
return
elif target_index is not E_StateIndices.DROP_OUT:
continue
elif not interval.contains(Setup.buffer_limit_code):
continue
# Found the interval that contains the buffer limit code.
# If the interval's size is alread 1, then there is nothing to be done
if interval.size() == 1: return
before_begin = interval.begin
before_end = BLC
after_begin = BLC + 1
after_end = interval.end
# Replace Entry with (max.) three intervals: before, buffer limit code, after
del TransitionMap[i]
if after_end > after_begin:
TransitionMap.insert(
i, (Interval(after_begin, after_end), E_StateIndices.DROP_OUT))
# "Target == E_StateIndices.RELOAD_PROCEDURE" => Buffer Limit Code
TransitionMap.insert(
i, (Interval(BLC, BLC + 1), E_StateIndices.RELOAD_PROCEDURE))
if before_end > before_begin and before_end > 0:
TransitionMap.insert(
i,
(Interval(before_begin, before_end), E_StateIndices.DROP_OUT))
return
# Any transition map, except for backward input position detection,
# must have a trigger on reload.
assert EngineType in [E_EngineTypes.BACKWARD_INPUT_POSITION, E_EngineTypes.INDENTATION_COUNTER], \
"Engine types other than 'backward input position detection' or 'indentation counter' must contain BLC.\n" \
"Found: %s" % repr(EngineType)
return
def get_trigger_sequence_for_interval(X):
# The interval either lies entirely >= 0x10000 or entirely < 0x10000
assert X.begin >= 0x10000 or X.end < 0x10000
# An interval below < 0x10000 remains the same
if X.end < 0x10000: return [ X ]
# In case that the interval >= 0x10000 it the value is split up into
# two values.
front_seq = unicode_to_utf16(X.begin)
back_seq = unicode_to_utf16(X.end - 1)
return [ Interval(front_seq[0], back_seq[0] + 1),
Interval(front_seq[1], back_seq[1] + 1) ]
def create_ALL_BUT_NEWLINE_state_machine():
global Setup
result = StateMachine()
# NOTE: Buffer control characters are supposed to be filtered out by the code
# generator.
trigger_set = NumberSet(Interval(ord("\n")).inverse())
if Setup.get_character_value_limit() != sys.maxint:
trigger_set.intersect_with(
Interval(0, Setup.get_character_value_limit()))
result.add_transition(result.init_state_index,
trigger_set,
AcceptanceF=True)
return result
def __create_database_file(TargetEncoding, TargetEncodingName):
"""Writes a database file for a given TargetEncodingName. The
TargetEncodingName is required to name the file where the
data is to be stored.
"""
encoder = codecs.getencoder(TargetEncoding)
prev_output = -1
db = []
bytes_per_char = -1
for input in range(0x110000):
output, n = __get_transformation(encoder, input)
if bytes_per_char == -1:
bytes_per_char = n
elif n != -1 and bytes_per_char != n:
print "# not a constant size byte format."
return False
# Detect discontinuity in the mapping
if prev_output == -1:
if output != -1:
input_interval = Interval(input)
target_interval_begin = output
elif output != prev_output + 1:
# If interval was valid, append it to the database
input_interval.end = input
db.append((input_interval, target_interval_begin))
# If interval ahead is valid, prepare an object for it
if output != -1:
input_interval = Interval(input)
target_interval_begin = output
prev_output = output
if prev_output != -1:
input_interval.end = input
db.append((input_interval, target_interval_begin))
fh = open_file_or_die(QUEX_CODEC_DB_PATH + "/%s.dat" % TargetEncoding, "wb")
fh.write("// Describes mapping from Unicode Code pointer to Character code in %s (%s)\n" \
% (TargetEncoding, TargetEncodingName))
fh.write("// [SourceInterval.begin] [SourceInterval.Size] [TargetInterval.begin] (all in hexidecimal)\n")
for i, t in db:
fh.write("0x%X %i 0x%X\n" % (i.begin, i.end - i.begin, t))
fh.close()
return True
def consider_interval(self, Begin, End):
if Begin > End:
raise RegularExpressionException("Character range: '-' requires character with 'lower code' to preceed\n" + \
"found range '%s-%s' which corresponds to %i-%i as unicode code points." % \
(utf8.map_unicode_to_utf8(Begin), utf8.map_unicode_to_utf8(End), Begin, End))
self.match_set.add_interval(Interval(Begin, End))
def get_contigous_intervals(X):
"""Split Unicode interval into intervals where all values
have the same utf16-byte sequence length. This is fairly
simple in comparison with utf8-byte sequence length: There
are only two lengths: 2 bytes and 2 x 2 bytes.
RETURNS: [X0, List1]
X0 = the sub-interval where all values are 1 word (2 byte)
utf16 encoded.
None => No such interval
List1 = list of contigous sub-intervals where coded as 2 words.
None => No such intervals
"""
global ForbiddenRange
if X.begin == -sys.maxint: X.begin = 0
if X.end == sys.maxint: X.end = 0x110000
assert X.end != X.begin # Empty intervals are nonsensical
assert X.end <= 0x110000 # Interval must lie in unicode range
assert not X.check_overlap(ForbiddenRange) # The 'forbidden range' is not to be covered.
if X.end <= 0x10000: return [X, None]
elif X.begin >= 0x10000: return [None, split_contigous_intervals_for_surrogates(X.begin, X.end)]
else: return [Interval(X.begin, 0x10000), split_contigous_intervals_for_surrogates(0x10000, X.end)]
def get_trigger_sequence_for_contigous_byte_range_interval(X, L):
front_sequence = unicode_to_utf8(X.begin)
back_sequence = unicode_to_utf8(X.end - 1)
# If the interval is contigous it must produce equal length utf8 sequences
# Let me play with 'list comprehensions' just one time
return [ Interval(front_sequence[i], back_sequence[i] + 1) for i in range(L) ]
def get_supported_unicode_character_set(CodecAlias=None, FileName=None, FH=-1, LineN=None):
assert CodecAlias is not None or FileName is not None
mapping_list = get_codec_transformation_info(CodecAlias, FileName, FH, LineN)
result = NumberSet()
for source_begin, source_end, target_begin in mapping_list:
result.add_interval(Interval(source_begin, source_end))
return result
def get_trigger_sequence_for_contigous_byte_range_interval(X, L):
front_sequence = unicode_to_utf8(X.begin)
back_sequence = unicode_to_utf8(X.end - 1)
# If the interval is contigous it must produce equal length utf8 sequences
return [
Interval(front_sequence[i], back_sequence[i] + 1) for i in range(L)
]
def do(section_list, fh):
"""Parses a codec information file. The described codec can only be
a 'static character length' encoding. That is every character in the
code occupies the same number of bytes.
RETURNS: [0] Set of characters in unicode which are covered by the
described codec.
[1] Range of values in the codec elements.
"""
source_set = NumberSet()
drain_set = NumberSet()
error_str = None
try:
while error_str is None:
skip_whitespace(fh)
source_begin = read_integer(fh)
if source_begin is None:
error_str = "Missing integer (source interval begin) in codec file."
continue
skip_whitespace(fh)
source_size = read_integer(fh)
if source_size is None:
error_str = "Missing integer (source interval size) in codec file."
continue
skip_whitespace(fh)
target_begin = read_integer(fh)
if target_begin is None:
error_str = "Missing integer (target interval begin) in codec file."
continue
source_end = source_begin + source_size
list.append(section_list, [source_begin, source_end, target_begin])
source_set.add_interval(Interval(source_begin, source_end))
drain_set.add_interval(
Interval(target_begin, target_begin + source_size))
except EndOfStreamException:
pass
return source_set, drain_set, error_str
def __add_case_fold(sm, Flags, trigger_set, start_state_idx, target_state_idx):
for interval in trigger_set.get_intervals(PromiseToTreatWellF=True):
for i in range(interval.begin, interval.end):
fold = ucs_case_fold.get_fold_set(i, Flags)
for x in fold:
if type(x) == list:
__add_intermediate_states(sm, x, start_state_idx, target_state_idx)
else:
trigger_set.add_interval(Interval(x, x+1))
def delete_transitions_on_character_list(self, CharacterCodeList):
for trigger_set in self.__db.values():
for char_code in CharacterCodeList:
if trigger_set.contains(char_code):
trigger_set.cut_interval(Interval(char_code,
char_code + 1))
self.delete_transitions_on_empty_trigger_sets()
def set_target(self, Character, NewTarget):
"""Set the target in the transition map for a given 'Character'.
"""
# Find the index of the interval which contains 'Character'
i = TransitionMap.bisect(self, Character)
if i is None:
self.insert(0, (Interval(Character), NewTarget))
self.sort()
return
# Split the found interval, if necessary, so that the map
# contains 'Character' --> 'NewTarget'.
interval, target = self[i]
assert interval.size() > 0
new_i = None
if target == NewTarget:
return # Nothing to be done
elif interval.size() == 1:
self[i] = (interval, NewTarget)
new_i = i
elif Character == interval.end - 1:
self.insert(i+1, (Interval(Character), NewTarget))
interval.end -= 1
new_i = i + 1
elif Character == interval.begin:
self.insert(i, (Interval(Character), NewTarget))
interval.begin += 1
new_i = i
else:
self.insert(i+1, (Interval(Character), NewTarget))
self.insert(i+2, (Interval(Character+1, interval.end), target))
interval.end = Character
new_i = i + 1
# Combine adjacent intervals which trigger to the same target.
self.combine_adjacents(new_i)
self.assert_continuity()
return
def parse_table(Filename,
IntervalColumnList=[],
NumberColumnList=[],
NumberListColumnList=[],
CommentF=False):
"""Columns in IntervalColumnList --> converted to Interval() objects
NumberColumnList --> converted to integers (hex numbers)
NumberListColumnList --> converted to integer list (hex numbers)
"""
fh = open_data_base_file(Filename)
record_set = []
for line in fh.readlines():
comment_idx = line.find("#")
comment = None
if comment_idx != -1:
comment = line[comment_idx + 1:]
line = line[:comment_idx]
if line == "" or line.isspace():
continue
# append content to record set
cells = map(lambda x: x.strip(), line.split(";"))
for i in IntervalColumnList:
fields = cells[i].split("..") # range: value0..value1
assert len(fields) in [1, 2]
if len(fields) == 2:
begin = int("0x" + fields[0], 16)
end = int("0x" + fields[1], 16) + 1
else:
begin = int("0x" + fields[0], 16)
end = int("0x" + fields[0], 16) + 1
cells[i] = Interval(begin, end)
for i in NumberColumnList:
cells[i] = int("0x" + cells[i], 16)
for i in NumberListColumnList:
nl = []
for n in cells[i].split():
nl.append(int("0x" + n, 16))
cells[i] = nl
# Sometimes, the comment is useful
if CommentF:
cells.append(comment)
record_set.append(cells)
# There is no need to decouple here, since the record_set is created
# each time that the function is called.
return record_set
def load_Composition_Exclusion(self):
# Column 0 contains what is interesting ...
table = parse_table("CompositionExclusions.txt", NumberColumnList=[0])
number_set = NumberSet()
for row in table:
begin = row[0]
number_set.quick_append_interval(Interval(begin, begin + 1))
number_set.clean()
self.db["CE"].code_point_db = number_set
def parse_character_set(Txt_or_File, PatternStringF=False):
sh, sh_ref, start_position = __prepare_text_or_file_stream(Txt_or_File)
try:
# -- parse regular expression, build state machine
character_set = charset_expression.snap_set_expression(
sh, blackboard.shorthand_db)
if character_set is None:
error_msg("No valid regular character set expression detected.",
sh_ref)
# -- character set is not supposed to contain buffer limit code
if character_set.contains(Setup.buffer_limit_code):
character_set.cut_interval(Interval(Setup.buffer_limit_code))
if character_set.contains(Setup.path_limit_code):
character_set.cut_interval(Interval(Setup.path_limit_code))
except RegularExpressionException, x:
error_msg("Regular expression parsing:\n" + x.message, sh_ref)
def get_all():
"""RETURNS:
A state machine that 'eats' absolutely everything, i.e.
.--- \Any ---.
| |
(0)--- \Any --->(( 0 ))<--------'
"""
result = StateMachine()
i = index.get()
state = State(AcceptanceF=True)
state.add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), i)
result.states[i] = state
result.get_init_state().add_transition(
NumberSet(Interval(-sys.maxint, sys.maxint)), i)
return result
def create_ALL_BUT_NEWLINE_state_machine(stream):
global Setup
result = StateMachine()
# NOTE: Buffer control characters are supposed to be filtered out by the code
# generator.
trigger_set = NumberSet(Interval(ord("\n"))).get_complement(Setup.buffer_codec.source_set)
if trigger_set.is_empty():
error_msg("The set of admissible characters contains only newline.\n"
"The '.' for 'all but newline' is an empty set.",
SourceRef.from_FileHandle(stream))
result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True)
return result
def get_any():
"""RETURNS:
A state machine that 'eats' any character, but only one.
(0)--- \Any --->(( 0 ))
"""
result = StateMachine()
result.add_transition(result.init_state_index,
NumberSet(Interval(-sys.maxint, sys.maxint)),
AcceptanceF=True)
return result
def __display_set(CharSet, cl):
if Setup.query_numeric_f: display = "hex"
else: display = "utf8"
CharSet.intersect_with(NumberSet(Interval(0, 0x110000)))
print "Characters:\n"
if Setup.query_interval_f:
__print_set_in_intervals(CharSet, display, 80)
elif Setup.query_unicode_names_f:
__print_set_character_names(CharSet, display, 80)
else:
__print_set_single_characters(CharSet, display, 80)
print
def __display_set(CharSet, cl):
if cl.search("--numeric"): display = "hex"
else: display = "utf8"
CharSet.intersect_with(NumberSet(Interval(0, 0x110000)))
print "Characters:\n",
if cl.search("--intervals"):
__print_set_in_intervals(CharSet, display, 80)
elif cl.search("--names"):
__print_set_character_names(CharSet, display, 80)
else:
__print_set_single_characters(CharSet, display, 80)
print
def arrange_trigger_map(trigger_map):
"""Arrange the trigger map: Sort, and insert 'drop-out-regions'
"""
# -- sort by interval
trigger_map.sort(lambda x, y: cmp(x[0].begin, y[0].begin))
# -- insert lower and upper 'drop-out-transitions'
if trigger_map[0][0].begin != -sys.maxint:
trigger_map.insert(0, [Interval(-sys.maxint, trigger_map[0][0].begin), E_StateIndices.DROP_OUT])
if trigger_map[-1][0].end != sys.maxint:
trigger_map.append([Interval(trigger_map[-1][0].end, sys.maxint), E_StateIndices.DROP_OUT])
# -- fill gaps
previous_end = -sys.maxint
i = 0
size = len(trigger_map)
while i < size:
interval = trigger_map[i][0]
if interval.begin != previous_end:
trigger_map.insert(i, [Interval(previous_end, interval.begin), E_StateIndices.DROP_OUT])
i += 1
size += 1
i += 1
previous_end = interval.end
def split_interval_according_to_utf8_byte_sequence_length(X):
"""Split Unicode interval into intervals where all values
have the same utf8-byte sequence length.
"""
global utf8_border
if X.begin == -sys.maxint: X.begin = 0
if X.end == sys.maxint: X.end = 0x110000
assert X.end <= 0x110000 # Interval must lie in unicode range
db = {}
current_begin = X.begin
LastL = len(unicode_to_utf8(X.end - 1)) # Length of utf8 sequence corresponding
# # the last value inside the interval.
while 1 + 1 == 2:
L = len(unicode_to_utf8(current_begin)) # Length of the first unicode in utf8
# Store the interval together with the required byte sequence length (as key)
current_end = utf8_border[L-1]
if L == LastL:
db[L] = Interval(current_begin, X.end)
break
db[L] = Interval(current_begin, current_end)
current_begin = current_end
return db
def __create_database_file(TargetEncoding, TargetEncodingName):
"""Writes a database file for a given TargetEncodingName. The
TargetEncodingName is required to name the file where the
data is to be stored.
"""
encoder = codecs.getencoder(TargetEncoding)
prev_output = -1
db = []
bytes_per_char = -1
for input in range(0x110000):
output, n = __get_transformation(encoder, input)
if bytes_per_char == -1:
bytes_per_char = n
elif n != -1 and bytes_per_char != n:
print "# not a constant size byte format."
return False
# Detect discontinuity in the mapping
if prev_output == -1:
if output != -1:
input_interval = Interval(input)
target_interval_begin = output
elif output != prev_output + 1:
# If interval was valid, append it to the database
input_interval.end = input
db.append((input_interval, target_interval_begin))
# If interval ahead is valid, prepare an object for it
if output != -1:
input_interval = Interval(input)
target_interval_begin = output
prev_output = output
if prev_output != -1:
input_interval.end = input
db.append((input_interval, target_interval_begin))
fh = open_file_or_die(QUEX_CODEC_DB_PATH + "/%s.dat" % TargetEncoding,
"wb")
fh.write("// Describes mapping from Unicode Code pointer to Character code in %s (%s)\n" \
% (TargetEncoding, TargetEncodingName))
fh.write(
"// [SourceInterval.begin] [SourceInterval.Size] [TargetInterval.begin] (all in hexidecimal)\n"
)
for i, t in db:
fh.write("0x%X %i 0x%X\n" % (i.begin, i.end - i.begin, t))
fh.close()
return True
def __delete_forbidden_character(sm, BLC):
"""The buffer limit code is something that **needs** to cause a drop out.
In the drop out handling, the buffer is reloaded.
Critical character is allowed at end of post context.
NOTE: This operation might result in orphaned states that have to
be deleted.
"""
for state in sm.states.values():
for target_state_index, trigger_set in state.transitions().get_map(
).items():
if trigger_set.contains(BLC):
trigger_set.cut_interval(Interval(BLC, BLC + 1))
# If the operation resulted in cutting the path to the target state, then delete it.
if trigger_set.is_empty():
state.transitions().delete_transitions_to_target(
target_state_index)
def convert_table_to_associative_map(table, ValueColumnIdx, ValueType,
KeyColumnIdx):
"""Produces a dictionary that maps from 'keys' to NumberSets. The
number sets represent the code points for which the key (property)
is valid.
ValueColumnIdx: Column that contains the character code interval or
string to which one wishes to map.
KeyColmnIdx: Column that contains the 'key' to be used for the map
self.db = database to contain the associative map.
"""
db = {}
if ValueType == "NumberSet":
for record in table:
key = record[KeyColumnIdx].strip()
key = key.replace(" ", "_")
value = record[ValueColumnIdx]
if type(value) == int: value = Interval(value)
db.setdefault(key, NumberSet()).quick_append_interval(value,
SortF=False)
elif ValueType == "number" or ValueType == "string":
for record in table:
key = record[KeyColumnIdx].strip()
key = key.replace(" ", "_")
value = record[ValueColumnIdx]
db[key] = value
else:
raise BaseException("ValueType = '%s' unknown.\n" % ValueType)
# if the content was a number set, it might be simplified, try it.
if ValueType == "NumberSet":
for key, number_set in db.items():
number_set.clean()
return db
def prepare_transition_map(TheState, TheAnalyzer, StateKeyStr):
"""Generate targets in the transition map which the code generation can
handle. The transition map will consist of pairs of
(Interval, TextTransitionCode)
objects.
NOTE: A word about the reload procedure.
Reload can end either with success (new data has been loaded), or failure
(no more data available). In case of success the **only** the transition
step has to be repeated. Nothing else is effected. Stored positions are
adapted automatically.
By convention we redo the transition map, in case of reload success and
jump to the state's drop-out in case of failure. There is no difference
here in the template state example.
"""
# Transition map of the 'skeleton'
if TheState.transition_map_empty_f:
# Transition Map Empty:
# This happens, for example, if there are only keywords and no
# 'overlaying' identifier pattern. But, in this case also, there
# must be something that catches the 'buffer limit code'.
# => Define an 'all drop out' trigger_map, and then later
# => Adapt the trigger map, so that the 'buffer limit' is an
# isolated single interval.
TheState.transition_map = [ (Interval(-sys.maxint, sys.maxint), MegaState_Target_DROP_OUT) ]
for i, info in enumerate(TheState.transition_map):
interval, target = info
new_target = prepare_target(target, TheState, TheAnalyzer.state_db, StateKeyStr)
TheState.transition_map[i] = (interval, new_target)
return