def add_transition(self,
StartStateIdx,
TriggerSet,
TargetStateIdx=None,
AcceptanceF=False):
"""Adds a transition from Start to Target based on a given Trigger.
TriggerSet can be of different types: ... see add_transition()
(see comment on 'State::add_transition)
RETURNS: The target state index.
"""
assert type(StartStateIdx) == long
# NOTE: The Transition Constructor is very tolerant, so no tests on TriggerSet()
# assert TriggerSet.__class__.__name__ == "NumberSet"
assert type(TargetStateIdx) == long or TargetStateIdx == None
assert type(AcceptanceF) == bool
# If target state is undefined (None) then a new one has to be created
if TargetStateIdx == None: TargetStateIdx = state_machine_index.get()
if self.states.has_key(StartStateIdx) == False:
self.states[StartStateIdx] = State()
if self.states.has_key(TargetStateIdx) == False:
self.states[TargetStateIdx] = State()
if AcceptanceF: self.states[TargetStateIdx].set_acceptance(True)
self.states[StartStateIdx].add_transition(TriggerSet, TargetStateIdx)
return TargetStateIdx
def create_new_state(self, AcceptanceF=False, StateIdx=None):
if StateIdx == None:
new_state_index = state_machine_index.get()
else:
new_state_index = StateIdx
self.states[new_state_index] = State(AcceptanceF)
return new_state_index
def create_new_state(self, AcceptanceF=False, StateIdx=None):
if StateIdx == None:
new_state_index = state_machine_index.get()
else:
new_state_index = StateIdx
self.states[new_state_index] = State(AcceptanceF)
return new_state_index
def get_character_set_skipper(TriggerSet, LanguageDB):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap()
transition_map.add_transition(TriggerSet, skipper_index)
iteration_code = transition_block.do(transition_map.get_trigger_map(),
skipper_index,
InitStateF=False,
DSM=None)
comment_str = LanguageDB["$comment"]("Skip any character in " +
TriggerSet.get_utf8_string())
# Line and column number counting
code_str = __set_skipper_lc_counting_replacements(
trigger_set_skipper_template, TriggerSet)
# The finishing touch
txt = blue_print(code_str, [
["$$DELIMITER_COMMENT$$", comment_str],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_START$$", LanguageDB["$label-def"]("$input", skipper_index)],
["$$GOTO_LOOP_START$$", LanguageDB["$goto"]("$input", skipper_index)],
[
"$$LOOP_REENTRANCE$$", LanguageDB["$label-def"]("$entry",
skipper_index)
],
["$$RESTART$$", LanguageDB["$label-def"]("$input", skipper_index)],
["$$DROP_OUT$$", LanguageDB["$label-def"]("$drop-out", skipper_index)],
[
"$$DROP_OUT_DIRECT$$", LanguageDB["$label-def"]("$drop-out-direct",
skipper_index)
],
["$$GOTO_LOOP_START$$", LanguageDB["$goto"]("$entry", skipper_index)],
["$$SKIPPER_INDEX$$", repr(skipper_index)],
["$$GOTO_TERMINAL_EOF$$", LanguageDB["$goto"]("$terminal-EOF")],
["$$GOTO_REENTRY_PREPARATION$$", LanguageDB["$goto"]("$re-start")],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$ON_TRIGGER_SET_TO_LOOP_START$$", iteration_code],
])
return blue_print(txt, [[
"$$GOTO_DROP_OUT$$", LanguageDB["$goto"]("$drop-out", skipper_index)
]])
def _do(CombinationList, SMD):
"""-- Returns generated code for all templates.
-- Sets the template_compression_db in SMD.
"""
global LanguageDB
assert type(CombinationList) == list
assert isinstance(SMD, StateMachineDecorator)
LanguageDB = Setup.language_db
# -- Collect all indices of states involved in templates
involved_state_index_list = set([])
# -- Generate 'TemplatedState's for each TemplateCombination
template_list = []
for combination in CombinationList:
assert isinstance(combination, templates.TemplateCombination)
# Two Scenarios for settings at state entry (last_acceptance_position, ...)
#
# (i) All state entries are uniform:
# -- Then, a representive state entry can be implemented at the
# template entry.
# -- Recursion happens to the template entry.
#
# (ii) One or more state entry are different (non-uniform):
# -- The particularities of each state entry need to be implemented
# at state entry.
# -- Recursion is routed to entries of involved states.
#
involved_state_list = combination.involved_state_list()
prototype = get_uniform_prototype(SMD, involved_state_list)
# -- create template state for combination object
# prototype == None, tells that there state entries differ and there
# is no representive state.
template = TemplateState(combination,
SMD.sm().get_id(), index.get(), prototype)
template_list.append(template)
# -- collect indices of involved states
involved_state_index_list.update(involved_state_list)
# -- transition target definition for each templated state
for template in template_list:
__transition_target_data_structures(template, SMD)
# -- template state entries
# -- template state
code = []
for template in template_list:
__templated_state_entries(code, template, SMD)
__template_state(code, template, SMD)
return code, involved_state_index_list
def create_state_machine(SM, StateSetList):
# If all states are of size one, this means, that there were no states that
# could have been combined. In this case a simple copy of the original
# state machine will do.
if filter(lambda state_set: len(state_set) != 1,
StateSetList.state_set_list) == []:
return SM.clone()
# Define a mapping from the state set to a new target state index
map_new_state_index = {}
for state_set_index in range(len(StateSetList.state_set_list)):
map_new_state_index[state_set_index] = state_machine_index.get()
# The state set that contains the initial state becomes the initial state of
# the new state machine.
state_set_containing_initial_state_i = StateSetList.map[
SM.init_state_index]
result = StateMachine(
map_new_state_index[state_set_containing_initial_state_i],
Core=SM.core())
# Ensure that each target state index has a state inside the state machine
for new_state_index in map_new_state_index.values():
result.create_new_state(StateIdx=new_state_index)
# Build up the state machine out of the remaining state sets
state_set_idx = -1L
for state_set in StateSetList.state_set_list:
state_set_idx += 1L
assert len(state_set) != 0, "State set of size '0'. List = " + repr(
StateSetList)
# The prototype: States in one set behave all equivalent with respect to target state sets
# thus only one state from the start set has to be considered.
prototype = SM.states[state_set[0]]
# The representive: shall represent the state set in the new state machine.
representive = result.states[map_new_state_index[state_set_idx]]
# The representive must have all transitions that the prototype has
for target_state_index, trigger_set in prototype.transitions().get_map(
).items():
target_state_set_index = StateSetList.map[target_state_index]
representive.add_transition(
trigger_set, map_new_state_index[target_state_set_index])
# Merge all core information of the states inside the state set.
# If one state set contains an acceptance state, then the result is 'acceptance'.
# (Note: The initial split separates acceptance states from those that are not
# acceptance states. There can be no state set containing acceptance and
# non-acceptance states)
# (Note, that the prototype's info has not been included yet, consider whole set)
for state_idx in state_set:
representive.merge(SM.states[state_idx])
return result
def _do(CombinationList, SMD):
"""-- Returns generated code for all templates.
-- Sets the template_compression_db in SMD.
"""
global LanguageDB
assert type(CombinationList) == list
assert isinstance(SMD, StateMachineDecorator)
LanguageDB = Setup.language_db
# -- Collect all indices of states involved in templates
involved_state_index_list = set([])
# -- Generate 'TemplatedState's for each TemplateCombination
template_list = []
for combination in CombinationList:
assert isinstance(combination, templates.TemplateCombination)
# Two Scenarios for settings at state entry (last_acceptance_position, ...)
#
# (i) All state entries are uniform:
# -- Then, a representive state entry can be implemented at the
# template entry.
# -- Recursion happens to the template entry.
#
# (ii) One or more state entry are different (non-uniform):
# -- The particularities of each state entry need to be implemented
# at state entry.
# -- Recursion is routed to entries of involved states.
#
involved_state_list = combination.involved_state_list()
prototype = get_uniform_prototype(SMD, involved_state_list)
# -- create template state for combination object
# prototype == None, tells that there state entries differ and there
# is no representive state.
template = TemplateState(combination, SMD.sm().get_id(), index.get(),
prototype)
template_list.append(template)
# -- collect indices of involved states
involved_state_index_list.update(involved_state_list)
# -- transition target definition for each templated state
for template in template_list:
__transition_target_data_structures(template, SMD)
# -- template state entries
# -- template state
code = []
for template in template_list:
__templated_state_entries(code, template, SMD)
__template_state(code, template, SMD)
return code, involved_state_index_list
def create_state_machine(SM, StateSetList):
# If all states are of size one, this means, that there were no states that
# could have been combined. In this case a simple copy of the original
# state machine will do.
if filter(lambda state_set: len(state_set) != 1, StateSetList.state_set_list) == []:
return SM.clone()
# Define a mapping from the state set to a new target state index
map_new_state_index = {}
for state_set_index in range(len(StateSetList.state_set_list)):
map_new_state_index[state_set_index] = state_machine_index.get()
# The state set that contains the initial state becomes the initial state of
# the new state machine.
state_set_containing_initial_state_i = StateSetList.map[SM.init_state_index]
result = StateMachine(map_new_state_index[state_set_containing_initial_state_i],
Core = SM.core())
# Ensure that each target state index has a state inside the state machine
for new_state_index in map_new_state_index.values():
result.create_new_state(StateIdx=new_state_index)
# Build up the state machine out of the remaining state sets
state_set_idx = -1L
for state_set in StateSetList.state_set_list:
state_set_idx += 1L
assert len(state_set) != 0, "State set of size '0'. List = " + repr(StateSetList)
# The prototype: States in one set behave all equivalent with respect to target state sets
# thus only one state from the start set has to be considered.
prototype = SM.states[state_set[0]]
# The representive: shall represent the state set in the new state machine.
representive = result.states[map_new_state_index[state_set_idx]]
# The representive must have all transitions that the prototype has
for target_state_index, trigger_set in prototype.transitions().get_map().items():
target_state_set_index = StateSetList.map[target_state_index]
representive.add_transition(trigger_set,
map_new_state_index[target_state_set_index])
# Merge all core information of the states inside the state set.
# If one state set contains an acceptance state, then the result is 'acceptance'.
# (Note: The initial split separates acceptance states from those that are not
# acceptance states. There can be no state set containing acceptance and
# non-acceptance states)
# (Note, that the prototype's info has not been included yet, consider whole set)
for state_idx in state_set:
representive.merge(SM.states[state_idx])
return result
def __init__(self, InitStateIndex=None, AcceptanceF=False, Core=None):
# print "##state_machine_init"
if InitStateIndex == None: self.init_state_index = state_machine_index.get()
else: self.init_state_index = InitStateIndex
# State Index => State (information about what triggers transition to what target state).
self.states = { self.init_state_index: State(AcceptanceF) }
# get a unique state machine id
id = state_machine_index.get_state_machine_id()
# Setup core information
if Core != None:
self.__core = deepcopy(Core)
self.__core.set_id(id)
else:
self.__core = StateMachineCoreInfo(id)
def get_character_set_skipper(TriggerSet, LanguageDB):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap()
transition_map.add_transition(TriggerSet, skipper_index)
iteration_code = transition_block.do(transition_map.get_trigger_map(), skipper_index, InitStateF=False, DSM=None)
comment_str = LanguageDB["$comment"]("Skip any character in " + TriggerSet.get_utf8_string())
# Line and column number counting
code_str = __set_skipper_lc_counting_replacements(trigger_set_skipper_template, TriggerSet)
# The finishing touch
txt = blue_print(code_str,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_START$$", LanguageDB["$label-def"]("$input", skipper_index)],
["$$GOTO_LOOP_START$$", LanguageDB["$goto"]("$input", skipper_index)],
["$$LOOP_REENTRANCE$$", LanguageDB["$label-def"]("$entry", skipper_index)],
["$$RESTART$$", LanguageDB["$label-def"]("$input", skipper_index)],
["$$DROP_OUT$$", LanguageDB["$label-def"]("$drop-out", skipper_index)],
["$$DROP_OUT_DIRECT$$", LanguageDB["$label-def"]("$drop-out-direct", skipper_index)],
["$$GOTO_LOOP_START$$", LanguageDB["$goto"]("$entry", skipper_index)],
["$$SKIPPER_INDEX$$", repr(skipper_index)],
["$$GOTO_TERMINAL_EOF$$", LanguageDB["$goto"]("$terminal-EOF")],
["$$GOTO_REENTRY_PREPARATION$$", LanguageDB["$goto"]("$re-start")],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$ON_TRIGGER_SET_TO_LOOP_START$$", iteration_code],
])
return blue_print(txt,
[["$$GOTO_DROP_OUT$$", LanguageDB["$goto"]("$drop-out", skipper_index)]])
def __init__(self, InitStateIndex=None, AcceptanceF=False, Core=None):
# print "##state_machine_init"
if InitStateIndex == None:
self.init_state_index = state_machine_index.get()
else:
self.init_state_index = InitStateIndex
# State Index => State (information about what triggers transition to what target state).
self.states = {self.init_state_index: State(AcceptanceF)}
# get a unique state machine id
id = state_machine_index.get_state_machine_id()
# Setup core information
if Core != None:
self.__core = deepcopy(Core)
self.__core.set_id(id)
else:
self.__core = StateMachineCoreInfo(id)
def clone(self):
"""Clone state machine, i.e. create a new one with the same behavior,
i.e. transitions, but with new unused state indices. This is used when
state machines are to be created that combine the behavior of more
then one state machine. E.g. see the function 'sequentialize'. Note:
the state ids SUCCESS and TERMINATION are not replaced by new ones.
RETURNS: cloned object if cloning successful
None if cloning not possible due to external state references
"""
replacement = {}
# (*) create the new state machine
# (this has to happen first, to get an init_state_index)
result = StateMachine()
# every target state has to appear as a start state (no external states)
# => it is enough to consider the start states and 'rename' them.
# if later a target state index appears, that is not in this set we
# return 'None' to indicate that this state machine cannot be cloned.
sorted_state_indices = self.states.keys()
sorted_state_indices.sort()
for state_idx in sorted_state_indices:
# NOTE: The constructor already delivered an init state index to 'result'.
# Thus self's init index has to be translated to result's init index.
if state_idx == self.init_state_index:
replacement[state_idx] = result.init_state_index
else:
replacement[state_idx] = state_machine_index.get()
# termination is a global state, it is not replaced by a new index
for state_idx, state in self.states.items():
new_state_idx = replacement[state_idx]
# print "##", state_idx, "-->", new_state_idx
result.states[new_state_idx] = self.states[state_idx].clone(
replacement)
result.__core = deepcopy(self.__core)
return result
def get(self, NameOrTerminalID):
"""NameOrTerminalID is something that identifies a position/address
in the code. This function returns a numeric id
for this address.
Exceptions are labels that are 'unique' inside a state machine
as defined by '__address_db_special'. For those the string itself
is returned.
"""
# Special addresses are not treated, but returned as string
if NameOrTerminalID in self.__special:
return NameOrTerminalID
# If the thing is known, return its id immediately
entry = self.__db.get(NameOrTerminalID)
if entry != None: return entry
# Generate unique id for the label: Use unique state index
entry = index.get()
self.__db[NameOrTerminalID] = entry
return entry
def get(self, NameOrTerminalID):
"""NameOrTerminalID is something that identifies a position/address
in the code. This function returns a numeric id
for this address.
Exceptions are labels that are 'unique' inside a state machine
as defined by '__address_db_special'. For those the string itself
is returned.
"""
# Special addresses are not treated, but returned as string
if NameOrTerminalID in self.__special:
return NameOrTerminalID
# If the thing is known, return its id immediately
entry = self.__db.get(NameOrTerminalID)
if entry != None: return entry
# Generate unique id for the label: Use unique state index
entry = index.get()
self.__db[NameOrTerminalID] = entry
return entry
def clone(self):
"""Clone state machine, i.e. create a new one with the same behavior,
i.e. transitions, but with new unused state indices. This is used when
state machines are to be created that combine the behavior of more
then one state machine. E.g. see the function 'sequentialize'. Note:
the state ids SUCCESS and TERMINATION are not replaced by new ones.
RETURNS: cloned object if cloning successful
None if cloning not possible due to external state references
"""
replacement = {}
# (*) create the new state machine
# (this has to happen first, to get an init_state_index)
result = StateMachine()
# every target state has to appear as a start state (no external states)
# => it is enough to consider the start states and 'rename' them.
# if later a target state index appears, that is not in this set we
# return 'None' to indicate that this state machine cannot be cloned.
sorted_state_indices = self.states.keys()
sorted_state_indices.sort()
for state_idx in sorted_state_indices:
# NOTE: The constructor already delivered an init state index to 'result'.
# Thus self's init index has to be translated to result's init index.
if state_idx == self.init_state_index:
replacement[state_idx] = result.init_state_index
else:
replacement[state_idx] = state_machine_index.get()
# termination is a global state, it is not replaced by a new index
for state_idx, state in self.states.items():
new_state_idx = replacement[state_idx]
# print "##", state_idx, "-->", new_state_idx
result.states[new_state_idx] = self.states[state_idx].clone(replacement)
result.__core = deepcopy(self.__core)
return result
def do(sm, CostCoefficient):
"""
sm: StateMachine object containing all states
NOTE: The init state is **never** part of a template
combination.
CostCoefficient: Coefficient that indicates how 'costy' it is differentiate
between target states when it is different in states that
are combined into a template. Meaningful range: 0 to 3.
RETURNS: List of template combinations.
"""
assert isinstance(sm, state_machine.StateMachine)
assert isinstance(CostCoefficient, (int, long, float))
trigger_map_db = TriggerMapDB(sm, CostCoefficient)
# Build templated combinations by finding best pairs, until there is no meaningful way to
# build any clusters. TemplateCombinations of states also take part in the race.
while 1 + 1 == 2:
i, i_trigger_map_db, k, k_trigger_map_db = trigger_map_db.pop_best_matching_pair(
)
if i == None: break
# Add new element: The combined pair
new_index = index.get()
trigger_map_db[new_index] = get_combined_trigger_map(
i_trigger_map_db, involved_state_list(i_trigger_map_db, i),
k_trigger_map_db, involved_state_list(k_trigger_map_db, k))
result = []
for state_index, combination in trigger_map_db.items():
if isinstance(combination, TemplateCombination):
result.append(combination)
return result
def do(sm, CostCoefficient):
"""
sm: StateMachine object containing all states
NOTE: The init state is **never** part of a template
combination.
CostCoefficient: Coefficient that indicates how 'costy' it is differentiate
between target states when it is different in states that
are combined into a template. Meaningful range: 0 to 3.
RETURNS: List of template combinations.
"""
assert isinstance(sm, state_machine.StateMachine)
assert isinstance(CostCoefficient, (int, long, float))
trigger_map_db = TriggerMapDB(sm, CostCoefficient)
# Build templated combinations by finding best pairs, until there is no meaningful way to
# build any clusters. TemplateCombinations of states also take part in the race.
while 1 + 1 == 2:
i, i_trigger_map_db, k, k_trigger_map_db = trigger_map_db.pop_best_matching_pair()
if i == None: break
# Add new element: The combined pair
new_index = index.get()
trigger_map_db[new_index] = get_combined_trigger_map(i_trigger_map_db,
involved_state_list(i_trigger_map_db, i),
k_trigger_map_db,
involved_state_list(k_trigger_map_db, k))
result = []
for state_index, combination in trigger_map_db.items():
if isinstance(combination, TemplateCombination): result.append(combination)
return result
def add_transition(self, StartStateIdx, TriggerSet, TargetStateIdx = None, AcceptanceF = False):
"""Adds a transition from Start to Target based on a given Trigger.
TriggerSet can be of different types: ... see add_transition()
(see comment on 'State::add_transition)
RETURNS: The target state index.
"""
assert type(StartStateIdx) == long
# NOTE: The Transition Constructor is very tolerant, so no tests on TriggerSet()
# assert TriggerSet.__class__.__name__ == "NumberSet"
assert type(TargetStateIdx) == long or TargetStateIdx == None
assert type(AcceptanceF) == bool
# If target state is undefined (None) then a new one has to be created
if TargetStateIdx == None: TargetStateIdx = state_machine_index.get()
if self.states.has_key(StartStateIdx) == False: self.states[StartStateIdx] = State()
if self.states.has_key(TargetStateIdx) == False: self.states[TargetStateIdx] = State()
if AcceptanceF: self.states[TargetStateIdx].set_acceptance(True)
self.states[StartStateIdx].add_transition(TriggerSet, TargetStateIdx)
return TargetStateIdx
def _do(PathList, SMD, UniformOnlyF):
"""-- Returns generated code for all templates.
-- Sets the template_compression_db in SMD.
"""
global LanguageDB
assert type(PathList) == list
assert isinstance(SMD, StateMachineDecorator)
LanguageDB = Setup.language_db
state_db = SMD.sm().states
SM_ID = SMD.sm().get_id()
def __equal(SkeletonA, SkeletonB):
if len(SkeletonA) != len(SkeletonB): return False
for key, trigger_set in SkeletonA.items():
if SkeletonB.has_key(key) == False: return False
if not trigger_set.is_equal(SkeletonB[key]): return False
return True
def __add_to_matching_path(path, path_db):
assert isinstance(path, paths.CharacterPath)
assert isinstance(path_db, dict)
prototype_state = state_db[path.sequence()[0][0]]
for index, path_list in path_db.items():
# If uniformity is required, only such paths can be combined
# where the states are uniform with each other. Assume that
# the states inside a path are 'uniform', so only one state
# of each as to be checked.
path_list_prototype_state = state_db[path_list[0].sequence()[0][0]]
if UniformOnlyF and not prototype_state.is_equivalent(path_list_prototype_state):
continue
for candidate in path_list:
# Compare the skeletons (remaining trigger maps)
if __equal(path.skeleton(), candidate.skeleton()):
path_list.append(path)
return True
return False
# -- Sort the paths according their skeleton. Paths with the
# same skeleton will use the same pathwalker.
path_db = {}
for candidate in PathList:
assert isinstance(candidate, paths.CharacterPath)
# Is there a path with the same skeleton?
if __add_to_matching_path(candidate, path_db): continue
# If there is no equivalent path, then add a new 'prototype'
path_walker_state_index = index.get()
path_db[path_walker_state_index] = [ candidate ]
# -- Create 'PathWalkerState' objects that can mimik state machine states.
# -- Collect all indices of states involved in paths
involved_state_index_list = set([])
path_walker_list = []
for state_index, path_list in path_db.items():
# Two Scenarios for settings at state entry (last_acceptance_position, ...)
#
# (i) All state entries are uniform:
# -- Then, a representive state entry can be implemented at the
# template entry.
# -- Recursion happens to the template entry.
#
# (ii) One or more state entry are different (non-uniform):
# -- The particularities of each state entry need to be implemented
# at state entry.
# -- Recursion is routed to entries of involved states.
#
# The last state in sequence is the end target state, which cannot be
# addressed inside the pathwalker. It is the first state after the path.
for path in path_list:
involved_state_list = map(lambda info: info[0], path.sequence())
involved_state_index_list.update(involved_state_list[:-1])
# -- Determine if all involved states are uniform
prototype = template_coder.get_uniform_prototype(SMD, involved_state_index_list)
path_walker_list.append(PathWalkerState(path_list, SM_ID, state_index, prototype))
for path in path_list:
final_state_index = map(lambda info: info[0], path.sequence())[-1]
# Allways be able to route to the end-state(?)
# -- Generate code for:
# -- related variables: paths for pathwalkers
# -- state entries
# -- the pathwalkers
code = []
router_code = []
for path_walker in path_walker_list:
__path_definition(path_walker, SMD)
if __state_entries(code, path_walker, SMD):
variable_db.require("path_end_state")
__path_walker(code, path_walker, SMD)
return code, involved_state_index_list
def do(Data):
"""The generated code is very similar to the 'skipper' code. It is to be executed
as soon as a 'real' newline arrived. Then it skips whitespace until the next
non-whitepace (also newline may trigger a 'stop').
Dependent on the setup the indentation is determined.
"""
IndentationSetup = Data["indentation_setup"]
assert IndentationSetup.__class__.__name__ == "IndentationSetup"
LanguageDB = Setup.language_db
Mode = None
if IndentationSetup.containing_mode_name() != "":
Mode = lexer_mode.mode_db[IndentationSetup.containing_mode_name()]
counter_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
trigger_map = []
# If the indentation consists only of spaces, than it is 'uniform' ...
if IndentationSetup.has_only_single_spaces():
# Count indentation/column at end of run;
# simply: current position - reference_p
character_set = IndentationSetup.space_db.values()[0]
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, counter_index])
# Reference Pointer: Define Variable, Initialize, determine how to subtact.
end_procedure = \
" me->counter._indentation = (size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer) - reference_p);\n"
else:
# Count the indentation/column during the 'run'
# Add the space counters
for count, character_set in IndentationSetup.space_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("space", count, counter_index)])
# Add the grid counters
for count, character_set in IndentationSetup.grid_db.items():
for interval in character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("grid", count, counter_index)])
# Reference Pointer: Not required.
# No subtraction 'current_position - reference_p'.
# (however, we pass 'reference_p' to indentation handler)
end_procedure = ""
# Bad character detection
if IndentationSetup.bad_character_set.get().is_empty() == False:
for interval in IndentationSetup.bad_character_set.get().get_intervals(PromiseToTreatWellF=True):
trigger_map.append([interval, IndentationCounter("bad", None, counter_index)])
# Since we do not use a 'TransitionMap', there are some things we need
# to do by hand.
arrange_trigger_map(trigger_map)
local_variable_db = { "reference_p" :
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0")
}
init_reference_p = " reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer);\n" + \
" me->counter._indentation = (QUEX_TYPE_INDENTATION)0;\n"
iteration_code = transition_block.do(trigger_map,
counter_index,
DSM=None,
GotoReload_Str="goto %s;" % get_label("$reload", counter_index))
comment_str = LanguageDB["$comment"]("Skip whitespace at line begin; count indentation.")
# NOTE: Line and column number counting is off
# -- No newline can occur
# -- column number = indentation at the end of the process
end_procedure += " __QUEX_IF_COUNT_COLUMNS_ADD(me->counter._indentation);\n"
if Mode == None or Mode.default_indentation_handler_sufficient():
end_procedure += " QUEX_NAME(on_indentation)(me, me->counter._indentation, reference_p);\n"
else:
# Definition of '%s_on_indentation' in mode_classes.py.
end_procedure += " QUEX_NAME(%s_on_indentation)(me, me->counter._indentation, reference_p);\n" \
% Mode.name
# The finishing touch
prolog = blue_print(prolog_txt,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$INIT_REFERENCE_POINTER$$", init_reference_p],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
])
# The finishing touch
epilog = blue_print(epilog_txt,
[
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("SkipDelimiter$$COUNTER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_REENTRANCE$$", get_label("$entry", counter_index)],
["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
["$$RELOAD$$", get_label("$reload", counter_index)],
["$$COUNTER_INDEX$$", repr(counter_index)],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$END_PROCEDURE$$", end_procedure],
["$$BAD_CHARACTER_HANDLING$$", get_bad_character_handler(Mode, IndentationSetup, counter_index)],
])
txt = [prolog]
txt.extend(iteration_code)
# txt.append(Address("$drop-out", counter_index))
txt.append("\n")
txt.append(epilog)
return txt, local_variable_db
def get_skipper(TriggerSet):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
global template_str
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap() # (don't worry about 'drop-out-ranges' etc.)
transition_map.add_transition(TriggerSet, skipper_index)
# On buffer limit code, the skipper must transit to a dedicated reloader
iteration_code = transition_block.do(transition_map.get_trigger_map(),
skipper_index,
DSM=None,
GotoReload_Str="goto %s;" % get_label("$reload", skipper_index))
comment_str = LanguageDB["$comment"]("Skip any character in " + TriggerSet.get_utf8_string())
# Line and column number counting
prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
epilog = __lc_counting_replacements(epilog_txt, TriggerSet)
prolog = blue_print(prolog,
[
["$$DELIMITER_COMMENT$$", comment_str],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
])
epilog = blue_print(epilog,
[
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_REENTRANCE$$", get_label("$entry", skipper_index)],
["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$DROP_OUT_DIRECT$$", get_label("$drop-out", skipper_index, U=True)],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$GOTO_TERMINAL_EOF$$", get_label("$terminal-EOF", U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
])
code = [ prolog ]
code.extend(iteration_code)
code.append(epilog)
local_variable_db = {}
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code, local_variable_db
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert type(EndSequence) == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
local_variable_db = {}
global template_str
LanguageDB = Setup.language_db
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str, delimiter_length_str, delimiter_comment_str = get_character_sequence(EndSequence)
delimiter_comment_str = LanguageDB["$comment"](" Delimiter: " + delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " " + LanguageDB["$input/get-offset"](i - 1) + "\n"
txt += " " + LanguageDB["$if !="]("Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " goto %s;" % get_label("$entry", skipper_index, U=True)
txt += " " + LanguageDB["$endif"]
delimiter_remainder_test_str = txt
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start", U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label(
"$terminal-direct", IndentationCounterTerminalID, U=True
)
if OnSkipRangeOpenStr != "":
on_skip_range_open_str = OnSkipRangeOpenStr
else:
on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)
# The main part
code_str = blue_print(
template_str,
[
["$$DELIMITER$$", delimiter_str],
["$$DELIMITER_LENGTH$$", delimiter_length_str],
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$WHILE_1_PLUS_1_EQUAL_2$$", LanguageDB["$loop-start-endless"]],
["$$END_WHILE$$", LanguageDB["$loop-end"]],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_ENTRY$$", get_label("$entry", skipper_index, U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
],
)
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(
code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]],
)
if reference_p_f:
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = Variable(
"reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING",
)
return code_str, local_variable_db
def get_range_skipper(EndSequence, LanguageDB, MissingClosingDelimiterAction=""):
assert EndSequence.__class__ == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str = ""
delimiter_comment_str = " Delimiter: "
for letter in EndSequence:
delimiter_comment_str += "'%s', " % utf8.map_unicode_to_utf8(letter)
delimiter_str += "0x%X, " % letter
delimiter_length_str = "%i" % len(EndSequence)
delimiter_comment_str = LanguageDB["$comment"](delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " " + LanguageDB["$input/get-offset"](i-1) + "\n"
txt += " " + LanguageDB["$if !="]("Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " " + LanguageDB["$goto"]("$entry", skipper_index) + "\n"
txt += " " + LanguageDB["$endif"]
delimiter_remainder_test_str = txt
# The main part
code_str = blue_print(range_skipper_template,
[["$$DELIMITER$$", delimiter_str],
["$$DELIMITER_LENGTH$$", delimiter_length_str],
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$WHILE_1_PLUS_1_EQUAL_2$$", LanguageDB["$loop-start-endless"]],
["$$END_WHILE$$", LanguageDB["$loop-end"]],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$BREAK$$", LanguageDB["$break"]],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", LanguageDB["$label-def"]("$entry", skipper_index)],
["$$DROP_OUT$$", LanguageDB["$label-def"]("$drop-out", skipper_index)],
["$$GOTO_ENTRY$$", LanguageDB["$goto"]("$entry", skipper_index)],
["$$GOTO_REENTRY_PREPARATION$$", LanguageDB["$goto"]("$re-start")],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$SET_INPUT_P_BEHIND_DELIMITER$$", LanguageDB["$input/add"](len(EndSequence)-1)],
["$$MISSING_CLOSING_DELIMITER$$", MissingClosingDelimiterAction],
])
# Line and column number counting
code_str = __range_skipper_lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_DROP_OUT$$", LanguageDB["$goto"]("$drop-out", skipper_index)]])
return code_str
def get_skipper(OpenerSequence, CloserSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert OpenerSequence.__class__ == list
assert len(OpenerSequence) >= 1
assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
assert CloserSequence.__class__ == list
assert len(CloserSequence) >= 1
assert map(type, CloserSequence) == [int] * len(CloserSequence)
assert OpenerSequence != CloserSequence
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
opener_str, opener_length_str, opener_comment_str = get_character_sequence(OpenerSequence)
closer_str, closer_length_str, closer_comment_str = get_character_sequence(CloserSequence)
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, CloserSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start", U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label("$terminal-direct",
IndentationCounterTerminalID, U=True)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)
local_variable_db = {
"counter": Variable("counter", "size_t", None, "0"),
"QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p":
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
}
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
code_str = blue_print(template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER$$", opener_str],
["$$OPENER_LENGTH$$", opener_length_str],
["$$OPENER_COMMENT$$", opener_comment_str],
["$$CLOSER$$", closer_str],
["$$CLOSER_LENGTH$$", closer_length_str],
["$$CLOSER_COMMENT$$", closer_comment_str],
#
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", get_label("$entry", skipper_index)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return code_str, local_variable_db
def get_skipper(TriggerSet):
"""This function implements simple 'skipping' in the sense of passing by
characters that belong to a given set of characters--the TriggerSet.
"""
global template_str
assert TriggerSet.__class__.__name__ == "NumberSet"
assert not TriggerSet.is_empty()
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
# Mini trigger map: [ trigger set ] --> loop start
# That means: As long as characters of the trigger set appear, we go to the loop start.
transition_map = TransitionMap(
) # (don't worry about 'drop-out-ranges' etc.)
transition_map.add_transition(TriggerSet, skipper_index)
# On buffer limit code, the skipper must transit to a dedicated reloader
iteration_code = transition_block.do(transition_map.get_trigger_map(),
skipper_index,
DSM=None,
GotoReload_Str="goto %s;" %
get_label("$reload", skipper_index))
comment_str = LanguageDB["$comment"]("Skip any character in " +
TriggerSet.get_utf8_string())
# Line and column number counting
prolog = __lc_counting_replacements(prolog_txt, TriggerSet)
epilog = __lc_counting_replacements(epilog_txt, TriggerSet)
prolog = blue_print(prolog, [
["$$DELIMITER_COMMENT$$", comment_str],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
])
epilog = blue_print(
epilog,
[
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB["$if =="]("SkipDelimiter$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$LOOP_REENTRANCE$$",
get_label("$entry", skipper_index)],
["$$INPUT_EQUAL_BUFFER_LIMIT_CODE$$", LanguageDB["$BLC"]],
["$$RELOAD$$", get_label("$reload", skipper_index)],
[
"$$DROP_OUT_DIRECT$$",
get_label("$drop-out", skipper_index, U=True)
],
["$$SKIPPER_INDEX$$", "%i" % skipper_index],
["$$GOTO_TERMINAL_EOF$$",
get_label("$terminal-EOF", U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_START$$", get_label("$start", U=True)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
])
code = [prolog]
code.extend(iteration_code)
code.append(epilog)
local_variable_db = {}
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code, local_variable_db
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert type(EndSequence) == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
local_variable_db = {}
global template_str
LanguageDB = Setup.language_db
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str, \
delimiter_length_str, \
delimiter_comment_str \
= get_character_sequence(EndSequence)
delimiter_comment_str = LanguageDB["$comment"](" Delimiter: "
+ delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " " + LanguageDB["$input/get-offset"](i-1) + "\n"
txt += " " + LanguageDB["$if !="]("Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " goto %s;" % get_label("$entry", skipper_index, U=True)
txt += " " + LanguageDB["$endif"]
delimiter_remainder_test_str = txt
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start", U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label("$terminal-direct", IndentationCounterTerminalID, U=True)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)
# The main part
code_str = blue_print(template_str,
[["$$DELIMITER$$", delimiter_str],
["$$DELIMITER_LENGTH$$", delimiter_length_str],
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$WHILE_1_PLUS_1_EQUAL_2$$", LanguageDB["$loop-start-endless"]],
["$$END_WHILE$$", LanguageDB["$loop-end"]],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_ENTRY$$", get_label("$entry", skipper_index, U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]])
if reference_p_f:
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code_str, local_variable_db
def get_skipper(OpenerSequence,
CloserSequence,
Mode=None,
IndentationCounterTerminalID=None,
OnSkipRangeOpenStr=""):
assert OpenerSequence.__class__ == list
assert len(OpenerSequence) >= 1
assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
assert CloserSequence.__class__ == list
assert len(CloserSequence) >= 1
assert map(type, CloserSequence) == [int] * len(CloserSequence)
assert OpenerSequence != CloserSequence
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
opener_str, opener_length_str, opener_comment_str = get_character_sequence(
OpenerSequence)
closer_str, closer_length_str, closer_comment_str = get_character_sequence(
CloserSequence)
if not end_delimiter_is_subset_of_indentation_counter_newline(
Mode, CloserSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start",
U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label(
"$terminal-direct", IndentationCounterTerminalID, U=True)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)
local_variable_db = {
"counter":
Variable("counter", "size_t", None, "0"),
"QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p":
Variable("reference_p", "QUEX_TYPE_CHARACTER_POSITION", None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
}
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
code_str = blue_print(
template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER$$", opener_str],
["$$OPENER_LENGTH$$", opener_length_str],
["$$OPENER_COMMENT$$", opener_comment_str],
["$$CLOSER$$", closer_str],
["$$CLOSER_LENGTH$$", closer_length_str],
["$$CLOSER_COMMENT$$", closer_comment_str],
#
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$GOTO_RELOAD$$",
get_label("$reload", skipper_index)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$",
get_label("$entry", skipper_index)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return code_str, local_variable_db
def get_range_skipper(EndSequence,
LanguageDB,
MissingClosingDelimiterAction=""):
assert EndSequence.__class__ == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str = ""
delimiter_comment_str = " Delimiter: "
for letter in EndSequence:
delimiter_comment_str += "'%s', " % utf8.map_unicode_to_utf8(letter)
delimiter_str += "0x%X, " % letter
delimiter_length_str = "%i" % len(EndSequence)
delimiter_comment_str = LanguageDB["$comment"](delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " " + LanguageDB["$input/get-offset"](i - 1) + "\n"
txt += " " + LanguageDB["$if !="](
"Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " " + LanguageDB["$goto"]("$entry",
skipper_index) + "\n"
txt += " " + LanguageDB["$endif"]
delimiter_remainder_test_str = txt
# The main part
code_str = blue_print(range_skipper_template, [
["$$DELIMITER$$", delimiter_str],
["$$DELIMITER_LENGTH$$", delimiter_length_str],
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$WHILE_1_PLUS_1_EQUAL_2$$", LanguageDB["$loop-start-endless"]],
["$$END_WHILE$$", LanguageDB["$loop-end"]],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")
],
["$$BREAK$$", LanguageDB["$break"]],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", LanguageDB["$label-def"]("$entry", skipper_index)],
["$$DROP_OUT$$", LanguageDB["$label-def"]("$drop-out", skipper_index)],
["$$GOTO_ENTRY$$", LanguageDB["$goto"]("$entry", skipper_index)],
["$$GOTO_REENTRY_PREPARATION$$", LanguageDB["$goto"]("$re-start")],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
[
"$$SET_INPUT_P_BEHIND_DELIMITER$$",
LanguageDB["$input/add"](len(EndSequence) - 1)
],
["$$MISSING_CLOSING_DELIMITER$$", MissingClosingDelimiterAction],
])
# Line and column number counting
code_str = __range_skipper_lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(code_str, [[
"$$SKIPPER_INDEX$$", __nice(skipper_index)
], ["$$GOTO_DROP_OUT$$", LanguageDB["$goto"]("$drop-out", skipper_index)]])
return code_str
