def do(SMD, UniformOnlyF):
"""UniformOnlyF --> Accept only uniform states in path.
(This must be done by the 'analyzer module' too.)
RETURNS: Array 'x'
x[0] transition target definitions in terms of a
local variable database
x[1] code for templates and state entries
x[2] involved_state_index_list
"""
assert isinstance(SMD, StateMachineDecorator)
# (1) Find possible state combinations
path_list = paths.do(SMD.sm(), UniformOnlyF)
for path in path_list:
involved_state_index_list = map(lambda info: info[0], path.sequence())
# (2) Implement code for template combinations
code, involved_state_index_list = _do(path_list, SMD, UniformOnlyF)
if len(involved_state_index_list) != 0:
variable_db.require("path_iterator")
transition_block.format_this(code)
return code, \
involved_state_index_list
def __path_definition(PathWalker, SMD):
"""Defines the transition targets for each involved state.
"""
PathWalkerID = PathWalker.core().state_index
PathList = PathWalker.path_list()
PathN = len(PathList)
def __get_state_list():
result = []
for path in PathList:
Sequence = path.sequence()
for state_index, dummy in Sequence[:-1]:
result.append("QUEX_LABEL(%i), " % get_address("$entry", state_index, R=True, U=True))
end_state_index = Sequence[-1][0]
# If 'end_state_index' belongs to a non-implemented dead-end-state
# => 'get_real_address()' returns address of immediately entered terminal
end_state_label = "QUEX_LABEL(%i), " % get_address("$entry", end_state_index, R=True, U=True)
result.append(end_state_label)
return result
memory = ["\n"]
memory_index = 0
for path in PathList:
Sequence = path.sequence()
L = len(Sequence)
# Last element of sequence contains only the 'end state'.
memory.append(" ")
sequence_str = []
for state_index, character in Sequence[:-1]:
memory.append("%i, " % character)
sequence_str.append(Interval(character).get_utf8_string())
memory.append("QUEX_SETTING_PATH_TERMINATION_CODE, ")
memory.append(LanguageDB["$comment"]("".join(sequence_str)) + "\n")
variable_db.require("path_%i",
Initial = "path_walker_%i_base + %i" % (PathWalkerID, memory_index),
Index = path.index())
memory_index += L
# (*) Path Walker Basis
# The 'base' must be defined before all --> PriorityF (see table in variable_db)
variable_db.require_array("path_walker_%i_base",
ElementN = memory_index + 1,
Initial = ["{"] + memory + ["\n }"],
Index = PathWalkerID)
# (*) The State Information for each path step
if not PathWalker.uniform_state_entries_f():
state_list = __get_state_list()
variable_db.require_array("path_walker_%i_state",
ElementN = len(state_list),
Initial = ["{"] + state_list + ["}"],
Index = PathWalkerID)
def __local_variable_definition(PostContextID_List, PreContextID_List, LanguageDB):
PostContextN = len(PostContextID_List)
variable_db.require("last_acceptance",
Initial="QUEX_LABEL(%i)" % get_address("$terminal-FAILURE"))
variable_db.require("last_acceptance_input_position")
variable_db.require_array("post_context_start_position",
ElementN = PostContextN,
Initial = "{ " + ("0, " * (PostContextN - 1) + "0") + "}")
variable_db.require("PostContextStartPositionN",
Initial = "(size_t)" + repr(PostContextN))
variable_db.require("input")
# -- pre-condition fulfillment flags
for sm_id in PreContextID_List:
variable_db.require("pre_context_%i_fulfilled_f", Index = sm_id)
return LanguageDB["$local-variable-defs"](variable_db.get())
def __local_variable_definition(PostContextID_List, PreContextID_List,
LanguageDB):
PostContextN = len(PostContextID_List)
variable_db.require("last_acceptance",
Initial="QUEX_LABEL(%i)" %
get_address("$terminal-FAILURE"))
variable_db.require("last_acceptance_input_position")
variable_db.require_array("post_context_start_position",
ElementN=PostContextN,
Initial="{ " + ("0, " *
(PostContextN - 1) + "0") + "}")
variable_db.require("PostContextStartPositionN",
Initial="(size_t)" + repr(PostContextN))
variable_db.require("input")
# -- pre-condition fulfillment flags
for sm_id in PreContextID_List:
variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id)
return LanguageDB["$local-variable-defs"](variable_db.get())
def do(SMD, CostCoefficient):
"""RETURNS: Array 'x'
x[0] transition target definitions in terms of a
local variable database
x[1] code for templates and state entries
x[2] state router for template targets
x[3] involved_state_index_list
"""
assert isinstance(SMD, StateMachineDecorator)
# (1) Find possible state combinations
combination_list = templates.do(SMD.sm(), CostCoefficient)
# (2) Implement code for template combinations
code, \
involved_state_index_list = _do(combination_list, SMD)
if len(involved_state_index_list) != 0:
variable_db.require("template_state_key")
return code, involved_state_index_list
def do(SMD, CostCoefficient):
"""RETURNS: Array 'x'
x[0] transition target definitions in terms of a
local variable database
x[1] code for templates and state entries
x[2] state router for template targets
x[3] involved_state_index_list
"""
assert isinstance(SMD, StateMachineDecorator)
# (1) Find possible state combinations
combination_list = templates.do(SMD.sm(), CostCoefficient)
# (2) Implement code for template combinations
code, \
involved_state_index_list = _do(combination_list, SMD)
if len(involved_state_index_list) != 0:
variable_db.require("template_state_key")
return code, involved_state_index_list
def analyzer_function_get(self, RequiredLocalVariablesDB):
routed_address_set = set([])
function_body = []
# (*) Core State Machine
# All pattern detectors combined in single forward analyzer
dsm = StateMachineDecorator(self.sm,
self.state_machine_name,
self.post_contexted_sm_id_list,
BackwardLexingF=False,
BackwardInputPositionDetectionF=False)
variable_db.init(RequiredLocalVariablesDB)
init_address_handling(dsm.get_direct_transition_to_terminal_db())
# (*) Pre Context State Machine
# All pre-context combined in single backward analyzer.
if self.pre_context_sm_list != []:
code = self.__backward_analyzer()
function_body.extend(code)
# -- now, consider core state machine
code = self.__forward_analyzer(dsm)
function_body.extend(code)
# At this point in time, the function body is completely defined
routed_address_set = get_address_set_subject_to_routing()
if len(routed_address_set) != 0 or is_label_referenced(
"$state-router"):
routed_state_info_list = state_router.get_info(
routed_address_set, dsm)
function_body.append(state_router.do(routed_state_info_list))
variable_db.require("target_state_index",
Condition_ComputedGoto=False)
if is_label_referenced("$reload-FORWARD") or is_label_referenced(
"$reload-BACKWARD"):
variable_db.require("target_state_else_index")
variable_db.require("target_state_index")
# Following function refers to the global 'variable_db'
variable_definitions = self.language_db["$variable-definitions"](
self.post_contexted_sm_id_list, self.pre_context_sm_id_list,
self.language_db)
# (*) Pack Pre-Context and Core State Machine into a single function
analyzer_function = self.language_db["$analyzer-func"](
self.state_machine_name,
self.analyzer_state_class_name,
self.stand_alone_analyzer_f,
variable_definitions,
function_body,
self.mode_name_list,
LanguageDB=self.language_db)
return get_plain_strings(analyzer_function)
def backward_detector_function_get(self, sm):
assert sm.get_orphaned_state_index_list() == []
dsm = StateMachineDecorator(sm,
"BACKWARD_DETECTOR_" + repr(sm.get_id()),
PostContextSM_ID_List = [],
BackwardLexingF = True,
BackwardInputPositionDetectionF = True)
variable_db.init()
init_address_handling(dsm.get_direct_transition_to_terminal_db())
function_body = state_machine_coder.do(dsm)
comment = []
if Setup.comment_state_machine_transitions_f:
comment = Setup.language_db["$ml-comment"]("BEGIN: BACKWARD DETECTOR STATE MACHINE\n" + \
sm.get_string(NormalizeF=False) + \
"\nEND: BACKWARD DETECTOR STATE MACHINE")
comment.append("\n")
# -- input position detectors simply the next 'catch' and return
terminal = []
terminal.append("\n")
terminal.append(" __quex_assert_no_passage();\n")
terminal.append(get_label("$terminal-general-bw") + ":\n")
terminal.append(" " + self.language_db["$input/seek_position"]("end_of_core_pattern_position") + "\n")
terminal.append(" " + self.language_db["$input/increment"] + "\n")
terminal.append(" return;\n")
routed_address_set = get_address_set_subject_to_routing()
state_router_txt = ""
if len(routed_address_set) != 0:
routed_state_info_list = state_router.get_info(routed_address_set, dsm)
state_router_txt = state_router.do(routed_state_info_list)
variable_db.require("target_state_index", Condition_ComputedGoto=False)
variable_db.require("input")
variable_db.require("end_of_core_pattern_position")
local_variable_definition = self.language_db["$local-variable-defs"](variable_db.get())
# Put all things together
txt = []
txt.append(bwd_prolog.replace("$$ID$$", repr(sm.get_id()).replace("L", "")))
txt.extend(local_variable_definition)
txt.extend(comment)
txt.extend(function_body)
txt.extend(terminal)
txt.append(state_router_txt)
txt.append(bwd_epilog.replace("$$INIT_STATE_ID$$", get_label_of_address(sm.init_state_index)))
return get_plain_strings(txt)
def analyzer_function_get(self, RequiredLocalVariablesDB):
routed_address_set = set([])
function_body = []
# (*) Core State Machine
# All pattern detectors combined in single forward analyzer
dsm = StateMachineDecorator(self.sm,
self.state_machine_name,
self.post_contexted_sm_id_list,
BackwardLexingF=False,
BackwardInputPositionDetectionF=False)
variable_db.init(RequiredLocalVariablesDB)
init_address_handling(dsm.get_direct_transition_to_terminal_db())
# (*) Pre Context State Machine
# All pre-context combined in single backward analyzer.
if self.pre_context_sm_list != []:
code = self.__backward_analyzer()
function_body.extend(code)
# -- now, consider core state machine
code = self.__forward_analyzer(dsm)
function_body.extend(code)
# At this point in time, the function body is completely defined
routed_address_set = get_address_set_subject_to_routing()
if len(routed_address_set) != 0 or is_label_referenced("$state-router"):
routed_state_info_list = state_router.get_info(routed_address_set, dsm)
function_body.append(state_router.do(routed_state_info_list))
variable_db.require("target_state_index", Condition_ComputedGoto=False)
if is_label_referenced("$reload-FORWARD") or is_label_referenced("$reload-BACKWARD"):
variable_db.require("target_state_else_index")
variable_db.require("target_state_index")
# Following function refers to the global 'variable_db'
variable_definitions = self.language_db["$variable-definitions"](self.post_contexted_sm_id_list,
self.pre_context_sm_id_list,
self.language_db)
# (*) Pack Pre-Context and Core State Machine into a single function
analyzer_function = self.language_db["$analyzer-func"](self.state_machine_name,
self.analyzer_state_class_name,
self.stand_alone_analyzer_f,
variable_definitions,
function_body,
self.mode_name_list,
LanguageDB=self.language_db)
return get_plain_strings(analyzer_function)
def _do(PathList, SMD, UniformOnlyF):
"""-- Returns generated code for all templates.
-- Sets the template_compression_db in SMD.
"""
global LanguageDB
assert type(PathList) == list
assert isinstance(SMD, StateMachineDecorator)
LanguageDB = Setup.language_db
state_db = SMD.sm().states
SM_ID = SMD.sm().get_id()
def __equal(SkeletonA, SkeletonB):
if len(SkeletonA) != len(SkeletonB): return False
for key, trigger_set in SkeletonA.items():
if SkeletonB.has_key(key) == False: return False
if not trigger_set.is_equal(SkeletonB[key]): return False
return True
def __add_to_matching_path(path, path_db):
assert isinstance(path, paths.CharacterPath)
assert isinstance(path_db, dict)
prototype_state = state_db[path.sequence()[0][0]]
for index, path_list in path_db.items():
# If uniformity is required, only such paths can be combined
# where the states are uniform with each other. Assume that
# the states inside a path are 'uniform', so only one state
# of each as to be checked.
path_list_prototype_state = state_db[path_list[0].sequence()[0][0]]
if UniformOnlyF and not prototype_state.is_equivalent(path_list_prototype_state):
continue
for candidate in path_list:
# Compare the skeletons (remaining trigger maps)
if __equal(path.skeleton(), candidate.skeleton()):
path_list.append(path)
return True
return False
# -- Sort the paths according their skeleton. Paths with the
# same skeleton will use the same pathwalker.
path_db = {}
for candidate in PathList:
assert isinstance(candidate, paths.CharacterPath)
# Is there a path with the same skeleton?
if __add_to_matching_path(candidate, path_db): continue
# If there is no equivalent path, then add a new 'prototype'
path_walker_state_index = index.get()
path_db[path_walker_state_index] = [ candidate ]
# -- Create 'PathWalkerState' objects that can mimik state machine states.
# -- Collect all indices of states involved in paths
involved_state_index_list = set([])
path_walker_list = []
for state_index, path_list in path_db.items():
# Two Scenarios for settings at state entry (last_acceptance_position, ...)
#
# (i) All state entries are uniform:
# -- Then, a representive state entry can be implemented at the
# template entry.
# -- Recursion happens to the template entry.
#
# (ii) One or more state entry are different (non-uniform):
# -- The particularities of each state entry need to be implemented
# at state entry.
# -- Recursion is routed to entries of involved states.
#
# The last state in sequence is the end target state, which cannot be
# addressed inside the pathwalker. It is the first state after the path.
for path in path_list:
involved_state_list = map(lambda info: info[0], path.sequence())
involved_state_index_list.update(involved_state_list[:-1])
# -- Determine if all involved states are uniform
prototype = template_coder.get_uniform_prototype(SMD, involved_state_index_list)
path_walker_list.append(PathWalkerState(path_list, SM_ID, state_index, prototype))
for path in path_list:
final_state_index = map(lambda info: info[0], path.sequence())[-1]
# Allways be able to route to the end-state(?)
# -- Generate code for:
# -- related variables: paths for pathwalkers
# -- state entries
# -- the pathwalkers
code = []
router_code = []
for path_walker in path_walker_list:
__path_definition(path_walker, SMD)
if __state_entries(code, path_walker, SMD):
variable_db.require("path_end_state")
__path_walker(code, path_walker, SMD)
return code, involved_state_index_list
def backward_detector_function_get(self, sm):
assert sm.get_orphaned_state_index_list() == []
dsm = StateMachineDecorator(sm,
"BACKWARD_DETECTOR_" + repr(sm.get_id()),
PostContextSM_ID_List=[],
BackwardLexingF=True,
BackwardInputPositionDetectionF=True)
variable_db.init()
init_address_handling(dsm.get_direct_transition_to_terminal_db())
function_body = state_machine_coder.do(dsm)
comment = []
if Setup.comment_state_machine_transitions_f:
comment = Setup.language_db["$ml-comment"]("BEGIN: BACKWARD DETECTOR STATE MACHINE\n" + \
sm.get_string(NormalizeF=False) + \
"\nEND: BACKWARD DETECTOR STATE MACHINE")
comment.append("\n")
# -- input position detectors simply the next 'catch' and return
terminal = []
terminal.append("\n")
terminal.append(" __quex_assert_no_passage();\n")
terminal.append(get_label("$terminal-general-bw") + ":\n")
terminal.append(" " + self.language_db["$input/seek_position"]
("end_of_core_pattern_position") + "\n")
terminal.append(" " + self.language_db["$input/increment"] + "\n")
terminal.append(" return;\n")
routed_address_set = get_address_set_subject_to_routing()
state_router_txt = ""
if len(routed_address_set) != 0:
routed_state_info_list = state_router.get_info(
routed_address_set, dsm)
state_router_txt = state_router.do(routed_state_info_list)
variable_db.require("target_state_index",
Condition_ComputedGoto=False)
variable_db.require("input")
variable_db.require("end_of_core_pattern_position")
local_variable_definition = self.language_db["$local-variable-defs"](
variable_db.get())
# Put all things together
txt = []
txt.append(
bwd_prolog.replace("$$ID$$",
repr(sm.get_id()).replace("L", "")))
txt.extend(local_variable_definition)
txt.extend(comment)
txt.extend(function_body)
txt.extend(terminal)
txt.append(state_router_txt)
txt.append(
bwd_epilog.replace("$$INIT_STATE_ID$$",
get_label_of_address(sm.init_state_index)))
return get_plain_strings(txt)
