def _get_source_code(CcFactory, analyzer, terminal_list):
"""RETURNS: String containing source code for the 'loop'.
-- The source code for the (looping) state machine.
-- The terminals which contain counting actions.
Also, it requests variable definitions as they are required.
"""
txt = []
txt.extend(
generator.do_analyzer(analyzer)
)
txt.extend(
generator.do_terminals(terminal_list, analyzer)
)
if analyzer.engine_type.subject_to_reload():
txt.extend(
generator.do_reload_procedure(analyzer)
)
if CcFactory.requires_reference_p():
variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
if Setup.buffer_codec.variable_character_sizes_f():
variable_db.require("character_begin_p")
return txt
def __code_pre_context_state_machine(self):
LanguageDB = self.language_db
if len(self.pre_context_sm_list) == 0: return []
assert len(self.pre_context_sm.get_orphaned_state_index_list()) == 0
txt = []
if Setup.comment_state_machine_f:
LanguageDB.ML_COMMENT(txt,
"BEGIN: PRE-CONTEXT STATE MACHINE\n" + \
self.pre_context_sm.get_string(NormalizeF=False) + \
"END: PRE-CONTEXT STATE MACHINE")
txt.append("\n") # For safety: New content may have to start in a newline, e.g. "#ifdef ..."
analyzer = analyzer_generator.do(self.pre_context_sm, E_EngineTypes.BACKWARD_PRE_CONTEXT)
msg = state_machine_coder.do(analyzer)
txt.extend(msg)
txt.append("\n%s" % LanguageDB.LABEL(E_StateIndices.END_OF_PRE_CONTEXT_CHECK))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % LanguageDB.INPUT_P_TO_LEXEME_START())
for sm_id in self.pre_context_sm_id_list:
variable_db.require("pre_context_%i_fulfilled_f", Index = sm_id)
return txt
def require_data(TState, TheAnalyzer):
"""Requires all variables which are necessary to implement the TemplateState.
NOTE: The target schemes are required from inside 'require_scheme_variable()'
in quex.engine.generator.mega_state.core.py
"""
variable_db.require("state_key")
def __code_pre_context_state_machine(self):
LanguageDB = self.language_db
if len(self.pre_context_sm_list) == 0: return []
assert len(self.pre_context_sm.get_orphaned_state_index_list()) == 0
txt = []
if Setup.comment_state_machine_f:
LanguageDB.ML_COMMENT(txt,
"BEGIN: PRE-CONTEXT STATE MACHINE\n" + \
self.pre_context_sm.get_string(NormalizeF=False) + \
"END: PRE-CONTEXT STATE MACHINE")
txt.append(
"\n"
) # For safety: New content may have to start in a newline, e.g. "#ifdef ..."
analyzer = analyzer_generator.do(self.pre_context_sm,
E_EngineTypes.BACKWARD_PRE_CONTEXT)
msg = state_machine_coder.do(analyzer)
txt.extend(msg)
txt.append("\n%s" %
LanguageDB.LABEL(E_StateIndices.END_OF_PRE_CONTEXT_CHECK))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % LanguageDB.INPUT_P_TO_LEXEME_START())
for sm_id in self.pre_context_sm_id_list:
variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id)
return txt
def require_position_registers(TheAnalyzer):
"""Require an array to store input positions. This has later to be
implemented as 'variables'. Position registers are exclusively used
for post-context restore. No other engine than FORWARD would require
those.
"""
if not TheAnalyzer.engine_type.is_FORWARD():
return
if TheAnalyzer.position_register_map is None:
position_register_n = 0
else:
position_register_n = len(
set(TheAnalyzer.position_register_map.itervalues()))
if position_register_n != 0:
initial_array = "{ " + ("0, " * (position_register_n - 1) + "0") + "}"
else:
# Implement a dummy array (except that there is no reload)
if Setup.buffer_based_analyzis_f: return
initial_array = "(void*)0"
variable_db.require_array("position",
ElementN=position_register_n,
Initial=initial_array)
variable_db.require("PositionRegisterN",
Initial="(size_t)%i" % position_register_n)
def do_pre_context(SM, PreContextSmIdList):
"""Pre-context detecting state machine (backward).
---------------------------------------------------------------------------
Micro actions are: pre-context fullfilled_f
DropOut --> Begin of 'main' state machine.
BLC --> ReloadStateBackward
EndOfStream --> 'error'
Variables (potentially) required:
pre_context_fulfilled_f[N] --> Array of flags for pre-context
indication.
RETURNS: [0] generated code text
[1] reload state BACKWARD, to be generated later.
"""
if SM is None:
return [], None
txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT)
txt.append(
"\n%s:" %
dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check()))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START())
for sm_id in PreContextSmIdList:
variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id)
variable_db.require("input")
return txt, analyzer
def require_position_registers(TheAnalyzer):
"""Require an array to store input positions. This has later to be
implemented as 'variables'. Position registers are exclusively used
for post-context restore. No other engine than FORWARD would require
those.
"""
if not TheAnalyzer.engine_type.is_FORWARD():
return
if TheAnalyzer.position_register_map is None:
position_register_n = 0
else:
position_register_n = len(set(TheAnalyzer.position_register_map.itervalues()))
if position_register_n != 0:
initial_array = "{ " + ("0, " * (position_register_n - 1) + "0") + "}"
else:
# Implement a dummy array (except that there is no reload)
if Setup.buffer_based_analyzis_f: return
initial_array = "(void*)0"
variable_db.require_array("position", ElementN = position_register_n,
Initial = initial_array)
variable_db.require("PositionRegisterN",
Initial = "(size_t)%i" % position_register_n)
def do_pre_context(SM, PreContextSmIdList):
"""Pre-context detecting state machine (backward).
---------------------------------------------------------------------------
Micro actions are: pre-context fullfilled_f
DropOut --> Begin of 'main' state machine.
BLC --> ReloadStateBackward
EndOfStream --> 'error'
Variables (potentially) required:
pre_context_fulfilled_f[N] --> Array of flags for pre-context
indication.
RETURNS: [0] generated code text
[1] reload state BACKWARD, to be generated later.
"""
if SM is None:
return [], None
txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT)
txt.append("\n%s:" % dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check()))
# -- set the input stream back to the real current position.
# during backward lexing the analyzer went backwards, so it needs to be reset.
txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START())
for sm_id in PreContextSmIdList:
variable_db.require("pre_context_%i_fulfilled_f", Index = sm_id)
variable_db.require("input")
return txt, analyzer
def require_data(TState, TheAnalyzer):
"""Requires all variables which are necessary to implement the TemplateState.
NOTE: The target schemes are required from inside 'require_scheme_variable()'
in quex.engine.generator.mega_state.core.py
"""
variable_db.require("state_key")
def do(self, RequiredLocalVariablesDB):
LanguageDB = Setup.language_db
# (*) Initialize the label and variable trackers
variable_db.init(RequiredLocalVariablesDB)
variable_db.require("input")
init_address_handling({})
# (*) Pre Context State Machine
# (If present: All pre-context combined in single backward analyzer.)
pre_context = self.__code_pre_context_state_machine()
# (*) Main State Machine -- try to match core patterns
main = self.__code_main_state_machine()
# (*) Backward input position detection
# (Seldomly present -- only for Pseudo-Ambiguous Post Contexts)
bipd = self.__code_backward_input_position_detection()
# (*) Determine required labels and variables
routed_address_set = get_address_set_subject_to_routing()
routed_address_set.add(get_address("$terminal-EOF", U=True))
routed_state_info_list = state_router_generator.get_info(routed_address_set)
state_router = [ state_router_generator.do(routed_state_info_list) ]
variable_db.require("target_state_index", Condition_ComputedGoto=False)
if is_label_referenced("$reload-FORWARD") or is_label_referenced("$reload-BACKWARD"):
variable_db.require("target_state_else_index")
variable_db.require("target_state_index")
# Following function refers to the global 'variable_db'
variable_definitions = self.language_db.VARIABLE_DEFINITIONS(variable_db)
function_body = []
function_body.extend(pre_context) # implementation of pre-contexts (if there are some)
function_body.extend(main) # main pattern matcher
function_body.extend(bipd) # (seldom != empty; only for pseudo-ambiguous post contexts)
function_body.extend(state_router) # route to state by index (only if no computed gotos)
# (*) Pack Pre-Context and Core State Machine into a single function
analyzer_function = self.language_db["$analyzer-func"](self.state_machine_name,
Setup,
variable_definitions,
function_body,
self.mode_name_list)
txt = [ LanguageDB["$header-definitions"](LanguageDB, self.on_after_match) ]
txt += get_plain_strings(analyzer_function)
for i, element in enumerate(txt):
if not isinstance(element, (str, unicode)):
print element.__class__.__name__
for k in range(max(0,i-10)):
print "before:", k, txt[k]
for k in range(i+1, min(i+10, len(txt))):
print "after: ", k, txt[k]
assert False
return txt
def do_reload_procedure(TheAnalyzer):
"""Lazy (delayed) code generation of the forward and backward reloaders.
Any state who needs reload may 'register' in a reloader. This registering may
happen after the code generation of forward or backward state machine.
"""
# Variables that tell where to go after reload success and reload failure
if TheAnalyzer is None: return []
elif not TheAnalyzer.engine_type.subject_to_reload(): return []
elif TheAnalyzer.reload_state is None: return []
elif TheAnalyzer.reload_state_extern_f: return []
variable_db.require("target_state_else_index") # upon reload failure
variable_db.require("target_state_index") # upon reload success
require_position_registers(TheAnalyzer)
return reload_state_coder.do(TheAnalyzer.reload_state)
def do_reload_procedure(TheAnalyzer):
"""Lazy (delayed) code generation of the forward and backward reloaders.
Any state who needs reload may 'register' in a reloader. This registering may
happen after the code generation of forward or backward state machine.
"""
# Variables that tell where to go after reload success and reload failure
if TheAnalyzer is None: return []
elif not TheAnalyzer.engine_type.subject_to_reload(): return []
elif TheAnalyzer.reload_state is None: return []
elif TheAnalyzer.reload_state_extern_f: return []
variable_db.require("target_state_else_index") # upon reload failure
variable_db.require("target_state_index") # upon reload success
require_position_registers(TheAnalyzer)
return reload_state_coder.do(TheAnalyzer.reload_state)
def __code_main_state_machine(self):
assert len(self.sm.get_orphaned_state_index_list()) == 0
LanguageDB = self.language_db
txt = []
# -- [optional] comment state machine transitions
if Setup.comment_state_machine_f:
LanguageDB.ML_COMMENT(txt,
"BEGIN: STATE MACHINE\n" + \
self.sm.get_string(NormalizeF=False) + \
"END: STATE MACHINE")
txt.append(
"\n"
) # For safety: New content may have to start in a newline, e.g. "#ifdef ..."
# -- implement the state machine itself
analyzer = analyzer_generator.do(self.sm, E_EngineTypes.FORWARD)
state_machine_code = state_machine_coder.do(analyzer)
txt.extend(state_machine_code)
lexeme_null_object_name = "QUEX_NAME(LexemeNullObject)"
if Setup.external_lexeme_null_object != "":
lexeme_null_object_name = Setup.external_lexeme_null_object
# -- terminal states: execution of pattern actions
terminal_code = LanguageDB["$terminal-code"](
self.state_machine_name, self.action_db, self.on_failure_action,
self.on_end_of_stream_action, self.pre_context_sm_id_list,
self.language_db, variable_db, self.on_after_match,
lexeme_null_object_name)
txt.extend(terminal_code)
N = len(set(analyzer.position_register_map.values()))
if len(analyzer.position_register_map) == 0:
variable_db.require("position", Initial="(void*)0x0", Type="void*")
variable_db.require("PositionRegisterN", Initial="(size_t)%i" % N)
else:
variable_db.require_array("position",
ElementN=N,
Initial="{ " + ("0, " * (N - 1) + "0") +
"}")
variable_db.require("PositionRegisterN", Initial="(size_t)%i" % N)
if analyzer.last_acceptance_variable_required():
variable_db.require("last_acceptance")
# -- reload definition (forward, backward, init state reload)
code = LanguageDB.RELOAD()
txt.extend(code)
return txt
def __code_main_state_machine(self):
assert len(self.sm.get_orphaned_state_index_list()) == 0
LanguageDB = self.language_db
txt = []
# -- [optional] comment state machine transitions
if Setup.comment_state_machine_f:
LanguageDB.ML_COMMENT(txt,
"BEGIN: STATE MACHINE\n" + \
self.sm.get_string(NormalizeF=False) + \
"END: STATE MACHINE")
txt.append("\n") # For safety: New content may have to start in a newline, e.g. "#ifdef ..."
# -- implement the state machine itself
analyzer = analyzer_generator.do(self.sm, E_EngineTypes.FORWARD)
state_machine_code = state_machine_coder.do(analyzer)
txt.extend(state_machine_code)
lexeme_null_object_name = "QUEX_NAME(LexemeNullObject)"
if Setup.external_lexeme_null_object != "":
lexeme_null_object_name = Setup.external_lexeme_null_object
# -- terminal states: execution of pattern actions
terminal_code = LanguageDB["$terminal-code"](self.state_machine_name,
self.action_db,
self.on_failure_action,
self.on_end_of_stream_action,
self.pre_context_sm_id_list,
self.language_db,
variable_db,
self.on_after_match,
lexeme_null_object_name)
txt.extend(terminal_code)
N = len(set(analyzer.position_register_map.values()))
if len(analyzer.position_register_map) == 0:
variable_db.require("position", Initial = "(void*)0x0", Type = "void*")
variable_db.require("PositionRegisterN", Initial = "(size_t)%i" % N)
else:
variable_db.require_array("position", ElementN = N,
Initial = "{ " + ("0, " * (N - 1) + "0") + "}")
variable_db.require("PositionRegisterN", Initial = "(size_t)%i" % N)
if analyzer.last_acceptance_variable_required():
variable_db.require("last_acceptance")
# -- reload definition (forward, backward, init state reload)
code = LanguageDB.RELOAD()
txt.extend(code)
return txt
def _get_source_code(CcFactory, analyzer, terminal_list):
"""RETURNS: String containing source code for the 'loop'.
-- The source code for the (looping) state machine.
-- The terminals which contain counting actions.
Also, it requests variable definitions as they are required.
"""
txt = []
txt.extend(generator.do_analyzer(analyzer))
txt.extend(generator.do_terminals(terminal_list, analyzer))
if analyzer.engine_type.subject_to_reload():
txt.extend(generator.do_reload_procedure(analyzer))
if CcFactory.requires_reference_p():
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
if Setup.buffer_codec.variable_character_sizes_f():
variable_db.require("character_begin_p")
return txt
def do_main(SM):
"""Main pattern matching state machine (forward).
---------------------------------------------------------------------------
Micro actions are: line/column number counting, position set/reset,
last acceptance setting/reset, lexeme start pointer set/reset, setting
terminating zero for lexeme/reset, setting last character.
DropOut --> FAILURE
BLC --> ReloadStateForward
EndOfStream --> END_OF_STREAM
Variables (potentially) required:
position, PositionRegisterN, last_acceptance, input.
"""
txt, analyzer = do_state_machine(SM, engine.Class_FORWARD())
if analyzer.last_acceptance_variable_required():
variable_db.require("last_acceptance")
return txt, analyzer
def do_main(SM):
"""Main pattern matching state machine (forward).
---------------------------------------------------------------------------
Micro actions are: line/column number counting, position set/reset,
last acceptance setting/reset, lexeme start pointer set/reset, setting
terminating zero for lexeme/reset, setting last character.
DropOut --> FAILURE
BLC --> ReloadStateForward
EndOfStream --> END_OF_STREAM
Variables (potentially) required:
position, PositionRegisterN, last_acceptance, input.
"""
txt, analyzer = do_state_machine(SM, engine.Class_FORWARD())
if analyzer.last_acceptance_variable_required():
variable_db.require("last_acceptance")
return txt, analyzer
def __character_sequences():
result = ["{\n"]
offset = 0
for path_id, path in enumerate(PWState.path_list):
# Commenting the transition sequence is not dangerous. 'COMMENT' eliminates
# comment delimiters if they would appear in the sequence_str.
# sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1])
# memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n")
# Last element of sequence contains only the 'end state'.
result.append(" ")
result.extend("%i, " % x.transition_char_to_next
for x in path[:-1])
result.append("QUEX_SETTING_PATH_TERMINATION_CODE,")
result.append("\n")
variable_db.require("path_walker_%i_path_%i",
Initial="path_walker_%i_path_base + %i" %
(PWState.index, offset),
Index=(PWState.index, path_id))
offset += len(path)
result.append(" }")
return offset, result
def __character_sequences():
result = ["{\n"]
offset = 0
for path_id, path in enumerate(PWState.path_list):
# Commenting the transition sequence is not dangerous. 'COMMENT' eliminates
# comment delimiters if they would appear in the sequence_str.
# sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1])
# memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n")
# Last element of sequence contains only the 'end state'.
result.append(" ")
result.extend("%i, " % x.transition_char_to_next for x in path[:-1])
result.append("QUEX_SETTING_PATH_TERMINATION_CODE,")
result.append("\n")
variable_db.require(
"path_walker_%i_path_%i",
Initial="path_walker_%i_path_base + %i" % (PWState.index, offset),
Index=(PWState.index, path_id),
)
offset += len(path)
result.append(" }")
return offset, result
def require_data(PWState, TheAnalyzer):
"""Defines the transition targets for each involved state.
"""
variable_db.require("path_iterator")
def __door_adr_sequences(PWState):
result = ["{\n"]
length = 0
for path_id, door_id_sequence in enumerate(PWState.door_id_sequence_list):
# NOTE: For all states in the path the 'from_state_index, to_state_index' can
# be determined, **except** for the FIRST state in the path. Thus for
# this state the 'door' cannot be determined in case that it is
# "not uniform_doors_f()".
#
# However, the only occasion where the FIRST state in the path may be
# used is reload during the FIRST state. The reload adapts the positions
# and acceptances are not changed. So, we can use the common entry
# to the first state as a reference here.
## print "#DoorID, Adr:", [(door_id, Lng.ADDRESS_BY_DOOR_ID(door_id)) for door_id in door_id_sequence]
result.append(" ")
result.append("/* Padding */0x0, ")
result.extend("QUEX_LABEL(%i), " % dial_db.get_address_by_door_id(door_id, RoutedF=True)
for door_id in door_id_sequence)
result.append("\n")
length += len(door_id_sequence) + 1 # 1 padding element
result.append(" }");
return length, result
def __character_sequences(PathList):
result = ["{\n"]
offset = 0
for path_id, step_list in enumerate(PathList):
## Commenting the transition sequence is not dangerous. 'COMMENT' eliminates
## comment delimiters if they would appear in the sequence_str.
## sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), step_list[:-1])
## memory.append(Lng.COMMENT("".join(sequence_str)) + "\n")
result.append(" ")
result.extend("%i, " % x.trigger for x in step_list[:-1])
result.append("QUEX_SETTING_PATH_TERMINATION_CODE,")
result.append("\n")
offset += len(step_list)
result.append(" }")
return offset, result
# (*) Path Walker Basis
# The 'base' must be defined before all --> PriorityF (see table in variable_db)
element_n, character_sequence_str = __character_sequences(PWState.path_list)
offset = 0
for path_id, step_list in enumerate(PWState.path_list):
variable_db.require("path_walker_%i_path_%i",
Initial = "&path_walker_%i_path_base[%i]" % (PWState.index, offset),
Index = (PWState.index, path_id))
offset += len(step_list)
variable_db.require_array("path_walker_%i_path_base",
ElementN = element_n,
Initial = character_sequence_str,
Index = PWState.index)
# (*) The State Information for each path step
if PWState.uniform_door_id is None:
element_n, door_adr_sequence_str = __door_adr_sequences(PWState)
variable_db.require_array("path_walker_%i_state_base",
ElementN = element_n,
Initial = door_adr_sequence_str,
Index = PWState.index)
# The path_iterator is incremented before the 'goto', thus
# 'path_iterator - (path_base + 1)' gives actually the correct offset.
# We define a variable for that, for elegance.
variable_db.require("path_walker_%i_reference",
Initial = "path_walker_%i_path_base" % PWState.index,
Index = (PWState.index))
def require_data(PWState, TheAnalyzer):
"""Defines the transition targets for each involved state.
"""
LanguageDB = Setup.language_db
variable_db.require("path_iterator")
def __door_adr_sequences():
result = ["{\n"]
offset = 0
for path_id, path in enumerate(PWState.path_list):
# NOTE: For all states in the path the 'from_state_index, to_state_index' can
# be determined, **except** for the FIRST state in the path. Thus for
# this state the 'door' cannot be determined in case that it is
# "not uniform_doors_f()".
#
# However, the only occasion where the FIRST state in the path may be
# used is reload during the FIRST state. The reload adapts the positions
# and acceptances are not changed. So, we can use the common entry
# to the first state as a reference here.
prev_state_index = path[0][0]
result.append(" ")
for state_index in (x[0] for x in path[1:]):
result.append(
"QUEX_LABEL(%i), " %
LanguageDB.ADDRESS(state_index, prev_state_index))
prev_state_index = state_index
result.append("/* Zero of Elegance */0x0,")
result.append("\n")
offset += len(path)
result.append(" }")
return offset, result
def __character_sequences():
result = ["{\n"]
offset = 0
for path_id, path in enumerate(PWState.path_list):
# Commenting the transition sequence is not dangerous. 'COMMENT' eliminates
# comment delimiters if they would appear in the sequence_str.
# sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1])
# memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n")
# Last element of sequence contains only the 'end state'.
result.append(" ")
result.extend("%i, " % x.transition_char_to_next
for x in path[:-1])
result.append("QUEX_SETTING_PATH_TERMINATION_CODE,")
result.append("\n")
variable_db.require("path_walker_%i_path_%i",
Initial="path_walker_%i_path_base + %i" %
(PWState.index, offset),
Index=(PWState.index, path_id))
offset += len(path)
result.append(" }")
return offset, result
# (*) Path Walker Basis
# The 'base' must be defined before all --> PriorityF (see table in variable_db)
element_n, character_sequence_str = __character_sequences()
variable_db.require_array("path_walker_%i_path_base",
ElementN=element_n,
Initial=character_sequence_str,
Index=PWState.index)
# (*) The State Information for each path step
if PWState.uniform_entry_door_id_along_all_paths is None:
element_n, state_sequence_str = __door_adr_sequences()
variable_db.require_array("path_walker_%i_state_base",
ElementN=element_n,
Initial=state_sequence_str,
Index=PWState.index)
# The path_iterator is incremented before the 'goto', thus
# 'path_iterator - (path_base + 1)' gives actually the correct offset.
# We define a variable for that, for elegance.
variable_db.require("path_walker_%i_reference",
Initial="path_walker_%i_path_base + 1" %
PWState.index,
Index=(PWState.index))
def do_analyzer(analyzer):
state_machine_code = state_machine_coder.do(analyzer)
Lng.REPLACE_INDENT(state_machine_code)
# Variable to store the current input
variable_db.require("input")
return state_machine_code
def do_analyzer(analyzer):
state_machine_code = state_machine_coder.do(analyzer)
Lng.REPLACE_INDENT(state_machine_code)
# Variable to store the current input
variable_db.require("input")
return state_machine_code
def do(self, RequiredLocalVariablesDB):
LanguageDB = Setup.language_db
# (*) Initialize the label and variable trackers
variable_db.init(RequiredLocalVariablesDB)
variable_db.require("input")
init_address_handling({})
# (*) Pre Context State Machine
# (If present: All pre-context combined in single backward analyzer.)
pre_context = self.__code_pre_context_state_machine()
# (*) Main State Machine -- try to match core patterns
main = self.__code_main_state_machine()
# (*) Backward input position detection
# (Seldomly present -- only for Pseudo-Ambiguous Post Contexts)
bipd = self.__code_backward_input_position_detection()
# (*) Determine required labels and variables
routed_address_set = get_address_set_subject_to_routing()
routed_address_set.add(get_address("$terminal-EOF", U=True))
routed_state_info_list = state_router_generator.get_info(
routed_address_set)
state_router = [state_router_generator.do(routed_state_info_list)]
variable_db.require("target_state_index", Condition_ComputedGoto=False)
if is_label_referenced("$reload-FORWARD") or is_label_referenced(
"$reload-BACKWARD"):
variable_db.require("target_state_else_index")
variable_db.require("target_state_index")
# Following function refers to the global 'variable_db'
variable_definitions = self.language_db.VARIABLE_DEFINITIONS(
variable_db)
function_body = []
function_body.extend(
pre_context) # implementation of pre-contexts (if there are some)
function_body.extend(main) # main pattern matcher
function_body.extend(
bipd) # (seldom != empty; only for pseudo-ambiguous post contexts)
function_body.extend(
state_router
) # route to state by index (only if no computed gotos)
# (*) Pack Pre-Context and Core State Machine into a single function
analyzer_function = self.language_db["$analyzer-func"](
self.state_machine_name, Setup, variable_definitions,
function_body, self.mode_name_list)
txt = [
LanguageDB["$header-definitions"](LanguageDB, self.on_after_match)
]
txt += get_plain_strings(analyzer_function)
for i, element in enumerate(txt):
if not isinstance(element, (str, unicode)):
print element.__class__.__name__
for k in range(max(0, i - 10)):
print "before:", k, txt[k]
for k in range(i + 1, min(i + 10, len(txt))):
print "after: ", k, txt[k]
assert False
return txt
def require_data(PWState, TheAnalyzer):
"""Defines the transition targets for each involved state.
"""
LanguageDB = Setup.language_db
variable_db.require("path_iterator")
def __door_adr_sequences():
result = ["{\n"]
offset = 0
for path_id, path in enumerate(PWState.path_list):
# NOTE: For all states in the path the 'from_state_index, to_state_index' can
# be determined, **except** for the FIRST state in the path. Thus for
# this state the 'door' cannot be determined in case that it is
# "not uniform_doors_f()".
#
# However, the only occasion where the FIRST state in the path may be
# used is reload during the FIRST state. The reload adapts the positions
# and acceptances are not changed. So, we can use the common entry
# to the first state as a reference here.
prev_state_index = path[0][0]
result.append(" ")
for state_index in (x[0] for x in path[1:]):
result.append("QUEX_LABEL(%i), " % LanguageDB.ADDRESS(state_index, prev_state_index))
prev_state_index = state_index
result.append("/* Zero of Elegance */0x0,")
result.append("\n")
offset += len(path)
result.append(" }")
return offset, result
def __character_sequences():
result = ["{\n"]
offset = 0
for path_id, path in enumerate(PWState.path_list):
# Commenting the transition sequence is not dangerous. 'COMMENT' eliminates
# comment delimiters if they would appear in the sequence_str.
# sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1])
# memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n")
# Last element of sequence contains only the 'end state'.
result.append(" ")
result.extend("%i, " % x.transition_char_to_next for x in path[:-1])
result.append("QUEX_SETTING_PATH_TERMINATION_CODE,")
result.append("\n")
variable_db.require(
"path_walker_%i_path_%i",
Initial="path_walker_%i_path_base + %i" % (PWState.index, offset),
Index=(PWState.index, path_id),
)
offset += len(path)
result.append(" }")
return offset, result
# (*) Path Walker Basis
# The 'base' must be defined before all --> PriorityF (see table in variable_db)
element_n, character_sequence_str = __character_sequences()
variable_db.require_array(
"path_walker_%i_path_base", ElementN=element_n, Initial=character_sequence_str, Index=PWState.index
)
# (*) The State Information for each path step
if PWState.uniform_entry_door_id_along_all_paths is None:
element_n, state_sequence_str = __door_adr_sequences()
variable_db.require_array(
"path_walker_%i_state_base", ElementN=element_n, Initial=state_sequence_str, Index=PWState.index
)
# The path_iterator is incremented before the 'goto', thus
# 'path_iterator - (path_base + 1)' gives actually the correct offset.
# We define a variable for that, for elegance.
variable_db.require(
"path_walker_%i_reference", Initial="path_walker_%i_path_base + 1" % PWState.index, Index=(PWState.index)
)
def require_data(PWState, TheAnalyzer):
"""Defines the transition targets for each involved state.
"""
variable_db.require("path_iterator")
def __door_adr_sequences(PWState):
result = ["{\n"]
length = 0
for path_id, door_id_sequence in enumerate(
PWState.door_id_sequence_list):
# NOTE: For all states in the path the 'from_state_index, to_state_index' can
# be determined, **except** for the FIRST state in the path. Thus for
# this state the 'door' cannot be determined in case that it is
# "not uniform_doors_f()".
#
# However, the only occasion where the FIRST state in the path may be
# used is reload during the FIRST state. The reload adapts the positions
# and acceptances are not changed. So, we can use the common entry
# to the first state as a reference here.
## print "#DoorID, Adr:", [(door_id, Lng.ADDRESS_BY_DOOR_ID(door_id)) for door_id in door_id_sequence]
result.append(" ")
result.append("/* Padding */0x0, ")
result.extend("QUEX_LABEL(%i), " %
dial_db.get_address_by_door_id(door_id, RoutedF=True)
for door_id in door_id_sequence)
result.append("\n")
length += len(door_id_sequence) + 1 # 1 padding element
result.append(" }")
return length, result
def __character_sequences(PathList):
result = ["{\n"]
offset = 0
for path_id, step_list in enumerate(PathList):
## Commenting the transition sequence is not dangerous. 'COMMENT' eliminates
## comment delimiters if they would appear in the sequence_str.
## sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), step_list[:-1])
## memory.append(Lng.COMMENT("".join(sequence_str)) + "\n")
result.append(" ")
result.extend("%i, " % x.trigger for x in step_list[:-1])
result.append("QUEX_SETTING_PATH_TERMINATION_CODE,")
result.append("\n")
offset += len(step_list)
result.append(" }")
return offset, result
# (*) Path Walker Basis
# The 'base' must be defined before all --> PriorityF (see table in variable_db)
element_n, character_sequence_str = __character_sequences(
PWState.path_list)
offset = 0
for path_id, step_list in enumerate(PWState.path_list):
variable_db.require("path_walker_%i_path_%i",
Initial="&path_walker_%i_path_base[%i]" %
(PWState.index, offset),
Index=(PWState.index, path_id))
offset += len(step_list)
variable_db.require_array("path_walker_%i_path_base",
ElementN=element_n,
Initial=character_sequence_str,
Index=PWState.index)
# (*) The State Information for each path step
if PWState.uniform_door_id is None:
element_n, door_adr_sequence_str = __door_adr_sequences(PWState)
variable_db.require_array("path_walker_%i_state_base",
ElementN=element_n,
Initial=door_adr_sequence_str,
Index=PWState.index)
# The path_iterator is incremented before the 'goto', thus
# 'path_iterator - (path_base + 1)' gives actually the correct offset.
# We define a variable for that, for elegance.
variable_db.require("path_walker_%i_reference",
Initial="path_walker_%i_path_base" % PWState.index,
Index=(PWState.index))
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName,
OnSkipRangeOpen, DoorIdAfter):
assert len(OpenerSequence) >= 1
assert len(CloserSequence) >= 1
assert OpenerSequence != CloserSequence
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
opener_str, opener_comment_str = get_character_sequence(OpenerSequence)
opener_length = len(OpenerSequence)
closer_str, closer_comment_str = get_character_sequence(CloserSequence)
closer_length = len(CloserSequence)
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require("counter")
variable_db.require_array("Skipper%i_Opener",
Initial="{ %s }" % opener_str,
ElementN=opener_length,
Index=skipper_index)
variable_db.require("Skipper%i_OpenerEnd",
"Skipper%i_Opener + (ptrdiff_t)%i" %
(skipper_index, opener_length),
Index=skipper_index)
variable_db.require("Skipper%i_Opener_it", "0x0", Index=skipper_index)
variable_db.require_array("Skipper%i_Closer",
Initial="{ %s }" % closer_str,
ElementN=closer_length,
Index=skipper_index)
variable_db.require("Skipper%i_CloserEnd",
"Skipper%i_Closer + (ptrdiff_t)%i" %
(skipper_index, closer_length),
Index=skipper_index)
variable_db.require("Skipper%i_Closer_it", "0x0", Index=skipper_index)
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
reload_door_id = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen,
CloserPattern,
NestedF=True)
code_str = blue_print(
template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER_LENGTH$$", "%i" % opener_length],
["$$INPUT_P_INCREMENT$$",
Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$",
Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$", Lng.LABEL(skipper_door_id)],
["$$RELOAD$$",
dial_db.get_label_by_door_id(reload_door_id)],
["$$GOTO_AFTER_END_OF_SKIPPING$$",
Lng.GOTO(DoorIdAfter)],
["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)],
["$$INPUT_P_TO_LEXEME_START$$",
Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$MARK_LEXEME_START$$",
Lng.LEXEME_START_SET()],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$",
line_column_counter_in_loop()],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return [code_str]
def get_skipper(EndSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter):
assert len(EndSequence) >= 1
global template_str
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
skipper_door_id = dial_db.new_door_id(skipper_index)
delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence)
end_sequence_transformed = transformation.do_sequence(EndSequence)
# Determine the $$DELIMITER$$
delimiter_length = len(end_sequence_transformed)
delimiter_comment_str = Lng.COMMENT(" Delimiter: %s" % delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = "".join(
" %s" % Lng.IF_GOTO(Lng.INPUT_P_DEREFERENCE(i-1), "!=",
"Skipper$$SKIPPER_INDEX$$[%i]" % i,
skipper_door_id, i == 1)
for i, letter in enumerate(EndSequence[1:], start=1)
)
delimiter_remainder_test_str = txt
door_id_reload = dial_db.new_door_id()
on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern)
# The main part
code_str = blue_print(template_str,
[
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()],
["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()],
["$$INPUT_GET$$", Lng.ACCESS_INPUT()],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", Lng.END_IF()],
["$$ENTRY$$", dial_db.get_label_by_door_id(skipper_door_id)],
["$$RELOAD$$", dial_db.get_label_by_door_id(door_id_reload)],
["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)],
["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)],
["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$", Lng.GOTO(door_id_reload)]])
if reference_p_f:
variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
variable_db.require_array("Skipper%i", Initial="{ %s }" % delimiter_str, ElementN=delimiter_length, Index=skipper_index)
variable_db.require("Skipper%iL", "%i" % delimiter_length, Index=skipper_index)
variable_db.require("text_end")
variable_db.require("input")
return [ code_str ]
