def _get_source_code(analyzer_list, terminal_list, ColumnNPerChunk,
AppendixSmExistF):
"""RETURNS: String containing source code for the 'loop'.
-- The source code for the (looping) state machine.
-- The terminals which contain counting actions.
Also, it requests variable definitions as they are required.
"""
txt = flatten_list_of_lists(
generator.do_analyzer(analyzer) for analyzer in analyzer_list
)
txt.extend(
generator.do_terminals(terminal_list, TheAnalyzer=None)
)
loop_analyzer = analyzer_list[0]
if loop_analyzer.engine_type.subject_to_reload():
txt.extend(
generator.do_reload_procedure(loop_analyzer)
)
if AppendixSmExistF or ColumnNPerChunk is not None:
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
if Setup.buffer_codec.variable_character_sizes_f():
variable_db.require("lexatom_begin_p")
return txt
def _get_source_code(CcFactory, analyzer, terminal_list):
"""RETURNS: String containing source code for the 'loop'.
-- The source code for the (looping) state machine.
-- The terminals which contain counting actions.
Also, it requests variable definitions as they are required.
"""
txt = []
txt.extend(
generator.do_analyzer(analyzer)
)
txt.extend(
generator.do_terminals(terminal_list, analyzer)
)
if analyzer.engine_type.subject_to_reload():
txt.extend(
generator.do_reload_procedure(analyzer)
)
if CcFactory.requires_reference_p():
variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
if Setup.buffer_codec.variable_character_sizes_f():
variable_db.require("character_begin_p")
return txt
def create_character_set_skipper_code(Language,
TestStr,
TriggerSet,
QuexBufferSize=1024,
InitialSkipF=True,
OnePassOnlyF=False,
CounterPrintF=True):
global dial_db
end_str = __prepare(Language)
analyzer_list, \
terminal_list, \
loop_map, \
required_register_set = character_set_skipper.do("M", LineColumnCount_Default(),
TriggerSet, FSM.reload_state,
dial_db)
loop_code = generator.do_analyzer_list(analyzer_list)
variable_db.require_registers(required_register_set)
loop_code.extend(
generator.do_terminals(terminal_list,
TheAnalyzer=None,
dial_db=dial_db))
if InitialSkipF: marker_char_list = TriggerSet.get_number_list()
else: marker_char_list = []
result = create_customized_analyzer_function(
Language,
TestStr,
loop_code,
QuexBufferSize,
CommentTestStrF=False,
ShowPositionF=False,
EndStr=end_str,
SkipUntilMarkerSet=marker_char_list,
LocalVariableDB=deepcopy(variable_db.get()),
ReloadF=True,
OnePassOnlyF=OnePassOnlyF,
CounterPrintF=CounterPrintF)
result = language_defines + result
result = result.replace("$$TEST_ANALYZER_DIR$$",
test_analyzer_dir(Language))
result = result.replace("$$COMPUTED_GOTOS_CHECK$$",
computed_gotos_check_str())
return result
def _get_source_code(CcFactory, analyzer, terminal_list):
"""RETURNS: String containing source code for the 'loop'.
-- The source code for the (looping) state machine.
-- The terminals which contain counting actions.
Also, it requests variable definitions as they are required.
"""
txt = []
txt.extend(generator.do_analyzer(analyzer))
txt.extend(generator.do_terminals(terminal_list, analyzer))
if analyzer.engine_type.subject_to_reload():
txt.extend(generator.do_reload_procedure(analyzer))
if CcFactory.requires_reference_p():
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
if Setup.buffer_codec.variable_character_sizes_f():
variable_db.require("character_begin_p")
return txt
def _get_source_code(analyzer_list, terminal_list, ColumnNPerChunk,
AppendixSmExistF):
"""RETURNS: String containing source code for the 'loop'.
-- The source code for the (looping) state machine.
-- The terminals which contain counting actions.
Also, it requests variable definitions as they are required.
"""
txt = flatten_list_of_lists(
generator.do_analyzer(analyzer) for analyzer in analyzer_list)
txt.extend(generator.do_terminals(terminal_list, TheAnalyzer=None))
loop_analyzer = analyzer_list[0]
if loop_analyzer.engine_type.subject_to_reload():
txt.extend(generator.do_reload_procedure(loop_analyzer))
if AppendixSmExistF or ColumnNPerChunk is not None:
variable_db.require("reference_p",
Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING")
if Setup.buffer_codec.variable_character_sizes_f():
variable_db.require("lexatom_begin_p")
return txt
def do_core(PatternList, TerminalDb, OnAfterMatchCode=None):
"""Produces main code for an analyzer function which can detect patterns given in
the 'PatternList' and has things to be done mentioned in 'TerminalDb'.
RETURN: Code implementing the lexical analyzer.
The code is not embedded in a function and required definitions are not provided.
This happens through function 'wrap_up()'.
"""
# Prepare the combined state machines and terminals
esms = EngineStateMachineSet(PatternList)
# (*) Pre Context State Machine
# (If present: All pre-context combined in single backward analyzer.)
pre_context, \
pre_analyzer = generator.do_pre_context(esms.pre_context_sm,
esms.pre_context_sm_id_list)
# assert all_isinstance(pre_context, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Backward input position detection
# (Seldomly present -- only for Pseudo-Ambiguous Post Contexts)
bipd = generator.do_backward_input_position_detectors(esms.bipd_sm_db)
# assert all_isinstance(bipd, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Main State Machine -- try to match core patterns
# Post-context handling is webbed into the main state machine.
main, \
main_analyzer = generator.do_main(esms.sm)
# assert all_isinstance(main, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Terminals
# (BEFORE 'Reload procedures' because some terminals may add entries
# to the reloader.)
terminals = generator.do_terminals(TerminalDb.values(),
main_analyzer)
# (*) Reload procedures
reload_procedure_fw = generator.do_reload_procedure(main_analyzer)
reload_procedure_bw = generator.do_reload_procedure(pre_analyzer)
# assert all_isinstance(reload_procedures, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Re-entry preparation
reentry_preparation = generator.do_reentry_preparation(esms.pre_context_sm_id_list,
OnAfterMatchCode)
# (*) State Router
# (Something that can goto a state address by an given integer value)
state_router = generator.do_state_router()
# assert all_isinstance(state_router, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Variable Definitions
# (Code that defines all required variables for the analyzer)
variable_definitions = generator.do_variable_definitions()
# assert all_isinstance(variable_definitions, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Putting it all together
function_body = []
function_body.extend(pre_context) # implementation of pre-contexts (if there are some)
function_body.extend(main) # main pattern matcher
function_body.extend(bipd) # (seldom != empty; only for pseudo-ambiguous post contexts)
function_body.extend(terminals)
function_body.extend(state_router) # route to state by index (only if no computed gotos)
function_body.extend(reload_procedure_fw)
function_body.extend(reload_procedure_bw)
function_body.extend(reentry_preparation)
return function_body, variable_definitions
def do_core(Mode):
"""Produces main code for an analyzer function which can detect patterns given in
the 'PatternList' and has things to be done mentioned in 'TerminalDb'.
RETURN: Code implementing the lexical analyzer.
The code is not embedded in a function and required definitions are not provided.
This happens through function 'wrap_up()'.
"""
# Prepare the combined state machines and terminals
TerminalDb = Mode.terminal_db
ReloadStateForward = Mode.reload_state_forward
OnAfterMatchCode = Mode.incidence_db.get_CodeTerminal(E_IncidenceIDs.AFTER_MATCH)
dial_db = Mode.dial_db
variable_db.require_registers(flatten_list_of_lists(
terminal.required_register_set()
for terminal in TerminalDb.itervalues()
))
# (*) Pre Context DFA
# (If present: All pre-context combined in single backward analyzer.)
Lng.debug_unit_name_set("Pre-Context:%s" % Mode.name)
pre_context, \
pre_analyzer = generator.do_pre_context(Mode.pre_context_sm_to_be_reversed_list,
Mode.pre_context_sm_id_list,
dial_db)
# assert all_isinstance(pre_context, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Backward input position detection
# (Seldomly present -- only for Pseudo-Ambiguous Post Contexts)
Lng.debug_unit_name_set("Backward-Input-Position-Detection:%s" % Mode.name)
bipd = generator.do_backward_read_position_detectors(Mode.bipd_sm_to_be_reversed_db,
dial_db)
# assert all_isinstance(bipd, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Main DFA -- try to match core patterns
# Post-context handling is webbed into the main state machine.
Lng.debug_unit_name_set("Core:%s" % Mode.name)
main, \
main_analyzer = generator.do_main(Mode.core_sm_list, ReloadStateForward,
dial_db)
Lng.debug_unit_name_set("Extra:%s" % Mode.name)
# assert all_isinstance(main, (IfDoorIdReferencedCode, int, str, unicode))
extra = generator.do_analyzer_list(Mode.extra_analyzer_list)
# (*) Terminals
# (BEFORE 'Reload procedures' because some terminals may add entries
# to the reloader.)
terminals = generator.do_terminals(TerminalDb.values(),
main_analyzer,
dial_db)
# (*) Reload procedures
reload_procedure_fw = generator.do_reload_procedure(main_analyzer)
reload_procedure_bw = generator.do_reload_procedure(pre_analyzer)
# assert all_isinstance(reload_procedures, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Re-entry preparation
Lng.debug_unit_name_set("Re-Entry-Preparation:%s" % Mode.name)
reentry_preparation = generator.do_reentry_preparation(Mode.pre_context_sm_id_list,
OnAfterMatchCode,
dial_db)
# (*) State Router
# (Something that can goto a state address by an given integer value)
state_router = generator.do_state_router(dial_db)
# assert all_isinstance(state_router, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Variable Definitions
# (Code that defines all required variables for the analyzer)
variable_db.require_registers(Mode.required_register_set)
variable_definitions = generator.do_variable_definitions()
# assert all_isinstance(variable_definitions, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Putting it all together
function_body = []
function_body.extend(pre_context) # implementation of pre-contexts (if there are some)
function_body.extend(main) # main pattern matcher
function_body.extend(extra) # extra state machines (from 'Loopers')
function_body.extend(bipd) # (seldom != empty; only for pseudo-ambiguous post contexts)
function_body.extend(terminals)
function_body.extend(state_router) # route to state by index (only if no computed gotos)
function_body.extend(reload_procedure_fw)
function_body.extend(reload_procedure_bw)
function_body.extend(reentry_preparation)
return function_body, variable_definitions
def create_indentation_handler_code(Language, TestStr, ISetup, BufferSize):
end_str = __prepare(Language)
class MiniIncidenceDb(dict):
def __init__(self):
self[E_IncidenceIDs.INDENTATION_BAD] = ""
def default_indentation_handler_f(self):
return True
mini_incidence_db = MiniIncidenceDb()
ca_map = LineColumnCount_Default()
counter_code = run_time_counter.get(ca_map, "M")
code = [
] # [ "%s\n" % Lng.LABEL(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER, dial_db)) ]
variable_db.init()
analyzer_list, \
terminal_list, \
required_register_set, \
run_time_counter_f = indentation_counter.do("M", ca_map, ISetup,
mini_incidence_db, FSM.reload_state,
dial_db)
loop_code = generator.do_analyzer_list(analyzer_list)
loop_code.extend(
generator.do_terminals(terminal_list,
TheAnalyzer=None,
dial_db=dial_db))
if not run_time_counter_f:
counter_code = None
code.extend(loop_code)
__require_variables(required_register_set)
main_txt = create_customized_analyzer_function(
Language,
TestStr,
code,
QuexBufferSize=BufferSize,
CommentTestStrF="",
ShowPositionF=True,
EndStr=end_str,
SkipUntilMarkerSet="behind newline",
LocalVariableDB=deepcopy(variable_db.get()),
IndentationSupportF=True,
ReloadF=True,
CounterPrintF=False,
BeforeCode=counter_code)
on_indentation_txt = indentation_handler.do(
AuxMode(), ["M", "M2"]).replace("$on_indentation",
"QUEX_NAME(M_on_indentation)")
Setup.analyzer_class_name = "TestAnalyzer"
Setup.analyzer_name_safe = "TestAnalyzer"
result = adapt.do(main_txt + on_indentation_txt,
test_analyzer_dir(Language))
result = language_defines + result
result = result.replace("$$TEST_ANALYZER_DIR$$",
test_analyzer_dir(Language))
result = result.replace("$$COMPUTED_GOTOS_CHECK$$",
computed_gotos_check_str())
return result
def create_nested_range_skipper_code(Language,
TestStr,
OpenerSequence,
CloserSequence,
QuexBufferSize=1024,
CommentTestStrF=False,
ShowPositionF=False):
assert QuexBufferSize >= len(CloserSequence) + 2
end_str = __prepare(Language)
sm_close = DFA.from_sequence(CloserSequence)
sm_open = DFA.from_sequence(OpenerSequence)
ca_map = LineColumnCount_Default()
closer_pattern = Pattern_Prep(sm_close,
PatternString="<skip range closer>",
Sr=SourceRef_VOID)
opener_pattern = Pattern_Prep(sm_open,
PatternString="<skip range opener>",
Sr=SourceRef_VOID)
door_id_exit = DoorID.continue_without_on_after_match(dial_db)
analyzer_list, \
terminal_list, \
required_register_set, \
run_time_counter_f = nested_range_skipper.do("MrUnitTest", ca_map,
OpenerPattern = opener_pattern,
CloserPattern = closer_pattern,
DoorIdExit = door_id_exit,
ReloadState = FSM.reload_state,
dial_db = dial_db)
loop_code = generator.do_analyzer_list(analyzer_list)
assert not run_time_counter_f
__require_variables(required_register_set)
loop_code.extend(
generator.do_terminals(terminal_list,
TheAnalyzer=None,
dial_db=dial_db))
code = []
if run_time_counter_f:
counter_code = run_time_counter.get(ca_map, "M")
code.extend(counter_code)
code.extend(loop_code)
result = create_customized_analyzer_function(
Language,
TestStr,
loop_code,
QuexBufferSize=QuexBufferSize,
CommentTestStrF=CommentTestStrF,
ShowPositionF=ShowPositionF,
EndStr=end_str,
SkipUntilMarkerSet=[],
LocalVariableDB=deepcopy(variable_db.get()),
DoorIdOnSkipRangeOpenF=True,
CounterPrintF="short")
result = language_defines + result
result = result.replace("$$TEST_ANALYZER_DIR$$",
test_analyzer_dir(Language))
result = result.replace("$$COMPUTED_GOTOS_CHECK$$",
computed_gotos_check_str())
return result
def get(CaMap, ModeName):
"""Implement the default counter for a given Counter Database.
In case the line and column number increment cannot be determined before-
hand, a something must be there that can count according to the rules given
in 'CaMap'. This function generates the code for a general counter
function which counts line and column number increments starting from the
begin of a lexeme to its end.
The implementation of the default counter is a direct function of the
'CaMap', i.e. the database telling how characters influence the
line and column number counting.
Multiple modes may have the same character counting behavior. If so,
then there's only one counter implemented while others refer to it.
---------------------------------------------------------------------------
RETURNS: function_name, string --> Function name and the implementation
of the character counter.
function_name, None --> The 'None' implementation indicates that
NO NEW counter is implemented. An
appropriate counter can be accessed
by the 'function name'.
---------------------------------------------------------------------------
"""
if not (Setup.count_line_number_f or Setup.count_column_number_f):
return ""
dial_db = DialDB()
mode_with_same_counter = DefaultCounterFunctionDB.get_mode_name(CaMap)
if mode_with_same_counter is not None:
# Use previously done implementation for this 'CaMap'
return __frame(Lng.DEFAULT_COUNTER_FUNCTION_NAME(ModeName),
[ Lng.DEFAULT_COUNTER_CALL(mode_with_same_counter) ],
None, None, None)
door_id_return = dial_db.new_door_id()
analyzer_list, \
terminal_list, \
dummy_loop_map, \
dummy_door_id_loop, \
required_register_set, \
dummy_run_time_counter_f = loop.do(CaMap,
OnLoopExitDoorId = door_id_return,
LexemeEndCheckF = True,
EngineType = engine.CHARACTER_COUNTER,
dial_db = dial_db,
ModeName = ModeName,
CutSignalLexatomsF = False)
code = generator.do_analyzer_list(analyzer_list)
code.extend(
generator.do_terminals(terminal_list, TheAnalyzer=None, dial_db=dial_db)
)
variable_db.require_registers(required_register_set)
implementation = __frame(Lng.DEFAULT_COUNTER_FUNCTION_NAME(ModeName), code,
Lng.INPUT_P(), door_id_return, dial_db)
DefaultCounterFunctionDB.enter(CaMap, ModeName)
return implementation
def do_core(PatternList, TerminalDb, OnAfterMatchCode=None):
"""Produces main code for an analyzer function which can detect patterns given in
the 'PatternList' and has things to be done mentioned in 'TerminalDb'.
RETURN: Code implementing the lexical analyzer.
The code is not embedded in a function and required definitions are not provided.
This happens through function 'wrap_up()'.
"""
# Prepare the combined state machines and terminals
esms = EngineStateMachineSet(PatternList)
# (*) Pre Context State Machine
# (If present: All pre-context combined in single backward analyzer.)
pre_context, \
pre_analyzer = generator.do_pre_context(esms.pre_context_sm,
esms.pre_context_sm_id_list)
# assert all_isinstance(pre_context, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Backward input position detection
# (Seldomly present -- only for Pseudo-Ambiguous Post Contexts)
bipd = generator.do_backward_input_position_detectors(esms.bipd_sm_db)
# assert all_isinstance(bipd, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Main State Machine -- try to match core patterns
# Post-context handling is webbed into the main state machine.
main, \
main_analyzer = generator.do_main(esms.sm)
# assert all_isinstance(main, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Terminals
# (BEFORE 'Reload procedures' because some terminals may add entries
# to the reloader.)
terminals = generator.do_terminals(TerminalDb.values(), main_analyzer)
# (*) Reload procedures
reload_procedure_fw = generator.do_reload_procedure(main_analyzer)
reload_procedure_bw = generator.do_reload_procedure(pre_analyzer)
# assert all_isinstance(reload_procedures, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Re-entry preparation
reentry_preparation = generator.do_reentry_preparation(
esms.pre_context_sm_id_list, OnAfterMatchCode)
# (*) State Router
# (Something that can goto a state address by an given integer value)
state_router = generator.do_state_router()
# assert all_isinstance(state_router, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Variable Definitions
# (Code that defines all required variables for the analyzer)
variable_definitions = generator.do_variable_definitions()
# assert all_isinstance(variable_definitions, (IfDoorIdReferencedCode, int, str, unicode))
# (*) Putting it all together
function_body = []
function_body.extend(
pre_context) # implementation of pre-contexts (if there are some)
function_body.extend(main) # main pattern matcher
function_body.extend(
bipd) # (seldom != empty; only for pseudo-ambiguous post contexts)
function_body.extend(terminals)
function_body.extend(
state_router) # route to state by index (only if no computed gotos)
function_body.extend(reload_procedure_fw)
function_body.extend(reload_procedure_bw)
function_body.extend(reentry_preparation)
return function_body, variable_definitions