def reentry_preparation(Lng, PreConditionIDList, OnAfterMatchCode):
"""Reentry preperation (without returning from the function."""
# (*) Unset all pre-context flags which may have possibly been set
unset_pre_context_flags_str = Lng.PRE_CONTEXT_RESET(PreConditionIDList)
if OnAfterMatchCode is not None:
on_after_match_str = Lng.SOURCE_REFERENCED(OnAfterMatchCode)
else:
on_after_match_str = ""
txt = [
"\n%s\n" % Lng.LABEL(DoorID.return_with_on_after_match()),
Lng.COMMENT("RETURN -- after executing 'on_after_match' code."),
on_after_match_str,
" %s\n\n" % Lng.PURE_RETURN,
#
"\n%s\n" % Lng.LABEL(DoorID.continue_with_on_after_match()),
Lng.COMMENT("CONTINUE -- after executing 'on_after_match' code."),
on_after_match_str,
#
"\n%s\n" % Lng.LABEL(DoorID.continue_without_on_after_match()),
Lng.COMMENT("CONTINUE -- without executing 'on_after_match' (e.g. on FAILURE)."), "\n",
#
__return_if_queue_full_or_simple_analyzer, "\n",
__return_if_mode_changed, "\n",
#
unset_pre_context_flags_str,
"\n%s\n" % Lng.GOTO(DoorID.global_reentry()),
]
return txt
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb,
IncidenceDb):
"""Generate a PPT for a range skipper.
"""
# -- door_id_after: Where to go after the closing character sequence matched:
# + Normally: To the begin of the analyzer. Start again.
# + End(Sequence) == newline of indentation counter.
# => goto indentation counter.
if _match_indentation_counter_newline_pattern(
OptionsDb.value("indentation"), data["closer_sequence"]):
door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)
else:
door_id_after = DoorID.continue_without_on_after_match()
if NestedF:
name = "SKIP NESTED RANGE"
code_generator_func = skip_nested_range.do
else:
name = "SKIP RANGE"
code_generator_func = skip_range.do
# -- data for code generation
my_data = deepcopy(data)
my_data["mode_name"] = ModeName
my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN]
my_data["door_id_after"] = door_id_after
my_data["counter_db"] = CounterDb
# -- terminal and code generator
priority = PatternPriority(MHI, i)
pattern = deepcopy(my_data["opener_pattern"])
pattern.set_incidence_id(dial_db.new_incidence_id())
pattern.set_pattern_string("<skip_range>")
code = CodeGenerated(code_generator_func, my_data, name)
return PPT(priority, pattern, code)
def do_match_failure(self, Code, ThePattern):
"""No pattern in the mode has matched. Line and column numbers are
still counted. But, no 'on_match' or 'on_after_match' action is
executed.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=False)
text = [
#Lng.IF_END_OF_FILE(),
# self.get_counter_text(None),
# Lng.GOTO(DoorID.continue_without_on_after_match()),
#Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False),
# Lng.INPUT_P_INCREMENT(),
#Lng.END_IF(),
self.get_counter_text(None),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_without_on_after_match()),
]
code = CodeTerminal(text,
SourceReference = Code.sr,
PureCode = Code.get_pure_code())
return Terminal(code, "FAILURE")
def reentry_preparation(Lng, PreConditionIDList, OnAfterMatchCode, dial_db):
"""Reentry preperation (without returning from the function."""
# (*) Unset all pre-context flags which may have possibly been set
unset_pre_context_flags_str = Lng.PRE_CONTEXT_RESET(PreConditionIDList)
on_after_match_str = Lng.SOURCE_REFERENCED(OnAfterMatchCode)
return [
"\n%s\n" % Lng.LABEL(DoorID.return_with_on_after_match(dial_db)),
Lng.COMMENT("RETURN -- after executing 'on_after_match' code."),
on_after_match_str,
" %s\n\n" % Lng.PURE_RETURN,
#
"\n%s\n" % Lng.LABEL(DoorID.continue_with_on_after_match(dial_db)),
Lng.COMMENT("CONTINUE -- after executing 'on_after_match' code."),
on_after_match_str,
#
"\n%s\n" % Lng.LABEL(DoorID.continue_without_on_after_match(dial_db)),
Lng.COMMENT(
"CONTINUE -- without executing 'on_after_match' (e.g. on FAILURE)."
),
"\n",
#
__assert_no_mode_change,
"\n",
__return_if_queue_full_or_simple_analyzer,
"\n",
unset_pre_context_flags_str,
"\n%s\n" % Lng.GOTO(DoorID.global_reentry(dial_db), dial_db),
]
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb, IncidenceDb):
"""Generate a PPT for a range skipper.
"""
# -- door_id_after: Where to go after the closing character sequence matched:
# + Normally: To the begin of the analyzer. Start again.
# + End(Sequence) == newline of indentation counter.
# => goto indentation counter.
if _match_indentation_counter_newline_pattern(OptionsDb.value("indentation"),
data["closer_sequence"]):
door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)
else:
door_id_after = DoorID.continue_without_on_after_match()
if NestedF:
name = "SKIP NESTED RANGE"
code_generator_func = skip_nested_range.do
else:
name = "SKIP RANGE"
code_generator_func = skip_range.do
# -- data for code generation
my_data = deepcopy(data)
my_data["mode_name"] = ModeName
my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN]
my_data["door_id_after"] = door_id_after
my_data["counter_db"] = CounterDb
# -- terminal and code generator
priority = PatternPriority(MHI, i)
pattern = deepcopy(my_data["opener_pattern"])
pattern.set_incidence_id(dial_db.new_incidence_id())
pattern.set_pattern_string("<skip_range>")
code = CodeGenerated(code_generator_func, my_data, name)
return PPT(priority, pattern, code)
def do_match_failure(self, Code, ThePattern):
"""No pattern in the mode has matched. Line and column numbers are
still counted. But, no 'on_match' or 'on_after_match' action is
executed.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=False)
text = [
#Lng.IF_END_OF_FILE(),
# self.get_counter_text(None),
# Lng.GOTO(DoorID.continue_without_on_after_match()),
#Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False),
# Lng.INPUT_P_INCREMENT(),
#Lng.END_IF(),
self.get_counter_text(None),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_without_on_after_match()),
]
code = CodeTerminal(text,
SourceReference=Code.sr,
PureCode=Code.get_pure_code())
return Terminal(code, "FAILURE")
def _range_skipper_door_id_exit(self, IndentationHandler, CloserPattern,
dial_db):
if self._match_indentation_counter_newline_pattern(
IndentationHandler, CloserPattern):
return DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER,
dial_db)
else:
return DoorID.continue_without_on_after_match(dial_db)
def _get_indentation_handler_terminal(ModeName, dial_db):
code = Lng.COMMAND_LIST([
Op.IndentationHandlerCall(ModeName),
Op.GotoDoorId(DoorID.continue_without_on_after_match(dial_db))
], dial_db)
incidence_id = dial.new_incidence_id()
terminal = Terminal(CodeTerminal(code),
"<CALL INDENTATION HANDLER>",
incidence_id,
dial_db=dial_db)
return DoorID.incidence(incidence_id, dial_db), terminal
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = StateMachine.from_sequence(CloserSequence)
sm.set_id(dial_db.new_incidence_id())
code = [ Lng.GOTO(DoorID.continue_without_on_after_match()) ]
terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>", sm.get_id())
return [ (sm, terminal) ]
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = StateMachine.from_sequence(CloserSequence)
sm.set_id(dial_db.new_incidence_id())
code = [Lng.GOTO(DoorID.continue_without_on_after_match())]
terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>",
sm.get_id())
return [(sm, terminal)]
def get_skipper(TheAnalyzer, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter, CounterDb):
"""
.---<---+----------<------+
| | |
| | not |
.------. | Closer[0] |
------------------------------>| Loop +--' |
| | | no
| | |
| | .-------------.
| +----->----| Closer[1-N] |------------> RESTART
| | | ? | yes
| | '-------------'
| |
| BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .-----------------.
.----| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'-----------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| SkipRangeOpen |
'---------------'
"""
psml = _get_state_machine_vs_terminal_list(CloserSequence,
CounterDb)
count_op_factory = CountInfoMap.from_LineColumnCount(CounterDb,
NumberSet_All(),
Lng.INPUT_P())
result, \
door_id_beyond = loop.do(count_op_factory,
OnLoopExit = [ Op.GotoDoorId(DoorID.continue_without_on_after_match()) ],
LexemeEndCheckF = False,
LexemeMaintainedF = False,
EngineType = engine.FORWARD,
ReloadStateExtern = TheAnalyzer.reload_state,
ParallelSmTerminalPairList = psml)
return result
def get_skipper(TheAnalyzer, CloserSequence, CloserPattern, ModeName,
OnSkipRangeOpen, DoorIdAfter, CounterDb):
"""
.---<---+----------<------+
| | |
| | not |
.------. | Closer[0] |
------------------------------>| Loop +--' |
| | | no
| | |
| | .-------------.
| +----->----| Closer[1-N] |------------> RESTART
| | | ? | yes
| | '-------------'
| |
| BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .-----------------.
.----| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'-----------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| SkipRangeOpen |
'---------------'
"""
psml = _get_state_machine_vs_terminal_list(CloserSequence, CounterDb)
count_op_factory = CountInfoMap.from_LineColumnCount(
CounterDb, NumberSet_All(), Lng.INPUT_P())
result, \
door_id_beyond = loop.do(count_op_factory,
OnLoopExit = [ Op.GotoDoorId(DoorID.continue_without_on_after_match()) ],
LexemeEndCheckF = False,
LexemeMaintainedF = False,
EngineType = engine.FORWARD,
ReloadStateExtern = TheAnalyzer.reload_state,
ParallelSmTerminalPairList = psml)
return result
def do(ModeName, CaMap, CharacterSet, ReloadState, dial_db):
"""Fast implementation of character set skipping machine.
________________________________________________________________________
As long as characters of a given character set appears it iterates:
input in Set
.--<---.
| |
.-------. |
--------->( SKIPPER )--+----->------> RESTART
'-------' input
not in Set
___________________________________________________________________________
Precisely, i.e. including counter and reload actions:
START
|
| .----------------------------------------------.
| |.-------------------------------------------. |
| ||.----------------------------------------. | |
| ||| | | |
| ||| .-DoorID(S, a)--. transition | | |
| || '-| gridstep(cn) | map | | |
| || '---------------'\ .------. | | |
| || .-DoorID(S, b)--. '->-| | | | |
| |'---| ln += 1 |--->-| '\t' +-->-----' | |
| | '---------------' | | | |
| | .-DoorID(S, c)--. | ' ' +-->-------' |
| '----| cn += 1 |--->-| | |
| '---------------' | '\n' +-->---------'
| | | .-DropOut ------.
| .-DoorID(S, 0)--. | else +-->---------------| on_exit |
'------>--| on_entry |--->-| | '---------------'
'---------------' | BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .-----------------.
.----| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'-----------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| End of Stream |
'---------------'
NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs,
then the single state may actually be split into a real state machine of
states.
"""
assert isinstance(CaMap, CountActionMap)
assert isinstance(CharacterSet, NumberSet)
if ReloadState: engine_type = ReloadState.engine_type
else: engine_type = None
on_loop_exit_door_id = DoorID.continue_without_on_after_match(dial_db)
analyzer_list, \
terminal_list, \
loop_map, \
door_id_loop, \
required_register_set, \
run_time_counter_f = loop.do(CaMap.pruned_clone(CharacterSet),
OnLoopExitDoorId = on_loop_exit_door_id,
EngineType = engine_type,
ReloadStateExtern = ReloadState,
dial_db = dial_db,
ModeName = ModeName)
assert not run_time_counter_f
return analyzer_list, \
terminal_list, \
loop_map, \
required_register_set
def do(Data, TheAnalyzer):
"""Fast implementation of character set skipping machine.
________________________________________________________________________
As long as characters of a given character set appears it iterates:
input in Set
.--<---.
| |
.-------. |
--------->( SKIPPER )--+----->------> RESTART
'-------' input
not in Set
___________________________________________________________________________
NOTE: The 'TerminalSkipRange' takes care that it transits immediately to
the indentation handler, if it ends on 'newline'. This is not necessary
for 'TerminalSkipCharacterSet'. Quex refuses to work on 'skip sets' when
they match common lexemes with the indentation handler.
___________________________________________________________________________
Precisely, i.e. including counter and reload actions:
START
|
| .----------------------------------------------.
| |.-------------------------------------------. |
| ||.----------------------------------------. | |
| ||| | | |
| ||| .-DoorID(S, a)--. transition | | |
| || '-| gridstep(cn) | map | | |
| || '---------------'\ .------. | | |
| || .-DoorID(S, b)--. '->-| | | | |
| |'---| ln += 1 |--->-| '\t' +-->-----' | |
| | '---------------' | | | |
| | .-DoorID(S, c)--. | ' ' +-->-------' |
| '----| cn += 1 |--->-| | |
| '---------------' | '\n' +-->---------'
| | | .-DropOut ------.
| .-DoorID(S, 0)--. | else +-->---------------| on_exit |
'------>--| on_entry |--->-| | '---------------'
'---------------' | BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .-----------------.
.----| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'-----------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| End of Stream |
'---------------'
NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs,
then the single state may actually be split into a real state machine of
states.
"""
counter_db = Data["counter_db"]
character_set = Data["character_set"]
if Setup.buffer_based_analyzis_f:
reload_state = None
else:
reload_state = TheAnalyzer.reload_state
result, \
door_id_beyond = loop.do(CountOpFactory.from_ParserDataLineColumn(counter_db, character_set, Lng.INPUT_P()),
AfterBeyond = [ Op.GotoDoorId(DoorID.continue_without_on_after_match()) ],
LexemeEndCheckF = False,
LexemeMaintainedF = False,
EngineType = engine.FORWARD,
ReloadStateExtern = reload_state)
assert isinstance(result, list)
return result
def do(Data, TheAnalyzer):
"""________________________________________________________________________
Counting whitespace at the beginning of a line.
.-----<----+----------<--------------+--<----.
| | count | | count = 0
| | whitespace | |
.---------. | | |
--------->| +--' | |
| | | |
| | | |
| | .------------. | |
| +----->----| suppressor |----' |
| | | + newline | |
| COUNTER | '------------' |
| | .---------. |
| +----->----| newline |---------------'
| | '---------'
| | .----------------.
| |----->----| on_indentation |---------> RESTART
'---------' else '----------------'
Generate an indentation counter. An indentation counter is entered upon
the detection of a newline (which is not followed by a newline suppressor).
Indentation Counter:
indentation = 0
column = 0
|
|<------------------------.
.-------------. |
| INDENTATION | indentation += count
| COUNTER | column += count
'-------------' |
| |
+-------- whitspace -->---'
|
Re-Enter
Analyzer
An indentation counter is a single state that iterates to itself as long
as whitespace occurs. During that iteration the column counter is adapted.
There are two types of adaption:
-- 'normal' adaption by a fixed delta. This adaption happens upon
normal space characters.
-- 'grid' adaption. When a grid character occurs, the column number
snaps to a value given by a grid size parameter.
When a newline occurs the indentation counter exits and restarts the
lexical analysis. If the newline is not followed by a newline suppressor
the analyzer will immediately be back to the indentation counter state.
___________________________________________________________________________
"""
counter_db = Data["counter_db"]
isetup = Data["indentation_setup"]
incidence_db = Data["incidence_db"]
default_ih_f = Data["default_indentation_handler_f"]
mode_name = Data["mode_name"]
sm_suppressed_newline = Data["sm_suppressed_newline"]
sm_newline = isetup.sm_newline.get()
sm_comment = isetup.sm_comment.get()
assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant()
assert sm_newline is None or sm_newline.is_DFA_compliant()
assert sm_comment is None or sm_comment.is_DFA_compliant()
# -- 'on_indentation' == 'on_beyond':
# A handler is called as soon as an indentation has been detected.
after_beyond = [
Op.IndentationHandlerCall(default_ih_f, mode_name),
Op.GotoDoorId(DoorID.continue_without_on_after_match())
]
# -- 'on_bad_indentation' is invoked if a character appeared that has been
# explicitly disallowed to be used as indentation.
bad_indentation_iid = dial_db.new_incidence_id()
if Setup.buffer_based_analyzis_f: reload_state = None
else: reload_state = TheAnalyzer.reload_state
sm_terminal_list = _get_state_machine_vs_terminal_list(sm_suppressed_newline,
isetup.sm_newline.get(),
isetup.sm_comment.get(),
counter_db)
# 'whitespace' --> normal counting
# 'bad' --> goto bad character indentation handler
# else --> non-whitespace detected => handle indentation
ccfactory = CountOpFactory.from_ParserDataIndentation(isetup,
counter_db,
Lng.INPUT_P(),
DoorID.incidence(bad_indentation_iid))
# (*) Generate Code
code, \
door_id_beyond = loop.do(ccfactory,
AfterBeyond = after_beyond,
EngineType = TheAnalyzer.engine_type,
ReloadStateExtern = reload_state,
LexemeMaintainedF = True,
ParallelSmTerminalPairList = sm_terminal_list)
_code_terminal_on_bad_indentation_character(code, isetup, mode_name, incidence_db,
bad_indentation_iid)
return code
def _analyzer_function(StateMachineName, Setup, variable_definitions,
function_body, ModeNameList=[]):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
if a stand-alone lexical engine is required (without the
complete mode-handling framework of quex).
"""
Lng = Setup.language_db
SingleModeAnalyzerF = Setup.single_mode_analyzer_f
mode_definition_str = ""
mode_undefinition_str = ""
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
mode_definition_str = "".join(
"# define %s%s (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name)
for name in ModeNameList
)
mode_undefinition_str = "".join(
"# undef %s\n" % name
for name in ModeNameList
)
function_signature_str = __function_signature.replace("$$STATE_MACHINE_NAME$$",
StateMachineName)
txt = [
"#include <quex/code_base/temporary_macros_on>\n",
function_signature_str,
#
# Macro definitions
#
"# ifdef self\n",
"# undef self\n",
"# endif\n",
"# define self (*((QUEX_TYPE_ANALYZER*)me))\n",
"/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n",
"# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router()),
mode_definition_str,
Lng.LEXEME_MACRO_SETUP(),
#
variable_definitions,
#
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
" || defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
#
# Entry to the actual function body
#
"%s\n" % Lng.LABEL(DoorID.global_reentry()),
" %s\n" % Lng.LEXEME_START_SET(),
" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n",
]
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.extend([
"\n",
" __quex_assert_no_passage();\n",
"\n",
" /* Following labels are referenced in macros. It cannot be detected\n"
" * whether the macros are applied in user code or not. To avoid compiler.\n"
" * warnings of unused labels, they are referenced in unreachable code. */\n"
" %s /* in RETURN */\n" % Lng.GOTO(DoorID.return_with_on_after_match()),
" %s /* in CONTINUE */\n" % Lng.GOTO(DoorID.continue_with_on_after_match()),
" %s /* in CONTINUE and skippers */\n" % Lng.GOTO(DoorID.continue_without_on_after_match()),
"# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n",
" %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router()),
"# endif\n",
"\n",
" /* Prevent compiler warning 'unused variable'. */\n",
" (void)QUEX_LEXEME_NULL;\n",
" (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n",
" /* target_state_index and target_state_else_index appear when \n",
" * QUEX_GOTO_STATE is used without computed goto-s. */\n",
" (void)target_state_index;\n",
" (void)target_state_else_index;\n",
#
# Macro undefinitions
#
lexeme_macro_clean_up,
mode_undefinition_str,
"# undef self\n",
"# undef QUEX_LABEL_STATE_ROUTER\n",
"}\n",
"#include <quex/code_base/temporary_macros_off>\n",
])
return txt
def do(Data, TheAnalyzer):
"""________________________________________________________________________
Counting whitespace at the beginning of a line.
.-----<----+----------<--------------+--<----.
| | count | | count = 0
| | whitespace | |
.---------. | | |
--------->| +--' | |
| | | |
| | | |
| | .------------. | |
| +----->----| suppressor |----' |
| | | + newline | |
| COUNTER | '------------' |
| | .---------. |
| +----->----| newline |---------------'
| | '---------'
| | .----------------.
| |----->----| on_indentation |---------> RESTART
'---------' else '----------------'
Generate an indentation counter. An indentation counter is entered upon
the detection of a newline (which is not followed by a newline suppressor).
Indentation Counter:
indentation = 0
column = 0
|
|<------------------------.
.-------------. |
| INDENTATION | indentation += count
| COUNTER | column += count
'-------------' |
| |
+-------- whitspace -->---'
|
Re-Enter
Analyzer
An indentation counter is a single state that iterates to itself as long
as whitespace occurs. During that iteration the column counter is adapted.
There are two types of adaption:
-- 'normal' adaption by a fixed delta. This adaption happens upon
normal space characters.
-- 'grid' adaption. When a grid character occurs, the column number
snaps to a value given by a grid size parameter.
When a newline occurs the indentation counter exits and restarts the
lexical analysis. If the newline is not followed by a newline suppressor
the analyzer will immediately be back to the indentation counter state.
___________________________________________________________________________
"""
counter_db = Data["counter_db"]
isetup = Data["indentation_setup"]
incidence_db = Data["incidence_db"]
default_ih_f = Data["default_indentation_handler_f"]
mode_name = Data["mode_name"]
sm_suppressed_newline = Data["sm_suppressed_newline"]
sm_newline = isetup.sm_newline.get()
sm_comment = isetup.sm_comment.get()
assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant(
)
assert sm_newline is None or sm_newline.is_DFA_compliant()
assert sm_comment is None or sm_comment.is_DFA_compliant()
# -- 'on_indentation' == 'on_beyond':
# A handler is called as soon as an indentation has been detected.
after_beyond = [
IndentationHandlerCall(default_ih_f, mode_name),
GotoDoorId(DoorID.continue_without_on_after_match())
]
# -- 'on_bad_indentation' is invoked if a character appeared that has been
# explicitly disallowed to be used as indentation.
bad_indentation_iid = dial_db.new_incidence_id()
if Setup.buffer_based_analyzis_f: reload_state = None
else: reload_state = TheAnalyzer.reload_state
sm_terminal_list = _get_state_machine_vs_terminal_list(
sm_suppressed_newline, isetup.sm_newline.get(),
isetup.sm_comment.get(), counter_db)
# 'whitespace' --> normal counting
# 'bad' --> goto bad character indentation handler
# else --> non-whitespace detected => handle indentation
ccfactory = CountCmdFactory.from_ParserDataIndentation(
isetup, counter_db, Lng.INPUT_P(),
DoorID.incidence(bad_indentation_iid))
# (*) Generate Code
code, \
door_id_beyond = loop.do(ccfactory,
AfterBeyond = after_beyond,
EngineType = TheAnalyzer.engine_type,
ReloadStateExtern = reload_state,
LexemeMaintainedF = True,
ParallelSmTerminalPairList = sm_terminal_list)
_code_terminal_on_bad_indentation_character(code, isetup, mode_name,
incidence_db,
bad_indentation_iid)
return code
def do(Data, TheAnalyzer):
"""Fast implementation of character set skipping machine.
________________________________________________________________________
As long as characters of a given character set appears it iterates:
input in Set
.--<---.
| |
.-------. |
--------->( SKIPPER )--+----->------> RESTART
'-------' input
not in Set
___________________________________________________________________________
NOTE: The 'TerminalSkipRange' takes care that it transits immediately to
the indentation handler, if it ends on 'newline'. This is not necessary
for 'TerminalSkipCharacterSet'. Quex refuses to work on 'skip sets' when
they match common lexemes with the indentation handler.
___________________________________________________________________________
Precisely, i.e. including counter and reload actions:
START
|
| .----------------------------------------------.
| |.-------------------------------------------. |
| ||.----------------------------------------. | |
| ||| | | |
| ||| .-DoorID(S, a)--. transition | | |
| || '-| gridstep(cn) | map | | |
| || '---------------'\ .------. | | |
| || .-DoorID(S, b)--. '->-| | | | |
| |'---| ln += 1 |--->-| '\t' +-->-----' | |
| | '---------------' | | | |
| | .-DoorID(S, c)--. | ' ' +-->-------' |
| '----| cn += 1 |--->-| | |
| '---------------' | '\n' +-->---------'
| | | .-DropOut ------.
| .-DoorID(S, 0)--. | else +-->---------------| on_exit |
'------>--| on_entry |--->-| | '---------------'
'---------------' | BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .-----------------.
.----| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'-----------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| End of Stream |
'---------------'
NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs,
then the single state may actually be split into a real state machine of
states.
"""
counter_db = Data["counter_db"]
character_set = Data["character_set"]
if Setup.buffer_based_analyzis_f:
reload_state = None
else:
reload_state = TheAnalyzer.reload_state
result, \
door_id_beyond = loop.do(CountCmdFactory.from_ParserDataLineColumn(counter_db, character_set, Lng.INPUT_P()),
AfterBeyond = [ GotoDoorId(DoorID.continue_without_on_after_match()) ],
LexemeEndCheckF = False,
LexemeMaintainedF = False,
EngineType = engine.FORWARD,
ReloadStateExtern = reload_state)
assert isinstance(result, list)
return result
def _analyzer_function(StateMachineName, Setup, variable_definitions,
function_body, dial_db, ModeNameList):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
"""
Lng = Setup.language_db
function_signature_str = __function_signature.replace(
"$$STATE_MACHINE_NAME$$", StateMachineName)
state_router_adr = DoorID.global_state_router(dial_db).related_address
txt = [
function_signature_str,
# Macro definitions
#
Lng.DEFINE_SELF("me"),
Lng.MODE_DEFINITION(ModeNameList),
"/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n",
"# define QUEX_LABEL_STATE_ROUTER %s\n" %
Lng.LABEL_STR_BY_ADR(state_router_adr),
Lng.DEFINE_LEXEME_VARIABLES(),
#
variable_definitions,
#
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
#
# Entry to the actual function body
#
"%s\n" % Lng.LABEL(DoorID.global_reentry(dial_db)),
" %s\n" % Lng.LEXEME_START_SET(),
" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n",
]
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.extend([
"\n",
" __quex_assert_no_passage();\n",
"\n",
" /* Following labels are referenced in macros. It cannot be detected\n"
" * whether the macros are applied in user code or not. To avoid compiler.\n"
" * warnings of unused labels, they are referenced in unreachable code. */\n"
" %s /* in RETURN */\n" %
Lng.GOTO(DoorID.return_with_on_after_match(dial_db), dial_db),
" %s /* in CONTINUE */\n" %
Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db),
" %s /* in CONTINUE and skippers */\n" %
Lng.GOTO(DoorID.continue_without_on_after_match(dial_db), dial_db),
"$$<not-computed-gotos>----------------------------------------------\n",
" %s /* in QUEX_GOTO_STATE */\n" %
Lng.GOTO(DoorID.global_state_router(dial_db), dial_db),
"$$------------------------------------------------------------------\n",
"\n",
" /* Prevent compiler warning 'unused variable'. */\n",
" (void)QUEX_NAME(LexemeNull);\n",
" /* target_state_index and target_state_else_index appear when \n",
" * QUEX_GOTO_STATE is used without computed goto-s. */\n",
" (void)target_state_index;\n",
" (void)target_state_else_index;\n",
#
# Macro undefinitions
#
Lng.UNDEFINE_LEXEME_VARIABLES(),
Lng.MODE_UNDEFINITION(ModeNameList),
"# undef self\n",
"# undef QUEX_LABEL_STATE_ROUTER\n",
"}\n",
])
return txt
def _analyzer_function(StateMachineName, Setup, variable_definitions,
function_body, ModeNameList=[]):
"""EngineClassName = name of the structure that contains the engine state.
if a mode of a complete quex environment is created, this
is the mode name. otherwise, any name can be chosen.
SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True
if a stand-alone lexical engine is required (without the
complete mode-handling framework of quex).
"""
Lng = Setup.language_db
SingleModeAnalyzerF = Setup.single_mode_analyzer_f
mode_definition_str = ""
mode_undefinition_str = ""
if len(ModeNameList) != 0 and not SingleModeAnalyzerF:
L = max(map(lambda name: len(name), ModeNameList))
mode_definition_str = "".join(
"# define %s%s (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name)
for name in ModeNameList
)
mode_undefinition_str = "".join(
"# undef %s\n" % name
for name in ModeNameList
)
function_signature_str = __function_signature.replace("$$STATE_MACHINE_NAME$$",
StateMachineName)
txt = [
"#include <quex/code_base/temporary_macros_on>\n",
function_signature_str,
#
# Macro definitions
#
"# ifdef self\n",
"# undef self\n",
"# endif\n",
"# define self (*((QUEX_TYPE_ANALYZER*)me))\n",
"/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n",
"# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router()),
mode_definition_str,
Lng.LEXEME_MACRO_SETUP(),
#
variable_definitions,
#
comment_on_post_context_position_init_str,
"# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n",
" || defined(QUEX_OPTION_ASSERTS)\n",
" me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n",
"# endif\n",
#
# Entry to the actual function body
#
"%s\n" % Lng.LABEL(DoorID.global_reentry()),
" %s\n" % Lng.LEXEME_START_SET(),
" QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n",
]
txt.extend(function_body)
# -- prevent the warning 'unused variable'
txt.extend([
"\n",
" __quex_assert_no_passage();\n",
"\n",
" /* Following labels are referenced in macros. It cannot be detected\n"
" * whether the macros are applied in user code or not. To avoid compiler.\n"
" * warnings of unused labels, they are referenced in unreachable code. */\n"
" %s /* in RETURN */\n" % Lng.GOTO(DoorID.return_with_on_after_match()),
" %s /* in CONTINUE */\n" % Lng.GOTO(DoorID.continue_with_on_after_match()),
" %s /* in CONTINUE and skippers */\n" % Lng.GOTO(DoorID.continue_without_on_after_match()),
"# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n",
" %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router()),
"# endif\n",
"\n",
" /* Prevent compiler warning 'unused variable'. */\n",
" (void)QUEX_LEXEME_NULL;\n",
" (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n",
" /* target_state_index and target_state_else_index appear when \n",
" * QUEX_GOTO_STATE is used without computed goto-s. */\n",
" (void)target_state_index;\n",
" (void)target_state_else_index;\n",
#
# Macro undefinitions
#
lexeme_macro_clean_up,
mode_undefinition_str,
"# undef self\n",
"# undef QUEX_LABEL_STATE_ROUTER\n",
"}\n",
"#include <quex/code_base/temporary_macros_off>\n",
])
return txt
