def reentry_preparation(Lng, PreConditionIDList, OnAfterMatchCode): """Reentry preperation (without returning from the function.""" # (*) Unset all pre-context flags which may have possibly been set unset_pre_context_flags_str = Lng.PRE_CONTEXT_RESET(PreConditionIDList) if OnAfterMatchCode is not None: on_after_match_str = Lng.SOURCE_REFERENCED(OnAfterMatchCode) else: on_after_match_str = "" txt = [ "\n%s\n" % Lng.LABEL(DoorID.return_with_on_after_match()), Lng.COMMENT("RETURN -- after executing 'on_after_match' code."), on_after_match_str, " %s\n\n" % Lng.PURE_RETURN, # "\n%s\n" % Lng.LABEL(DoorID.continue_with_on_after_match()), Lng.COMMENT("CONTINUE -- after executing 'on_after_match' code."), on_after_match_str, # "\n%s\n" % Lng.LABEL(DoorID.continue_without_on_after_match()), Lng.COMMENT("CONTINUE -- without executing 'on_after_match' (e.g. on FAILURE)."), "\n", # __return_if_queue_full_or_simple_analyzer, "\n", __return_if_mode_changed, "\n", # unset_pre_context_flags_str, "\n%s\n" % Lng.GOTO(DoorID.global_reentry()), ] return txt
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb, IncidenceDb): """Generate a PPT for a range skipper. """ # -- door_id_after: Where to go after the closing character sequence matched: # + Normally: To the begin of the analyzer. Start again. # + End(Sequence) == newline of indentation counter. # => goto indentation counter. if _match_indentation_counter_newline_pattern( OptionsDb.value("indentation"), data["closer_sequence"]): door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER) else: door_id_after = DoorID.continue_without_on_after_match() if NestedF: name = "SKIP NESTED RANGE" code_generator_func = skip_nested_range.do else: name = "SKIP RANGE" code_generator_func = skip_range.do # -- data for code generation my_data = deepcopy(data) my_data["mode_name"] = ModeName my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN] my_data["door_id_after"] = door_id_after my_data["counter_db"] = CounterDb # -- terminal and code generator priority = PatternPriority(MHI, i) pattern = deepcopy(my_data["opener_pattern"]) pattern.set_incidence_id(dial_db.new_incidence_id()) pattern.set_pattern_string("<skip_range>") code = CodeGenerated(code_generator_func, my_data, name) return PPT(priority, pattern, code)
def do_match_failure(self, Code, ThePattern): """No pattern in the mode has matched. Line and column numbers are still counted. But, no 'on_match' or 'on_after_match' action is executed. """ lexeme_begin_f, \ terminating_zero_f, \ adorned_code = self.__adorn_user_code(Code, MatchF=False) text = [ #Lng.IF_END_OF_FILE(), # self.get_counter_text(None), # Lng.GOTO(DoorID.continue_without_on_after_match()), #Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False), # Lng.INPUT_P_INCREMENT(), #Lng.END_IF(), self.get_counter_text(None), # adorned_code, # Lng.GOTO(DoorID.continue_without_on_after_match()), ] code = CodeTerminal(text, SourceReference = Code.sr, PureCode = Code.get_pure_code()) return Terminal(code, "FAILURE")
def reentry_preparation(Lng, PreConditionIDList, OnAfterMatchCode, dial_db): """Reentry preperation (without returning from the function.""" # (*) Unset all pre-context flags which may have possibly been set unset_pre_context_flags_str = Lng.PRE_CONTEXT_RESET(PreConditionIDList) on_after_match_str = Lng.SOURCE_REFERENCED(OnAfterMatchCode) return [ "\n%s\n" % Lng.LABEL(DoorID.return_with_on_after_match(dial_db)), Lng.COMMENT("RETURN -- after executing 'on_after_match' code."), on_after_match_str, " %s\n\n" % Lng.PURE_RETURN, # "\n%s\n" % Lng.LABEL(DoorID.continue_with_on_after_match(dial_db)), Lng.COMMENT("CONTINUE -- after executing 'on_after_match' code."), on_after_match_str, # "\n%s\n" % Lng.LABEL(DoorID.continue_without_on_after_match(dial_db)), Lng.COMMENT( "CONTINUE -- without executing 'on_after_match' (e.g. on FAILURE)." ), "\n", # __assert_no_mode_change, "\n", __return_if_queue_full_or_simple_analyzer, "\n", unset_pre_context_flags_str, "\n%s\n" % Lng.GOTO(DoorID.global_reentry(dial_db), dial_db), ]
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb, IncidenceDb): """Generate a PPT for a range skipper. """ # -- door_id_after: Where to go after the closing character sequence matched: # + Normally: To the begin of the analyzer. Start again. # + End(Sequence) == newline of indentation counter. # => goto indentation counter. if _match_indentation_counter_newline_pattern(OptionsDb.value("indentation"), data["closer_sequence"]): door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER) else: door_id_after = DoorID.continue_without_on_after_match() if NestedF: name = "SKIP NESTED RANGE" code_generator_func = skip_nested_range.do else: name = "SKIP RANGE" code_generator_func = skip_range.do # -- data for code generation my_data = deepcopy(data) my_data["mode_name"] = ModeName my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN] my_data["door_id_after"] = door_id_after my_data["counter_db"] = CounterDb # -- terminal and code generator priority = PatternPriority(MHI, i) pattern = deepcopy(my_data["opener_pattern"]) pattern.set_incidence_id(dial_db.new_incidence_id()) pattern.set_pattern_string("<skip_range>") code = CodeGenerated(code_generator_func, my_data, name) return PPT(priority, pattern, code)
def do_match_failure(self, Code, ThePattern): """No pattern in the mode has matched. Line and column numbers are still counted. But, no 'on_match' or 'on_after_match' action is executed. """ lexeme_begin_f, \ terminating_zero_f, \ adorned_code = self.__adorn_user_code(Code, MatchF=False) text = [ #Lng.IF_END_OF_FILE(), # self.get_counter_text(None), # Lng.GOTO(DoorID.continue_without_on_after_match()), #Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False), # Lng.INPUT_P_INCREMENT(), #Lng.END_IF(), self.get_counter_text(None), # adorned_code, # Lng.GOTO(DoorID.continue_without_on_after_match()), ] code = CodeTerminal(text, SourceReference=Code.sr, PureCode=Code.get_pure_code()) return Terminal(code, "FAILURE")
def _range_skipper_door_id_exit(self, IndentationHandler, CloserPattern, dial_db): if self._match_indentation_counter_newline_pattern( IndentationHandler, CloserPattern): return DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER, dial_db) else: return DoorID.continue_without_on_after_match(dial_db)
def _get_indentation_handler_terminal(ModeName, dial_db): code = Lng.COMMAND_LIST([ Op.IndentationHandlerCall(ModeName), Op.GotoDoorId(DoorID.continue_without_on_after_match(dial_db)) ], dial_db) incidence_id = dial.new_incidence_id() terminal = Terminal(CodeTerminal(code), "<CALL INDENTATION HANDLER>", incidence_id, dial_db=dial_db) return DoorID.incidence(incidence_id, dial_db), terminal
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb): """Additionally to all characters, the loop shall walk along the 'closer'. If the closer matches, the range skipping exits. Characters need to be counted properly. RETURNS: list(state machine, terminal) The list contains only one single element. """ sm = StateMachine.from_sequence(CloserSequence) sm.set_id(dial_db.new_incidence_id()) code = [ Lng.GOTO(DoorID.continue_without_on_after_match()) ] terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>", sm.get_id()) return [ (sm, terminal) ]
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb): """Additionally to all characters, the loop shall walk along the 'closer'. If the closer matches, the range skipping exits. Characters need to be counted properly. RETURNS: list(state machine, terminal) The list contains only one single element. """ sm = StateMachine.from_sequence(CloserSequence) sm.set_id(dial_db.new_incidence_id()) code = [Lng.GOTO(DoorID.continue_without_on_after_match())] terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>", sm.get_id()) return [(sm, terminal)]
def get_skipper(TheAnalyzer, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter, CounterDb): """ .---<---+----------<------+ | | | | | not | .------. | Closer[0] | ------------------------------>| Loop +--' | | | | no | | | | | .-------------. | +----->----| Closer[1-N] |------------> RESTART | | | ? | yes | | '-------------' | | | BLC +-->-. .->-| | \ Reload State .-DoorID(S, 1)--./ '------' \ .-----------------. .----| after_reload | \ .---------------. | | '---------------' '---------| before_reload | | | '---------------' | '-----------------------------------------------------| | success '------------------' | failure | .---------------. | SkipRangeOpen | '---------------' """ psml = _get_state_machine_vs_terminal_list(CloserSequence, CounterDb) count_op_factory = CountInfoMap.from_LineColumnCount(CounterDb, NumberSet_All(), Lng.INPUT_P()) result, \ door_id_beyond = loop.do(count_op_factory, OnLoopExit = [ Op.GotoDoorId(DoorID.continue_without_on_after_match()) ], LexemeEndCheckF = False, LexemeMaintainedF = False, EngineType = engine.FORWARD, ReloadStateExtern = TheAnalyzer.reload_state, ParallelSmTerminalPairList = psml) return result
def get_skipper(TheAnalyzer, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter, CounterDb): """ .---<---+----------<------+ | | | | | not | .------. | Closer[0] | ------------------------------>| Loop +--' | | | | no | | | | | .-------------. | +----->----| Closer[1-N] |------------> RESTART | | | ? | yes | | '-------------' | | | BLC +-->-. .->-| | \ Reload State .-DoorID(S, 1)--./ '------' \ .-----------------. .----| after_reload | \ .---------------. | | '---------------' '---------| before_reload | | | '---------------' | '-----------------------------------------------------| | success '------------------' | failure | .---------------. | SkipRangeOpen | '---------------' """ psml = _get_state_machine_vs_terminal_list(CloserSequence, CounterDb) count_op_factory = CountInfoMap.from_LineColumnCount( CounterDb, NumberSet_All(), Lng.INPUT_P()) result, \ door_id_beyond = loop.do(count_op_factory, OnLoopExit = [ Op.GotoDoorId(DoorID.continue_without_on_after_match()) ], LexemeEndCheckF = False, LexemeMaintainedF = False, EngineType = engine.FORWARD, ReloadStateExtern = TheAnalyzer.reload_state, ParallelSmTerminalPairList = psml) return result
def do(ModeName, CaMap, CharacterSet, ReloadState, dial_db): """Fast implementation of character set skipping machine. ________________________________________________________________________ As long as characters of a given character set appears it iterates: input in Set .--<---. | | .-------. | --------->( SKIPPER )--+----->------> RESTART '-------' input not in Set ___________________________________________________________________________ Precisely, i.e. including counter and reload actions: START | | .----------------------------------------------. | |.-------------------------------------------. | | ||.----------------------------------------. | | | ||| | | | | ||| .-DoorID(S, a)--. transition | | | | || '-| gridstep(cn) | map | | | | || '---------------'\ .------. | | | | || .-DoorID(S, b)--. '->-| | | | | | |'---| ln += 1 |--->-| '\t' +-->-----' | | | | '---------------' | | | | | | .-DoorID(S, c)--. | ' ' +-->-------' | | '----| cn += 1 |--->-| | | | '---------------' | '\n' +-->---------' | | | .-DropOut ------. | .-DoorID(S, 0)--. | else +-->---------------| on_exit | '------>--| on_entry |--->-| | '---------------' '---------------' | BLC +-->-. .->-| | \ Reload State .-DoorID(S, 1)--./ '------' \ .-----------------. .----| after_reload | \ .---------------. | | '---------------' '---------| before_reload | | | '---------------' | '-----------------------------------------------------| | success '------------------' | failure | .---------------. | End of Stream | '---------------' NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs, then the single state may actually be split into a real state machine of states. """ assert isinstance(CaMap, CountActionMap) assert isinstance(CharacterSet, NumberSet) if ReloadState: engine_type = ReloadState.engine_type else: engine_type = None on_loop_exit_door_id = DoorID.continue_without_on_after_match(dial_db) analyzer_list, \ terminal_list, \ loop_map, \ door_id_loop, \ required_register_set, \ run_time_counter_f = loop.do(CaMap.pruned_clone(CharacterSet), OnLoopExitDoorId = on_loop_exit_door_id, EngineType = engine_type, ReloadStateExtern = ReloadState, dial_db = dial_db, ModeName = ModeName) assert not run_time_counter_f return analyzer_list, \ terminal_list, \ loop_map, \ required_register_set
def do(Data, TheAnalyzer): """Fast implementation of character set skipping machine. ________________________________________________________________________ As long as characters of a given character set appears it iterates: input in Set .--<---. | | .-------. | --------->( SKIPPER )--+----->------> RESTART '-------' input not in Set ___________________________________________________________________________ NOTE: The 'TerminalSkipRange' takes care that it transits immediately to the indentation handler, if it ends on 'newline'. This is not necessary for 'TerminalSkipCharacterSet'. Quex refuses to work on 'skip sets' when they match common lexemes with the indentation handler. ___________________________________________________________________________ Precisely, i.e. including counter and reload actions: START | | .----------------------------------------------. | |.-------------------------------------------. | | ||.----------------------------------------. | | | ||| | | | | ||| .-DoorID(S, a)--. transition | | | | || '-| gridstep(cn) | map | | | | || '---------------'\ .------. | | | | || .-DoorID(S, b)--. '->-| | | | | | |'---| ln += 1 |--->-| '\t' +-->-----' | | | | '---------------' | | | | | | .-DoorID(S, c)--. | ' ' +-->-------' | | '----| cn += 1 |--->-| | | | '---------------' | '\n' +-->---------' | | | .-DropOut ------. | .-DoorID(S, 0)--. | else +-->---------------| on_exit | '------>--| on_entry |--->-| | '---------------' '---------------' | BLC +-->-. .->-| | \ Reload State .-DoorID(S, 1)--./ '------' \ .-----------------. .----| after_reload | \ .---------------. | | '---------------' '---------| before_reload | | | '---------------' | '-----------------------------------------------------| | success '------------------' | failure | .---------------. | End of Stream | '---------------' NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs, then the single state may actually be split into a real state machine of states. """ counter_db = Data["counter_db"] character_set = Data["character_set"] if Setup.buffer_based_analyzis_f: reload_state = None else: reload_state = TheAnalyzer.reload_state result, \ door_id_beyond = loop.do(CountOpFactory.from_ParserDataLineColumn(counter_db, character_set, Lng.INPUT_P()), AfterBeyond = [ Op.GotoDoorId(DoorID.continue_without_on_after_match()) ], LexemeEndCheckF = False, LexemeMaintainedF = False, EngineType = engine.FORWARD, ReloadStateExtern = reload_state) assert isinstance(result, list) return result
def do(Data, TheAnalyzer): """________________________________________________________________________ Counting whitespace at the beginning of a line. .-----<----+----------<--------------+--<----. | | count | | count = 0 | | whitespace | | .---------. | | | --------->| +--' | | | | | | | | | | | | .------------. | | | +----->----| suppressor |----' | | | | + newline | | | COUNTER | '------------' | | | .---------. | | +----->----| newline |---------------' | | '---------' | | .----------------. | |----->----| on_indentation |---------> RESTART '---------' else '----------------' Generate an indentation counter. An indentation counter is entered upon the detection of a newline (which is not followed by a newline suppressor). Indentation Counter: indentation = 0 column = 0 | |<------------------------. .-------------. | | INDENTATION | indentation += count | COUNTER | column += count '-------------' | | | +-------- whitspace -->---' | Re-Enter Analyzer An indentation counter is a single state that iterates to itself as long as whitespace occurs. During that iteration the column counter is adapted. There are two types of adaption: -- 'normal' adaption by a fixed delta. This adaption happens upon normal space characters. -- 'grid' adaption. When a grid character occurs, the column number snaps to a value given by a grid size parameter. When a newline occurs the indentation counter exits and restarts the lexical analysis. If the newline is not followed by a newline suppressor the analyzer will immediately be back to the indentation counter state. ___________________________________________________________________________ """ counter_db = Data["counter_db"] isetup = Data["indentation_setup"] incidence_db = Data["incidence_db"] default_ih_f = Data["default_indentation_handler_f"] mode_name = Data["mode_name"] sm_suppressed_newline = Data["sm_suppressed_newline"] sm_newline = isetup.sm_newline.get() sm_comment = isetup.sm_comment.get() assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant() assert sm_newline is None or sm_newline.is_DFA_compliant() assert sm_comment is None or sm_comment.is_DFA_compliant() # -- 'on_indentation' == 'on_beyond': # A handler is called as soon as an indentation has been detected. after_beyond = [ Op.IndentationHandlerCall(default_ih_f, mode_name), Op.GotoDoorId(DoorID.continue_without_on_after_match()) ] # -- 'on_bad_indentation' is invoked if a character appeared that has been # explicitly disallowed to be used as indentation. bad_indentation_iid = dial_db.new_incidence_id() if Setup.buffer_based_analyzis_f: reload_state = None else: reload_state = TheAnalyzer.reload_state sm_terminal_list = _get_state_machine_vs_terminal_list(sm_suppressed_newline, isetup.sm_newline.get(), isetup.sm_comment.get(), counter_db) # 'whitespace' --> normal counting # 'bad' --> goto bad character indentation handler # else --> non-whitespace detected => handle indentation ccfactory = CountOpFactory.from_ParserDataIndentation(isetup, counter_db, Lng.INPUT_P(), DoorID.incidence(bad_indentation_iid)) # (*) Generate Code code, \ door_id_beyond = loop.do(ccfactory, AfterBeyond = after_beyond, EngineType = TheAnalyzer.engine_type, ReloadStateExtern = reload_state, LexemeMaintainedF = True, ParallelSmTerminalPairList = sm_terminal_list) _code_terminal_on_bad_indentation_character(code, isetup, mode_name, incidence_db, bad_indentation_iid) return code
def _analyzer_function(StateMachineName, Setup, variable_definitions, function_body, ModeNameList=[]): """EngineClassName = name of the structure that contains the engine state. if a mode of a complete quex environment is created, this is the mode name. otherwise, any name can be chosen. SingleModeAnalyzerF = False if a mode for a quex engine is to be created. True if a stand-alone lexical engine is required (without the complete mode-handling framework of quex). """ Lng = Setup.language_db SingleModeAnalyzerF = Setup.single_mode_analyzer_f mode_definition_str = "" mode_undefinition_str = "" if len(ModeNameList) != 0 and not SingleModeAnalyzerF: L = max(map(lambda name: len(name), ModeNameList)) mode_definition_str = "".join( "# define %s%s (QUEX_NAME(%s))\n" % (name, " " * (L- len(name)), name) for name in ModeNameList ) mode_undefinition_str = "".join( "# undef %s\n" % name for name in ModeNameList ) function_signature_str = __function_signature.replace("$$STATE_MACHINE_NAME$$", StateMachineName) txt = [ "#include <quex/code_base/temporary_macros_on>\n", function_signature_str, # # Macro definitions # "# ifdef self\n", "# undef self\n", "# endif\n", "# define self (*((QUEX_TYPE_ANALYZER*)me))\n", "/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n", "# define QUEX_LABEL_STATE_ROUTER %s\n" % dial_db.get_label_by_door_id(DoorID.global_state_router()), mode_definition_str, Lng.LEXEME_MACRO_SETUP(), # variable_definitions, # comment_on_post_context_position_init_str, "# if defined(QUEX_OPTION_AUTOMATIC_ANALYSIS_CONTINUATION_ON_MODE_CHANGE) \\\n", " || defined(QUEX_OPTION_ASSERTS)\n", " me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n", "# endif\n", # # Entry to the actual function body # "%s\n" % Lng.LABEL(DoorID.global_reentry()), " %s\n" % Lng.LEXEME_START_SET(), " QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n", ] txt.extend(function_body) # -- prevent the warning 'unused variable' txt.extend([ "\n", " __quex_assert_no_passage();\n", "\n", " /* Following labels are referenced in macros. It cannot be detected\n" " * whether the macros are applied in user code or not. To avoid compiler.\n" " * warnings of unused labels, they are referenced in unreachable code. */\n" " %s /* in RETURN */\n" % Lng.GOTO(DoorID.return_with_on_after_match()), " %s /* in CONTINUE */\n" % Lng.GOTO(DoorID.continue_with_on_after_match()), " %s /* in CONTINUE and skippers */\n" % Lng.GOTO(DoorID.continue_without_on_after_match()), "# if ! defined(QUEX_OPTION_COMPUTED_GOTOS)\n", " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router()), "# endif\n", "\n", " /* Prevent compiler warning 'unused variable'. */\n", " (void)QUEX_LEXEME_NULL;\n", " (void)QUEX_NAME_TOKEN(DumpedTokenIdObject);\n", " /* target_state_index and target_state_else_index appear when \n", " * QUEX_GOTO_STATE is used without computed goto-s. */\n", " (void)target_state_index;\n", " (void)target_state_else_index;\n", # # Macro undefinitions # lexeme_macro_clean_up, mode_undefinition_str, "# undef self\n", "# undef QUEX_LABEL_STATE_ROUTER\n", "}\n", "#include <quex/code_base/temporary_macros_off>\n", ]) return txt
def do(Data, TheAnalyzer): """________________________________________________________________________ Counting whitespace at the beginning of a line. .-----<----+----------<--------------+--<----. | | count | | count = 0 | | whitespace | | .---------. | | | --------->| +--' | | | | | | | | | | | | .------------. | | | +----->----| suppressor |----' | | | | + newline | | | COUNTER | '------------' | | | .---------. | | +----->----| newline |---------------' | | '---------' | | .----------------. | |----->----| on_indentation |---------> RESTART '---------' else '----------------' Generate an indentation counter. An indentation counter is entered upon the detection of a newline (which is not followed by a newline suppressor). Indentation Counter: indentation = 0 column = 0 | |<------------------------. .-------------. | | INDENTATION | indentation += count | COUNTER | column += count '-------------' | | | +-------- whitspace -->---' | Re-Enter Analyzer An indentation counter is a single state that iterates to itself as long as whitespace occurs. During that iteration the column counter is adapted. There are two types of adaption: -- 'normal' adaption by a fixed delta. This adaption happens upon normal space characters. -- 'grid' adaption. When a grid character occurs, the column number snaps to a value given by a grid size parameter. When a newline occurs the indentation counter exits and restarts the lexical analysis. If the newline is not followed by a newline suppressor the analyzer will immediately be back to the indentation counter state. ___________________________________________________________________________ """ counter_db = Data["counter_db"] isetup = Data["indentation_setup"] incidence_db = Data["incidence_db"] default_ih_f = Data["default_indentation_handler_f"] mode_name = Data["mode_name"] sm_suppressed_newline = Data["sm_suppressed_newline"] sm_newline = isetup.sm_newline.get() sm_comment = isetup.sm_comment.get() assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant( ) assert sm_newline is None or sm_newline.is_DFA_compliant() assert sm_comment is None or sm_comment.is_DFA_compliant() # -- 'on_indentation' == 'on_beyond': # A handler is called as soon as an indentation has been detected. after_beyond = [ IndentationHandlerCall(default_ih_f, mode_name), GotoDoorId(DoorID.continue_without_on_after_match()) ] # -- 'on_bad_indentation' is invoked if a character appeared that has been # explicitly disallowed to be used as indentation. bad_indentation_iid = dial_db.new_incidence_id() if Setup.buffer_based_analyzis_f: reload_state = None else: reload_state = TheAnalyzer.reload_state sm_terminal_list = _get_state_machine_vs_terminal_list( sm_suppressed_newline, isetup.sm_newline.get(), isetup.sm_comment.get(), counter_db) # 'whitespace' --> normal counting # 'bad' --> goto bad character indentation handler # else --> non-whitespace detected => handle indentation ccfactory = CountCmdFactory.from_ParserDataIndentation( isetup, counter_db, Lng.INPUT_P(), DoorID.incidence(bad_indentation_iid)) # (*) Generate Code code, \ door_id_beyond = loop.do(ccfactory, AfterBeyond = after_beyond, EngineType = TheAnalyzer.engine_type, ReloadStateExtern = reload_state, LexemeMaintainedF = True, ParallelSmTerminalPairList = sm_terminal_list) _code_terminal_on_bad_indentation_character(code, isetup, mode_name, incidence_db, bad_indentation_iid) return code
def do(Data, TheAnalyzer): """Fast implementation of character set skipping machine. ________________________________________________________________________ As long as characters of a given character set appears it iterates: input in Set .--<---. | | .-------. | --------->( SKIPPER )--+----->------> RESTART '-------' input not in Set ___________________________________________________________________________ NOTE: The 'TerminalSkipRange' takes care that it transits immediately to the indentation handler, if it ends on 'newline'. This is not necessary for 'TerminalSkipCharacterSet'. Quex refuses to work on 'skip sets' when they match common lexemes with the indentation handler. ___________________________________________________________________________ Precisely, i.e. including counter and reload actions: START | | .----------------------------------------------. | |.-------------------------------------------. | | ||.----------------------------------------. | | | ||| | | | | ||| .-DoorID(S, a)--. transition | | | | || '-| gridstep(cn) | map | | | | || '---------------'\ .------. | | | | || .-DoorID(S, b)--. '->-| | | | | | |'---| ln += 1 |--->-| '\t' +-->-----' | | | | '---------------' | | | | | | .-DoorID(S, c)--. | ' ' +-->-------' | | '----| cn += 1 |--->-| | | | '---------------' | '\n' +-->---------' | | | .-DropOut ------. | .-DoorID(S, 0)--. | else +-->---------------| on_exit | '------>--| on_entry |--->-| | '---------------' '---------------' | BLC +-->-. .->-| | \ Reload State .-DoorID(S, 1)--./ '------' \ .-----------------. .----| after_reload | \ .---------------. | | '---------------' '---------| before_reload | | | '---------------' | '-----------------------------------------------------| | success '------------------' | failure | .---------------. | End of Stream | '---------------' NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs, then the single state may actually be split into a real state machine of states. """ counter_db = Data["counter_db"] character_set = Data["character_set"] if Setup.buffer_based_analyzis_f: reload_state = None else: reload_state = TheAnalyzer.reload_state result, \ door_id_beyond = loop.do(CountCmdFactory.from_ParserDataLineColumn(counter_db, character_set, Lng.INPUT_P()), AfterBeyond = [ GotoDoorId(DoorID.continue_without_on_after_match()) ], LexemeEndCheckF = False, LexemeMaintainedF = False, EngineType = engine.FORWARD, ReloadStateExtern = reload_state) assert isinstance(result, list) return result
def _analyzer_function(StateMachineName, Setup, variable_definitions, function_body, dial_db, ModeNameList): """EngineClassName = name of the structure that contains the engine state. if a mode of a complete quex environment is created, this is the mode name. otherwise, any name can be chosen. """ Lng = Setup.language_db function_signature_str = __function_signature.replace( "$$STATE_MACHINE_NAME$$", StateMachineName) state_router_adr = DoorID.global_state_router(dial_db).related_address txt = [ function_signature_str, # Macro definitions # Lng.DEFINE_SELF("me"), Lng.MODE_DEFINITION(ModeNameList), "/* 'QUEX_GOTO_STATE' requires 'QUEX_LABEL_STATE_ROUTER' */\n", "# define QUEX_LABEL_STATE_ROUTER %s\n" % Lng.LABEL_STR_BY_ADR(state_router_adr), Lng.DEFINE_LEXEME_VARIABLES(), # variable_definitions, # comment_on_post_context_position_init_str, "# if defined(QUEX_OPTION_ASSERTS)\n", " me->DEBUG_analyzer_function_at_entry = me->current_analyzer_function;\n", "# endif\n", # # Entry to the actual function body # "%s\n" % Lng.LABEL(DoorID.global_reentry(dial_db)), " %s\n" % Lng.LEXEME_START_SET(), " QUEX_LEXEME_TERMINATING_ZERO_UNDO(&me->buffer);\n", ] txt.extend(function_body) # -- prevent the warning 'unused variable' txt.extend([ "\n", " __quex_assert_no_passage();\n", "\n", " /* Following labels are referenced in macros. It cannot be detected\n" " * whether the macros are applied in user code or not. To avoid compiler.\n" " * warnings of unused labels, they are referenced in unreachable code. */\n" " %s /* in RETURN */\n" % Lng.GOTO(DoorID.return_with_on_after_match(dial_db), dial_db), " %s /* in CONTINUE */\n" % Lng.GOTO(DoorID.continue_with_on_after_match(dial_db), dial_db), " %s /* in CONTINUE and skippers */\n" % Lng.GOTO(DoorID.continue_without_on_after_match(dial_db), dial_db), "$$<not-computed-gotos>----------------------------------------------\n", " %s /* in QUEX_GOTO_STATE */\n" % Lng.GOTO(DoorID.global_state_router(dial_db), dial_db), "$$------------------------------------------------------------------\n", "\n", " /* Prevent compiler warning 'unused variable'. */\n", " (void)QUEX_NAME(LexemeNull);\n", " /* target_state_index and target_state_else_index appear when \n", " * QUEX_GOTO_STATE is used without computed goto-s. */\n", " (void)target_state_index;\n", " (void)target_state_else_index;\n", # # Macro undefinitions # Lng.UNDEFINE_LEXEME_VARIABLES(), Lng.MODE_UNDEFINITION(ModeNameList), "# undef self\n", "# undef QUEX_LABEL_STATE_ROUTER\n", "}\n", ]) return txt