def _get_source_code(CcFactory, analyzer, terminal_list): """RETURNS: String containing source code for the 'loop'. -- The source code for the (looping) state machine. -- The terminals which contain counting actions. Also, it requests variable definitions as they are required. """ txt = [] txt.extend( generator.do_analyzer(analyzer) ) txt.extend( generator.do_terminals(terminal_list, analyzer) ) if analyzer.engine_type.subject_to_reload(): txt.extend( generator.do_reload_procedure(analyzer) ) if CcFactory.requires_reference_p(): variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") if Setup.buffer_codec.variable_character_sizes_f(): variable_db.require("character_begin_p") return txt
def __code_pre_context_state_machine(self): LanguageDB = self.language_db if len(self.pre_context_sm_list) == 0: return [] assert len(self.pre_context_sm.get_orphaned_state_index_list()) == 0 txt = [] if Setup.comment_state_machine_f: LanguageDB.ML_COMMENT(txt, "BEGIN: PRE-CONTEXT STATE MACHINE\n" + \ self.pre_context_sm.get_string(NormalizeF=False) + \ "END: PRE-CONTEXT STATE MACHINE") txt.append("\n") # For safety: New content may have to start in a newline, e.g. "#ifdef ..." analyzer = analyzer_generator.do(self.pre_context_sm, E_EngineTypes.BACKWARD_PRE_CONTEXT) msg = state_machine_coder.do(analyzer) txt.extend(msg) txt.append("\n%s" % LanguageDB.LABEL(E_StateIndices.END_OF_PRE_CONTEXT_CHECK)) # -- set the input stream back to the real current position. # during backward lexing the analyzer went backwards, so it needs to be reset. txt.append(" %s\n" % LanguageDB.INPUT_P_TO_LEXEME_START()) for sm_id in self.pre_context_sm_id_list: variable_db.require("pre_context_%i_fulfilled_f", Index = sm_id) return txt
def require_data(TState, TheAnalyzer): """Requires all variables which are necessary to implement the TemplateState. NOTE: The target schemes are required from inside 'require_scheme_variable()' in quex.engine.generator.mega_state.core.py """ variable_db.require("state_key")
def __code_pre_context_state_machine(self): LanguageDB = self.language_db if len(self.pre_context_sm_list) == 0: return [] assert len(self.pre_context_sm.get_orphaned_state_index_list()) == 0 txt = [] if Setup.comment_state_machine_f: LanguageDB.ML_COMMENT(txt, "BEGIN: PRE-CONTEXT STATE MACHINE\n" + \ self.pre_context_sm.get_string(NormalizeF=False) + \ "END: PRE-CONTEXT STATE MACHINE") txt.append( "\n" ) # For safety: New content may have to start in a newline, e.g. "#ifdef ..." analyzer = analyzer_generator.do(self.pre_context_sm, E_EngineTypes.BACKWARD_PRE_CONTEXT) msg = state_machine_coder.do(analyzer) txt.extend(msg) txt.append("\n%s" % LanguageDB.LABEL(E_StateIndices.END_OF_PRE_CONTEXT_CHECK)) # -- set the input stream back to the real current position. # during backward lexing the analyzer went backwards, so it needs to be reset. txt.append(" %s\n" % LanguageDB.INPUT_P_TO_LEXEME_START()) for sm_id in self.pre_context_sm_id_list: variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id) return txt
def require_position_registers(TheAnalyzer): """Require an array to store input positions. This has later to be implemented as 'variables'. Position registers are exclusively used for post-context restore. No other engine than FORWARD would require those. """ if not TheAnalyzer.engine_type.is_FORWARD(): return if TheAnalyzer.position_register_map is None: position_register_n = 0 else: position_register_n = len( set(TheAnalyzer.position_register_map.itervalues())) if position_register_n != 0: initial_array = "{ " + ("0, " * (position_register_n - 1) + "0") + "}" else: # Implement a dummy array (except that there is no reload) if Setup.buffer_based_analyzis_f: return initial_array = "(void*)0" variable_db.require_array("position", ElementN=position_register_n, Initial=initial_array) variable_db.require("PositionRegisterN", Initial="(size_t)%i" % position_register_n)
def do_pre_context(SM, PreContextSmIdList): """Pre-context detecting state machine (backward). --------------------------------------------------------------------------- Micro actions are: pre-context fullfilled_f DropOut --> Begin of 'main' state machine. BLC --> ReloadStateBackward EndOfStream --> 'error' Variables (potentially) required: pre_context_fulfilled_f[N] --> Array of flags for pre-context indication. RETURNS: [0] generated code text [1] reload state BACKWARD, to be generated later. """ if SM is None: return [], None txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT) txt.append( "\n%s:" % dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check())) # -- set the input stream back to the real current position. # during backward lexing the analyzer went backwards, so it needs to be reset. txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START()) for sm_id in PreContextSmIdList: variable_db.require("pre_context_%i_fulfilled_f", Index=sm_id) variable_db.require("input") return txt, analyzer
def require_position_registers(TheAnalyzer): """Require an array to store input positions. This has later to be implemented as 'variables'. Position registers are exclusively used for post-context restore. No other engine than FORWARD would require those. """ if not TheAnalyzer.engine_type.is_FORWARD(): return if TheAnalyzer.position_register_map is None: position_register_n = 0 else: position_register_n = len(set(TheAnalyzer.position_register_map.itervalues())) if position_register_n != 0: initial_array = "{ " + ("0, " * (position_register_n - 1) + "0") + "}" else: # Implement a dummy array (except that there is no reload) if Setup.buffer_based_analyzis_f: return initial_array = "(void*)0" variable_db.require_array("position", ElementN = position_register_n, Initial = initial_array) variable_db.require("PositionRegisterN", Initial = "(size_t)%i" % position_register_n)
def do_pre_context(SM, PreContextSmIdList): """Pre-context detecting state machine (backward). --------------------------------------------------------------------------- Micro actions are: pre-context fullfilled_f DropOut --> Begin of 'main' state machine. BLC --> ReloadStateBackward EndOfStream --> 'error' Variables (potentially) required: pre_context_fulfilled_f[N] --> Array of flags for pre-context indication. RETURNS: [0] generated code text [1] reload state BACKWARD, to be generated later. """ if SM is None: return [], None txt, analyzer = do_state_machine(SM, engine.BACKWARD_PRE_CONTEXT) txt.append("\n%s:" % dial_db.get_label_by_door_id(DoorID.global_end_of_pre_context_check())) # -- set the input stream back to the real current position. # during backward lexing the analyzer went backwards, so it needs to be reset. txt.append(" %s\n" % Lng.INPUT_P_TO_LEXEME_START()) for sm_id in PreContextSmIdList: variable_db.require("pre_context_%i_fulfilled_f", Index = sm_id) variable_db.require("input") return txt, analyzer
def require_data(TState, TheAnalyzer): """Requires all variables which are necessary to implement the TemplateState. NOTE: The target schemes are required from inside 'require_scheme_variable()' in quex.engine.generator.mega_state.core.py """ variable_db.require("state_key")
def do(self, RequiredLocalVariablesDB): LanguageDB = Setup.language_db # (*) Initialize the label and variable trackers variable_db.init(RequiredLocalVariablesDB) variable_db.require("input") init_address_handling({}) # (*) Pre Context State Machine # (If present: All pre-context combined in single backward analyzer.) pre_context = self.__code_pre_context_state_machine() # (*) Main State Machine -- try to match core patterns main = self.__code_main_state_machine() # (*) Backward input position detection # (Seldomly present -- only for Pseudo-Ambiguous Post Contexts) bipd = self.__code_backward_input_position_detection() # (*) Determine required labels and variables routed_address_set = get_address_set_subject_to_routing() routed_address_set.add(get_address("$terminal-EOF", U=True)) routed_state_info_list = state_router_generator.get_info(routed_address_set) state_router = [ state_router_generator.do(routed_state_info_list) ] variable_db.require("target_state_index", Condition_ComputedGoto=False) if is_label_referenced("$reload-FORWARD") or is_label_referenced("$reload-BACKWARD"): variable_db.require("target_state_else_index") variable_db.require("target_state_index") # Following function refers to the global 'variable_db' variable_definitions = self.language_db.VARIABLE_DEFINITIONS(variable_db) function_body = [] function_body.extend(pre_context) # implementation of pre-contexts (if there are some) function_body.extend(main) # main pattern matcher function_body.extend(bipd) # (seldom != empty; only for pseudo-ambiguous post contexts) function_body.extend(state_router) # route to state by index (only if no computed gotos) # (*) Pack Pre-Context and Core State Machine into a single function analyzer_function = self.language_db["$analyzer-func"](self.state_machine_name, Setup, variable_definitions, function_body, self.mode_name_list) txt = [ LanguageDB["$header-definitions"](LanguageDB, self.on_after_match) ] txt += get_plain_strings(analyzer_function) for i, element in enumerate(txt): if not isinstance(element, (str, unicode)): print element.__class__.__name__ for k in range(max(0,i-10)): print "before:", k, txt[k] for k in range(i+1, min(i+10, len(txt))): print "after: ", k, txt[k] assert False return txt
def do_reload_procedure(TheAnalyzer): """Lazy (delayed) code generation of the forward and backward reloaders. Any state who needs reload may 'register' in a reloader. This registering may happen after the code generation of forward or backward state machine. """ # Variables that tell where to go after reload success and reload failure if TheAnalyzer is None: return [] elif not TheAnalyzer.engine_type.subject_to_reload(): return [] elif TheAnalyzer.reload_state is None: return [] elif TheAnalyzer.reload_state_extern_f: return [] variable_db.require("target_state_else_index") # upon reload failure variable_db.require("target_state_index") # upon reload success require_position_registers(TheAnalyzer) return reload_state_coder.do(TheAnalyzer.reload_state)
def do_reload_procedure(TheAnalyzer): """Lazy (delayed) code generation of the forward and backward reloaders. Any state who needs reload may 'register' in a reloader. This registering may happen after the code generation of forward or backward state machine. """ # Variables that tell where to go after reload success and reload failure if TheAnalyzer is None: return [] elif not TheAnalyzer.engine_type.subject_to_reload(): return [] elif TheAnalyzer.reload_state is None: return [] elif TheAnalyzer.reload_state_extern_f: return [] variable_db.require("target_state_else_index") # upon reload failure variable_db.require("target_state_index") # upon reload success require_position_registers(TheAnalyzer) return reload_state_coder.do(TheAnalyzer.reload_state)
def __code_main_state_machine(self): assert len(self.sm.get_orphaned_state_index_list()) == 0 LanguageDB = self.language_db txt = [] # -- [optional] comment state machine transitions if Setup.comment_state_machine_f: LanguageDB.ML_COMMENT(txt, "BEGIN: STATE MACHINE\n" + \ self.sm.get_string(NormalizeF=False) + \ "END: STATE MACHINE") txt.append( "\n" ) # For safety: New content may have to start in a newline, e.g. "#ifdef ..." # -- implement the state machine itself analyzer = analyzer_generator.do(self.sm, E_EngineTypes.FORWARD) state_machine_code = state_machine_coder.do(analyzer) txt.extend(state_machine_code) lexeme_null_object_name = "QUEX_NAME(LexemeNullObject)" if Setup.external_lexeme_null_object != "": lexeme_null_object_name = Setup.external_lexeme_null_object # -- terminal states: execution of pattern actions terminal_code = LanguageDB["$terminal-code"]( self.state_machine_name, self.action_db, self.on_failure_action, self.on_end_of_stream_action, self.pre_context_sm_id_list, self.language_db, variable_db, self.on_after_match, lexeme_null_object_name) txt.extend(terminal_code) N = len(set(analyzer.position_register_map.values())) if len(analyzer.position_register_map) == 0: variable_db.require("position", Initial="(void*)0x0", Type="void*") variable_db.require("PositionRegisterN", Initial="(size_t)%i" % N) else: variable_db.require_array("position", ElementN=N, Initial="{ " + ("0, " * (N - 1) + "0") + "}") variable_db.require("PositionRegisterN", Initial="(size_t)%i" % N) if analyzer.last_acceptance_variable_required(): variable_db.require("last_acceptance") # -- reload definition (forward, backward, init state reload) code = LanguageDB.RELOAD() txt.extend(code) return txt
def __code_main_state_machine(self): assert len(self.sm.get_orphaned_state_index_list()) == 0 LanguageDB = self.language_db txt = [] # -- [optional] comment state machine transitions if Setup.comment_state_machine_f: LanguageDB.ML_COMMENT(txt, "BEGIN: STATE MACHINE\n" + \ self.sm.get_string(NormalizeF=False) + \ "END: STATE MACHINE") txt.append("\n") # For safety: New content may have to start in a newline, e.g. "#ifdef ..." # -- implement the state machine itself analyzer = analyzer_generator.do(self.sm, E_EngineTypes.FORWARD) state_machine_code = state_machine_coder.do(analyzer) txt.extend(state_machine_code) lexeme_null_object_name = "QUEX_NAME(LexemeNullObject)" if Setup.external_lexeme_null_object != "": lexeme_null_object_name = Setup.external_lexeme_null_object # -- terminal states: execution of pattern actions terminal_code = LanguageDB["$terminal-code"](self.state_machine_name, self.action_db, self.on_failure_action, self.on_end_of_stream_action, self.pre_context_sm_id_list, self.language_db, variable_db, self.on_after_match, lexeme_null_object_name) txt.extend(terminal_code) N = len(set(analyzer.position_register_map.values())) if len(analyzer.position_register_map) == 0: variable_db.require("position", Initial = "(void*)0x0", Type = "void*") variable_db.require("PositionRegisterN", Initial = "(size_t)%i" % N) else: variable_db.require_array("position", ElementN = N, Initial = "{ " + ("0, " * (N - 1) + "0") + "}") variable_db.require("PositionRegisterN", Initial = "(size_t)%i" % N) if analyzer.last_acceptance_variable_required(): variable_db.require("last_acceptance") # -- reload definition (forward, backward, init state reload) code = LanguageDB.RELOAD() txt.extend(code) return txt
def _get_source_code(CcFactory, analyzer, terminal_list): """RETURNS: String containing source code for the 'loop'. -- The source code for the (looping) state machine. -- The terminals which contain counting actions. Also, it requests variable definitions as they are required. """ txt = [] txt.extend(generator.do_analyzer(analyzer)) txt.extend(generator.do_terminals(terminal_list, analyzer)) if analyzer.engine_type.subject_to_reload(): txt.extend(generator.do_reload_procedure(analyzer)) if CcFactory.requires_reference_p(): variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") if Setup.buffer_codec.variable_character_sizes_f(): variable_db.require("character_begin_p") return txt
def do_main(SM): """Main pattern matching state machine (forward). --------------------------------------------------------------------------- Micro actions are: line/column number counting, position set/reset, last acceptance setting/reset, lexeme start pointer set/reset, setting terminating zero for lexeme/reset, setting last character. DropOut --> FAILURE BLC --> ReloadStateForward EndOfStream --> END_OF_STREAM Variables (potentially) required: position, PositionRegisterN, last_acceptance, input. """ txt, analyzer = do_state_machine(SM, engine.Class_FORWARD()) if analyzer.last_acceptance_variable_required(): variable_db.require("last_acceptance") return txt, analyzer
def do_main(SM): """Main pattern matching state machine (forward). --------------------------------------------------------------------------- Micro actions are: line/column number counting, position set/reset, last acceptance setting/reset, lexeme start pointer set/reset, setting terminating zero for lexeme/reset, setting last character. DropOut --> FAILURE BLC --> ReloadStateForward EndOfStream --> END_OF_STREAM Variables (potentially) required: position, PositionRegisterN, last_acceptance, input. """ txt, analyzer = do_state_machine(SM, engine.Class_FORWARD()) if analyzer.last_acceptance_variable_required(): variable_db.require("last_acceptance") return txt, analyzer
def __character_sequences(): result = ["{\n"] offset = 0 for path_id, path in enumerate(PWState.path_list): # Commenting the transition sequence is not dangerous. 'COMMENT' eliminates # comment delimiters if they would appear in the sequence_str. # sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1]) # memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n") # Last element of sequence contains only the 'end state'. result.append(" ") result.extend("%i, " % x.transition_char_to_next for x in path[:-1]) result.append("QUEX_SETTING_PATH_TERMINATION_CODE,") result.append("\n") variable_db.require("path_walker_%i_path_%i", Initial="path_walker_%i_path_base + %i" % (PWState.index, offset), Index=(PWState.index, path_id)) offset += len(path) result.append(" }") return offset, result
def __character_sequences(): result = ["{\n"] offset = 0 for path_id, path in enumerate(PWState.path_list): # Commenting the transition sequence is not dangerous. 'COMMENT' eliminates # comment delimiters if they would appear in the sequence_str. # sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1]) # memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n") # Last element of sequence contains only the 'end state'. result.append(" ") result.extend("%i, " % x.transition_char_to_next for x in path[:-1]) result.append("QUEX_SETTING_PATH_TERMINATION_CODE,") result.append("\n") variable_db.require( "path_walker_%i_path_%i", Initial="path_walker_%i_path_base + %i" % (PWState.index, offset), Index=(PWState.index, path_id), ) offset += len(path) result.append(" }") return offset, result
def require_data(PWState, TheAnalyzer): """Defines the transition targets for each involved state. """ variable_db.require("path_iterator") def __door_adr_sequences(PWState): result = ["{\n"] length = 0 for path_id, door_id_sequence in enumerate(PWState.door_id_sequence_list): # NOTE: For all states in the path the 'from_state_index, to_state_index' can # be determined, **except** for the FIRST state in the path. Thus for # this state the 'door' cannot be determined in case that it is # "not uniform_doors_f()". # # However, the only occasion where the FIRST state in the path may be # used is reload during the FIRST state. The reload adapts the positions # and acceptances are not changed. So, we can use the common entry # to the first state as a reference here. ## print "#DoorID, Adr:", [(door_id, Lng.ADDRESS_BY_DOOR_ID(door_id)) for door_id in door_id_sequence] result.append(" ") result.append("/* Padding */0x0, ") result.extend("QUEX_LABEL(%i), " % dial_db.get_address_by_door_id(door_id, RoutedF=True) for door_id in door_id_sequence) result.append("\n") length += len(door_id_sequence) + 1 # 1 padding element result.append(" }"); return length, result def __character_sequences(PathList): result = ["{\n"] offset = 0 for path_id, step_list in enumerate(PathList): ## Commenting the transition sequence is not dangerous. 'COMMENT' eliminates ## comment delimiters if they would appear in the sequence_str. ## sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), step_list[:-1]) ## memory.append(Lng.COMMENT("".join(sequence_str)) + "\n") result.append(" ") result.extend("%i, " % x.trigger for x in step_list[:-1]) result.append("QUEX_SETTING_PATH_TERMINATION_CODE,") result.append("\n") offset += len(step_list) result.append(" }") return offset, result # (*) Path Walker Basis # The 'base' must be defined before all --> PriorityF (see table in variable_db) element_n, character_sequence_str = __character_sequences(PWState.path_list) offset = 0 for path_id, step_list in enumerate(PWState.path_list): variable_db.require("path_walker_%i_path_%i", Initial = "&path_walker_%i_path_base[%i]" % (PWState.index, offset), Index = (PWState.index, path_id)) offset += len(step_list) variable_db.require_array("path_walker_%i_path_base", ElementN = element_n, Initial = character_sequence_str, Index = PWState.index) # (*) The State Information for each path step if PWState.uniform_door_id is None: element_n, door_adr_sequence_str = __door_adr_sequences(PWState) variable_db.require_array("path_walker_%i_state_base", ElementN = element_n, Initial = door_adr_sequence_str, Index = PWState.index) # The path_iterator is incremented before the 'goto', thus # 'path_iterator - (path_base + 1)' gives actually the correct offset. # We define a variable for that, for elegance. variable_db.require("path_walker_%i_reference", Initial = "path_walker_%i_path_base" % PWState.index, Index = (PWState.index))
def require_data(PWState, TheAnalyzer): """Defines the transition targets for each involved state. """ LanguageDB = Setup.language_db variable_db.require("path_iterator") def __door_adr_sequences(): result = ["{\n"] offset = 0 for path_id, path in enumerate(PWState.path_list): # NOTE: For all states in the path the 'from_state_index, to_state_index' can # be determined, **except** for the FIRST state in the path. Thus for # this state the 'door' cannot be determined in case that it is # "not uniform_doors_f()". # # However, the only occasion where the FIRST state in the path may be # used is reload during the FIRST state. The reload adapts the positions # and acceptances are not changed. So, we can use the common entry # to the first state as a reference here. prev_state_index = path[0][0] result.append(" ") for state_index in (x[0] for x in path[1:]): result.append( "QUEX_LABEL(%i), " % LanguageDB.ADDRESS(state_index, prev_state_index)) prev_state_index = state_index result.append("/* Zero of Elegance */0x0,") result.append("\n") offset += len(path) result.append(" }") return offset, result def __character_sequences(): result = ["{\n"] offset = 0 for path_id, path in enumerate(PWState.path_list): # Commenting the transition sequence is not dangerous. 'COMMENT' eliminates # comment delimiters if they would appear in the sequence_str. # sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1]) # memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n") # Last element of sequence contains only the 'end state'. result.append(" ") result.extend("%i, " % x.transition_char_to_next for x in path[:-1]) result.append("QUEX_SETTING_PATH_TERMINATION_CODE,") result.append("\n") variable_db.require("path_walker_%i_path_%i", Initial="path_walker_%i_path_base + %i" % (PWState.index, offset), Index=(PWState.index, path_id)) offset += len(path) result.append(" }") return offset, result # (*) Path Walker Basis # The 'base' must be defined before all --> PriorityF (see table in variable_db) element_n, character_sequence_str = __character_sequences() variable_db.require_array("path_walker_%i_path_base", ElementN=element_n, Initial=character_sequence_str, Index=PWState.index) # (*) The State Information for each path step if PWState.uniform_entry_door_id_along_all_paths is None: element_n, state_sequence_str = __door_adr_sequences() variable_db.require_array("path_walker_%i_state_base", ElementN=element_n, Initial=state_sequence_str, Index=PWState.index) # The path_iterator is incremented before the 'goto', thus # 'path_iterator - (path_base + 1)' gives actually the correct offset. # We define a variable for that, for elegance. variable_db.require("path_walker_%i_reference", Initial="path_walker_%i_path_base + 1" % PWState.index, Index=(PWState.index))
def do_analyzer(analyzer): state_machine_code = state_machine_coder.do(analyzer) Lng.REPLACE_INDENT(state_machine_code) # Variable to store the current input variable_db.require("input") return state_machine_code
def do_analyzer(analyzer): state_machine_code = state_machine_coder.do(analyzer) Lng.REPLACE_INDENT(state_machine_code) # Variable to store the current input variable_db.require("input") return state_machine_code
def do(self, RequiredLocalVariablesDB): LanguageDB = Setup.language_db # (*) Initialize the label and variable trackers variable_db.init(RequiredLocalVariablesDB) variable_db.require("input") init_address_handling({}) # (*) Pre Context State Machine # (If present: All pre-context combined in single backward analyzer.) pre_context = self.__code_pre_context_state_machine() # (*) Main State Machine -- try to match core patterns main = self.__code_main_state_machine() # (*) Backward input position detection # (Seldomly present -- only for Pseudo-Ambiguous Post Contexts) bipd = self.__code_backward_input_position_detection() # (*) Determine required labels and variables routed_address_set = get_address_set_subject_to_routing() routed_address_set.add(get_address("$terminal-EOF", U=True)) routed_state_info_list = state_router_generator.get_info( routed_address_set) state_router = [state_router_generator.do(routed_state_info_list)] variable_db.require("target_state_index", Condition_ComputedGoto=False) if is_label_referenced("$reload-FORWARD") or is_label_referenced( "$reload-BACKWARD"): variable_db.require("target_state_else_index") variable_db.require("target_state_index") # Following function refers to the global 'variable_db' variable_definitions = self.language_db.VARIABLE_DEFINITIONS( variable_db) function_body = [] function_body.extend( pre_context) # implementation of pre-contexts (if there are some) function_body.extend(main) # main pattern matcher function_body.extend( bipd) # (seldom != empty; only for pseudo-ambiguous post contexts) function_body.extend( state_router ) # route to state by index (only if no computed gotos) # (*) Pack Pre-Context and Core State Machine into a single function analyzer_function = self.language_db["$analyzer-func"]( self.state_machine_name, Setup, variable_definitions, function_body, self.mode_name_list) txt = [ LanguageDB["$header-definitions"](LanguageDB, self.on_after_match) ] txt += get_plain_strings(analyzer_function) for i, element in enumerate(txt): if not isinstance(element, (str, unicode)): print element.__class__.__name__ for k in range(max(0, i - 10)): print "before:", k, txt[k] for k in range(i + 1, min(i + 10, len(txt))): print "after: ", k, txt[k] assert False return txt
def require_data(PWState, TheAnalyzer): """Defines the transition targets for each involved state. """ LanguageDB = Setup.language_db variable_db.require("path_iterator") def __door_adr_sequences(): result = ["{\n"] offset = 0 for path_id, path in enumerate(PWState.path_list): # NOTE: For all states in the path the 'from_state_index, to_state_index' can # be determined, **except** for the FIRST state in the path. Thus for # this state the 'door' cannot be determined in case that it is # "not uniform_doors_f()". # # However, the only occasion where the FIRST state in the path may be # used is reload during the FIRST state. The reload adapts the positions # and acceptances are not changed. So, we can use the common entry # to the first state as a reference here. prev_state_index = path[0][0] result.append(" ") for state_index in (x[0] for x in path[1:]): result.append("QUEX_LABEL(%i), " % LanguageDB.ADDRESS(state_index, prev_state_index)) prev_state_index = state_index result.append("/* Zero of Elegance */0x0,") result.append("\n") offset += len(path) result.append(" }") return offset, result def __character_sequences(): result = ["{\n"] offset = 0 for path_id, path in enumerate(PWState.path_list): # Commenting the transition sequence is not dangerous. 'COMMENT' eliminates # comment delimiters if they would appear in the sequence_str. # sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), path[:-1]) # memory.append(LanguageDB.COMMENT("".join(sequence_str)) + "\n") # Last element of sequence contains only the 'end state'. result.append(" ") result.extend("%i, " % x.transition_char_to_next for x in path[:-1]) result.append("QUEX_SETTING_PATH_TERMINATION_CODE,") result.append("\n") variable_db.require( "path_walker_%i_path_%i", Initial="path_walker_%i_path_base + %i" % (PWState.index, offset), Index=(PWState.index, path_id), ) offset += len(path) result.append(" }") return offset, result # (*) Path Walker Basis # The 'base' must be defined before all --> PriorityF (see table in variable_db) element_n, character_sequence_str = __character_sequences() variable_db.require_array( "path_walker_%i_path_base", ElementN=element_n, Initial=character_sequence_str, Index=PWState.index ) # (*) The State Information for each path step if PWState.uniform_entry_door_id_along_all_paths is None: element_n, state_sequence_str = __door_adr_sequences() variable_db.require_array( "path_walker_%i_state_base", ElementN=element_n, Initial=state_sequence_str, Index=PWState.index ) # The path_iterator is incremented before the 'goto', thus # 'path_iterator - (path_base + 1)' gives actually the correct offset. # We define a variable for that, for elegance. variable_db.require( "path_walker_%i_reference", Initial="path_walker_%i_path_base + 1" % PWState.index, Index=(PWState.index) )
def require_data(PWState, TheAnalyzer): """Defines the transition targets for each involved state. """ variable_db.require("path_iterator") def __door_adr_sequences(PWState): result = ["{\n"] length = 0 for path_id, door_id_sequence in enumerate( PWState.door_id_sequence_list): # NOTE: For all states in the path the 'from_state_index, to_state_index' can # be determined, **except** for the FIRST state in the path. Thus for # this state the 'door' cannot be determined in case that it is # "not uniform_doors_f()". # # However, the only occasion where the FIRST state in the path may be # used is reload during the FIRST state. The reload adapts the positions # and acceptances are not changed. So, we can use the common entry # to the first state as a reference here. ## print "#DoorID, Adr:", [(door_id, Lng.ADDRESS_BY_DOOR_ID(door_id)) for door_id in door_id_sequence] result.append(" ") result.append("/* Padding */0x0, ") result.extend("QUEX_LABEL(%i), " % dial_db.get_address_by_door_id(door_id, RoutedF=True) for door_id in door_id_sequence) result.append("\n") length += len(door_id_sequence) + 1 # 1 padding element result.append(" }") return length, result def __character_sequences(PathList): result = ["{\n"] offset = 0 for path_id, step_list in enumerate(PathList): ## Commenting the transition sequence is not dangerous. 'COMMENT' eliminates ## comment delimiters if they would appear in the sequence_str. ## sequence_str = imap(lambda x: Interval(x[1]).get_utf8_string(), step_list[:-1]) ## memory.append(Lng.COMMENT("".join(sequence_str)) + "\n") result.append(" ") result.extend("%i, " % x.trigger for x in step_list[:-1]) result.append("QUEX_SETTING_PATH_TERMINATION_CODE,") result.append("\n") offset += len(step_list) result.append(" }") return offset, result # (*) Path Walker Basis # The 'base' must be defined before all --> PriorityF (see table in variable_db) element_n, character_sequence_str = __character_sequences( PWState.path_list) offset = 0 for path_id, step_list in enumerate(PWState.path_list): variable_db.require("path_walker_%i_path_%i", Initial="&path_walker_%i_path_base[%i]" % (PWState.index, offset), Index=(PWState.index, path_id)) offset += len(step_list) variable_db.require_array("path_walker_%i_path_base", ElementN=element_n, Initial=character_sequence_str, Index=PWState.index) # (*) The State Information for each path step if PWState.uniform_door_id is None: element_n, door_adr_sequence_str = __door_adr_sequences(PWState) variable_db.require_array("path_walker_%i_state_base", ElementN=element_n, Initial=door_adr_sequence_str, Index=PWState.index) # The path_iterator is incremented before the 'goto', thus # 'path_iterator - (path_base + 1)' gives actually the correct offset. # We define a variable for that, for elegance. variable_db.require("path_walker_%i_reference", Initial="path_walker_%i_path_base" % PWState.index, Index=(PWState.index))
def get_skipper(OpenerSequence, CloserSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter): assert len(OpenerSequence) >= 1 assert len(CloserSequence) >= 1 assert OpenerSequence != CloserSequence skipper_index = sm_index.get() skipper_door_id = dial_db.new_door_id(skipper_index) opener_str, opener_comment_str = get_character_sequence(OpenerSequence) opener_length = len(OpenerSequence) closer_str, closer_comment_str = get_character_sequence(CloserSequence) closer_length = len(CloserSequence) variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") variable_db.require("counter") variable_db.require_array("Skipper%i_Opener", Initial="{ %s }" % opener_str, ElementN=opener_length, Index=skipper_index) variable_db.require("Skipper%i_OpenerEnd", "Skipper%i_Opener + (ptrdiff_t)%i" % (skipper_index, opener_length), Index=skipper_index) variable_db.require("Skipper%i_Opener_it", "0x0", Index=skipper_index) variable_db.require_array("Skipper%i_Closer", Initial="{ %s }" % closer_str, ElementN=closer_length, Index=skipper_index) variable_db.require("Skipper%i_CloserEnd", "Skipper%i_Closer + (ptrdiff_t)%i" % (skipper_index, closer_length), Index=skipper_index) variable_db.require("Skipper%i_Closer_it", "0x0", Index=skipper_index) reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n" if CloserSequence[-1] == ord('\n'): end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n" end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n" else: end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \ " - reference_p));\n" reload_door_id = dial_db.new_door_id() on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern, NestedF=True) code_str = blue_print( template_str, [ ["$$SKIPPER_INDEX$$", __nice(skipper_index)], # ["$$OPENER_LENGTH$$", "%i" % opener_length], ["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", Lng.ACCESS_INPUT()], [ "$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]") ], ["$$ENDIF$$", Lng.END_IF()], ["$$ENTRY$$", Lng.LABEL(skipper_door_id)], ["$$RELOAD$$", dial_db.get_label_by_door_id(reload_door_id)], ["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)], ["$$GOTO_RELOAD$$", Lng.GOTO(reload_door_id)], ["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)], ["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open], # ["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def], ["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop()], ["$$LC_COUNT_END_PROCEDURE$$", end_procedure], ["$$LC_COUNT_BEFORE_RELOAD$$", before_reload], ["$$LC_COUNT_AFTER_RELOAD$$", after_reload], ]) return [code_str]
def get_skipper(EndSequence, CloserPattern, ModeName, OnSkipRangeOpen, DoorIdAfter): assert len(EndSequence) >= 1 global template_str # Name the $$SKIPPER$$ skipper_index = sm_index.get() skipper_door_id = dial_db.new_door_id(skipper_index) delimiter_str, delimiter_comment_str = get_character_sequence(EndSequence) end_sequence_transformed = transformation.do_sequence(EndSequence) # Determine the $$DELIMITER$$ delimiter_length = len(end_sequence_transformed) delimiter_comment_str = Lng.COMMENT(" Delimiter: %s" % delimiter_comment_str) # Determine the check for the tail of the delimiter delimiter_remainder_test_str = "" if len(EndSequence) != 1: txt = "".join( " %s" % Lng.IF_GOTO(Lng.INPUT_P_DEREFERENCE(i-1), "!=", "Skipper$$SKIPPER_INDEX$$[%i]" % i, skipper_door_id, i == 1) for i, letter in enumerate(EndSequence[1:], start=1) ) delimiter_remainder_test_str = txt door_id_reload = dial_db.new_door_id() on_skip_range_open = get_on_skip_range_open(OnSkipRangeOpen, CloserPattern) # The main part code_str = blue_print(template_str, [ ["$$DELIMITER_COMMENT$$", delimiter_comment_str], ["$$INPUT_P_INCREMENT$$", Lng.INPUT_P_INCREMENT()], ["$$INPUT_P_DECREMENT$$", Lng.INPUT_P_DECREMENT()], ["$$INPUT_GET$$", Lng.ACCESS_INPUT()], ["$$IF_INPUT_EQUAL_DELIMITER_0$$", Lng.IF_INPUT("==", "Skipper$$SKIPPER_INDEX$$[0]")], ["$$ENDIF$$", Lng.END_IF()], ["$$ENTRY$$", dial_db.get_label_by_door_id(skipper_door_id)], ["$$RELOAD$$", dial_db.get_label_by_door_id(door_id_reload)], ["$$GOTO_ENTRY$$", Lng.GOTO(skipper_door_id)], ["$$INPUT_P_TO_LEXEME_START$$", Lng.INPUT_P_TO_LEXEME_START()], # When things were skipped, no change to acceptance flags or modes has # happend. One can jump immediately to the start without re-entry preparation. ["$$GOTO_AFTER_END_OF_SKIPPING$$", Lng.GOTO(DoorIdAfter)], ["$$MARK_LEXEME_START$$", Lng.LEXEME_START_SET()], ["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str], ["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open], ]) # Line and column number counting code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence) # The finishing touch code_str = blue_print(code_str, [["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", Lng.GOTO(door_id_reload)]]) if reference_p_f: variable_db.require("reference_p", Condition="QUEX_OPTION_COLUMN_NUMBER_COUNTING") variable_db.require_array("Skipper%i", Initial="{ %s }" % delimiter_str, ElementN=delimiter_length, Index=skipper_index) variable_db.require("Skipper%iL", "%i" % delimiter_length, Index=skipper_index) variable_db.require("text_end") variable_db.require("input") return [ code_str ]