def get_sm_list(FSM0, FSM1, FSM2): # SPECIALITIES: -- sm0 and sm1 have an intersection between their second # transition. # -- sm1 transits further upon acceptance. # -- sm2 has only one transition. # Generate DFA that does not have any intersection with # the loop transitions. sm0 = DFA() si = sm0.add_transition(sm0.init_state_index, FSM0) si = sm0.add_transition(si, NS_A, AcceptanceF=True) sm0.states[si].mark_acceptance_id(dial.new_incidence_id()) sm1 = DFA() si0 = sm1.add_transition(sm1.init_state_index, FSM1) si = sm1.add_transition(si0, NS_A, AcceptanceF=True) iid1 = dial.new_incidence_id() sm1.states[si].mark_acceptance_id(iid1) si = sm1.add_transition(si, NS_B, si0) sm1.states[si].mark_acceptance_id(iid1) sm2 = DFA() si = sm2.add_transition(sm2.init_state_index, FSM2, AcceptanceF=True) sm2.states[si].mark_acceptance_id(dial.new_incidence_id()) return [sm0, sm1, sm2]
def test(LoopMap, ColumnNPerCodeUnit): global dial_db Setup.buffer_encoding.source_set = NumberSet_All() # Generate sample state machines from what the loop map tells. appendix_sm_list = _get_appendix_sm_list(LoopMap) UserOnLoopExitDoorId = dial_db.new_door_id() events = loop.LoopEvents(ColumnNPerCodeUnit, None, UserOnLoopExitDoorId) config = loop.LoopConfig(ColumnNPerCodeUnit = ColumnNPerCodeUnit, LexemeEndCheckF = False, EngineType = engine.FORWARD, ReloadStateExtern = None, UserOnLoopExitDoorId = UserOnLoopExitDoorId, dial_db = dial_db, OnReloadFailureDoorId = None, ModeName = "M", Events = events) config.iid_loop_after_appendix_drop_out = dial.new_incidence_id() loop_sm = DFA.from_IncidenceIdMap( (lei.character_set, lei.iid_couple_terminal) for lei in LoopMap ) analyzer_list, \ door_id_loop = analyzer_construction.do(loop_sm, appendix_sm_list, config, True) print_this(analyzer_list)
def _add_pair(psml, SmOriginal, Name, dial_db): """Add a state machine-terminal pair to 'psml'. A terminal is generated which transits to 'INDENTATION_HANDLER'. The state machine is cloned for safety. """ class IC_MiniTerminal(loop.MiniTerminal): def __init__(self, Name, IncidenceId): loop.MiniTerminal.__init__(self, None, Name, IncidenceId) def get_code(self, LoopStateMachineId): return [ Lng.GOTO( DoorID.state_machine_entry(LoopStateMachineId, dial_db), dial_db) ] if SmOriginal is None: return # Disconnect from machines being used elsewhere. sm = SmOriginal.clone(StateMachineId=dial.new_incidence_id()) terminal = IC_MiniTerminal(Name, sm.get_id()) # TRY: terminal.set_requires_goto_loop_entry_f() # INSTEAD: GOTO 'INDENTATION_HANDLER' psml.append((sm, terminal))
def test(NsCaList, SM_list=[]): global dial_db Setup.buffer_encoding.source_set = NumberSet_All() ca_map = CountActionMap.from_list(NsCaList) iid_loop_exit = dial.new_incidence_id() for sm in SM_list: sm.mark_state_origins() class Pseudo(loop.LoopConfig): def __init__(self, TheDialDb): self.dial_db = TheDialDb self.column_number_per_code_unit = None self.lexeme_end_check_f = False self.mode_name = "TestMode" loop_map, \ appendix_sm_list, \ lcci_db = loop._get_loop_map(Pseudo(dial_db), ca_map, SM_list, iid_loop_exit, NumberSet_All()) print print print general_checks(loop_map, appendix_sm_list) print_this(loop_map, appendix_sm_list)
def _get_indentation_handler_terminal(ModeName, dial_db): code = Lng.COMMAND_LIST([ Op.IndentationHandlerCall(ModeName), Op.GotoDoorId(DoorID.continue_without_on_after_match(dial_db)) ], dial_db) incidence_id = dial.new_incidence_id() terminal = Terminal(CodeTerminal(code), "<CALL INDENTATION HANDLER>", incidence_id, dial_db=dial_db) return DoorID.incidence(incidence_id, dial_db), terminal
def _prepare_skip_character_set(self, MHI, Loopers, CaMap, ReloadState): """MHI = Mode hierarchie index.""" if Loopers.skip is None: return [], [], [] skipped_character_set, \ pattern_str, \ aux_source_reference = Loopers.combined_skip(CaMap) new_analyzer_list, \ new_terminal_list, \ loop_map, \ required_register_set = skip_character_set.do(self.terminal_factory.mode_name, CaMap, skipped_character_set, ReloadState, self.terminal_factory.dial_db) self.required_register_set.update(required_register_set) extra_terminal_list = [ self._terminal_goto_to_looper(new_analyzer_list, E_IncidenceIDs.SKIP, "<skip>", required_register_set) ] extra_terminal_list.extend(t for t in new_terminal_list) # Any skipped character must enter the skipper entry. goto_code = [ Op.GotoDoorId( DoorID.incidence(E_IncidenceIDs.SKIP, self.terminal_factory.dial_db)) ] new_ppt_list = [] for lei in loop_map: new_incidence_id = dial.new_incidence_id() pattern = Pattern.from_character_set(lei.character_set, new_incidence_id, PatternString="<skip>", Sr=aux_source_reference) # There is no reference pointer => Add directly count_code = lei.aux_count_action.get_OpList(None) code = Lng.COMMAND_LIST(count_code + goto_code, self.terminal_factory.dial_db) terminal = Terminal(CodeTerminal(code), "ENTER SKIP:", new_incidence_id, dial_db=self.terminal_factory.dial_db) new_ppt_list.append( PPT(PatternPriority(MHI, new_incidence_id), pattern, terminal)) return new_ppt_list, new_analyzer_list, extra_terminal_list
def _get_state_machine_vs_terminal_list(CloserPattern, dial_db, DoorIdExit): """Additionally to all characters, the loop shall walk along the 'closer'. If the closer matches, the range skipping exits. Characters need to be counted properly. RETURNS: list(state machine, terminal) The list contains only one single element. """ sm = CloserPattern.get_cloned_sm(StateMachineId=dial.new_incidence_id()) code = [Lng.GOTO(DoorIdExit, dial_db)] mini_terminal = loop.MiniTerminal(code, "<SKIP RANGE TERMINATED>", sm.get_id()) return [(sm, mini_terminal)]
def __init__(self, ColumnNPerCodeUnit, LexemeEndCheckF, EngineType, ReloadStateExtern, UserOnLoopExitDoorId, dial_db, OnReloadFailureDoorId, ModeName, Events): """ColumnNPerCodeUnit is None => no constant relationship between column number and code unit. """ self.mode_name = ModeName self.column_number_per_code_unit = ColumnNPerCodeUnit self.lexeme_end_check_f = LexemeEndCheckF self.reload_state_extern = ReloadStateExtern self.engine_type = EngineType self.dial_db = dial_db self.door_id_on_reload_failure = OnReloadFailureDoorId self.door_id_on_loop_exit_user_code = UserOnLoopExitDoorId self.events = Events self.loop_state_machine_id = None self.__appendix_dfa_present_f = False self.loop_state_machine_id = index.get_state_machine_id() self.iid_loop_exit = dial.new_incidence_id() self.iid_loop_after_appendix_drop_out = dial.new_incidence_id() self.__run_time_counter_required_f = False
def _get_state_machine_vs_terminal_list(CloserPattern, OpenerPattern, DoorIdExit, dial_db): """Additionally to all characters, the loop shall walk along the 'closer'. If the closer matches, the range skipping exits. Characters need to be counted properly. RETURNS: [0] list(state machine, terminal) [1] incidence id of auxiliary terminal that goto-s to the loop entry. The auxiliary terminal is necessary since the DoorID of the loop entry cannot be known beforehand. """ # DoorID of loop entry cannot be known beforehand. # => generate an intermediate door_id from where the loop is entered. iid_aux_reentry = dial.new_incidence_id() door_id_aux_reentry = dial.DoorID.incidence(iid_aux_reentry, dial_db) # Opener Pattern Reaction opener_op_list = [ Op.Increment(E_R.Counter), Op.GotoDoorId(door_id_aux_reentry) ] # Closer Pattern Reaction closer_op_list = [ Op.Decrement(E_R.Counter), Op.GotoDoorIdIfCounterEqualZero(DoorIdExit), Op.GotoDoorId(door_id_aux_reentry) ] def sm_terminal_pair(Pattern, Name, OpList, dial_db): sm = Pattern.get_cloned_sm(StateMachineId=dial.new_incidence_id()) terminal = loop.MiniTerminal(Lng.COMMAND_LIST(OpList, dial_db), Name, sm.get_id()) return sm, terminal smt_list = [ sm_terminal_pair(OpenerPattern, "<SKIP NESTED RANGE OPENER>", opener_op_list, dial_db), sm_terminal_pair(CloserPattern, "<SKIP NESTED RANGE CLOSER>", closer_op_list, dial_db) ] return smt_list, iid_aux_reentry
def _get_LoopMapEntry_list_plain(loop_config, CaMap, L_pure): """RETURNS: list of LoopMapEntry-s. The list defines the loop behavior for characters which are not transits to appendix state machines. The LoopMapEntry-s are setup as below: [0] Character set to trigger to a terminal. [1] CountAction. [2] IncidenceId of the CountAction. [3] 'None' indicating: no appendix sm, no 'goto couple state'. """ assert L_pure is not None return [ LoopMapEntry(character_set, dial.new_incidence_id(), Code=loop_config.cmd_list_CA_GotoLoopEntry(ca), CA=ca) for character_set, ca in CaMap.iterable_in_sub_set(L_pure) ]
def _adapt_pattern_id_to_priority(ppt_list): """Ensure that the incidence-id of each pattern fits the position in priorization sequence. That is, early entries in the list MUST have lower incidence id than later entries. NOTE: For MHI-s < 0, i.e. loopers, this functions basically asserts that the incidence ids of all patterns are aligned with their position in the list. """ if len(ppt_list) < 1: return ppt_list.sort(key=attrgetter("priority")) prev_incidence_id = ppt_list[0].pattern.incidence_id for i, ppt in enumerate(ppt_list[1:], start=1): priority, pattern, terminal = ppt # NOT: Mode Hierarchy Index < 0 == 'Entry into Looper' # Those patterns are NOT SUPPOSED to match common lexemes # with each other or other lexemes in the mode! # => They are not subject to repriorization! if priority.mode_hierarchy_index < 0: continue if pattern.incidence_id > prev_incidence_id: prev_incidence_id = pattern.incidence_id continue # Generate a new, cloned pattern. So that other related modes are not effected. new_incidence_id = dial.new_incidence_id( ) # new id > any older id. new_pattern = pattern.clone_with_new_incidence_id(new_incidence_id) if terminal is not None: new_terminal = terminal.clone(new_incidence_id) else: new_terminal = None new_ppt = PPT(priority, new_pattern, new_terminal) ppt_list[i] = new_ppt prev_incidence_id = new_incidence_id
print for i, analyzer in enumerate(AnalyzerList): print "--( %i: init si = %i )-------------------------\n" % (i, analyzer.init_state_index) print analyzer print_drop_out(analyzer) NS_A = NumberSet.from_range(0x600, 0x601) # UTF8: D8 80 => 216, 128 NS_B = NumberSet.from_range(0x601, 0x602) # UTF8: D8 81 => 216, 129 NS_C = NumberSet.from_range(0x640, 0x641) # UTF8: D9 80 => 217, 128 appendix_sm_id = 4711L if "loop" in sys.argv: loop_map = loop.LoopMap([ TestLME(NS_A, dial.new_incidence_id(), None), ]) column_n_per_code_unit = 5 elif "appendix" in sys.argv: loop_map = loop.LoopMap([ TestLME(NS_A, dial.new_incidence_id(), appendix_sm_id), # appendix_sm_id ]) column_n_per_code_unit = 5 elif "appendix-wot" in sys.argv: loop_map = loop.LoopMap([ TestLME(NS_A, dial.new_incidence_id(), None), ]) column_n_per_code_unit = 5 elif "non-const" in sys.argv: loop_map = loop.LoopMap([ TestLME(NS_A, dial.new_incidence_id(), None),
def sm_terminal_pair(Pattern, Name, OpList, dial_db): sm = Pattern.get_cloned_sm(StateMachineId=dial.new_incidence_id()) terminal = loop.MiniTerminal(Lng.COMMAND_LIST(OpList, dial_db), Name, sm.get_id()) return sm, terminal
def do(loop_config, CaMap, SmList): """Perform separation: Parallel state machine ----> first transition + appendix state machine Appendix Sm-Id --> Original Sm-Id The 'first transition' is mounted on the loop state machine triggering an acceptance that causes a transit to the appendix state machine. RETURNS: list of LoopMapEntry-s """ # ESSENTIAL: Delimiter state machines shall never match on a common lexeme! _assert_no_intersections(SmList) assert all(sm.get_id() is not None for sm in SmList) loop_map_1, \ original_iid_db = split_first_transition(SmList) # loop_map_1: list of [0] first transition character set # [1] appendix sm with first transition removed # # original_iid_db: appendix sm id --> original sm id appendix_cmd_list_db = loop_config.get_appendix_terminal_cmd_list_db( CaMap, [sm for cs, sm in loop_map_1], original_iid_db) # appendix_cmd_list_db: appendix sm id --> CmdList(count action, # goto original terminal) loop_map_2 = \ split_first_character_set_for_distinct_count_actions(CaMap, loop_map_1) # loop_map_2: list of [0] character set where all elements # require same count actions # [1] count action # [2] appendix sm # For a 'state transition' it is required that all character sets # in the list are disjoint. Thus, any intersection must build its # on entry. Thus, some entries might have more than one appendix. loop_map_3 = combine_intersecting_character_sets(loop_map_2) # loop_map_2: list of [0] character set no character set intersects # with any other. # [1] count action # [2] list of (appendix sm) # A transition can only enter one state machine, so all appendix # state machines related to the same character set must be combined. loop_map_4, \ combined_appendix_sm_list_raw = combine_appendix_sm_lists(loop_map_3) # loop_map_4: list of [0] (disjoint) character set # [1] count action for character set # [2] state machine id of related combined appendix sm # combined_appendix_sm_lists: list of all generated (combined) appendix sm-s loop_map_5, \ combined_appendix_sm_list = determine_CmdLists(loop_config, loop_map_4, combined_appendix_sm_list_raw, original_iid_db) # loop_map_5: list of [0] (disjoint) character set # [1] CmdList = (count action, goto terminal/appendix sm), # combined_appendix_sm_list: contains only those combined appendix state machines # that do have transitions loop_map_6 = [ LoopMapEntry(character_set, IidCoupleTerminal=dial.new_incidence_id(), Code=cmd_list) for character_set, cmd_list in loop_map_5 ] # loop_map_6: list of LoopMapEntry-s # There must be a command list for any acceptance in the appendix # state machines. all_acceptance_id_set = flatten_list_of_lists( sm.acceptance_id_set() for sm in combined_appendix_sm_list) assert all(iid in appendix_cmd_list_db for iid in all_acceptance_id_set) return loop_map_6, combined_appendix_sm_list, appendix_cmd_list_db
ca_list = get_ca_list(0x10, 0x60) sm_list = get_sm_list(NumberSet.from_range(0x10, 0x40), NumberSet.from_range(0x20, 0x50), NumberSet.from_range(0x30, 0x60)) # Test for each 'sm' in 'sm_list' is superfluous. # It is done in 'AppendixNoI'. test(ca_list, sm_list) elif "Split" in sys.argv: # A first transition of a state machine is separated into two, because # it is covered by more than one different count action. NS1 = NumberSet.from_range(0x10, 0x20) NS2 = NumberSet.from_range(0x20, 0x30) NS3 = NumberSet.from_range(0x30, 0x40) NS4 = NumberSet.from_range(0x40, 0x50) ca_list = [ (NS1, CountAction(E_CharacterCountType.COLUMN, 1)), (NS2, CountAction(E_CharacterCountType.COLUMN, 2)), (NS3, CountAction(E_CharacterCountType.COLUMN, 3)), (NS4, CountAction(E_CharacterCountType.COLUMN, 4)), ] sm = DFA() si = sm.init_state_index iid = dial.new_incidence_id() ti0 = sm.add_transition(si, NumberSet.from_range(0x1A, 0x4B)) ac0 = sm.add_transition(ti0, NS_A, AcceptanceF=True) test(ca_list, [sm])
def _prepare_indentation_counter(self, MHI, Loopers, CaMap, ReloadState): """Prepare indentation counter. An indentation counter is implemented by the following: 'newline' pattern --> triggers as soon as an UNSUPPRESSED newline occurs. --> entry to the INDENTATION COUNTER. 'suppressed newline' --> INDENTATION COUNTER is NOT triggered. The supressed newline pattern is longer (and has precedence) over the newline pattern. With the suppressed newline it is possible to write lines which overstep the newline (see backslahs in Python, for example). RETURNS: List of: [0] newline PPT and [1] optionally the PPT of the newline suppressor. The primary pattern action pair list is to be the head of all pattern action pairs. MHI = Mode hierarchie index defining the priority of the current mode. """ ISetup = Loopers.indentation_handler if ISetup is None: return [], [], [] check_indentation_setup(ISetup) # Isolate the pattern objects, so alternatively, # they may be treated in 'indentation_counter'. pattern_newline = ISetup.pattern_newline.clone() if ISetup.pattern_suppressed_newline: pattern_suppressed_newline = ISetup.pattern_suppressed_newline.clone( ) else: pattern_suppressed_newline = None new_analyzer_list, \ new_terminal_list, \ required_register_set, \ run_time_counter_f = indentation_counter.do(self.terminal_factory.mode_name, CaMap, ISetup, self.terminal_factory.incidence_db, ReloadState, self.terminal_factory.dial_db) self.terminal_factory.run_time_counter_required_f |= run_time_counter_f self.required_register_set.update(required_register_set) # 'newline' triggers --> indentation counter pattern = pattern_newline.finalize(CaMap) pattern = pattern.clone_with_new_incidence_id( E_IncidenceIDs.INDENTATION_HANDLER) terminal = self._terminal_goto_to_looper(new_analyzer_list, None, "<indentation handler>", required_register_set, Pattern=pattern) new_ppt_list = [PPT(PatternPriority(MHI, 0), pattern, terminal)] if pattern_suppressed_newline is not None: # 'newline-suppressor' causes following 'newline' to be ignored. # => next line not subject to new indentation counting. new_incidence_id = dial.new_incidence_id() pattern = pattern_suppressed_newline.finalize(CaMap) pattern = pattern.clone_with_new_incidence_id(new_incidence_id) door_id = DoorID.global_reentry(self.terminal_factory.dial_db) code = CodeTerminal( [Lng.GOTO(door_id, self.terminal_factory.dial_db)]) new_ppt_list.append( PPT( PatternPriority(MHI, 1), pattern, self.terminal_factory.do_plain( code, pattern, "INDENTATION COUNTER SUPPRESSED_NEWLINE: "))) return new_ppt_list, new_analyzer_list, new_terminal_list