def get_sm_list(FSM0, FSM1, FSM2):
# SPECIALITIES: -- sm0 and sm1 have an intersection between their second
# transition.
# -- sm1 transits further upon acceptance.
# -- sm2 has only one transition.
# Generate DFA that does not have any intersection with
# the loop transitions.
sm0 = DFA()
si = sm0.add_transition(sm0.init_state_index, FSM0)
si = sm0.add_transition(si, NS_A, AcceptanceF=True)
sm0.states[si].mark_acceptance_id(dial.new_incidence_id())
sm1 = DFA()
si0 = sm1.add_transition(sm1.init_state_index, FSM1)
si = sm1.add_transition(si0, NS_A, AcceptanceF=True)
iid1 = dial.new_incidence_id()
sm1.states[si].mark_acceptance_id(iid1)
si = sm1.add_transition(si, NS_B, si0)
sm1.states[si].mark_acceptance_id(iid1)
sm2 = DFA()
si = sm2.add_transition(sm2.init_state_index, FSM2, AcceptanceF=True)
sm2.states[si].mark_acceptance_id(dial.new_incidence_id())
return [sm0, sm1, sm2]
def test(LoopMap, ColumnNPerCodeUnit):
global dial_db
Setup.buffer_encoding.source_set = NumberSet_All()
# Generate sample state machines from what the loop map tells.
appendix_sm_list = _get_appendix_sm_list(LoopMap)
UserOnLoopExitDoorId = dial_db.new_door_id()
events = loop.LoopEvents(ColumnNPerCodeUnit, None, UserOnLoopExitDoorId)
config = loop.LoopConfig(ColumnNPerCodeUnit = ColumnNPerCodeUnit,
LexemeEndCheckF = False,
EngineType = engine.FORWARD,
ReloadStateExtern = None,
UserOnLoopExitDoorId = UserOnLoopExitDoorId,
dial_db = dial_db,
OnReloadFailureDoorId = None,
ModeName = "M",
Events = events)
config.iid_loop_after_appendix_drop_out = dial.new_incidence_id()
loop_sm = DFA.from_IncidenceIdMap(
(lei.character_set, lei.iid_couple_terminal) for lei in LoopMap
)
analyzer_list, \
door_id_loop = analyzer_construction.do(loop_sm, appendix_sm_list, config, True)
print_this(analyzer_list)
def _add_pair(psml, SmOriginal, Name, dial_db):
"""Add a state machine-terminal pair to 'psml'. A terminal is generated
which transits to 'INDENTATION_HANDLER'. The state machine is cloned
for safety.
"""
class IC_MiniTerminal(loop.MiniTerminal):
def __init__(self, Name, IncidenceId):
loop.MiniTerminal.__init__(self, None, Name, IncidenceId)
def get_code(self, LoopStateMachineId):
return [
Lng.GOTO(
DoorID.state_machine_entry(LoopStateMachineId, dial_db),
dial_db)
]
if SmOriginal is None: return
# Disconnect from machines being used elsewhere.
sm = SmOriginal.clone(StateMachineId=dial.new_incidence_id())
terminal = IC_MiniTerminal(Name, sm.get_id())
# TRY: terminal.set_requires_goto_loop_entry_f()
# INSTEAD: GOTO 'INDENTATION_HANDLER'
psml.append((sm, terminal))
def test(NsCaList, SM_list=[]):
global dial_db
Setup.buffer_encoding.source_set = NumberSet_All()
ca_map = CountActionMap.from_list(NsCaList)
iid_loop_exit = dial.new_incidence_id()
for sm in SM_list:
sm.mark_state_origins()
class Pseudo(loop.LoopConfig):
def __init__(self, TheDialDb):
self.dial_db = TheDialDb
self.column_number_per_code_unit = None
self.lexeme_end_check_f = False
self.mode_name = "TestMode"
loop_map, \
appendix_sm_list, \
lcci_db = loop._get_loop_map(Pseudo(dial_db),
ca_map,
SM_list,
iid_loop_exit,
NumberSet_All())
print
print
print
general_checks(loop_map, appendix_sm_list)
print_this(loop_map, appendix_sm_list)
def _get_indentation_handler_terminal(ModeName, dial_db):
code = Lng.COMMAND_LIST([
Op.IndentationHandlerCall(ModeName),
Op.GotoDoorId(DoorID.continue_without_on_after_match(dial_db))
], dial_db)
incidence_id = dial.new_incidence_id()
terminal = Terminal(CodeTerminal(code),
"<CALL INDENTATION HANDLER>",
incidence_id,
dial_db=dial_db)
return DoorID.incidence(incidence_id, dial_db), terminal
def _prepare_skip_character_set(self, MHI, Loopers, CaMap, ReloadState):
"""MHI = Mode hierarchie index."""
if Loopers.skip is None: return [], [], []
skipped_character_set, \
pattern_str, \
aux_source_reference = Loopers.combined_skip(CaMap)
new_analyzer_list, \
new_terminal_list, \
loop_map, \
required_register_set = skip_character_set.do(self.terminal_factory.mode_name,
CaMap, skipped_character_set,
ReloadState,
self.terminal_factory.dial_db)
self.required_register_set.update(required_register_set)
extra_terminal_list = [
self._terminal_goto_to_looper(new_analyzer_list,
E_IncidenceIDs.SKIP, "<skip>",
required_register_set)
]
extra_terminal_list.extend(t for t in new_terminal_list)
# Any skipped character must enter the skipper entry.
goto_code = [
Op.GotoDoorId(
DoorID.incidence(E_IncidenceIDs.SKIP,
self.terminal_factory.dial_db))
]
new_ppt_list = []
for lei in loop_map:
new_incidence_id = dial.new_incidence_id()
pattern = Pattern.from_character_set(lei.character_set,
new_incidence_id,
PatternString="<skip>",
Sr=aux_source_reference)
# There is no reference pointer => Add directly
count_code = lei.aux_count_action.get_OpList(None)
code = Lng.COMMAND_LIST(count_code + goto_code,
self.terminal_factory.dial_db)
terminal = Terminal(CodeTerminal(code),
"ENTER SKIP:",
new_incidence_id,
dial_db=self.terminal_factory.dial_db)
new_ppt_list.append(
PPT(PatternPriority(MHI, new_incidence_id), pattern, terminal))
return new_ppt_list, new_analyzer_list, extra_terminal_list
def _get_state_machine_vs_terminal_list(CloserPattern, dial_db, DoorIdExit):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = CloserPattern.get_cloned_sm(StateMachineId=dial.new_incidence_id())
code = [Lng.GOTO(DoorIdExit, dial_db)]
mini_terminal = loop.MiniTerminal(code, "<SKIP RANGE TERMINATED>",
sm.get_id())
return [(sm, mini_terminal)]
def __init__(self, ColumnNPerCodeUnit, LexemeEndCheckF, EngineType,
ReloadStateExtern, UserOnLoopExitDoorId, dial_db,
OnReloadFailureDoorId, ModeName, Events):
"""ColumnNPerCodeUnit is None => no constant relationship between
column number and code unit.
"""
self.mode_name = ModeName
self.column_number_per_code_unit = ColumnNPerCodeUnit
self.lexeme_end_check_f = LexemeEndCheckF
self.reload_state_extern = ReloadStateExtern
self.engine_type = EngineType
self.dial_db = dial_db
self.door_id_on_reload_failure = OnReloadFailureDoorId
self.door_id_on_loop_exit_user_code = UserOnLoopExitDoorId
self.events = Events
self.loop_state_machine_id = None
self.__appendix_dfa_present_f = False
self.loop_state_machine_id = index.get_state_machine_id()
self.iid_loop_exit = dial.new_incidence_id()
self.iid_loop_after_appendix_drop_out = dial.new_incidence_id()
self.__run_time_counter_required_f = False
def _get_state_machine_vs_terminal_list(CloserPattern, OpenerPattern,
DoorIdExit, dial_db):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: [0] list(state machine, terminal)
[1] incidence id of auxiliary terminal that goto-s to the
loop entry.
The auxiliary terminal is necessary since the DoorID of the loop entry
cannot be known beforehand.
"""
# DoorID of loop entry cannot be known beforehand.
# => generate an intermediate door_id from where the loop is entered.
iid_aux_reentry = dial.new_incidence_id()
door_id_aux_reentry = dial.DoorID.incidence(iid_aux_reentry, dial_db)
# Opener Pattern Reaction
opener_op_list = [
Op.Increment(E_R.Counter),
Op.GotoDoorId(door_id_aux_reentry)
]
# Closer Pattern Reaction
closer_op_list = [
Op.Decrement(E_R.Counter),
Op.GotoDoorIdIfCounterEqualZero(DoorIdExit),
Op.GotoDoorId(door_id_aux_reentry)
]
def sm_terminal_pair(Pattern, Name, OpList, dial_db):
sm = Pattern.get_cloned_sm(StateMachineId=dial.new_incidence_id())
terminal = loop.MiniTerminal(Lng.COMMAND_LIST(OpList, dial_db), Name,
sm.get_id())
return sm, terminal
smt_list = [
sm_terminal_pair(OpenerPattern, "<SKIP NESTED RANGE OPENER>",
opener_op_list, dial_db),
sm_terminal_pair(CloserPattern, "<SKIP NESTED RANGE CLOSER>",
closer_op_list, dial_db)
]
return smt_list, iid_aux_reentry
def _get_LoopMapEntry_list_plain(loop_config, CaMap, L_pure):
"""RETURNS: list of LoopMapEntry-s.
The list defines the loop behavior for characters which are not transits
to appendix state machines. The LoopMapEntry-s are setup as below:
[0] Character set to trigger to a terminal.
[1] CountAction.
[2] IncidenceId of the CountAction.
[3] 'None' indicating: no appendix sm, no 'goto couple state'.
"""
assert L_pure is not None
return [
LoopMapEntry(character_set,
dial.new_incidence_id(),
Code=loop_config.cmd_list_CA_GotoLoopEntry(ca),
CA=ca)
for character_set, ca in CaMap.iterable_in_sub_set(L_pure)
]
def _adapt_pattern_id_to_priority(ppt_list):
"""Ensure that the incidence-id of each pattern fits the position in
priorization sequence. That is, early entries in the list MUST have
lower incidence id than later entries.
NOTE: For MHI-s < 0, i.e. loopers, this functions basically asserts
that the incidence ids of all patterns are aligned with their
position in the list.
"""
if len(ppt_list) < 1: return
ppt_list.sort(key=attrgetter("priority"))
prev_incidence_id = ppt_list[0].pattern.incidence_id
for i, ppt in enumerate(ppt_list[1:], start=1):
priority, pattern, terminal = ppt
# NOT: Mode Hierarchy Index < 0 == 'Entry into Looper'
# Those patterns are NOT SUPPOSED to match common lexemes
# with each other or other lexemes in the mode!
# => They are not subject to repriorization!
if priority.mode_hierarchy_index < 0: continue
if pattern.incidence_id > prev_incidence_id:
prev_incidence_id = pattern.incidence_id
continue
# Generate a new, cloned pattern. So that other related modes are not effected.
new_incidence_id = dial.new_incidence_id(
) # new id > any older id.
new_pattern = pattern.clone_with_new_incidence_id(new_incidence_id)
if terminal is not None:
new_terminal = terminal.clone(new_incidence_id)
else:
new_terminal = None
new_ppt = PPT(priority, new_pattern, new_terminal)
ppt_list[i] = new_ppt
prev_incidence_id = new_incidence_id
print
for i, analyzer in enumerate(AnalyzerList):
print "--( %i: init si = %i )-------------------------\n" % (i, analyzer.init_state_index)
print analyzer
print_drop_out(analyzer)
NS_A = NumberSet.from_range(0x600, 0x601) # UTF8: D8 80 => 216, 128
NS_B = NumberSet.from_range(0x601, 0x602) # UTF8: D8 81 => 216, 129
NS_C = NumberSet.from_range(0x640, 0x641) # UTF8: D9 80 => 217, 128
appendix_sm_id = 4711L
if "loop" in sys.argv:
loop_map = loop.LoopMap([
TestLME(NS_A, dial.new_incidence_id(), None),
])
column_n_per_code_unit = 5
elif "appendix" in sys.argv:
loop_map = loop.LoopMap([
TestLME(NS_A, dial.new_incidence_id(), appendix_sm_id), # appendix_sm_id
])
column_n_per_code_unit = 5
elif "appendix-wot" in sys.argv:
loop_map = loop.LoopMap([
TestLME(NS_A, dial.new_incidence_id(), None),
])
column_n_per_code_unit = 5
elif "non-const" in sys.argv:
loop_map = loop.LoopMap([
TestLME(NS_A, dial.new_incidence_id(), None),
def sm_terminal_pair(Pattern, Name, OpList, dial_db):
sm = Pattern.get_cloned_sm(StateMachineId=dial.new_incidence_id())
terminal = loop.MiniTerminal(Lng.COMMAND_LIST(OpList, dial_db), Name,
sm.get_id())
return sm, terminal
def do(loop_config, CaMap, SmList):
"""Perform separation:
Parallel state machine ----> first transition
+ appendix state machine
Appendix Sm-Id --> Original Sm-Id
The 'first transition' is mounted on the loop state machine triggering an
acceptance that causes a transit to the appendix state machine.
RETURNS: list of LoopMapEntry-s
"""
# ESSENTIAL: Delimiter state machines shall never match on a common lexeme!
_assert_no_intersections(SmList)
assert all(sm.get_id() is not None for sm in SmList)
loop_map_1, \
original_iid_db = split_first_transition(SmList)
# loop_map_1: list of [0] first transition character set
# [1] appendix sm with first transition removed
#
# original_iid_db: appendix sm id --> original sm id
appendix_cmd_list_db = loop_config.get_appendix_terminal_cmd_list_db(
CaMap, [sm for cs, sm in loop_map_1], original_iid_db)
# appendix_cmd_list_db: appendix sm id --> CmdList(count action,
# goto original terminal)
loop_map_2 = \
split_first_character_set_for_distinct_count_actions(CaMap,
loop_map_1)
# loop_map_2: list of [0] character set where all elements
# require same count actions
# [1] count action
# [2] appendix sm
# For a 'state transition' it is required that all character sets
# in the list are disjoint. Thus, any intersection must build its
# on entry. Thus, some entries might have more than one appendix.
loop_map_3 = combine_intersecting_character_sets(loop_map_2)
# loop_map_2: list of [0] character set no character set intersects
# with any other.
# [1] count action
# [2] list of (appendix sm)
# A transition can only enter one state machine, so all appendix
# state machines related to the same character set must be combined.
loop_map_4, \
combined_appendix_sm_list_raw = combine_appendix_sm_lists(loop_map_3)
# loop_map_4: list of [0] (disjoint) character set
# [1] count action for character set
# [2] state machine id of related combined appendix sm
# combined_appendix_sm_lists: list of all generated (combined) appendix sm-s
loop_map_5, \
combined_appendix_sm_list = determine_CmdLists(loop_config, loop_map_4,
combined_appendix_sm_list_raw,
original_iid_db)
# loop_map_5: list of [0] (disjoint) character set
# [1] CmdList = (count action, goto terminal/appendix sm),
# combined_appendix_sm_list: contains only those combined appendix state machines
# that do have transitions
loop_map_6 = [
LoopMapEntry(character_set,
IidCoupleTerminal=dial.new_incidence_id(),
Code=cmd_list) for character_set, cmd_list in loop_map_5
]
# loop_map_6: list of LoopMapEntry-s
# There must be a command list for any acceptance in the appendix
# state machines.
all_acceptance_id_set = flatten_list_of_lists(
sm.acceptance_id_set() for sm in combined_appendix_sm_list)
assert all(iid in appendix_cmd_list_db for iid in all_acceptance_id_set)
return loop_map_6, combined_appendix_sm_list, appendix_cmd_list_db
ca_list = get_ca_list(0x10, 0x60)
sm_list = get_sm_list(NumberSet.from_range(0x10, 0x40),
NumberSet.from_range(0x20, 0x50),
NumberSet.from_range(0x30, 0x60))
# Test for each 'sm' in 'sm_list' is superfluous.
# It is done in 'AppendixNoI'.
test(ca_list, sm_list)
elif "Split" in sys.argv:
# A first transition of a state machine is separated into two, because
# it is covered by more than one different count action.
NS1 = NumberSet.from_range(0x10, 0x20)
NS2 = NumberSet.from_range(0x20, 0x30)
NS3 = NumberSet.from_range(0x30, 0x40)
NS4 = NumberSet.from_range(0x40, 0x50)
ca_list = [
(NS1, CountAction(E_CharacterCountType.COLUMN, 1)),
(NS2, CountAction(E_CharacterCountType.COLUMN, 2)),
(NS3, CountAction(E_CharacterCountType.COLUMN, 3)),
(NS4, CountAction(E_CharacterCountType.COLUMN, 4)),
]
sm = DFA()
si = sm.init_state_index
iid = dial.new_incidence_id()
ti0 = sm.add_transition(si, NumberSet.from_range(0x1A, 0x4B))
ac0 = sm.add_transition(ti0, NS_A, AcceptanceF=True)
test(ca_list, [sm])
def _prepare_indentation_counter(self, MHI, Loopers, CaMap, ReloadState):
"""Prepare indentation counter. An indentation counter is implemented by
the following:
'newline' pattern --> triggers as soon as an UNSUPPRESSED newline occurs.
--> entry to the INDENTATION COUNTER.
'suppressed newline' --> INDENTATION COUNTER is NOT triggered.
The supressed newline pattern is longer (and has precedence) over the
newline pattern. With the suppressed newline it is possible to write
lines which overstep the newline (see backslahs in Python, for example).
RETURNS: List of:
[0] newline PPT and
[1] optionally the PPT of the newline suppressor.
The primary pattern action pair list is to be the head of all pattern
action pairs.
MHI = Mode hierarchie index defining the priority of the current mode.
"""
ISetup = Loopers.indentation_handler
if ISetup is None: return [], [], []
check_indentation_setup(ISetup)
# Isolate the pattern objects, so alternatively,
# they may be treated in 'indentation_counter'.
pattern_newline = ISetup.pattern_newline.clone()
if ISetup.pattern_suppressed_newline:
pattern_suppressed_newline = ISetup.pattern_suppressed_newline.clone(
)
else:
pattern_suppressed_newline = None
new_analyzer_list, \
new_terminal_list, \
required_register_set, \
run_time_counter_f = indentation_counter.do(self.terminal_factory.mode_name,
CaMap,
ISetup,
self.terminal_factory.incidence_db,
ReloadState,
self.terminal_factory.dial_db)
self.terminal_factory.run_time_counter_required_f |= run_time_counter_f
self.required_register_set.update(required_register_set)
# 'newline' triggers --> indentation counter
pattern = pattern_newline.finalize(CaMap)
pattern = pattern.clone_with_new_incidence_id(
E_IncidenceIDs.INDENTATION_HANDLER)
terminal = self._terminal_goto_to_looper(new_analyzer_list,
None,
"<indentation handler>",
required_register_set,
Pattern=pattern)
new_ppt_list = [PPT(PatternPriority(MHI, 0), pattern, terminal)]
if pattern_suppressed_newline is not None:
# 'newline-suppressor' causes following 'newline' to be ignored.
# => next line not subject to new indentation counting.
new_incidence_id = dial.new_incidence_id()
pattern = pattern_suppressed_newline.finalize(CaMap)
pattern = pattern.clone_with_new_incidence_id(new_incidence_id)
door_id = DoorID.global_reentry(self.terminal_factory.dial_db)
code = CodeTerminal(
[Lng.GOTO(door_id, self.terminal_factory.dial_db)])
new_ppt_list.append(
PPT(
PatternPriority(MHI, 1), pattern,
self.terminal_factory.do_plain(
code, pattern,
"INDENTATION COUNTER SUPPRESSED_NEWLINE: ")))
return new_ppt_list, new_analyzer_list, new_terminal_list
