def get_setup(L0, L1, FSM0, FSM1, FSM2):
# SPECIALITIES: -- sm0 and sm1 have an intersection between their second
# transition.
# -- sm1 transits further upon acceptance.
# -- sm2 has only one transition.
ci_list = [
CountInfo(dial_db.new_incidence_id(), NumberSet.from_range(L0, L1),
CountAction(E_CharacterCountType.COLUMN, 0)),
]
# Generate State Machine that does not have any intersection with
# the loop transitions.
sm0 = StateMachine()
si = sm0.add_transition(sm0.init_state_index, FSM0)
si = sm0.add_transition(si, NS_A, AcceptanceF=True)
sm0.states[si].mark_acceptance_id(dial_db.new_incidence_id())
sm1 = StateMachine()
si0 = sm1.add_transition(sm1.init_state_index, FSM1)
si = sm1.add_transition(si0, NS_A, AcceptanceF=True)
iid1 = dial_db.new_incidence_id()
sm1.states[si].mark_acceptance_id(iid1)
si = sm1.add_transition(si, NS_B, si0)
sm1.states[si].mark_acceptance_id(iid1)
sm2 = StateMachine()
si = sm2.add_transition(sm2.init_state_index, FSM2, AcceptanceF=True)
sm2.states[si].mark_acceptance_id(dial_db.new_incidence_id())
return ci_list, [sm0, sm1, sm2]
def get_setup(L0, L1, FSM0, FSM1, FSM2):
# SPECIALITIES: -- sm0 and sm1 have an intersection between their second
# transition.
# -- sm1 transits further upon acceptance.
# -- sm2 has only one transition.
ci_list = [
CountInfo(dial_db.new_incidence_id(), NumberSet.from_range(L0, L1),
CountAction(E_CharacterCountType.COLUMN, 0)),
]
# Generate State Machine that does not have any intersection with
# the loop transitions.
sm0 = StateMachine()
si = sm0.add_transition(sm0.init_state_index, FSM0)
si = sm0.add_transition(si, NS_A, AcceptanceF=True)
sm0.states[si].mark_acceptance_id(dial_db.new_incidence_id())
sm1 = StateMachine()
si0 = sm1.add_transition(sm1.init_state_index, FSM1)
si = sm1.add_transition(si0, NS_A, AcceptanceF=True)
iid1 = dial_db.new_incidence_id()
sm1.states[si].mark_acceptance_id(iid1)
si = sm1.add_transition(si, NS_B, si0)
sm1.states[si].mark_acceptance_id(iid1)
sm2 = StateMachine()
si = sm2.add_transition(sm2.init_state_index, FSM2, AcceptanceF=True)
sm2.states[si].mark_acceptance_id(dial_db.new_incidence_id())
return ci_list, [sm0, sm1, sm2]
def _determine_LoopMapEntry(sm_db, CharacterSet, CA, AppendixSmList):
appendix_sm_id = combined(sm_db, appendix_sm_list)
has_transitions_f = sm_db[appendix_sm_id].get_init_state().has_transitions()
if has_transitions_f:
# There is an appendix after the first transition.
# => goto the appendix state machine
return LoopMapEntry(CharacterSet, CA, dial_db.new_incidence_id(),
appendix_sm_id, True)
else:
# There is NO appendix after the first transition.
# => directly goto to terminal of the matched state machine.
appendix_sm_id = min(sm.get_id() for sm in AppendixSmList)
return LoopMapEntry(CharacterSet, CA, dial_db.new_incidence_id(),
appendix_sm_id, False)
def _determine_LoopMapEntry(sm_db, CharacterSet, CA, AppendixSmList):
appendix_sm_id = combined(sm_db, appendix_sm_list)
has_transitions_f = sm_db[appendix_sm_id].get_init_state(
).has_transitions()
if has_transitions_f:
# There is an appendix after the first transition.
# => goto the appendix state machine
return LoopMapEntry(CharacterSet, CA, dial_db.new_incidence_id(),
appendix_sm_id, True)
else:
# There is NO appendix after the first transition.
# => directly goto to terminal of the matched state machine.
appendix_sm_id = min(sm.get_id() for sm in AppendixSmList)
return LoopMapEntry(CharacterSet, CA, dial_db.new_incidence_id(),
appendix_sm_id, False)
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
Op.LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(SmComment,
CounterDb,
CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Op.Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Op.Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb, IncidenceDb):
"""Generate a PPT for a range skipper.
"""
# -- door_id_after: Where to go after the closing character sequence matched:
# + Normally: To the begin of the analyzer. Start again.
# + End(Sequence) == newline of indentation counter.
# => goto indentation counter.
if _match_indentation_counter_newline_pattern(OptionsDb.value("indentation"),
data["closer_sequence"]):
door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)
else:
door_id_after = DoorID.continue_without_on_after_match()
if NestedF:
name = "SKIP NESTED RANGE"
code_generator_func = skip_nested_range.do
else:
name = "SKIP RANGE"
code_generator_func = skip_range.do
# -- data for code generation
my_data = deepcopy(data)
my_data["mode_name"] = ModeName
my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN]
my_data["door_id_after"] = door_id_after
my_data["counter_db"] = CounterDb
# -- terminal and code generator
priority = PatternPriority(MHI, i)
pattern = deepcopy(my_data["opener_pattern"])
pattern.set_incidence_id(dial_db.new_incidence_id())
pattern.set_pattern_string("<skip_range>")
code = CodeGenerated(code_generator_func, my_data, name)
return PPT(priority, pattern, code)
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
Op.LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
Op.GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def PPT_range_skipper(NestedF, MHI, i, data, ModeName, OptionsDb, CounterDb,
IncidenceDb):
"""Generate a PPT for a range skipper.
"""
# -- door_id_after: Where to go after the closing character sequence matched:
# + Normally: To the begin of the analyzer. Start again.
# + End(Sequence) == newline of indentation counter.
# => goto indentation counter.
if _match_indentation_counter_newline_pattern(
OptionsDb.value("indentation"), data["closer_sequence"]):
door_id_after = DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)
else:
door_id_after = DoorID.continue_without_on_after_match()
if NestedF:
name = "SKIP NESTED RANGE"
code_generator_func = skip_nested_range.do
else:
name = "SKIP RANGE"
code_generator_func = skip_range.do
# -- data for code generation
my_data = deepcopy(data)
my_data["mode_name"] = ModeName
my_data["on_skip_range_open"] = IncidenceDb[E_IncidenceIDs.SKIP_RANGE_OPEN]
my_data["door_id_after"] = door_id_after
my_data["counter_db"] = CounterDb
# -- terminal and code generator
priority = PatternPriority(MHI, i)
pattern = deepcopy(my_data["opener_pattern"])
pattern.set_incidence_id(dial_db.new_incidence_id())
pattern.set_pattern_string("<skip_range>")
code = CodeGenerated(code_generator_func, my_data, name)
return PPT(priority, pattern, code)
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def from_LineColumnCount(CounterDb, CharacterSet, InputPName):
"""Use NumberSet_All() if all characters shall be used.
"""
cmap = [
CountInfo(dial_db.new_incidence_id(), intersection, info)
for intersection, info in CounterDb.count_command_map.column_grid_line_iterable_pruned(CharacterSet)
]
column_n_per_code_unit = CounterDb.get_column_number_per_code_unit(CharacterSet)
return CountInfoMap(cmap, CharacterSet, column_n_per_code_unit)
def from_ParserDataLineColumn(CounterDb, CharacterSet, InputPName):
"""Use NumberSet_All() if all characters shall be used.
"""
cmap = [
CountInfo(dial_db.new_incidence_id(), info.cc_type, info.value, intersection)
for intersection, info in CounterDb.count_command_map.column_grid_line_iterable_pruned(CharacterSet)
]
ColumnNPerChunk = CounterDb.count_command_map.get_column_number_per_chunk(CharacterSet)
return CountOpFactory(cmap, ColumnNPerChunk, InputPName, CharacterSet)
def test(ci_list, SM_list=[]):
Setup.buffer_codec.source_set = NumberSet_All()
ci_map = CountInfoMap(ci_list, NumberSet.from_range(0, 100))
iid_loop_exit = dial_db.new_incidence_id()
loop_map, appendix_sm_list = loop._get_loop_map(ci_map, SM_list, iid_loop_exit)
print
print
print
general_checks(loop_map, appendix_sm_list)
print_this(loop_map, appendix_sm_list)
def _get_state_machine_and_terminal(Sequence, Name, OpList):
"""Create state machine that detects the 'Sequence', names the terminal
with 'Name', and implements the 'CmdList' in the terminal.
RETURNS: (state machine, terminal)
"""
sm = StateMachine.from_sequence(Sequence)
sm.set_id(dial_db.new_incidence_id())
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(OpList)), Name, sm.get_id())
terminal.set_requires_goto_loop_entry_f() # --> Goto Loop Entry
return sm, terminal
def from_LineColumnCount(CounterDb, CharacterSet, InputPName):
"""Use NumberSet_All() if all characters shall be used.
"""
cmap = [
CountInfo(dial_db.new_incidence_id(), intersection, info)
for intersection, info in CounterDb.count_command_map.
column_grid_line_iterable_pruned(CharacterSet)
]
column_n_per_code_unit = CounterDb.get_column_number_per_code_unit(
CharacterSet)
return CountInfoMap(cmap, CharacterSet, column_n_per_code_unit)
def test(ci_list, SM_list=[]):
Setup.buffer_codec.source_set = NumberSet_All()
ci_map = CountInfoMap(ci_list, NumberSet.from_range(0, 100))
iid_loop_exit = dial_db.new_incidence_id()
loop_map, appendix_sm_list = loop._get_loop_map(ci_map, SM_list,
iid_loop_exit)
print
print
print
general_checks(loop_map, appendix_sm_list)
print_this(loop_map, appendix_sm_list)
def _get_state_machine_and_terminal(Sequence, Name, OpList):
"""Create state machine that detects the 'Sequence', names the terminal
with 'Name', and implements the 'CmdList' in the terminal.
RETURNS: (state machine, terminal)
"""
sm = StateMachine.from_sequence(Sequence)
sm.set_id(dial_db.new_incidence_id())
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(OpList)), Name,
sm.get_id())
terminal.set_requires_goto_loop_entry_f() # --> Goto Loop Entry
return sm, terminal
def incidence_id_db_get(cls, CA):
"""RETURNS: Unique incidence id for a given count action.
Same count actions will have the same incidence id. Initialize the
'incidence_id_db' with 'CountAction.incidence_id_db.clear()' before
generating a new set of count action incidence ids.
"""
key = (CA.cc_type, CA.value)
incidence_id = CA.incidence_id_db.get(key)
if incidence_id is None:
incidence_id = dial_db.new_incidence_id()
CA.incidence_id_db[key] = incidence_id
return incidence_id
def incidence_id_db_get(cls, CA):
"""RETURNS: Unique incidence id for a given count action.
Same count actions will have the same incidence id. Initialize the
'incidence_id_db' with 'CountAction.incidence_id_db.clear()' before
generating a new set of count action incidence ids.
"""
key = (CA.cc_type, CA.value)
incidence_id = CA.incidence_id_db.get(key)
if incidence_id is None:
incidence_id = dial_db.new_incidence_id()
CA.incidence_id_db[key] = incidence_id
return incidence_id
def from_ParserDataLineColumn(CounterDb, CharacterSet, InputPName):
"""Use NumberSet_All() if all characters shall be used.
"""
cmap = [
CountInfo(dial_db.new_incidence_id(), info.cc_type, info.value,
intersection) for intersection, info in CounterDb.
count_command_map.column_grid_line_iterable_pruned(CharacterSet)
]
ColumnNPerChunk = CounterDb.count_command_map.get_column_number_per_chunk(
CharacterSet)
return CountOpFactory(cmap, ColumnNPerChunk, InputPName, CharacterSet)
def from_ParserDataIndentation(ISetup, CounterDb, InputPName, DoorIdBad):
"""Return a factory that produces 'column' and 'grid' counting incidence_id-maps.
"""
result = CountOpFactory.from_ParserDataLineColumn(
CounterDb, ISetup.whitespace_character_set.get(), InputPName)
# Up to now, the '__map' contains only character sets which intersect with the
# defined whitespace. Add the 'bad indentation characters'.
bad_character_set = ISetup.bad_character_set.get()
if bad_character_set is not None:
result.__map.append(
CountInfo(dial_db.new_incidence_id(), E_CharacterCountType.BAD,
None, bad_character_set))
result.door_id_on_bad_indentation = DoorIdBad
return result
def from_IndentationCount(ISetup, CounterDb, InputPName, DoorIdBad):
"""Return a factory that produces 'column' and 'grid' counting incidence_id-maps.
"""
column_n_per_code_unit = CounterDb.get_column_number_per_code_unit()
result = CountInfoMap.from_LineColumnCount(
ISetup.whitespace_character_set.get(), column_n_per_code_unit)
# Up to now, the '__map' contains only character sets which intersect with the
# defined whitespace. Add the 'bad indentation characters'.
bad_character_set = ISetup.bad_character_set.get()
if bad_character_set is not None:
result.__map.append(
CountInfo(dial_db.new_incidence_id(), bad_character_set,
CountAction(E_CharacterCountType.BAD, DoorIdBad)))
return result
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = StateMachine.from_sequence(CloserSequence)
sm.set_id(dial_db.new_incidence_id())
code = [ Lng.GOTO(DoorID.continue_without_on_after_match()) ]
terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>", sm.get_id())
return [ (sm, terminal) ]
def from_CountOpFactory(ccfactory, LexemeMaintainedF, ParallelSmList=None):
"""The function 'get_terminal_list' returns a list of termins which
fit the incidence ids of the counting actions.
"""
beyond_iid = dial_db.new_incidence_id()
# Build a state machine based on (character set, incidence_id) pairs.
return CharacterSetStateMachine(
ccfactory.get_incidence_id_map(beyond_iid),
LexemeMaintainedF,
ParallelSmList,
OnBegin = ccfactory.on_begin,
OnEnd = ccfactory.on_end,
OnBeforeReload = ccfactory.on_before_reload,
OnAfterReload = ccfactory.on_after_reload,
BeyondIid = beyond_iid)
def from_ParserDataIndentation(ISetup, CounterDb, InputPName, DoorIdBad):
"""Return a factory that produces 'column' and 'grid' counting incidence_id-maps.
"""
result = CountOpFactory.from_ParserDataLineColumn(CounterDb,
ISetup.whitespace_character_set.get(),
InputPName)
# Up to now, the '__map' contains only character sets which intersect with the
# defined whitespace. Add the 'bad indentation characters'.
bad_character_set = ISetup.bad_character_set.get()
if bad_character_set is not None:
result.__map.append(
CountInfo(dial_db.new_incidence_id(), E_CharacterCountType.BAD, None,
bad_character_set)
)
result.door_id_on_bad_indentation = DoorIdBad
return result
def _get_state_machine_vs_terminal_list(CloserSequence, CounterDb):
"""Additionally to all characters, the loop shall walk along the 'closer'.
If the closer matches, the range skipping exits. Characters need to be
counted properly.
RETURNS: list(state machine, terminal)
The list contains only one single element.
"""
sm = StateMachine.from_sequence(CloserSequence)
sm.set_id(dial_db.new_incidence_id())
code = [Lng.GOTO(DoorID.continue_without_on_after_match())]
terminal = Terminal(CodeTerminal(code), "<SKIP RANGE TERMINATED>",
sm.get_id())
return [(sm, terminal)]
def from_IndentationCount(ISetup, CounterDb, InputPName, DoorIdBad):
"""Return a factory that produces 'column' and 'grid' counting incidence_id-maps.
"""
column_n_per_code_unit = CounterDb.get_column_number_per_code_unit()
result = CountInfoMap.from_LineColumnCount(ISetup.whitespace_character_set.get(),
column_n_per_code_unit)
# Up to now, the '__map' contains only character sets which intersect with the
# defined whitespace. Add the 'bad indentation characters'.
bad_character_set = ISetup.bad_character_set.get()
if bad_character_set is not None:
result.__map.append(
CountInfo(dial_db.new_incidence_id(), bad_character_set,
CountAction(E_CharacterCountType.BAD, DoorIdBad))
)
return result
def _pattern_delete_and_reprioritize(ppt_list, BaseModeSequence):
"""Performs the deletion and repriorization according the DELETE and PRIORITY-MARK
commands in the mode. This may change the order and the incidence ids of the
mode's patterns. Thus, the ppt_list needs to be resorted and new incidence_id-s
need to be assigned.
"""
# -- Delete and reprioritize
history_deletion = _pattern_deletion(ppt_list, BaseModeSequence)
history_reprioritization = _pattern_reprioritization(ppt_list, BaseModeSequence)
# -- Re-sort and re-assign new incidence id which reflect the new order.
ppt_list.sort(key=attrgetter("priority"))
for priority, pattern, terminal in ppt_list:
if pattern.incidence_id() in IncidenceDB.unmutable_incidence_ids:
continue
new_pattern_id = dial_db.new_incidence_id() # new id > any older id.
pattern.set_incidence_id(new_pattern_id)
return history_deletion, history_reprioritization
def _add_pair(psml, SmOriginal, Name):
"""Add a state machine-terminal pair to 'psml'. A terminal is generated
which transits to 'INDENTATION_HANDLER'. The state machine is cloned
for safety.
"""
if SmOriginal is None: return
incidence_id = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
sm = SmOriginal.clone(StateMachineId=incidence_id)
code = [Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))]
terminal = Terminal(CodeTerminal(code), Name, incidence_id)
# TRY: terminal.set_requires_goto_loop_entry_f()
# INSTEAD: GOTO 'INDENTATION_HANDLER'
psml.append((sm, terminal))
def _pattern_delete_and_reprioritize(ppt_list, BaseModeSequence):
"""Performs the deletion and repriorization according the DELETE and PRIORITY-MARK
commands in the mode. This may change the order and the incidence ids of the
mode's patterns. Thus, the ppt_list needs to be resorted and new incidence_id-s
need to be assigned.
"""
# -- Delete and reprioritize
history_deletion = _pattern_deletion(ppt_list, BaseModeSequence)
history_reprioritization = _pattern_reprioritization(ppt_list, BaseModeSequence)
# -- Re-sort and re-assign new incidence id which reflect the new order.
ppt_list.sort(key=attrgetter("priority"))
for priority, pattern, terminal in ppt_list:
if pattern.incidence_id() in IncidenceDB.unmutable_incidence_ids:
continue
new_pattern_id = dial_db.new_incidence_id() # new id > any older id.
pattern.set_incidence_id(new_pattern_id)
return history_deletion, history_reprioritization
def do(TheCountMap,
OnLoopExit,
LexemeEndCheckF=False,
EngineType=None,
ReloadStateExtern=None,
LexemeMaintainedF=False,
ParallelSmTerminalPairList=None):
"""Generates a structure that 'loops' quickly over incoming characters.
Loop continues
.---------( ++i )-----+--------<-------------------. at AFTER position of
| .------. | | the first lexatom 'ir'.
'--->| | | |
| pure |-->[ Terminals A ] |
| L |-->[ Terminals B ] |
| |-->[ Terminals C ] |
+------+ |
| | ( i = ir )
| LaF |-->[ Terminals A ]-->-. | drop-out
| |-->[ Terminals B ]-->. \ |
| |-->[ Terminals C ]-->( ir = i )--[ StateMachine ]-->[ Terminals X ]
| | \
+------+ '-->[ Terminals Y ]
| Else |----> Exit
'------'
The terminals may contain a 'lexeme end check', that ensures that the
borders of a lexeme are not exceeded. The loop therefore ends:
(i) when a character appears, that is not a loop character.
(ii) one of the appendix state machine exits.
(iii) [Optional] if the lexeme end is reached.
At the end of the iteration, the input pointer points to (the begin of) the
first lexatom behind what is treated.
[i][i][i]..................[i][i][X][....
|
input_p
During the 'loop' possible line/column count commands may be applied.
"""
parallel_terminal_list, \
parallel_sm_list = _extract_state_machines_and_terminals(ParallelSmTerminalPairList)
iid_loop_exit = dial_db.new_incidence_id()
iid_loop_after_appendix_drop_out = dial_db.new_incidence_id()
assert EngineType is not None
event_handler = LoopEventHandlers(
TheCountMap.column_number_per_code_unit(),
LexemeEndCheckF,
LexemeMaintainedF,
EngineType,
ReloadStateExtern,
UserOnLoopExit=OnLoopExit)
# LoopMap: Associate characters with the reactions on their occurrence ____
#
loop_map, \
appendix_sm_list = _get_loop_map(TheCountMap, parallel_sm_list,
iid_loop_exit)
# Loop represented by Analyzer-s and Terminal-s ___________________________
#
analyzer_list, \
door_id_loop, \
appendix_sm_exist_f = _get_analyzer_list(loop_map, event_handler, appendix_sm_list,
iid_loop_after_appendix_drop_out)
if not appendix_sm_exist_f:
iid_loop_after_appendix_drop_out = None
terminal_list = _get_terminal_list(loop_map, event_handler,
parallel_terminal_list, door_id_loop,
iid_loop_exit,
iid_loop_after_appendix_drop_out)
# Generate Code ___________________________________________________________
#
txt = _get_source_code(analyzer_list, terminal_list,
TheCountMap.column_number_per_code_unit(),
appendix_sm_exist_f)
return txt, DoorID.incidence(iid_loop_exit)
def do(TheCountMap, OnLoopExit, LexemeEndCheckF=False, EngineType=None,
ReloadStateExtern=None, LexemeMaintainedF=False,
ParallelSmTerminalPairList=None):
"""Generates a structure that 'loops' quickly over incoming characters.
Loop continues
.---------( ++i )-----+--------<-------------------. at AFTER position of
| .------. | | the first lexatom 'ir'.
'--->| | | |
| pure |-->[ Terminals A ] |
| L |-->[ Terminals B ] |
| |-->[ Terminals C ] |
+------+ |
| | ( i = ir )
| LaF |-->[ Terminals A ]-->-. | drop-out
| |-->[ Terminals B ]-->. \ |
| |-->[ Terminals C ]-->( ir = i )--[ StateMachine ]-->[ Terminals X ]
| | \
+------+ '-->[ Terminals Y ]
| Else |----> Exit
'------'
The terminals may contain a 'lexeme end check', that ensures that the
borders of a lexeme are not exceeded. The loop therefore ends:
(i) when a character appears, that is not a loop character.
(ii) one of the appendix state machine exits.
(iii) [Optional] if the lexeme end is reached.
At the end of the iteration, the input pointer points to (the begin of) the
first lexatom behind what is treated.
[i][i][i]..................[i][i][X][....
|
input_p
During the 'loop' possible line/column count commands may be applied.
"""
parallel_terminal_list, \
parallel_sm_list = _extract_state_machines_and_terminals(ParallelSmTerminalPairList)
iid_loop_exit = dial_db.new_incidence_id()
iid_loop_after_appendix_drop_out = dial_db.new_incidence_id()
assert EngineType is not None
event_handler = LoopEventHandlers(TheCountMap.column_number_per_code_unit(),
LexemeEndCheckF, LexemeMaintainedF,
EngineType, ReloadStateExtern,
UserOnLoopExit=OnLoopExit)
# LoopMap: Associate characters with the reactions on their occurrence ____
#
loop_map, \
appendix_sm_list = _get_loop_map(TheCountMap, parallel_sm_list,
iid_loop_exit)
# Loop represented by Analyzer-s and Terminal-s ___________________________
#
analyzer_list, \
door_id_loop, \
appendix_sm_exist_f = _get_analyzer_list(loop_map, event_handler, appendix_sm_list,
iid_loop_after_appendix_drop_out)
if not appendix_sm_exist_f:
iid_loop_after_appendix_drop_out = None
terminal_list = _get_terminal_list(loop_map, event_handler, parallel_terminal_list,
door_id_loop,
iid_loop_exit,
iid_loop_after_appendix_drop_out)
# Generate Code ___________________________________________________________
#
txt = _get_source_code(analyzer_list, terminal_list,
TheCountMap.column_number_per_code_unit(),
appendix_sm_exist_f)
return txt, DoorID.incidence(iid_loop_exit)
ci_list, sm_list = get_setup(0x10, 0x60,
NumberSet.from_range(0x10, 0x40),
NumberSet.from_range(0x20, 0x50),
NumberSet.from_range(0x30, 0x60))
# Test for each 'sm' in 'sm_list' is superfluous.
# It is done in 'AppendixNoI'.
test(ci_list, sm_list)
elif "Split" in sys.argv:
# A first transition of a state machine is separated into two, because
# it is covered by more than one different count action.
NS1 = NumberSet.from_range(0x10, 0x20)
NS2 = NumberSet.from_range(0x20, 0x30)
NS3 = NumberSet.from_range(0x30, 0x40)
NS4 = NumberSet.from_range(0x40, 0x50)
ci_list = [
CountInfo(dial_db.new_incidence_id(), NS1, CountAction(E_CharacterCountType.COLUMN, 1)),
CountInfo(dial_db.new_incidence_id(), NS2, CountAction(E_CharacterCountType.COLUMN, 2)),
CountInfo(dial_db.new_incidence_id(), NS3, CountAction(E_CharacterCountType.COLUMN, 3)),
CountInfo(dial_db.new_incidence_id(), NS4, CountAction(E_CharacterCountType.COLUMN, 4))
]
sm = StateMachine()
si = sm.init_state_index
iid = dial_db.new_incidence_id()
ti0 = sm.add_transition(si, NumberSet.from_range(0x1A, 0x4B))
ac0 = sm.add_transition(ti0, NS_A, AcceptanceF=True)
test(ci_list, [sm])
def do(Data, TheAnalyzer):
"""________________________________________________________________________
Counting whitespace at the beginning of a line.
.-----<----+----------<--------------+--<----.
| | count | | count = 0
| | whitespace | |
.---------. | | |
--------->| +--' | |
| | | |
| | | |
| | .------------. | |
| +----->----| suppressor |----' |
| | | + newline | |
| COUNTER | '------------' |
| | .---------. |
| +----->----| newline |---------------'
| | '---------'
| | .----------------.
| |----->----| on_indentation |---------> RESTART
'---------' else '----------------'
Generate an indentation counter. An indentation counter is entered upon
the detection of a newline (which is not followed by a newline suppressor).
Indentation Counter:
indentation = 0
column = 0
|
|<------------------------.
.-------------. |
| INDENTATION | indentation += count
| COUNTER | column += count
'-------------' |
| |
+-------- whitspace -->---'
|
Re-Enter
Analyzer
An indentation counter is a single state that iterates to itself as long
as whitespace occurs. During that iteration the column counter is adapted.
There are two types of adaption:
-- 'normal' adaption by a fixed delta. This adaption happens upon
normal space characters.
-- 'grid' adaption. When a grid character occurs, the column number
snaps to a value given by a grid size parameter.
When a newline occurs the indentation counter exits and restarts the
lexical analysis. If the newline is not followed by a newline suppressor
the analyzer will immediately be back to the indentation counter state.
___________________________________________________________________________
"""
counter_db = Data["counter_db"]
isetup = Data["indentation_setup"]
incidence_db = Data["incidence_db"]
default_ih_f = Data["default_indentation_handler_f"]
mode_name = Data["mode_name"]
sm_suppressed_newline = Data["sm_suppressed_newline"]
sm_newline = isetup.sm_newline.get()
sm_comment = isetup.sm_comment.get()
assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant()
assert sm_newline is None or sm_newline.is_DFA_compliant()
assert sm_comment is None or sm_comment.is_DFA_compliant()
# -- 'on_indentation' == 'on_beyond':
# A handler is called as soon as an indentation has been detected.
after_beyond = [
Op.IndentationHandlerCall(default_ih_f, mode_name),
Op.GotoDoorId(DoorID.continue_without_on_after_match())
]
# -- 'on_bad_indentation' is invoked if a character appeared that has been
# explicitly disallowed to be used as indentation.
bad_indentation_iid = dial_db.new_incidence_id()
if Setup.buffer_based_analyzis_f: reload_state = None
else: reload_state = TheAnalyzer.reload_state
sm_terminal_list = _get_state_machine_vs_terminal_list(sm_suppressed_newline,
isetup.sm_newline.get(),
isetup.sm_comment.get(),
counter_db)
# 'whitespace' --> normal counting
# 'bad' --> goto bad character indentation handler
# else --> non-whitespace detected => handle indentation
ccfactory = CountOpFactory.from_ParserDataIndentation(isetup,
counter_db,
Lng.INPUT_P(),
DoorID.incidence(bad_indentation_iid))
# (*) Generate Code
code, \
door_id_beyond = loop.do(ccfactory,
AfterBeyond = after_beyond,
EngineType = TheAnalyzer.engine_type,
ReloadStateExtern = reload_state,
LexemeMaintainedF = True,
ParallelSmTerminalPairList = sm_terminal_list)
_code_terminal_on_bad_indentation_character(code, isetup, mode_name, incidence_db,
bad_indentation_iid)
return code
NumberSet.from_range(0x20, 0x50),
NumberSet.from_range(0x30, 0x60))
# Test for each 'sm' in 'sm_list' is superfluous.
# It is done in 'AppendixNoI'.
test(ci_list, sm_list)
elif "Split" in sys.argv:
# A first transition of a state machine is separated into two, because
# it is covered by more than one different count action.
NS1 = NumberSet.from_range(0x10, 0x20)
NS2 = NumberSet.from_range(0x20, 0x30)
NS3 = NumberSet.from_range(0x30, 0x40)
NS4 = NumberSet.from_range(0x40, 0x50)
ci_list = [
CountInfo(dial_db.new_incidence_id(), NS1,
CountAction(E_CharacterCountType.COLUMN, 1)),
CountInfo(dial_db.new_incidence_id(), NS2,
CountAction(E_CharacterCountType.COLUMN, 2)),
CountInfo(dial_db.new_incidence_id(), NS3,
CountAction(E_CharacterCountType.COLUMN, 3)),
CountInfo(dial_db.new_incidence_id(), NS4,
CountAction(E_CharacterCountType.COLUMN, 4))
]
sm = StateMachine()
si = sm.init_state_index
iid = dial_db.new_incidence_id()
ti0 = sm.add_transition(si, NumberSet.from_range(0x1A, 0x4B))
ac0 = sm.add_transition(ti0, NS_A, AcceptanceF=True)
def do(Data, TheAnalyzer):
"""________________________________________________________________________
Counting whitespace at the beginning of a line.
.-----<----+----------<--------------+--<----.
| | count | | count = 0
| | whitespace | |
.---------. | | |
--------->| +--' | |
| | | |
| | | |
| | .------------. | |
| +----->----| suppressor |----' |
| | | + newline | |
| COUNTER | '------------' |
| | .---------. |
| +----->----| newline |---------------'
| | '---------'
| | .----------------.
| |----->----| on_indentation |---------> RESTART
'---------' else '----------------'
Generate an indentation counter. An indentation counter is entered upon
the detection of a newline (which is not followed by a newline suppressor).
Indentation Counter:
indentation = 0
column = 0
|
|<------------------------.
.-------------. |
| INDENTATION | indentation += count
| COUNTER | column += count
'-------------' |
| |
+-------- whitspace -->---'
|
Re-Enter
Analyzer
An indentation counter is a single state that iterates to itself as long
as whitespace occurs. During that iteration the column counter is adapted.
There are two types of adaption:
-- 'normal' adaption by a fixed delta. This adaption happens upon
normal space characters.
-- 'grid' adaption. When a grid character occurs, the column number
snaps to a value given by a grid size parameter.
When a newline occurs the indentation counter exits and restarts the
lexical analysis. If the newline is not followed by a newline suppressor
the analyzer will immediately be back to the indentation counter state.
___________________________________________________________________________
"""
counter_db = Data["counter_db"]
isetup = Data["indentation_setup"]
incidence_db = Data["incidence_db"]
default_ih_f = Data["default_indentation_handler_f"]
mode_name = Data["mode_name"]
sm_suppressed_newline = Data["sm_suppressed_newline"]
sm_newline = isetup.sm_newline.get()
sm_comment = isetup.sm_comment.get()
assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant(
)
assert sm_newline is None or sm_newline.is_DFA_compliant()
assert sm_comment is None or sm_comment.is_DFA_compliant()
# -- 'on_indentation' == 'on_beyond':
# A handler is called as soon as an indentation has been detected.
after_beyond = [
IndentationHandlerCall(default_ih_f, mode_name),
GotoDoorId(DoorID.continue_without_on_after_match())
]
# -- 'on_bad_indentation' is invoked if a character appeared that has been
# explicitly disallowed to be used as indentation.
bad_indentation_iid = dial_db.new_incidence_id()
if Setup.buffer_based_analyzis_f: reload_state = None
else: reload_state = TheAnalyzer.reload_state
sm_terminal_list = _get_state_machine_vs_terminal_list(
sm_suppressed_newline, isetup.sm_newline.get(),
isetup.sm_comment.get(), counter_db)
# 'whitespace' --> normal counting
# 'bad' --> goto bad character indentation handler
# else --> non-whitespace detected => handle indentation
ccfactory = CountCmdFactory.from_ParserDataIndentation(
isetup, counter_db, Lng.INPUT_P(),
DoorID.incidence(bad_indentation_iid))
# (*) Generate Code
code, \
door_id_beyond = loop.do(ccfactory,
AfterBeyond = after_beyond,
EngineType = TheAnalyzer.engine_type,
ReloadStateExtern = reload_state,
LexemeMaintainedF = True,
ParallelSmTerminalPairList = sm_terminal_list)
_code_terminal_on_bad_indentation_character(code, isetup, mode_name,
incidence_db,
bad_indentation_iid)
return code
