def __prepare_before_and_after_reload(self, OnBeforeReload, OnAfterReload):
"""The 'lexeme_start_p' restricts the amount of data which is loaded
into the buffer upon reload--if the lexeme needs to be maintained. If
the lexeme does not need to be maintained, then the whole buffer can
be refilled.
For this, the 'lexeme_start_p' is set to the input pointer.
EXCEPTION: Variable character sizes. There, the 'lexeme_start_p' is used
to mark the begin of the current letter. However, letters are short, so
the drawback is tiny.
RETURNS: [0] on_before_reload
[1] on_after_reload
"""
if Setup.buffer_codec.variable_character_sizes_f():
if not self.maintain_lexeme_f:
on_before_reload = [ Op.Assign(E_R.LexemeStartP, E_R.CharacterBeginP) ]
on_after_reload = [ Op.Assign(E_R.CharacterBeginP, E_R.LexemeStartP) ]
else:
assert False
# Here, the character begin p needs to be adapted to what has been reloaded.
on_before_reload = [ ] # LexemeBegin is enough.
on_after_reload = [ ]
else:
on_before_reload = [ Op.Assign(E_R.LexemeStartP, E_R.InputP) ]
on_after_reload = [ ] # Op.Assign(E_R.InputP, E_R.LexemeStartP) ]
self.on_before_reload = concatinate(on_before_reload,OnBeforeReload)
self.on_after_reload = concatinate(on_after_reload, OnAfterReload)
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
Op.AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Op.Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Op.Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def __prepare_positioning_before_and_after_reload(MaintainLexemeF):
"""The 'lexeme_start_p' restricts the amount of data which is loaded
into the buffer upon reload--if the lexeme needs to be maintained. If
the lexeme does not need to be maintained, then the whole buffer can
be refilled.
For this, the 'lexeme_start_p' is set to the input pointer.
EXCEPTION: Variable character sizes. There, the 'lexeme_start_p' is used
to mark the begin of the current letter. However, letters are short, so
the drawback is tiny.
RETURNS: [0] on_before_reload
[1] on_after_reload
"""
if Setup.buffer_codec.variable_character_sizes_f():
if MaintainLexemeF:
on_before_reload = [
Op.Assign(E_R.LexemeStartP, E_R.CharacterBeginP)
]
on_after_reload = [
Op.Assign(E_R.CharacterBeginP, E_R.LexemeStartP)
]
else:
# Here, the character begin p needs to be adapted to what has been reloaded.
on_before_reload = [] # Begin of lexeme is enough.
on_after_reload = []
else:
on_before_reload = [Op.Assign(E_R.LexemeStartP, E_R.InputP)]
on_after_reload = [] # Op.Assign(E_R.InputP, E_R.LexemeStartP) ]
return on_before_reload, on_after_reload
def on_loop_after_appendix_drop_out(self, DoorIdLoop, ColumnNPerCodeUnit):
# Upon drop-out, the input position is set to where the apendix
# started. Then, the loop is re-entered.
op_list = []
if ColumnNPerCodeUnit is not None:
op_list.append(
Op.Assign(E_R.CountReferenceP,
E_R.LoopRestartP,
Condition="COLUMN"))
op_list.extend([
Op.Assign(E_R.InputP, E_R.LoopRestartP),
Op.GotoDoorId(DoorIdLoop)
])
return op_list
def __prepare_begin_and_putback(self, OnBegin, OnEnd):
"""If we deal with variable character sizes, the begin of the letter is stored
in 'character_begin_p'. To reset the input pointer 'input_p = character_begin_p'
is applied.
"""
if not Setup.buffer_codec.variable_character_sizes_f():
# 1 character == 1 chunk
# => rest to last character: 'input_p = input_p - 1'
self.on_step = []
self.on_putback = [ Op.InputPDecrement() ]
else:
# 1 character == variable number of chunks
# => store begin of character in 'lexeme_start_p'
# => rest to last character: 'input_p = lexeme_start_p'
self.on_step = [ Op.Assign(E_R.CharacterBeginP, E_R.InputP) ]
self.on_putback = [ Op.Assign(E_R.InputP, E_R.CharacterBeginP) ]
self.on_begin = concatinate(self.on_step, OnBegin)
self.on_end = concatinate(self.on_putback, OnEnd)
def __prepare_positioning_at_loop_begin_and_exit():
"""With codecs of dynamic character sizes (UTF8), the pointer to the
first letter is stored in 'lexatom_begin_p'. To reset the input
pointer 'input_p = lexatom_begin_p' is applied.
"""
if not Setup.buffer_codec.variable_character_sizes_f():
# 1 character == 1 code unit
# => reset to last character: 'input_p = input_p - 1'
on_loop_exit = [] # Op.Decrement(E_R.InputP) ]
on_loop_reentry = []
else:
# 1 character == variable number of code units
# => store begin of character in 'lexeme_start_p'
# => reset to last character: 'input_p = lexeme_start_p'
on_loop_exit = [Op.Assign(E_R.InputP, E_R.CharacterBeginP)]
on_loop_reentry = [Op.Assign(E_R.CharacterBeginP, E_R.InputP)]
return on_loop_reentry, on_loop_exit
def get_couple_terminal_cmd_list(self, CA, CombinedAppendixSm,
OriginalIidDb):
"""CA: Count action for the character set of the loop
entry.
CombinedAppendixSm: (combined) appendix sm where to jump upon
triggering of the character set.
OriginalIidDb: Appendix sm id --> original iid of the state machine
where it came from.
RETURNS: [0] CmdList -- for loop count action and transition to appendix
state machine or entry into according terminal.
[1] True, if there is a transition to an appendix state machine
False, if not
"""
transition_to_appendix_f = CombinedAppendixSm.get_init_state(
).has_transitions()
acceptance_id_set = CombinedAppendixSm.acceptance_id_set()
combined_appendix_sm_id = CombinedAppendixSm.get_id()
cmd_list = []
if CA is not None:
cmd_list.extend(CA.get_OpList(self.column_number_per_code_unit))
if not transition_to_appendix_f:
# NO appendix after first transition => jump to appendix terminal.
assert len(acceptance_id_set) == 1
iid_original = OriginalIidDb[acceptance_id_set.pop()]
target_door_id = DoorID.incidence(iid_original, self.dial_db)
transition_to_appendix_f = False
else:
# appendix after first transition. => jump to appendix state machine.
self.__appendix_dfa_present_f = True
cmd_list.extend([
Op.Assign(E_R.LoopRestartP, E_R.InputP),
Op.Assign(E_R.LexemeStartP, E_R.InputP)
])
target_door_id = DoorID.state_machine_entry(
combined_appendix_sm_id, self.dial_db)
cmd_list.append(Op.GotoDoorId(target_door_id))
return cmd_list, transition_to_appendix_f
def get_count_code(self, LCCI):
if LCCI is None: return []
run_time_counter_required_f, \
cmd_list = SmLineColumnCountInfo.get_OpList(LCCI, ModeName=self.mode_name)
if run_time_counter_required_f:
self.__run_time_counter_required_f = True
# The content to be counted starts where the appendix started.
# * Begin of counting at 'loop restart pointer'.
# * Run-time counting can ONLY work, if the lexeme start pointer
# is at position of appendix begin.
cmd_list[:0] = [Op.Assign(E_R.LexemeStartP, E_R.LoopRestartP)]
if self.column_number_per_code_unit is not None:
# If the reference counting is applied, the reference pointer
# must be set right behind the last counted character.
cmd_list.append(
Op.Assign(E_R.CountReferenceP, E_R.InputP, Condition="COLUMN"))
return cmd_list
def get_loop_terminal_code(self, TheLoopMapEntry, DoorIdLoop,
DoorIdLoopExit):
"""RETURNS: A loop terminal.
A terminal: (i) Counts,
(ii) checks possibly for the lexeme end, and
(iii)a either re-enters the loop, or
(iii)b transits to an appendix state machine (couple terminal).
"""
IncidenceId = TheLoopMapEntry.incidence_id
AppendixSmId = TheLoopMapEntry.appendix_sm_id
TheCountAction = TheLoopMapEntry.count_action
code = []
if TheCountAction is not None:
code.extend(
TheCountAction.get_OpList(self.column_number_per_code_unit))
if AppendixSmId is not None:
if not lei.appendix_sm_has_transitions_f:
# If there is no appendix, directly goto to the terminal.
code.extend([Op.GotoDoorId(DoorID.incidence_id(AppendixSmId))])
else:
assert not self.lexeme_end_check_f
# Couple Terminal: transit to appendix state machine.
code.extend([
Op.Assign(E_R.ReferenceP, E_R.InputP),
Op.GotoDoorId(DoorID.state_machine_entry(AppendixSmId))
])
elif not self.lexeme_end_check_f:
# Loop Terminal: directly re-enter loop.
code.append(Op.GotoDoorId(DoorIdLoop))
else:
# Check Terminal: check against lexeme end before re-entering loop.
code.append(
Op.GotoDoorIdIfInputPNotEqualPointer(DoorIdLoop,
E_R.LexemeEnd))
if self.column_number_per_code_unit is not None \
and TheCountAction is not None \
and TheCountAction.cc_type == E_CharacterCountType.COLUMN:
# With reference counting, no column counting while looping.
# => Do it now, before leaving.
code.append(
Op.ColumnCountReferencePDeltaAdd(
E_R.InputP, self.column_number_per_code_unit, False))
code.append(Op.GotoDoorId(DoorIdLoopExit))
return Terminal(CodeTerminal(Lng.COMMAND_LIST(code)),
"<LOOP TERMINAL %i>" % IncidenceId, IncidenceId)
def __prepare_positioning_at_loop_begin_and_exit(ColumnNPerCodeUnit):
"""With encodings of dynamic character sizes (UTF8), the pointer to the
first letter is stored in 'character_begin_p'. To reset the input
pointer 'input_p = character_begin_p' is applied.
"""
if Setup.buffer_encoding.variable_character_sizes_f():
# 1 character == variable number of code units
# => Begin of character must be stored upon entry
# and restored upon exit.
entry = [Op.Assign(E_R.LoopRestartP, E_R.InputP)]
reentry = [Op.Assign(E_R.LoopRestartP, E_R.InputP)]
exit = [Op.Assign(E_R.InputP, E_R.LoopRestartP)]
else:
# 1 character == 1 code unit
# => reset to last character: 'input_p = input_p - 1'
entry = []
reentry = []
exit = [Op.Decrement(E_R.InputP)]
if ColumnNPerCodeUnit is not None:
entry.append(
Op.Assign(E_R.CountReferenceP, E_R.InputP, Condition="COLUMN"))
return entry, reentry, exit
def __prepare_positioning_before_and_after_reload(self, AppendixSmF=False):
"""When new content is loaded into the buffer, the positions of the
pointers must be adapted, so that they point to the same byte as
before the reload. This function determines the pointer adaptions for
all required pointers depending on the 'circumstances'.
AppendixSmF: True --> generate for appendix state machines
False --> generate for main loop
The considered pointers are:
CountReferenceP --> required for counting with 'delta reference'.
LoopRestartP --> Position where to restart loop or 'character':
* if the appendix state machine drops-out.
* if current multi byte character is incomplete.
If the lexeme is not to be maintained, the amount of data to be reloaded
is maximized by setting the lexeme start pointer to the read pointer.
! TODO: The 'Buffer_reload()' function should simply take an array of
! position pointers. Then this function would be (almost) superfluous.
RETURNS: [0] on_before_reload
[1] on_after_reload
"""
if Setup.buffer_encoding.variable_character_sizes_f():
maintain_loop_restart_p = True
elif AppendixSmF:
maintain_loop_restart_p = True
else:
maintain_loop_restart_p = False
# NOTE: The 'CountReferenceP' is not subject to reload, since a
# 'delta addition' is done right before reload. After reload,
# the pointer is set to the input pointer.
before = []
after = []
# Before Reload:
# Just get the 'lexeme_start_p' out of the way, so that anything
# is filled from 'read_p'.
before.append(Op.Assign(E_R.LexemeStartP, E_R.InputP))
if maintain_loop_restart_p:
before.extend([
Op.Assign(E_R.InputPBeforeReload, E_R.InputP),
Op.PointerAssignMin(E_R.LexemeStartP, E_R.LexemeStartP,
E_R.LoopRestartP)
])
# After Reload:
if maintain_loop_restart_p:
after.extend([
Op.AssignPointerDifference(E_R.PositionDelta, E_R.InputP,
E_R.InputPBeforeReload),
Op.PointerAdd(E_R.LoopRestartP, E_R.PositionDelta)
])
# Make sure, that the lexeme start pointer makes sense:
# => begin of input
after.append(Op.Assign(E_R.LexemeStartP, E_R.InputP))
return before, after
def on_loop_after_appendix_drop_out(self, DoorIdLoop):
# 'CharacterBeginP' has been assigned in the 'Couple Terminal'.
# (see ".get_loop_terminal_code()").
return Lng.COMMAND_LIST(
[Op.Assign(E_R.InputP, E_R.ReferenceP),
Op.GotoDoorId(DoorIdLoop)])