def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def create_DropOut(self, SM_State):
if SM_State.is_acceptance():
incidence_id = self.__incidence_id_of_bipd
return CommandList(
QuexDebug('pattern %i: backward input position detected\\n' %
incidence_id), InputPIncrement(),
GotoDoorId(DoorID.bipd_return(incidence_id)))
else:
return CommandList(QuexAssertNoPassage())
def _command(self, CC_Type, Parameter):
if self.column_count_per_chunk is None:
if CC_Type == E_CharacterCountType.BAD:
return [GotoDoorId(self.door_id_on_bad_indentation)]
elif CC_Type == E_CharacterCountType.COLUMN:
return [
ColumnCountAdd(Parameter),
]
elif CC_Type == E_CharacterCountType.GRID:
return [
ColumnCountGridAdd(Parameter),
]
elif CC_Type == E_CharacterCountType.LINE:
return [
LineCountAdd(Parameter),
AssignConstant(E_R.Column, 1),
]
else:
if CC_Type == E_CharacterCountType.BAD:
return [
ColumnCountReferencePDeltaAdd(E_R.InputP,
self.column_count_per_chunk,
False),
ColumnCountReferencePSet(E_R.InputP),
GotoDoorId(self.door_id_on_bad_indentation)
]
elif CC_Type == E_CharacterCountType.COLUMN:
return []
elif CC_Type == E_CharacterCountType.GRID:
return [
ColumnCountReferencePDeltaAdd(E_R.InputP,
self.column_count_per_chunk,
True),
ColumnCountGridAdd(Parameter),
ColumnCountReferencePSet(E_R.InputP)
]
elif CC_Type == E_CharacterCountType.LINE:
return [
LineCountAdd(Parameter),
AssignConstant(E_R.Column, 1),
ColumnCountReferencePSet(E_R.InputP)
]
def get(CCFactory, Name):
"""Implement the default counter for a given Counter Database.
In case the line and column number increment cannot be determined before-
hand, a something must be there that can count according to the rules given
in 'CCFactory'. This function generates the code for a general counter
function which counts line and column number increments starting from the
begin of a lexeme to its end.
The implementation of the default counter is a direct function of the
'CCFactory', i.e. the database telling how characters influence the
line and column number counting.
Multiple modes may have the same character counting behavior. If so,
then there's only one counter implemented while others refer to it.
---------------------------------------------------------------------------
RETURNS: function_name, string --> Function name and the implementation
of the character counter.
function_name, None --> The 'None' implementation indicates that
NO NEW counter is implemented. An
appropriate counter can be accessed
by the 'function name'.
---------------------------------------------------------------------------
"""
function_name = DefaultCounterFunctionDB.get_function_name(CCFactory)
if function_name is not None:
return function_name, None # Implementation has been done before.
function_name = Lng.DEFAULT_COUNTER_FUNCTION_NAME(Name)
door_id_return = dial_db.new_door_id()
code, \
door_id_beyond = loop.do(CCFactory,
AfterBeyond = [ GotoDoorId(door_id_return) ],
LexemeEndCheckF = True,
EngineType = engine.CHARACTER_COUNTER)
implementation = __frame(function_name, Lng.INPUT_P(), code, door_id_return,
door_id_beyond)
DefaultCounterFunctionDB.enter(CCFactory, function_name)
return function_name, implementation
def get_LexemeEndCheck_appendix(ccfactory, CC_Type):
if not LexemeEndCheckF:
return [GotoDoorId(door_id_loop)]
#
# .---------------. ,----------. no
# --->| Count Command |-------< LexemeEnd? >------> DoorIdOk
# '---------------' '----+-----'
# | yes
# .---------------.
# | Lexeme End |
# | Count Command |----> DoorIdOnLexemeEnd
# '---------------'
#
elif ccfactory.requires_reference_p(
) and CC_Type == E_CharacterCountType.COLUMN:
return [
GotoDoorIdIfInputPNotEqualPointer(door_id_loop, E_R.LexemeEnd),
ColumnCountReferencePDeltaAdd(
E_R.InputP, ccfactory.column_count_per_chunk, False),
] + AfterBeyond
else:
return [
GotoDoorIdIfInputPNotEqualPointer(door_id_loop, E_R.LexemeEnd),
] + AfterBeyond
def do(Data, TheAnalyzer):
"""________________________________________________________________________
Counting whitespace at the beginning of a line.
.-----<----+----------<--------------+--<----.
| | count | | count = 0
| | whitespace | |
.---------. | | |
--------->| +--' | |
| | | |
| | | |
| | .------------. | |
| +----->----| suppressor |----' |
| | | + newline | |
| COUNTER | '------------' |
| | .---------. |
| +----->----| newline |---------------'
| | '---------'
| | .----------------.
| |----->----| on_indentation |---------> RESTART
'---------' else '----------------'
Generate an indentation counter. An indentation counter is entered upon
the detection of a newline (which is not followed by a newline suppressor).
Indentation Counter:
indentation = 0
column = 0
|
|<------------------------.
.-------------. |
| INDENTATION | indentation += count
| COUNTER | column += count
'-------------' |
| |
+-------- whitspace -->---'
|
Re-Enter
Analyzer
An indentation counter is a single state that iterates to itself as long
as whitespace occurs. During that iteration the column counter is adapted.
There are two types of adaption:
-- 'normal' adaption by a fixed delta. This adaption happens upon
normal space characters.
-- 'grid' adaption. When a grid character occurs, the column number
snaps to a value given by a grid size parameter.
When a newline occurs the indentation counter exits and restarts the
lexical analysis. If the newline is not followed by a newline suppressor
the analyzer will immediately be back to the indentation counter state.
___________________________________________________________________________
"""
counter_db = Data["counter_db"]
isetup = Data["indentation_setup"]
incidence_db = Data["incidence_db"]
default_ih_f = Data["default_indentation_handler_f"]
mode_name = Data["mode_name"]
sm_suppressed_newline = Data["sm_suppressed_newline"]
sm_newline = isetup.sm_newline.get()
sm_comment = isetup.sm_comment.get()
assert sm_suppressed_newline is None or sm_suppressed_newline.is_DFA_compliant(
)
assert sm_newline is None or sm_newline.is_DFA_compliant()
assert sm_comment is None or sm_comment.is_DFA_compliant()
# -- 'on_indentation' == 'on_beyond':
# A handler is called as soon as an indentation has been detected.
after_beyond = [
IndentationHandlerCall(default_ih_f, mode_name),
GotoDoorId(DoorID.continue_without_on_after_match())
]
# -- 'on_bad_indentation' is invoked if a character appeared that has been
# explicitly disallowed to be used as indentation.
bad_indentation_iid = dial_db.new_incidence_id()
if Setup.buffer_based_analyzis_f: reload_state = None
else: reload_state = TheAnalyzer.reload_state
sm_terminal_list = _get_state_machine_vs_terminal_list(
sm_suppressed_newline, isetup.sm_newline.get(),
isetup.sm_comment.get(), counter_db)
# 'whitespace' --> normal counting
# 'bad' --> goto bad character indentation handler
# else --> non-whitespace detected => handle indentation
ccfactory = CountCmdFactory.from_ParserDataIndentation(
isetup, counter_db, Lng.INPUT_P(),
DoorID.incidence(bad_indentation_iid))
# (*) Generate Code
code, \
door_id_beyond = loop.do(ccfactory,
AfterBeyond = after_beyond,
EngineType = TheAnalyzer.engine_type,
ReloadStateExtern = reload_state,
LexemeMaintainedF = True,
ParallelSmTerminalPairList = sm_terminal_list)
_code_terminal_on_bad_indentation_character(code, isetup, mode_name,
incidence_db,
bad_indentation_iid)
return code
def do(Data, TheAnalyzer):
"""Fast implementation of character set skipping machine.
________________________________________________________________________
As long as characters of a given character set appears it iterates:
input in Set
.--<---.
| |
.-------. |
--------->( SKIPPER )--+----->------> RESTART
'-------' input
not in Set
___________________________________________________________________________
NOTE: The 'TerminalSkipRange' takes care that it transits immediately to
the indentation handler, if it ends on 'newline'. This is not necessary
for 'TerminalSkipCharacterSet'. Quex refuses to work on 'skip sets' when
they match common lexemes with the indentation handler.
___________________________________________________________________________
Precisely, i.e. including counter and reload actions:
START
|
| .----------------------------------------------.
| |.-------------------------------------------. |
| ||.----------------------------------------. | |
| ||| | | |
| ||| .-DoorID(S, a)--. transition | | |
| || '-| gridstep(cn) | map | | |
| || '---------------'\ .------. | | |
| || .-DoorID(S, b)--. '->-| | | | |
| |'---| ln += 1 |--->-| '\t' +-->-----' | |
| | '---------------' | | | |
| | .-DoorID(S, c)--. | ' ' +-->-------' |
| '----| cn += 1 |--->-| | |
| '---------------' | '\n' +-->---------'
| | | .-DropOut ------.
| .-DoorID(S, 0)--. | else +-->---------------| on_exit |
'------>--| on_entry |--->-| | '---------------'
'---------------' | BLC +-->-.
.->-| | \ Reload State
.-DoorID(S, 1)--./ '------' \ .-----------------.
.----| after_reload | \ .---------------. |
| '---------------' '---------| before_reload | |
| '---------------' |
'-----------------------------------------------------| |
success '------------------'
| failure
|
.---------------.
| End of Stream |
'---------------'
NOTE: If dynamic character size codings, such as UTF8, are used as engine codecs,
then the single state may actually be split into a real state machine of
states.
"""
counter_db = Data["counter_db"]
character_set = Data["character_set"]
if Setup.buffer_based_analyzis_f:
reload_state = None
else:
reload_state = TheAnalyzer.reload_state
result, \
door_id_beyond = loop.do(CountCmdFactory.from_ParserDataLineColumn(counter_db, character_set, Lng.INPUT_P()),
AfterBeyond = [ GotoDoorId(DoorID.continue_without_on_after_match()) ],
LexemeEndCheckF = False,
LexemeMaintainedF = False,
EngineType = engine.FORWARD,
ReloadStateExtern = reload_state)
assert isinstance(result, list)
return result
def create_DropOut(self, SM_State):
return CommandList(GotoDoorId(
DoorID.global_end_of_pre_context_check()))