def do_end_of_stream(self, Code, ThePattern):
"""End of Stream: The terminating zero has been reached and no further
content can be loaded.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# No indentation handler => Empty string.
text = [
Lng.DEFAULT_COUNTER_CALL(),
self.txt_indentation_handler_call,
#
adorned_code,
#
Lng.ML_COMMENT(
"End of Stream FORCES a return from the lexical analyzer, so that no\n"
"tokens can be filled after the termination token."
),
Lng.GOTO(DoorID.return_with_on_after_match()),
]
code = CodeTerminal(text,
SourceReference = Code.sr,
PureCode = Code.get_pure_code())
return Terminal(code, "END_OF_STREAM")
def do_match_pattern(self, Code, ThePattern):
"""A pattern has matched."""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=True)
# IMPORTANT: Terminals can be entered by any kind of 'GOTO'. In order to
# be on the safe side, BIPD should be started from within the
# terminal itself. Otherwise, it may be missed due to some
# coding negligence.
text = []
if ThePattern.bipd_sm is not None:
TerminalFactory.do_bipd_entry_and_return(text, ThePattern)
text.extend([
self.get_counter_text(ThePattern),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_with_on_after_match())
])
code = CodeTerminal(text,
SourceReference = Code.sr,
PureCode = Code.get_pure_code(),
LexemeRelevanceF = True,
LexemeBeginF = lexeme_begin_f,
LexemeTerminatingZeroF = terminating_zero_f)
name = TerminalFactory.name_pattern_match_terminal(ThePattern.pattern_string())
return Terminal(code, name)
def do_match_failure(self, Code, ThePattern):
"""No pattern in the mode has matched. Line and column numbers are
still counted. But, no 'on_match' or 'on_after_match' action is
executed.
"""
lexeme_begin_f, \
terminating_zero_f, \
adorned_code = self.__adorn_user_code(Code, MatchF=False)
text = [
#Lng.IF_END_OF_FILE(),
# self.get_counter_text(None),
# Lng.GOTO(DoorID.continue_without_on_after_match()),
#Lng.IF_INPUT_P_EQUAL_LEXEME_START_P(FirstF=False),
# Lng.INPUT_P_INCREMENT(),
#Lng.END_IF(),
self.get_counter_text(None),
#
adorned_code,
#
Lng.GOTO(DoorID.continue_without_on_after_match()),
]
code = CodeTerminal(text,
SourceReference = Code.sr,
PureCode = Code.get_pure_code())
return Terminal(code, "FAILURE")
def _add_newline(psml, SmNewlineOriginal):
"""Add a pair (newline state machine, terminal on newline) to 'psml'.
When a newline occurs, the column count can be set to 1 and the line number
is incremented. Then the indentation counting restarts.
"""
assert SmNewlineOriginal is not None
# Disconnect from machines being used elsewhere.
SmNewline = SmNewlineOriginal.clone()
SmNewline.set_id(dial_db.new_incidence_id())
# The SmNewline has been used before in the main state machine with a
# different incidence id. It is essential to clone!
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER))
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION NEWLINE>")
terminal.set_incidence_id(SmNewline.get_id())
psml.append((SmNewline, terminal))
def _add_comment(psml, SmCommentOriginal, CounterDb):
"""On matching the comment state machine goto a terminal that does the
following:
"""
if SmCommentOriginal is None: return
comment_skip_iid = dial_db.new_incidence_id()
# Disconnect from machines being used elsewhere.
SmComment = SmCommentOriginal.clone()
SmComment.set_id(comment_skip_iid)
if SmComment.last_character_set().contains_only(ord('\n')):
code = Lng.COMMAND_LIST([
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
])
else:
count_info = CountInfo.from_StateMachine(
SmComment, CounterDb, CodecTrafoInfo=Setup.buffer_codec)
code = [
Lng.COMMAND(Assign(E_R.ReferenceP, E_R.LexemeStartP)),
CounterDb.do_CountInfo(count_info),
Lng.COMMAND(Assign(E_R.LexemeStartP, E_R.ReferenceP))
]
code.append(Lng.GOTO(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)))
terminal = Terminal(CodeTerminal(code), "INDENTATION COMMENT")
terminal.set_incidence_id(comment_skip_iid)
psml.append((SmComment, terminal))
def _add_suppressed_newline(psml, SmSuppressedNewlineOriginal):
"""Add a pair (suppressed newline, terminal on suppressed newline to 'psml'.
A suppresed newline is not like a newline--the next line is considered as
being appended to the current line. Nevertheless the line number needs to
incremented, just the column number is not reset to 1. Then, it continues
with indentation counting.
"""
if SmSuppressedNewlineOriginal is None:
return
# Disconnect from machines being used elsewhere.
SmSuppressedNewline = SmSuppressedNewlineOriginal.clone()
SmSuppressedNewline.set_id(dial_db.new_incidence_id())
# The parser MUST ensure that if there is a newline suppressor, there MUST
# be a newline being defined.
cl = [
LineCountAdd(1),
AssignConstant(E_R.Column, 1),
GotoDoorId(DoorID.incidence(E_IncidenceIDs.INDENTATION_HANDLER)),
]
terminal = Terminal(CodeTerminal(Lng.COMMAND_LIST(cl)),
"<INDENTATION SUPPRESSED NEWLINE>")
terminal.set_incidence_id(SmSuppressedNewline.get_id())
psml.append((SmSuppressedNewline, terminal))
def _get_terminal(X, get_appendix):
cl = self._command(X.cc_type, X.parameter)
appendix = get_appendix(self, X.cc_type)
terminal = Terminal(CodeTerminal(
Lng.COMMAND_LIST(chain(cl, appendix))),
Name="%s" % X.cc_type)
terminal.set_incidence_id(X.incidence_id)
return terminal
def __get_terminal_beyond(OnBeyond, BeyondIid):
"""Generate Terminal to be executed upon exit from the 'loop'.
BeyondIid -- 'Beyond Incidence Id', that is the incidencen id if of
the terminal to be generated.
"""
code_on_beyond = CodeTerminal(Lng.COMMAND_LIST(OnBeyond))
result = Terminal(code_on_beyond,
"<BEYOND>") # Put last considered character back
result.set_incidence_id(BeyondIid)
return result
def PPT_character_set_skipper(MHI, character_set, incidence_id, CounterDb, goto_terminal_str, Sr):
"""Generate a PPT for a character set skipper. That is,
-- A PatternPriority based on a given MHI and the specified incidence id.
-- A Pattern to be webbed into the lexical analyzer state machine.
-- A Terminal implementing the character set skipper.
"""
priority = PatternPriority(MHI, incidence_id)
pattern = Pattern.from_character_set(character_set)
pattern.set_pattern_string("<skip>")
pattern.set_source_reference(Sr)
code = CodeTerminal([ goto_terminal_str ], Sr)
return PPT(priority, pattern, code)
def do_plain(self, Code, ThePattern):
"""Plain source code text as generated by quex."""
text = [
self.get_counter_text(ThePattern)
]
text.extend(
Code.get_code()
)
code = CodeTerminal(text,
SourceReference = Code.sr,
PureCode = Code.get_code())
if ThePattern is None: name = "<no name>"
else: name = ThePattern.pattern_string()
return Terminal(code, name)
def PPT_indentation_handler_suppressed_newline(MHI, SmSuppressedNewline):
"""Generate a PPT for suppressed newline, that is:
-- its PatternPriority.
-- the Pattern object.
-- the Terminal object.
The terminal simply jumpts to the re-entry of the lexical analyzer.
"""
assert SmSuppressedNewline is not None
pattern = Pattern(SmSuppressedNewline,
PatternString="<indentation suppressed newline>")
code = CodeTerminal([Lng.GOTO(DoorID.global_reentry())])
# terminal = terminal_factory.do(E_TerminalType.PLAIN, code)
# terminal.set_name("INDENTATION COUNTER: SUPPRESSED_NEWLINE")
return PPT(PatternPriority(MHI, 1), pattern, code)
def extract_terminal_db(self, factory, ReloadRequiredF):
"""SpecialTerminals: END_OF_STREAM
FAILURE
CODEC_ERROR
...
"""
result = {}
for incidence_id, code_fragment in self.iteritems():
if incidence_id not in IncidenceDB.terminal_type_db:
continue
elif incidence_id == E_IncidenceIDs.END_OF_STREAM \
and not ReloadRequiredF:
continue
terminal_type = IncidenceDB.terminal_type_db[incidence_id]
code_terminal = CodeTerminal(code_fragment.get_code())
assert terminal_type not in result
terminal = factory.do(terminal_type, code_terminal)
terminal.set_incidence_id(incidence_id)
result[incidence_id] = terminal
return result
def get_CodeTerminal(self, IncidenceId):
if IncidenceId not in self:
return CodeTerminal([""])
else:
return CodeTerminal(self[IncidenceId].get_code(), LexemeRelevanceF=True)