def create_range_skipper_code(Language, TestStr, CloserSequence, QuexBufferSize=1024,
CommentTestStrF=False, ShowPositionF=False):
assert QuexBufferSize >= len(CloserSequence) + 2
end_str = __prepare(Language)
door_id_on_skip_range_open = dial_db.new_door_id()
data = {
"closer_sequence": CloserSequence,
"closer_pattern": Pattern(StateMachine.from_sequence(CloserSequence),
PatternString="<skip range closer>"),
"mode_name": "MrUnitTest",
"on_skip_range_open": CodeFragment([end_str]),
"door_id_after": DoorID.continue_without_on_after_match(),
}
skipper_code = range_skipper.do(data, Analyzer)
__require_variables()
return create_customized_analyzer_function(Language, TestStr, skipper_code,
QuexBufferSize, CommentTestStrF, ShowPositionF, end_str,
MarkerCharList = [],
LocalVariableDB = deepcopy(variable_db.get()),
DoorIdOnSkipRangeOpen=door_id_on_skip_range_open)
def do(UTF8_String_or_Stream, PatternDict,
AllowNothingIsNecessaryF = False, SpecialTerminator=None):
global SPECIAL_TERMINATOR
assert type(AllowNothingIsNecessaryF) == bool
assert type(PatternDict) == dict
# SPECIAL_TERMINATOR --> if string is not only to be terminated by ' '
SPECIAL_TERMINATOR = SpecialTerminator
def __ensure_whitespace_follows(InitialPos, stream):
tmp = stream.read(1)
if tmp == "" or tmp.isspace() or tmp == SPECIAL_TERMINATOR:
stream.seek(-1, 1)
return
end_position = stream.tell() - 1
stream.seek(InitialPos)
pattern_str = stream.read(end_position - InitialPos)
error_msg("Pattern definition '%s' not followed by whitespace.\n" % pattern_str + \
"Found subsequent character '%s'." % tmp,
stream)
if type(UTF8_String_or_Stream) == str: stream = StringIO(UTF8_String_or_Stream)
else: stream = UTF8_String_or_Stream
if PatternDict is None: PatternDict = {}
initial_position = stream.tell()
# -- check for the begin of line condition (BOL)
if check(stream, '^'): begin_of_line_f = True
else: begin_of_line_f = False
# -- MAIN: transform the pattern into a state machine
pre, core, post = snap_conditional_expression(stream, PatternDict)
if core is None:
stream.seek(initial_position)
return None
# -- check for end of line condition (EOL)
# -- check for terminating whitespace
end_of_line_f = False
if check(stream, '$'): end_of_line_f = True
__ensure_whitespace_follows(initial_position, stream)
pattern = Pattern(CoreSM = core,
BeginOfLineF = begin_of_line_f,
PreContextSM = pre,
EndOfLineF = end_of_line_f,
PostContextSM = post,
Sr = SourceRef.from_FileHandle(stream),
PatternString = read_pattern_string(stream, initial_position),
AllowNothingIsNecessaryF = AllowNothingIsNecessaryF)
return pattern
def PPT_indentation_handler_newline(MHI, data, ISetup, CounterDb):
"""Generate a PPT for newline, that is:
-- its PatternPriority.
-- the Pattern object.
-- the Terminal object.
The terminal object contains a generator which generates the INDENTATION
COUNTER.
"""
sm = ISetup.sm_newline.get()
pattern = Pattern(sm, PatternString="<indentation newline>",
Sr = ISetup.sm_newline.sr)
code = CodeGenerated(indentation_counter.do, data,
"INDENTATION COUNTER: NEWLINE")
pattern.set_incidence_id(E_IncidenceIDs.INDENTATION_HANDLER)
return PPT(PatternPriority(MHI, 0), pattern, code)
def PPT_indentation_handler_newline(MHI, data, ISetup, CounterDb):
"""Generate a PPT for newline, that is:
-- its PatternPriority.
-- the Pattern object.
-- the Terminal object.
The terminal object contains a generator which generates the INDENTATION
COUNTER.
"""
sm = ISetup.sm_newline.get()
pattern = Pattern(sm, PatternString="<indentation newline>",
Sr = ISetup.sm_newline.sr)
code = CodeGenerated(indentation_counter.do, data,
"INDENTATION COUNTER: NEWLINE")
pattern.set_incidence_id(E_IncidenceIDs.INDENTATION_HANDLER)
return PPT(PatternPriority(MHI, 0), pattern, code)
def PPT_character_set_skipper(MHI, character_set, incidence_id, CounterDb, goto_terminal_str, Sr):
"""Generate a PPT for a character set skipper. That is,
-- A PatternPriority based on a given MHI and the specified incidence id.
-- A Pattern to be webbed into the lexical analyzer state machine.
-- A Terminal implementing the character set skipper.
"""
priority = PatternPriority(MHI, incidence_id)
pattern = Pattern.from_character_set(character_set)
pattern.set_pattern_string("<skip>")
pattern.set_source_reference(Sr)
code = CodeTerminal([ goto_terminal_str ], Sr)
return PPT(priority, pattern, code)
def PPT_character_set_skipper(MHI, character_set, incidence_id, CounterDb, goto_terminal_str, Sr):
"""Generate a PPT for a character set skipper. That is,
-- A PatternPriority based on a given MHI and the specified incidence id.
-- A Pattern to be webbed into the lexical analyzer state machine.
-- A Terminal implementing the character set skipper.
"""
priority = PatternPriority(MHI, incidence_id)
pattern = Pattern.from_character_set(character_set)
pattern.set_pattern_string("<skip>")
pattern.set_source_reference(Sr)
code = CodeTerminal([ goto_terminal_str ], Sr)
return PPT(priority, pattern, code)
def PPT_indentation_handler_suppressed_newline(MHI, SmSuppressedNewline):
"""Generate a PPT for suppressed newline, that is:
-- its PatternPriority.
-- the Pattern object.
-- the Terminal object.
The terminal simply jumpts to the re-entry of the lexical analyzer.
"""
assert SmSuppressedNewline is not None
pattern = Pattern(SmSuppressedNewline,
PatternString="<indentation suppressed newline>")
code = CodeTerminal([Lng.GOTO(DoorID.global_reentry())])
# terminal = terminal_factory.do(E_TerminalType.PLAIN, code)
# terminal.set_name("INDENTATION COUNTER: SUPPRESSED_NEWLINE")
return PPT(PatternPriority(MHI, 1), pattern, code)
def get_Pattern(ValueList):
return Pattern.from_character_set(
NumberSet([Interval(ord(x)) for x in ValueList]))
def get_pattern_object(SM):
if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
else: result = SM
result = hopcroft.do(result, CreateNewStateMachineF=False)
return Pattern(result, AllowStateMachineTrafoF=True)
def get_Pattern(ValueList):
return Pattern.from_character_set(NumberSet([ Interval(ord(x)) for x in ValueList ]))