def snap_term(stream, PatternDict):
"""term: primary
primary term
"""
__debug_entry("term", stream)
# -- primary
result = snap_primary(stream, PatternDict)
__debug_print("##primary(in term):", result)
if result == None: return __debug_exit(None, stream)
position_1 = stream.tell()
# -- optional 'term'
result_2 = snap_term(stream, PatternDict)
__debug_print("##term(in term):", result_2)
if result_2 == None:
stream.seek(position_1)
return __debug_exit(result, stream)
## print "##1:", result.get_string(NormalizeF=False)
## print "##2:", result_2.get_string(NormalizeF=False)
result = sequentialize.do([result, result_2],
MountToFirstStateMachineF=True,
CloneRemainingStateMachinesF=False)
return __debug_exit(construct.beautify(result), stream)
def snap_term(stream, PatternDict):
"""term: primary
primary term
"""
__debug_entry("term", stream)
# -- primary
result = snap_primary(stream, PatternDict)
__debug_print("##primary(in term):", result)
if result == None: return __debug_exit(None, stream)
position_1 = stream.tell()
# -- optional 'term'
result_2 = snap_term(stream, PatternDict)
__debug_print("##term(in term):", result_2)
if result_2 == None:
stream.seek(position_1)
return __debug_exit(result, stream)
result = sequentialize.do([result, result_2],
MountToFirstStateMachineF=True,
CloneRemainingStateMachinesF=False)
return __debug_exit(__beautify(result), stream)
def mount(the_state_machine, PostConditionSM):
"""This function mounts a post condition to a state machine with
a mechanism that is able to handle the pseudo ambigous post-
condition. Note, that this mechanism can also treat 'normal'
post-conditions. However, it is slightly less efficient.
core- post-
-----0000000000000111111111--------------
(1) |-------------------->
acceptance
(2) <-------|
reset input position
The first step is performed by 'normal' lexing. The second step
via the backward detector, which is basically an inverse state
machine of the post-condition.
NOTE: This function does **not** return a state machine that is
necessarily deterministic. Run nfa_to_dfa on the result
of this function.
NOTE: This function is very similar to the function that mounts
a pre-condition to a state machine. The only major difference
is that the post condition is actually webbed into the
state machine for forward lexing. For backward lexing
a reference is stored that points to the backward detecting
state machine.
"""
assert the_state_machine.__class__.__name__ == "StateMachine"
assert PostConditionSM.__class__.__name__ == "StateMachine"
# -- state machines with no states are senseless here.
assert not the_state_machine.is_empty()
assert not PostConditionSM.is_empty()
# -- trivial pre-conditions should be added last, for simplicity
# (*) concatinate the two state machines:
# -- deletes acceptance states of the core pattern
# -- leaves acceptance states of the post condition
sequentialize.do([the_state_machine, PostConditionSM],
MountToFirstStateMachineF=True)
# (*) get the state machine that can go backwards from the acceptance
# state of the post condition to the start of the post-condition.
# The start of the post condition is at the same time the end
# of the core pattern.
backward_detector_sm = __get_inverse_state_machine_that_finds_end_of_core_expression(
PostConditionSM)
backward_detector_sm_id = backward_detector_sm.get_id()
# NOTE: We do not need to mark any origins in the backward detector,
# since it is not concerned with acceptance states. Its only
# task is to reset the input stream.
# NOTE: It is not necessary that the state machine directly refers to
# the backward detector. The origins of the acceptance state will do so.
acceptance_state_list = the_state_machine.get_acceptance_state_list()
assert len(acceptance_state_list) != 0, \
"error: mounting pseudo-ambiguous post condition:\n" + \
"error: no acceptance state in sequentialized state machine."
# (*) Create origin data, in case where there is none yet create new one.
# (Do not delete, otherwise existing information gets lost.)
for state in acceptance_state_list:
state.core().set_post_context_backward_detector_sm_id(
backward_detector_sm_id)
# At the end of the post condition, the input positions needs to be stored. Before
# we can go backwards, we need to know where the post condition actually ended.
state.core().set_store_input_position_f(True)
the_state_machine.core(
).set_post_context_backward_input_position_detector_sm(
backward_detector_sm)
# We cannot do a NFA to DFA and Hopcroft Optimization, because otherwise we
# would create a new state machine. This function, though, is considered to
# 'mount' something on an existing state machine, i.e. change the object
# that is referenced by the first function argument 'the_state_machine'.
return the_state_machine
def mount(the_state_machine, PostConditionSM):
"""This function mounts a post condition to a state machine with
a mechanism that is able to handle the pseudo ambigous post-
condition. Note, that this mechanism can also treat 'normal'
post-conditions. However, it is slightly less efficient.
core- post-
-----0000000000000111111111--------------
(1) |-------------------->
acceptance
(2) <-------|
reset input position
The first step is performed by 'normal' lexing. The second step
via the backward detector, which is basically an inverse state
machine of the post-condition.
NOTE: This function does **not** return a state machine that is
necessarily deterministic. Run nfa_to_dfa on the result
of this function.
NOTE: This function is very similar to the function that mounts
a pre-condition to a state machine. The only major difference
is that the post condition is actually webbed into the
state machine for forward lexing. For backward lexing
a reference is stored that points to the backward detecting
state machine.
"""
assert the_state_machine.__class__.__name__ == "StateMachine"
assert PostConditionSM.__class__.__name__ == "StateMachine"
# -- state machines with no states are senseless here.
assert not the_state_machine.is_empty()
assert not PostConditionSM.is_empty()
# -- trivial pre-conditions should be added last, for simplicity
# (*) concatinate the two state machines:
# -- deletes acceptance states of the core pattern
# -- leaves acceptance states of the post condition
sequentialize.do([the_state_machine, PostConditionSM], MountToFirstStateMachineF=True)
# (*) get the state machine that can go backwards from the acceptance
# state of the post condition to the start of the post-condition.
# The start of the post condition is at the same time the end
# of the core pattern.
backward_detector_sm = __get_inverse_state_machine_that_finds_end_of_core_expression(PostConditionSM)
backward_detector_sm_id = backward_detector_sm.get_id()
# NOTE: We do not need to mark any origins in the backward detector,
# since it is not concerned with acceptance states. Its only
# task is to reset the input stream.
# NOTE: It is not necessary that the state machine directly refers to
# the backward detector. The origins of the acceptance state will do so.
acceptance_state_list = the_state_machine.get_acceptance_state_list()
assert len(acceptance_state_list) != 0, \
"error: mounting pseudo-ambiguous post condition:\n" + \
"error: no acceptance state in sequentialized state machine."
# (*) Create origin data, in case where there is none yet create new one.
# (Do not delete, otherwise existing information gets lost.)
for state in acceptance_state_list:
state.core().set_post_context_backward_detector_sm_id(backward_detector_sm_id)
# At the end of the post condition, the input positions needs to be stored. Before
# we can go backwards, we need to know where the post condition actually ended.
state.core().set_store_input_position_f(True)
the_state_machine.core().set_post_context_backward_input_position_detector_sm(backward_detector_sm)
# We cannot do a NFA to DFA and Hopcroft Optimization, because otherwise we
# would create a new state machine. This function, though, is considered to
# 'mount' something on an existing state machine, i.e. change the object
# that is referenced by the first argument.
return the_state_machine
def parse_mode_option(fh, new_mode):
LanguageDB = Setup.language_db
def fit_state_machine(SM):
if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
else: result = SM
result = hopcroft.do(result, CreateNewStateMachineF=False)
return result
identifier = read_option_start(fh)
if identifier == None: return False
verify_word_in_list(identifier, lexer_mode.mode_option_info_db.keys(),
"mode option", fh.name,
get_current_line_info_number(fh))
if identifier == "skip":
# A skipper 'eats' characters at the beginning of a pattern that belong
# to a specified set of characters. A useful application is most probably
# the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
# implement a very effective way to skip these regions.
pattern_str, trigger_set = regular_expression.parse_character_set(
fh, PatternStringF=True)
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'." % identifier,
fh)
if trigger_set.is_empty():
error_msg("Empty trigger set for skipper." % identifier, fh)
# TriggerSet skipping is implemented the following way: As soon as one element of the
# trigger set appears, the state machine enters the 'trigger set skipper section'.
# Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
# NOTE: The correspondent CodeFragment for skipping is created in 'implement_skippers(...)'
pattern_sm = StateMachine()
pattern_sm.add_transition(pattern_sm.init_state_index,
trigger_set,
AcceptanceF=True)
# Skipper code is to be generated later
action = GeneratedCode(skip_character_set.do,
FileName=fh.name,
LineN=get_current_line_info_number(fh))
action.data["character_set"] = trigger_set
pattern_sm = fit_state_machine(pattern_sm)
# For skippers line and column counting detection is not really a topic
# It is done in the skipper itself.
pattern_sm.side_info = SideInfo()
new_mode.add_match(pattern_str, action, pattern_sm)
return True
elif identifier in ["skip_range", "skip_nested_range"]:
# A non-nesting skipper can contain a full fledged regular expression as opener,
# since it only effects the trigger. Not so the nested range skipper-see below.
# -- opener
skip_whitespace(fh)
if identifier == "skip_nested_range":
# Nested range state machines only accept 'strings' not state machines
opener_str, opener_sequence = parse_string_constant(
fh, "Opener pattern for 'skip_nested_range'")
opener_sm = StateMachine()
idx = opener_sm.init_state_index
for letter in opener_sequence:
idx = opener_sm.add_transition(idx, letter)
opener_sm.states[idx].set_acceptance(True)
else:
opener_str, opener_sm = regular_expression.parse(fh)
# For 'range skipping' the opener sequence is not needed, only the opener state
# machine is webbed into the pattern matching state machine.
opener_sequence = None
skip_whitespace(fh)
# -- closer
closer_str, closer_sequence = parse_string_constant(
fh, "Closing pattern for 'skip_range' or 'skip_nested_range'")
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'" % identifier,
fh)
# Skipper code is to be generated later
generator_function = {
"skip_range": skip_range.do,
"skip_nested_range": skip_nested_range.do,
}[identifier]
action = GeneratedCode(generator_function,
FileName=fh.name,
LineN=get_current_line_info_number(fh))
action.data["opener_sequence"] = opener_sequence
action.data["closer_sequence"] = closer_sequence
action.data["mode_name"] = new_mode.name
fit_state_machine(opener_sm)
# For skippers line and column counting detection is not really a topic
# It is done in the skipper itself.
opener_sm.side_info = SideInfo()
new_mode.add_match(opener_str, action, opener_sm)
return True
elif identifier == "indentation":
value = indentation_setup.do(fh)
# Enter 'Newline' and 'Suppressed Newline' as matches into the engine.
# Similar to skippers, the indentation count is then triggered by the newline.
# -- Suppressed Newline = Suppressor followed by Newline,
# then newline does not trigger indentation counting.
suppressed_newline_pattern = ""
if value.newline_suppressor_state_machine.get() != None:
suppressed_newline_pattern = \
"(" + value.newline_suppressor_state_machine.pattern_str + ")" \
+ "(" + value.newline_state_machine.pattern_str + ")"
suppressed_newline_sm = \
sequentialize.do([value.newline_suppressor_state_machine.get(),
value.newline_state_machine.get()])
FileName = value.newline_suppressor_state_machine.file_name
LineN = value.newline_suppressor_state_machine.line_n
# Go back to start.
code_fragment = UserCodeFragment(
"goto %s;" % get_label("$start", U=True), FileName, LineN)
suppressed_newline_sm = fit_state_machine(suppressed_newline_sm)
# Analyze pattern for constant number of newlines, characters, etc.
suppressed_newline_sm.side_info = SideInfo(
character_counter.get_newline_n(suppressed_newline_sm),
character_counter.get_character_n(suppressed_newline_sm))
new_mode.add_match(suppressed_newline_pattern,
code_fragment,
suppressed_newline_sm,
Comment="indentation newline suppressor")
# When there is an empty line, then there shall be no indentation count on it.
# Here comes the trick:
#
# Let newline
# be defined as: newline ([space]* newline])*
#
# This way empty lines are eating away before the indentation count is activated.
# -- 'space'
x0 = StateMachine()
x0.add_transition(x0.init_state_index,
value.indentation_count_character_set(),
AcceptanceF=True)
# -- '[space]*'
x1 = repeat.do(x0)
# -- '[space]* newline'
x2 = sequentialize.do([x1, value.newline_state_machine.get()])
# -- '([space]* newline)*'
x3 = repeat.do(x2)
# -- 'newline ([space]* newline)*'
x4 = sequentialize.do([value.newline_state_machine.get(), x3])
# -- nfa to dfa; hopcroft optimization
sm = hopcroft.do(nfa_to_dfa.do(x4), CreateNewStateMachineF=False)
FileName = value.newline_state_machine.file_name
LineN = value.newline_state_machine.line_n
action = GeneratedCode(indentation_counter.do, FileName, LineN)
action.data["indentation_setup"] = value
sm = fit_state_machine(sm)
sm.side_info = SideInfo(character_counter.get_newline_n(sm),
character_counter.get_character_n(sm))
new_mode.add_match(value.newline_state_machine.pattern_str,
action,
sm,
Comment="indentation newline")
# Announce the mode to which the setup belongs
value.set_containing_mode_name(new_mode.name)
else:
value = read_option_value(fh)
# The 'verify_word_in_list()' call must have ensured that the following holds
assert lexer_mode.mode_option_info_db.has_key(identifier)
# Is the option of the appropriate value?
option_info = lexer_mode.mode_option_info_db[identifier]
if option_info.domain != None and value not in option_info.domain:
error_msg("Tried to set value '%s' for option '%s'. " % (Value, Option) + \
"Though, possible for this option are only: %s." % repr(oi.domain)[1:-1], fh)
# Finally, set the option
new_mode.add_option(identifier, value)
return True
def parse_mode_option(fh, new_mode):
LanguageDB = Setup.language_db
def fit_state_machine(SM):
if not SM.is_DFA_compliant(): result = nfa_to_dfa.do(SM)
else: result = SM
result = hopcroft.do(result, CreateNewStateMachineF=False)
return result
identifier = read_option_start(fh)
if identifier == None: return False
verify_word_in_list(identifier, lexer_mode.mode_option_info_db.keys(),
"mode option", fh.name, get_current_line_info_number(fh))
if identifier == "skip":
# A skipper 'eats' characters at the beginning of a pattern that belong
# to a specified set of characters. A useful application is most probably
# the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
# implement a very effective way to skip these regions.
pattern_str, trigger_set = regular_expression.parse_character_set(fh, PatternStringF=True)
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'." % identifier, fh)
if trigger_set.is_empty():
error_msg("Empty trigger set for skipper." % identifier, fh)
# TriggerSet skipping is implemented the following way: As soon as one element of the
# trigger set appears, the state machine enters the 'trigger set skipper section'.
# Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
# NOTE: The correspondent CodeFragment for skipping is created in 'implement_skippers(...)'
pattern_sm = StateMachine()
pattern_sm.add_transition(pattern_sm.init_state_index, trigger_set, AcceptanceF=True)
# Skipper code is to be generated later
action = GeneratedCode(skip_character_set.do,
FileName = fh.name,
LineN = get_current_line_info_number(fh))
action.data["character_set"] = trigger_set
pattern_sm = fit_state_machine(pattern_sm)
# For skippers line and column counting detection is not really a topic
# It is done in the skipper itself.
pattern_sm.side_info = SideInfo()
new_mode.add_match(pattern_str, action, pattern_sm)
return True
elif identifier in ["skip_range", "skip_nested_range"]:
# A non-nesting skipper can contain a full fledged regular expression as opener,
# since it only effects the trigger. Not so the nested range skipper-see below.
# -- opener
skip_whitespace(fh)
if identifier == "skip_nested_range":
# Nested range state machines only accept 'strings' not state machines
opener_str, opener_sequence = parse_string_constant(fh, "Opener pattern for 'skip_nested_range'")
opener_sm = StateMachine()
idx = opener_sm.init_state_index
for letter in opener_sequence:
idx = opener_sm.add_transition(idx, letter)
opener_sm.states[idx].set_acceptance(True)
else:
opener_str, opener_sm = regular_expression.parse(fh)
# For 'range skipping' the opener sequence is not needed, only the opener state
# machine is webbed into the pattern matching state machine.
opener_sequence = None
skip_whitespace(fh)
# -- closer
closer_str, closer_sequence = parse_string_constant(fh, "Closing pattern for 'skip_range' or 'skip_nested_range'")
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'" % identifier, fh)
# Skipper code is to be generated later
generator_function = {
"skip_range": skip_range.do,
"skip_nested_range": skip_nested_range.do,
}[identifier]
action = GeneratedCode(generator_function,
FileName = fh.name,
LineN = get_current_line_info_number(fh))
action.data["opener_sequence"] = opener_sequence
action.data["closer_sequence"] = closer_sequence
action.data["mode_name"] = new_mode.name
fit_state_machine(opener_sm)
# For skippers line and column counting detection is not really a topic
# It is done in the skipper itself.
opener_sm.side_info = SideInfo()
new_mode.add_match(opener_str, action, opener_sm)
return True
elif identifier == "indentation":
value = indentation_setup.do(fh)
# Enter 'Newline' and 'Suppressed Newline' as matches into the engine.
# Similar to skippers, the indentation count is then triggered by the newline.
# -- Suppressed Newline = Suppressor followed by Newline,
# then newline does not trigger indentation counting.
suppressed_newline_pattern = ""
if value.newline_suppressor_state_machine.get() != None:
suppressed_newline_pattern = \
"(" + value.newline_suppressor_state_machine.pattern_str + ")" \
+ "(" + value.newline_state_machine.pattern_str + ")"
suppressed_newline_sm = \
sequentialize.do([value.newline_suppressor_state_machine.get(),
value.newline_state_machine.get()])
FileName = value.newline_suppressor_state_machine.file_name
LineN = value.newline_suppressor_state_machine.line_n
# Go back to start.
code_fragment = UserCodeFragment("goto %s;" % get_label("$start", U=True), FileName, LineN)
suppressed_newline_sm = fit_state_machine(suppressed_newline_sm)
# Analyze pattern for constant number of newlines, characters, etc.
suppressed_newline_sm.side_info = SideInfo(
character_counter.get_newline_n(suppressed_newline_sm),
character_counter.get_character_n(suppressed_newline_sm))
new_mode.add_match(suppressed_newline_pattern, code_fragment, suppressed_newline_sm,
Comment="indentation newline suppressor")
# When there is an empty line, then there shall be no indentation count on it.
# Here comes the trick:
#
# Let newline
# be defined as: newline ([space]* newline])*
#
# This way empty lines are eating away before the indentation count is activated.
# -- 'space'
x0 = StateMachine()
x0.add_transition(x0.init_state_index, value.indentation_count_character_set(),
AcceptanceF=True)
# -- '[space]*'
x1 = repeat.do(x0)
# -- '[space]* newline'
x2 = sequentialize.do([x1, value.newline_state_machine.get()])
# -- '([space]* newline)*'
x3 = repeat.do(x2)
# -- 'newline ([space]* newline)*'
x4 = sequentialize.do([value.newline_state_machine.get(), x3])
# -- nfa to dfa; hopcroft optimization
sm = hopcroft.do(nfa_to_dfa.do(x4), CreateNewStateMachineF=False)
FileName = value.newline_state_machine.file_name
LineN = value.newline_state_machine.line_n
action = GeneratedCode(indentation_counter.do, FileName, LineN)
action.data["indentation_setup"] = value
sm = fit_state_machine(sm)
sm.side_info = SideInfo(character_counter.get_newline_n(sm),
character_counter.get_character_n(sm))
new_mode.add_match(value.newline_state_machine.pattern_str,
action, sm, Comment="indentation newline")
# Announce the mode to which the setup belongs
value.set_containing_mode_name(new_mode.name)
else:
value = read_option_value(fh)
# The 'verify_word_in_list()' call must have ensured that the following holds
assert lexer_mode.mode_option_info_db.has_key(identifier)
# Is the option of the appropriate value?
option_info = lexer_mode.mode_option_info_db[identifier]
if option_info.domain != None and value not in option_info.domain:
error_msg("Tried to set value '%s' for option '%s'. " % (Value, Option) + \
"Though, possible for this option are only: %s." % repr(oi.domain)[1:-1], fh)
# Finally, set the option
new_mode.add_option(identifier, value)
return True
def snap_non_control_characters(stream):
"""Snaps any 'non_control_character' using UTF8 encoding from the given string. Note, that
in UTF8 a character may consist of more than one byte. Creates a state machine
that contains solely one trigger for each character to a acceptance state.
This function **concatinates** incoming characters, but **repetition** has preceedence
over concatination, so it checks after each character whether it is followed by
a repetition ('*', '+', '?', '{..}'). In such a case, the repetition of the character
is appended.
"""
__debug_entry("non-control characters", stream)
result = StateMachine()
state_index = result.init_state_index
# (*) read first character
position = stream.tell()
char_code = utf8.__read_one_utf8_code_from_stream(stream)
while char_code != 0xFF:
# (1) check against occurence of control characters
# this needs to come **before** the backslashed character interpretation.
# NOTE: A backslashed character can be a whitespace (for example '\n').
# (check against 0xFF to avoid overflow in function 'chr()')
if char_code < 0xFF \
and (chr(char_code) in CONTROL_CHARACTERS or chr(char_code).isspace()):
stream.seek(-1, 1)
break
# (2) treat backslashed characters
if char_code == ord('\\'):
stream.seek(-1, 1)
trigger_set = character_set_expression.snap_property_set(stream)
if trigger_set == None:
stream.seek(1, 1) # snap_property_set() leaves tream right before '\\'
char_code = snap_backslashed_character.do(stream)
if char_code == None:
raise RegularExpressionException("Backslash followed by unrecognized character code.")
trigger_set = char_code
else:
trigger_set = char_code
# (3) read next character
position = stream.tell()
next_char_code = utf8.__read_one_utf8_code_from_stream(stream)
# -- check for repetition (repetition has preceedence over concatination)
if next_char_code in [ord("+"), ord("*"), ord("?"), ord("{")]:
# (*) create state machine that consist of a single transition
tmp = StateMachine()
tmp.add_transition(tmp.init_state_index, trigger_set, AcceptanceF=True)
# -- repeat the single character state machine
stream.seek(position)
tmp_repeated = __snap_repetition_range(tmp, stream)
# -- append it to the result (last state must be set to acceptance for concatenation)
result.states[state_index].set_acceptance()
result = sequentialize.do([result, tmp_repeated], MountToFirstStateMachineF=True)
# as soon as there is repetition there might be more than one acceptance
# state and thus simple concatination via 'add_transition' fails.
# let us return and check treat the remaining chars
# at the next call to this function.
return __debug_exit(result, stream)
else:
# (*) add new transition from current state to a new state triggering
# on the given character.
state_index = result.add_transition(state_index, trigger_set)
char_code = next_char_code
# last character in the chain triggers an 'acceptance state'
result.states[state_index].set_acceptance()
return __debug_exit(result, stream)
def snap_non_control_characters(stream):
"""Snaps any 'non_control_character' using UTF8 encoding from the given string. Note, that
in UTF8 a character may consist of more than one byte. Creates a state machine
that contains solely one trigger for each character to a acceptance state.
This function **concatinates** incoming characters, but **repetition** has preceedence
over concatination, so it checks after each character whether it is followed by
a repetition ('*', '+', '?', '{..}'). In such a case, the repetition of the character
is appended.
"""
__debug_entry("non-control characters", stream)
result = StateMachine()
state_index = result.init_state_index
# (*) read first character
position = stream.tell()
char_code = utf8.__read_one_utf8_code_from_stream(stream)
while char_code != 0xFF:
# (1) check against occurence of control characters
# this needs to come **before** the backslashed character interpretation.
# NOTE: A backslashed character can be a whitespace (for example '\n').
# (check against 0xFF to avoid overflow in function 'chr()')
if char_code < 0xFF \
and (chr(char_code) in CONTROL_CHARACTERS or chr(char_code).isspace()):
stream.seek(-1, 1)
break
# (2) treat backslashed characters
if char_code == ord('\\'):
stream.seek(-1, 1)
trigger_set = character_set_expression.snap_property_set(stream)
if trigger_set == None:
stream.seek(
1, 1) # snap_property_set() leaves tream right before '\\'
char_code = snap_backslashed_character.do(stream)
if char_code == None:
raise RegularExpressionException(
"Backslash followed by unrecognized character code.")
trigger_set = char_code
else:
trigger_set = char_code
# (3) read next character
position = stream.tell()
next_char_code = utf8.__read_one_utf8_code_from_stream(stream)
# -- check for repetition (repetition has preceedence over concatination)
if next_char_code in [ord("+"), ord("*"), ord("?"), ord("{")]:
# (*) create state machine that consist of a single transition
tmp = StateMachine()
tmp.add_transition(tmp.init_state_index,
trigger_set,
AcceptanceF=True)
# -- repeat the single character state machine
stream.seek(position)
tmp_repeated = __snap_repetition_range(tmp, stream)
# -- append it to the result (last state must be set to acceptance for concatenation)
result.states[state_index].set_acceptance()
result = sequentialize.do([result, tmp_repeated],
MountToFirstStateMachineF=True)
# as soon as there is repetition there might be more than one acceptance
# state and thus simple concatination via 'add_transition' fails.
# let us return and check treat the remaining chars
# at the next call to this function.
return __debug_exit(result, stream)
else:
# (*) add new transition from current state to a new state triggering
# on the given character.
state_index = result.add_transition(state_index, trigger_set)
char_code = next_char_code
# last character in the chain triggers an 'acceptance state'
result.states[state_index].set_acceptance()
return __debug_exit(result, stream)