def snap_expression(stream, PatternDict):
"""expression: term
term | expression
"""
__debug_entry("expression", stream)
# -- term
result = snap_term(stream, PatternDict)
if result is None:
return __debug_exit(None, stream)
# -- optional '|'
if not check(stream, '|'):
return __debug_exit(result, stream)
position_1 = stream.tell()
__debug_print("'|' (in expression)")
# -- expression
result_2 = snap_expression(stream, PatternDict)
__debug_print("expression(in expression):", result_2)
if result_2 is None:
stream.seek(position_1)
return __debug_exit(result, stream)
result = parallelize.do([result, result_2], CloneF=True) # CloneF = False (shold be!)
return __debug_exit(beautifier.do(result), stream)
def snap_term(stream, PatternDict):
"""term: primary
primary term
"""
__debug_entry("term", stream)
# -- primary
result = snap_primary(stream, PatternDict)
__debug_print("##primary(in term):", result)
if result is None: return __debug_exit(None, stream)
position_1 = stream.tell()
# -- optional 'term'
result_2 = snap_term(stream, PatternDict)
__debug_print("##term(in term):", result_2)
if result_2 is None:
stream.seek(position_1)
return __debug_exit(result, stream)
## print "##1:", result.get_string(NormalizeF=False)
## print "##2:", result_2.get_string(NormalizeF=False)
result = sequentialize.do([result, result_2],
MountToFirstStateMachineF=True,
CloneRemainingStateMachinesF=False)
return __debug_exit(beautifier.do(result), stream)
def snap_expression(stream, PatternDict):
"""expression: term
term | expression
"""
__debug_entry("expression", stream)
# -- term
result = snap_term(stream, PatternDict)
if result is None:
return __debug_exit(None, stream)
# -- optional '|'
if not check(stream, '|'):
return __debug_exit(result, stream)
position_1 = stream.tell()
__debug_print("'|' (in expression)")
# -- expression
result_2 = snap_expression(stream, PatternDict)
__debug_print("expression(in expression):", result_2)
if result_2 is None:
stream.seek(position_1)
return __debug_exit(result, stream)
result = parallelize.do([result, result_2],
CloneF=True) # CloneF = False (shold be!)
return __debug_exit(beautifier.do(result), stream)
def snap_set_list(stream, set_operation_name, PatternDict):
__debug_entry("set_list", stream)
skip_whitespace(stream)
if stream.read(1) != "(":
error.log(
"Missing opening bracket '%s' operation." % set_operation_name,
stream)
set_list = []
while 1 + 1 == 2:
skip_whitespace(stream)
result = snap_set_term(stream, PatternDict)
if result is None:
error.log(
"Missing set expression list after %s operation." %
set_operation_name, stream)
set_list.append(result)
skip_whitespace(stream)
tmp = stream.read(1)
if tmp != ",":
if tmp != ")":
stream.seek(-1, 1)
error.log(
"Missing closing ')' or argument seperator ',' in %s operation."
% set_operation_name, stream)
return __debug_exit(set_list, stream)
def snap_set_expression(stream, PatternDict):
assert stream.__class__.__name__ == "StringIO" \
or stream.__class__.__name__ == "file"
__debug_entry("set_expression", stream)
result = snap_property_set(stream)
if result is not None: return result
x = stream.read(2)
if x == "\\C":
return snap_case_folded_pattern(stream, PatternDict, NumberSetF=True)
elif x == "[:":
result = snap_set_term(stream, PatternDict)
skip_whitespace(stream)
x = stream.read(2)
if x != ":]":
raise RegularExpressionException("Missing closing ':]' for character set expression.\n" + \
"found: '%s'" % x)
elif x[0] == "[":
stream.seek(-1, 1)
result = traditional_character_set.do(stream)
elif x[0] == "{":
stream.seek(-1, 1)
result = snap_replacement(stream, PatternDict, StateMachineF=False)
else:
result = None
return __debug_exit(result, stream)
def snap_term(stream, PatternDict):
"""term: primary
primary term
"""
__debug_entry("term", stream)
# -- primary
result = snap_primary(stream, PatternDict)
__debug_print("##primary(in term):", result)
if result is None: return __debug_exit(None, stream)
position_1 = stream.tell()
# -- optional 'term'
result_2 = snap_term(stream, PatternDict)
__debug_print("##term(in term):", result_2)
if result_2 is None:
stream.seek(position_1)
return __debug_exit(result, stream)
## print "##1:", result.get_string(NormalizeF=False)
## print "##2:", result_2.get_string(NormalizeF=False)
result = sequentialize.do([result, result_2],
MountToFirstStateMachineF=True,
CloneRemainingStateMachinesF=False)
return __debug_exit(beautifier.do(result), stream)
def snap_set_list(stream, set_operation_name, PatternDict):
__debug_entry("set_list", stream)
skip_whitespace(stream)
if stream.read(1) != "(":
raise RegularExpressionException(
"Missing opening bracket '%s' operation." % set_operation_name)
set_list = []
while 1 + 1 == 2:
skip_whitespace(stream)
result = snap_set_term(stream, PatternDict)
if result is None:
raise RegularExpressionException(
"Missing set expression list after '%s' operation." %
set_operation_name)
set_list.append(result)
skip_whitespace(stream)
tmp = stream.read(1)
if tmp != ",":
if tmp != ")":
stream.seek(-1, 1)
raise RegularExpressionException(
"Missing closing ')' after after '%s' operation." %
set_operation_name)
return __debug_exit(set_list, stream)
def snap_set_term(stream, PatternDict):
global special_character_set_db
__debug_entry("set_term", stream)
operation_list = ["union", "intersection", "difference", "inverse"]
character_set_list = special_character_set_db.keys()
skip_whitespace(stream)
position = stream.tell()
# if there is no following '(', then enter the 'snap_expression' block below
word = read_identifier(stream)
if word in operation_list:
set_list = snap_set_list(stream, word, PatternDict)
# if an error occurs during set_list parsing, an exception is thrown about syntax error
L = len(set_list)
result = set_list[0]
if word == "inverse":
# The inverse of multiple sets, is to be the inverse of the union of these sets.
if L > 1:
for character_set in set_list[1:]:
result.unite_with(character_set)
return __debug_exit(
result.get_complement(Setup.buffer_codec.source_set), stream)
if L < 2:
raise RegularExpressionException("Regular Expression: A %s operation needs at least\n" % word + \
"two sets to operate on them.")
if word == "union":
for set in set_list[1:]:
result.unite_with(set)
elif word == "intersection":
for set in set_list[1:]:
result.intersect_with(set)
elif word == "difference":
for set in set_list[1:]:
result.subtract(set)
elif word in character_set_list:
reg_expr = special_character_set_db[word]
result = traditional_character_set.do_string(reg_expr)
elif word != "":
error.verify_word_in_list(word, character_set_list + operation_list,
"Unknown keyword '%s'." % word, stream)
else:
stream.seek(position)
result = snap_set_expression(stream, PatternDict)
return __debug_exit(result, stream)
def snap_set_term(stream, PatternDict):
global special_character_set_db
__debug_entry("set_term", stream)
operation_list = [ "union", "intersection", "difference", "inverse"]
character_set_list = special_character_set_db.keys()
skip_whitespace(stream)
position = stream.tell()
# if there is no following '(', then enter the 'snap_expression' block below
word = read_identifier(stream)
if word in operation_list:
set_list = snap_set_list(stream, word, PatternDict)
# if an error occurs during set_list parsing, an exception is thrown about syntax error
L = len(set_list)
result = set_list[0]
if word == "inverse":
# The inverse of multiple sets, is to be the inverse of the union of these sets.
if L > 1:
for character_set in set_list[1:]:
result.unite_with(character_set)
return __debug_exit(result.get_complement(Setup.buffer_codec.source_set), stream)
if L < 2:
raise RegularExpressionException("Regular Expression: A %s operation needs at least\n" % word + \
"two sets to operate on them.")
if word == "union":
for set in set_list[1:]:
result.unite_with(set)
elif word == "intersection":
for set in set_list[1:]:
result.intersect_with(set)
elif word == "difference":
for set in set_list[1:]:
result.subtract(set)
elif word in character_set_list:
reg_expr = special_character_set_db[word]
result = traditional_character_set.do_string(reg_expr)
elif word != "":
verify_word_in_list(word, character_set_list + operation_list,
"Unknown keyword '%s'." % word, stream)
else:
stream.seek(position)
result = snap_set_expression(stream, PatternDict)
return __debug_exit(result, stream)
def snap_non_control_character(stream, PatternDict):
__debug_entry("non-control characters", stream)
# (*) read first character
char_code = utf8.__read_one_utf8_code_from_stream(stream)
if char_code is None:
error_msg("Character could not be interpreted as UTF8 code or End of File reached prematurely.",
stream)
result = StateMachine()
result.add_transition(result.init_state_index, char_code, AcceptanceF=True)
return __debug_exit(result, stream)
def snap_non_control_character(stream, PatternDict):
__debug_entry("non-control characters", stream)
# (*) read first character
char_code = utf8.__read_one_utf8_code_from_stream(stream)
if char_code is None:
error_msg("Character could not be interpreted as UTF8 code or End of File reached prematurely.",
stream)
result = StateMachine()
result.add_transition(result.init_state_index, char_code, AcceptanceF=True)
return __debug_exit(result, stream)
def snap_conditional_expression(stream, PatternDict):
"""conditional expression: expression
expression / expression = post conditioned expression
expression / expression / = pre conditioned expression
expression / expression / expression = pre and post conditioned expression
TODO: <- ($8592) for pre-conditions
-> ($8594) for post-conditions
RETURNS: pre_context, core_pattern, post_context
"""
__debug_entry("conditional expression", stream)
# -- expression
pattern_0 = snap_expression(stream, PatternDict)
if pattern_0 is None:
return None, None, None
# -- '/'
if not check(stream, '/'):
# (1) expression without pre and post condition
# pattern_0 is already beautified by 'snap_expression()'
return None, pattern_0, None
# -- expression
pattern_1 = snap_expression(stream, PatternDict)
if pattern_1 is None:
return None, pattern_0, None
# -- '/'
if not check(stream, '/'):
# (2) expression with only a post condition
# NOTE: setup_post_context() marks state origins!
return None, pattern_0, pattern_1
# -- expression
pattern_2 = snap_expression(stream, PatternDict)
if pattern_2 is None:
# (3) expression with only a pre condition
# NOTE: setup_pre_context() marks the state origins!
return pattern_0, pattern_1, None
else:
# (4) expression with post and pre-condition
return pattern_0, pattern_1, pattern_2
def snap_conditional_expression(stream, PatternDict):
"""conditional expression: expression
expression / expression = post conditioned expression
expression / expression / = pre conditioned expression
expression / expression / expression = pre and post conditioned expression
TODO: <- ($8592) for pre-conditions
-> ($8594) for post-conditions
RETURNS: pre_context, core_pattern, post_context
"""
__debug_entry("conditional expression", stream)
# -- expression
pattern_0 = snap_expression(stream, PatternDict)
if pattern_0 is None:
return None, None, None
# -- '/'
if not check(stream, '/'):
# (1) expression without pre and post condition
# pattern_0 is already beautified by 'snap_expression()'
return None, pattern_0, None
# -- expression
pattern_1 = snap_expression(stream, PatternDict)
if pattern_1 is None:
return None, pattern_0, None
# -- '/'
if not check(stream, '/'):
# (2) expression with only a post condition
# NOTE: setup_post_context() marks state origins!
return None, pattern_0, pattern_1
# -- expression
pattern_2 = snap_expression(stream, PatternDict)
if pattern_2 is None:
# (3) expression with only a pre condition
# NOTE: setup_pre_context() marks the state origins!
return pattern_0, pattern_1, None
else:
# (4) expression with post and pre-condition
return pattern_0, pattern_1, pattern_2
def snap_set_list(stream, set_operation_name, PatternDict):
__debug_entry("set_list", stream)
skip_whitespace(stream)
if stream.read(1) != "(":
raise RegularExpressionException("Missing opening bracket '%s' operation." % set_operation_name)
set_list = []
while 1 + 1 == 2:
skip_whitespace(stream)
result = snap_set_term(stream, PatternDict)
if result is None:
raise RegularExpressionException("Missing set expression list after '%s' operation." % set_operation_name)
set_list.append(result)
skip_whitespace(stream)
tmp = stream.read(1)
if tmp != ",":
if tmp != ")":
stream.seek(-1, 1)
raise RegularExpressionException("Missing closing ')' after after '%s' operation." % set_operation_name)
return __debug_exit(set_list, stream)
def snap_primary(stream, PatternDict):
"""primary: " non_double_quote * " = character string
[ non_rect_bracket_close ] = set of characters
{ identifier } = pattern replacement
( expression )
non_control_character+ = lonely characters
primary repetition_cmd
"""
__debug_entry("primary", stream)
x = stream.read(1)
lookahead = stream.read(1)
if x != "" and lookahead != "": stream.seek(-1, 1)
if x == "": return __debug_exit(None, stream)
# -- 'primary' primary
if x == "\"": result = snap_character_string.do(stream)
elif x == "[":
stream.seek(-1, 1)
result = character_set_expression.do(stream, PatternDict)
elif x == "{":
result = snap_replacement(stream, PatternDict)
elif x == ".":
result = create_ALL_BUT_NEWLINE_state_machine()
elif x == "(":
result = snap_bracketed_expression(stream, PatternDict)
elif x.isspace():
# a lonestanding space ends the regular expression
stream.seek(-1, 1)
return __debug_exit(None, stream)
elif x in ["*", "+", "?"]:
raise RegularExpressionException(
"lonely operator '%s' without expression proceeding." % x)
elif x == "\\":
if lookahead == "C":
stream.read(1)
result = snap_case_folded_pattern(stream, PatternDict)
elif lookahead == "R":
result = get_expression_in_brackets(stream, PatternDict,
"reverse operator",
"R").get_inverse()
elif lookahead == "A":
result = get_expression_in_brackets(stream, PatternDict,
"anti-pattern operator", "A")
result.transform_to_anti_pattern()
else:
stream.seek(-1, 1)
trigger_set = character_set_expression.snap_property_set(stream)
if trigger_set is None:
stream.seek(
1, 1) # snap_property_set() leaves tream right before '\\'
char_code = snap_backslashed_character.do(stream)
if char_code is None:
raise RegularExpressionException(
"Backslash followed by unrecognized character code.")
trigger_set = char_code
result = StateMachine()
result.add_transition(result.init_state_index,
trigger_set,
AcceptanceF=True)
elif x not in CONTROL_CHARACTERS:
# NOTE: The '\' is not inside the control characters---for a reason.
# It is used to define for example character codes using '\x' etc.
stream.seek(-1, 1)
result = snap_non_control_character(stream, PatternDict)
else:
# NOTE: This includes the '$' sign which means 'end of line'
# because the '$' sign is in CONTROL_CHARACTERS, but is not checked
# against. Thus, it it good to leave here on '$' because the
# '$' sign is handled on the very top level.
# this is not a valid primary
stream.seek(-1, 1)
return __debug_exit(None, stream)
# -- optional repetition command?
result_repeated = __snap_repetition_range(result, stream)
if result_repeated is not None: result = result_repeated
return __debug_exit(beautifier.do(result), stream)
def snap_primary(stream, PatternDict):
"""primary: " non_double_quote * " = character string
[ non_rect_bracket_close ] = set of characters
{ identifier } = pattern replacement
( expression )
non_control_character+ = lonely characters
primary repetition_cmd
"""
global SPECIAL_TERMINATOR
__debug_entry("primary", stream)
x = stream.read(1)
if x == "": return __debug_exit(None, stream)
# -- 'primary' primary
if x == "\"": result = snap_character_string.do(stream)
elif x == "[":
stream.seek(-1, 1);
result = snap_character_set_expression(stream, PatternDict)
elif x == "{": result = snap_replacement(stream, PatternDict)
elif x == ".": result = create_ALL_BUT_NEWLINE_state_machine(stream)
elif x == "(": result = snap_bracketed_expression(stream, PatternDict)
elif x.isspace():
# a lonestanding space ends the regular expression
stream.seek(-1, 1)
return __debug_exit(None, stream)
elif x in ["*", "+", "?"]:
raise RegularExpressionException("lonely operator '%s' without expression proceeding." % x)
elif x == "\\":
result = snap_command(stream, PatternDict)
if result is None:
stream.seek(-1, 1)
trigger_set = snap_property_set(stream)
if trigger_set is None:
# snap the '\'
stream.read(1)
char_code = snap_backslashed_character.do(stream)
if char_code is None:
raise RegularExpressionException("Backslash followed by unrecognized character code.")
trigger_set = char_code
result = StateMachine()
result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True)
elif x not in CONTROL_CHARACTERS and x != SPECIAL_TERMINATOR:
# NOTE: The '\' is not inside the control characters---for a reason.
# It is used to define for example character codes using '\x' etc.
stream.seek(-1, 1)
result = snap_non_control_character(stream, PatternDict)
else:
# NOTE: This includes the '$' sign which means 'end of line'
# because the '$' sign is in CONTROL_CHARACTERS, but is not checked
# against. Thus, it it good to leave here on '$' because the
# '$' sign is handled on the very top level.
# this is not a valid primary
stream.seek(-1, 1)
return __debug_exit(None, stream)
# -- optional repetition command?
result_repeated = __snap_repetition_range(result, stream)
if result_repeated is not None: result = result_repeated
# There's something going wrong with pseudo-ambigous post context
# if we do not clean-up here. TODO: Investigate why?
# See tests in generator/TEST directory.
return __debug_exit(beautifier.do(result), stream)
def snap_primary(stream, PatternDict):
"""primary: " non_double_quote * " = character string
[ non_rect_bracket_close ] = set of characters
{ identifier } = pattern replacement
( expression )
non_control_character+ = lonely characters
primary repetition_cmd
"""
__debug_entry("primary", stream)
x = stream.read(1); lookahead = stream.read(1);
if x != "" and lookahead != "": stream.seek(-1, 1)
if x == "": return __debug_exit(None, stream)
# -- 'primary' primary
if x == "\"": result = snap_character_string.do(stream)
elif x == "[":
stream.seek(-1, 1);
result = character_set_expression.do(stream, PatternDict)
elif x == "{": result = snap_replacement(stream, PatternDict)
elif x == ".": result = create_ALL_BUT_NEWLINE_state_machine()
elif x == "(": result = snap_bracketed_expression(stream, PatternDict)
elif x.isspace():
# a lonestanding space ends the regular expression
stream.seek(-1, 1)
return __debug_exit(None, stream)
elif x in ["*", "+", "?"]:
raise RegularExpressionException("lonely operator '%s' without expression proceeding." % x)
elif x == "\\":
if lookahead == "C":
stream.read(1)
result = snap_case_folded_pattern(stream, PatternDict)
elif lookahead == "R":
result = get_expression_in_brackets(stream, PatternDict, "reverse operator", "R").get_inverse()
elif lookahead == "A":
result = get_expression_in_brackets(stream, PatternDict, "anti-pattern operator", "A")
result.transform_to_anti_pattern()
else:
stream.seek(-1, 1)
trigger_set = character_set_expression.snap_property_set(stream)
if trigger_set is None:
stream.seek(1, 1) # snap_property_set() leaves tream right before '\\'
char_code = snap_backslashed_character.do(stream)
if char_code is None:
raise RegularExpressionException("Backslash followed by unrecognized character code.")
trigger_set = char_code
result = StateMachine()
result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True)
elif x not in CONTROL_CHARACTERS:
# NOTE: The '\' is not inside the control characters---for a reason.
# It is used to define for example character codes using '\x' etc.
stream.seek(-1, 1)
result = snap_non_control_character(stream, PatternDict)
else:
# NOTE: This includes the '$' sign which means 'end of line'
# because the '$' sign is in CONTROL_CHARACTERS, but is not checked
# against. Thus, it it good to leave here on '$' because the
# '$' sign is handled on the very top level.
# this is not a valid primary
stream.seek(-1, 1)
return __debug_exit(None, stream)
# -- optional repetition command?
result_repeated = __snap_repetition_range(result, stream)
if result_repeated is not None: result = result_repeated
return __debug_exit(beautifier.do(result), stream)
