def snap_expression(stream, PatternDict): """expression: term term | expression """ __debug_entry("expression", stream) # -- term result = snap_term(stream, PatternDict) if result is None: return __debug_exit(None, stream) # -- optional '|' if not check(stream, '|'): return __debug_exit(result, stream) position_1 = stream.tell() __debug_print("'|' (in expression)") # -- expression result_2 = snap_expression(stream, PatternDict) __debug_print("expression(in expression):", result_2) if result_2 is None: stream.seek(position_1) return __debug_exit(result, stream) result = parallelize.do([result, result_2], CloneF=True) # CloneF = False (shold be!) return __debug_exit(beautifier.do(result), stream)
def snap_term(stream, PatternDict): """term: primary primary term """ __debug_entry("term", stream) # -- primary result = snap_primary(stream, PatternDict) __debug_print("##primary(in term):", result) if result is None: return __debug_exit(None, stream) position_1 = stream.tell() # -- optional 'term' result_2 = snap_term(stream, PatternDict) __debug_print("##term(in term):", result_2) if result_2 is None: stream.seek(position_1) return __debug_exit(result, stream) ## print "##1:", result.get_string(NormalizeF=False) ## print "##2:", result_2.get_string(NormalizeF=False) result = sequentialize.do([result, result_2], MountToFirstStateMachineF=True, CloneRemainingStateMachinesF=False) return __debug_exit(beautifier.do(result), stream)
def snap_set_list(stream, set_operation_name, PatternDict): __debug_entry("set_list", stream) skip_whitespace(stream) if stream.read(1) != "(": error.log( "Missing opening bracket '%s' operation." % set_operation_name, stream) set_list = [] while 1 + 1 == 2: skip_whitespace(stream) result = snap_set_term(stream, PatternDict) if result is None: error.log( "Missing set expression list after %s operation." % set_operation_name, stream) set_list.append(result) skip_whitespace(stream) tmp = stream.read(1) if tmp != ",": if tmp != ")": stream.seek(-1, 1) error.log( "Missing closing ')' or argument seperator ',' in %s operation." % set_operation_name, stream) return __debug_exit(set_list, stream)
def snap_set_expression(stream, PatternDict): assert stream.__class__.__name__ == "StringIO" \ or stream.__class__.__name__ == "file" __debug_entry("set_expression", stream) result = snap_property_set(stream) if result is not None: return result x = stream.read(2) if x == "\\C": return snap_case_folded_pattern(stream, PatternDict, NumberSetF=True) elif x == "[:": result = snap_set_term(stream, PatternDict) skip_whitespace(stream) x = stream.read(2) if x != ":]": raise RegularExpressionException("Missing closing ':]' for character set expression.\n" + \ "found: '%s'" % x) elif x[0] == "[": stream.seek(-1, 1) result = traditional_character_set.do(stream) elif x[0] == "{": stream.seek(-1, 1) result = snap_replacement(stream, PatternDict, StateMachineF=False) else: result = None return __debug_exit(result, stream)
def snap_set_list(stream, set_operation_name, PatternDict): __debug_entry("set_list", stream) skip_whitespace(stream) if stream.read(1) != "(": raise RegularExpressionException( "Missing opening bracket '%s' operation." % set_operation_name) set_list = [] while 1 + 1 == 2: skip_whitespace(stream) result = snap_set_term(stream, PatternDict) if result is None: raise RegularExpressionException( "Missing set expression list after '%s' operation." % set_operation_name) set_list.append(result) skip_whitespace(stream) tmp = stream.read(1) if tmp != ",": if tmp != ")": stream.seek(-1, 1) raise RegularExpressionException( "Missing closing ')' after after '%s' operation." % set_operation_name) return __debug_exit(set_list, stream)
def snap_set_term(stream, PatternDict): global special_character_set_db __debug_entry("set_term", stream) operation_list = ["union", "intersection", "difference", "inverse"] character_set_list = special_character_set_db.keys() skip_whitespace(stream) position = stream.tell() # if there is no following '(', then enter the 'snap_expression' block below word = read_identifier(stream) if word in operation_list: set_list = snap_set_list(stream, word, PatternDict) # if an error occurs during set_list parsing, an exception is thrown about syntax error L = len(set_list) result = set_list[0] if word == "inverse": # The inverse of multiple sets, is to be the inverse of the union of these sets. if L > 1: for character_set in set_list[1:]: result.unite_with(character_set) return __debug_exit( result.get_complement(Setup.buffer_codec.source_set), stream) if L < 2: raise RegularExpressionException("Regular Expression: A %s operation needs at least\n" % word + \ "two sets to operate on them.") if word == "union": for set in set_list[1:]: result.unite_with(set) elif word == "intersection": for set in set_list[1:]: result.intersect_with(set) elif word == "difference": for set in set_list[1:]: result.subtract(set) elif word in character_set_list: reg_expr = special_character_set_db[word] result = traditional_character_set.do_string(reg_expr) elif word != "": error.verify_word_in_list(word, character_set_list + operation_list, "Unknown keyword '%s'." % word, stream) else: stream.seek(position) result = snap_set_expression(stream, PatternDict) return __debug_exit(result, stream)
def snap_set_term(stream, PatternDict): global special_character_set_db __debug_entry("set_term", stream) operation_list = [ "union", "intersection", "difference", "inverse"] character_set_list = special_character_set_db.keys() skip_whitespace(stream) position = stream.tell() # if there is no following '(', then enter the 'snap_expression' block below word = read_identifier(stream) if word in operation_list: set_list = snap_set_list(stream, word, PatternDict) # if an error occurs during set_list parsing, an exception is thrown about syntax error L = len(set_list) result = set_list[0] if word == "inverse": # The inverse of multiple sets, is to be the inverse of the union of these sets. if L > 1: for character_set in set_list[1:]: result.unite_with(character_set) return __debug_exit(result.get_complement(Setup.buffer_codec.source_set), stream) if L < 2: raise RegularExpressionException("Regular Expression: A %s operation needs at least\n" % word + \ "two sets to operate on them.") if word == "union": for set in set_list[1:]: result.unite_with(set) elif word == "intersection": for set in set_list[1:]: result.intersect_with(set) elif word == "difference": for set in set_list[1:]: result.subtract(set) elif word in character_set_list: reg_expr = special_character_set_db[word] result = traditional_character_set.do_string(reg_expr) elif word != "": verify_word_in_list(word, character_set_list + operation_list, "Unknown keyword '%s'." % word, stream) else: stream.seek(position) result = snap_set_expression(stream, PatternDict) return __debug_exit(result, stream)
def snap_non_control_character(stream, PatternDict): __debug_entry("non-control characters", stream) # (*) read first character char_code = utf8.__read_one_utf8_code_from_stream(stream) if char_code is None: error_msg("Character could not be interpreted as UTF8 code or End of File reached prematurely.", stream) result = StateMachine() result.add_transition(result.init_state_index, char_code, AcceptanceF=True) return __debug_exit(result, stream)
def snap_conditional_expression(stream, PatternDict): """conditional expression: expression expression / expression = post conditioned expression expression / expression / = pre conditioned expression expression / expression / expression = pre and post conditioned expression TODO: <- ($8592) for pre-conditions -> ($8594) for post-conditions RETURNS: pre_context, core_pattern, post_context """ __debug_entry("conditional expression", stream) # -- expression pattern_0 = snap_expression(stream, PatternDict) if pattern_0 is None: return None, None, None # -- '/' if not check(stream, '/'): # (1) expression without pre and post condition # pattern_0 is already beautified by 'snap_expression()' return None, pattern_0, None # -- expression pattern_1 = snap_expression(stream, PatternDict) if pattern_1 is None: return None, pattern_0, None # -- '/' if not check(stream, '/'): # (2) expression with only a post condition # NOTE: setup_post_context() marks state origins! return None, pattern_0, pattern_1 # -- expression pattern_2 = snap_expression(stream, PatternDict) if pattern_2 is None: # (3) expression with only a pre condition # NOTE: setup_pre_context() marks the state origins! return pattern_0, pattern_1, None else: # (4) expression with post and pre-condition return pattern_0, pattern_1, pattern_2
def snap_set_list(stream, set_operation_name, PatternDict): __debug_entry("set_list", stream) skip_whitespace(stream) if stream.read(1) != "(": raise RegularExpressionException("Missing opening bracket '%s' operation." % set_operation_name) set_list = [] while 1 + 1 == 2: skip_whitespace(stream) result = snap_set_term(stream, PatternDict) if result is None: raise RegularExpressionException("Missing set expression list after '%s' operation." % set_operation_name) set_list.append(result) skip_whitespace(stream) tmp = stream.read(1) if tmp != ",": if tmp != ")": stream.seek(-1, 1) raise RegularExpressionException("Missing closing ')' after after '%s' operation." % set_operation_name) return __debug_exit(set_list, stream)
def snap_primary(stream, PatternDict): """primary: " non_double_quote * " = character string [ non_rect_bracket_close ] = set of characters { identifier } = pattern replacement ( expression ) non_control_character+ = lonely characters primary repetition_cmd """ __debug_entry("primary", stream) x = stream.read(1) lookahead = stream.read(1) if x != "" and lookahead != "": stream.seek(-1, 1) if x == "": return __debug_exit(None, stream) # -- 'primary' primary if x == "\"": result = snap_character_string.do(stream) elif x == "[": stream.seek(-1, 1) result = character_set_expression.do(stream, PatternDict) elif x == "{": result = snap_replacement(stream, PatternDict) elif x == ".": result = create_ALL_BUT_NEWLINE_state_machine() elif x == "(": result = snap_bracketed_expression(stream, PatternDict) elif x.isspace(): # a lonestanding space ends the regular expression stream.seek(-1, 1) return __debug_exit(None, stream) elif x in ["*", "+", "?"]: raise RegularExpressionException( "lonely operator '%s' without expression proceeding." % x) elif x == "\\": if lookahead == "C": stream.read(1) result = snap_case_folded_pattern(stream, PatternDict) elif lookahead == "R": result = get_expression_in_brackets(stream, PatternDict, "reverse operator", "R").get_inverse() elif lookahead == "A": result = get_expression_in_brackets(stream, PatternDict, "anti-pattern operator", "A") result.transform_to_anti_pattern() else: stream.seek(-1, 1) trigger_set = character_set_expression.snap_property_set(stream) if trigger_set is None: stream.seek( 1, 1) # snap_property_set() leaves tream right before '\\' char_code = snap_backslashed_character.do(stream) if char_code is None: raise RegularExpressionException( "Backslash followed by unrecognized character code.") trigger_set = char_code result = StateMachine() result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True) elif x not in CONTROL_CHARACTERS: # NOTE: The '\' is not inside the control characters---for a reason. # It is used to define for example character codes using '\x' etc. stream.seek(-1, 1) result = snap_non_control_character(stream, PatternDict) else: # NOTE: This includes the '$' sign which means 'end of line' # because the '$' sign is in CONTROL_CHARACTERS, but is not checked # against. Thus, it it good to leave here on '$' because the # '$' sign is handled on the very top level. # this is not a valid primary stream.seek(-1, 1) return __debug_exit(None, stream) # -- optional repetition command? result_repeated = __snap_repetition_range(result, stream) if result_repeated is not None: result = result_repeated return __debug_exit(beautifier.do(result), stream)
def snap_primary(stream, PatternDict): """primary: " non_double_quote * " = character string [ non_rect_bracket_close ] = set of characters { identifier } = pattern replacement ( expression ) non_control_character+ = lonely characters primary repetition_cmd """ global SPECIAL_TERMINATOR __debug_entry("primary", stream) x = stream.read(1) if x == "": return __debug_exit(None, stream) # -- 'primary' primary if x == "\"": result = snap_character_string.do(stream) elif x == "[": stream.seek(-1, 1); result = snap_character_set_expression(stream, PatternDict) elif x == "{": result = snap_replacement(stream, PatternDict) elif x == ".": result = create_ALL_BUT_NEWLINE_state_machine(stream) elif x == "(": result = snap_bracketed_expression(stream, PatternDict) elif x.isspace(): # a lonestanding space ends the regular expression stream.seek(-1, 1) return __debug_exit(None, stream) elif x in ["*", "+", "?"]: raise RegularExpressionException("lonely operator '%s' without expression proceeding." % x) elif x == "\\": result = snap_command(stream, PatternDict) if result is None: stream.seek(-1, 1) trigger_set = snap_property_set(stream) if trigger_set is None: # snap the '\' stream.read(1) char_code = snap_backslashed_character.do(stream) if char_code is None: raise RegularExpressionException("Backslash followed by unrecognized character code.") trigger_set = char_code result = StateMachine() result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True) elif x not in CONTROL_CHARACTERS and x != SPECIAL_TERMINATOR: # NOTE: The '\' is not inside the control characters---for a reason. # It is used to define for example character codes using '\x' etc. stream.seek(-1, 1) result = snap_non_control_character(stream, PatternDict) else: # NOTE: This includes the '$' sign which means 'end of line' # because the '$' sign is in CONTROL_CHARACTERS, but is not checked # against. Thus, it it good to leave here on '$' because the # '$' sign is handled on the very top level. # this is not a valid primary stream.seek(-1, 1) return __debug_exit(None, stream) # -- optional repetition command? result_repeated = __snap_repetition_range(result, stream) if result_repeated is not None: result = result_repeated # There's something going wrong with pseudo-ambigous post context # if we do not clean-up here. TODO: Investigate why? # See tests in generator/TEST directory. return __debug_exit(beautifier.do(result), stream)
def snap_primary(stream, PatternDict): """primary: " non_double_quote * " = character string [ non_rect_bracket_close ] = set of characters { identifier } = pattern replacement ( expression ) non_control_character+ = lonely characters primary repetition_cmd """ __debug_entry("primary", stream) x = stream.read(1); lookahead = stream.read(1); if x != "" and lookahead != "": stream.seek(-1, 1) if x == "": return __debug_exit(None, stream) # -- 'primary' primary if x == "\"": result = snap_character_string.do(stream) elif x == "[": stream.seek(-1, 1); result = character_set_expression.do(stream, PatternDict) elif x == "{": result = snap_replacement(stream, PatternDict) elif x == ".": result = create_ALL_BUT_NEWLINE_state_machine() elif x == "(": result = snap_bracketed_expression(stream, PatternDict) elif x.isspace(): # a lonestanding space ends the regular expression stream.seek(-1, 1) return __debug_exit(None, stream) elif x in ["*", "+", "?"]: raise RegularExpressionException("lonely operator '%s' without expression proceeding." % x) elif x == "\\": if lookahead == "C": stream.read(1) result = snap_case_folded_pattern(stream, PatternDict) elif lookahead == "R": result = get_expression_in_brackets(stream, PatternDict, "reverse operator", "R").get_inverse() elif lookahead == "A": result = get_expression_in_brackets(stream, PatternDict, "anti-pattern operator", "A") result.transform_to_anti_pattern() else: stream.seek(-1, 1) trigger_set = character_set_expression.snap_property_set(stream) if trigger_set is None: stream.seek(1, 1) # snap_property_set() leaves tream right before '\\' char_code = snap_backslashed_character.do(stream) if char_code is None: raise RegularExpressionException("Backslash followed by unrecognized character code.") trigger_set = char_code result = StateMachine() result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True) elif x not in CONTROL_CHARACTERS: # NOTE: The '\' is not inside the control characters---for a reason. # It is used to define for example character codes using '\x' etc. stream.seek(-1, 1) result = snap_non_control_character(stream, PatternDict) else: # NOTE: This includes the '$' sign which means 'end of line' # because the '$' sign is in CONTROL_CHARACTERS, but is not checked # against. Thus, it it good to leave here on '$' because the # '$' sign is handled on the very top level. # this is not a valid primary stream.seek(-1, 1) return __debug_exit(None, stream) # -- optional repetition command? result_repeated = __snap_repetition_range(result, stream) if result_repeated is not None: result = result_repeated return __debug_exit(beautifier.do(result), stream)