def snap_property_set(stream): position = stream.tell() x = stream.read(2) if x == "\\P": stream.seek(position) return property.do(stream) elif x == "\\N": stream.seek(position) return property.do_shortcut(stream, "N", "na") # UCS Property: Name elif x == "\\G": stream.seek(position) return property.do_shortcut(stream, "G", "gc") # UCS Property: General_Category elif x == "\\E": skip_whitespace(stream) if check(stream, "{") == False: error_msg("Missing '{' after '\\E'.", stream) encoding_name = __snap_until(stream, "}").strip() return codec_db.get_supported_unicode_character_set(encoding_name, stream) else: stream.seek(position) return None
def snap_property_set(stream): position = stream.tell() x = stream.read(2) if x == "\\P": stream.seek(position) return property.do(stream) elif x == "\\N": stream.seek(position) return property.do_shortcut(stream, "N", "na") # UCS Property: Name elif x == "\\G": stream.seek(position) return property.do_shortcut(stream, "G", "gc") # UCS Property: General_Category elif x == "\\E": skip_whitespace(stream) if check(stream, "{") == False: error_msg("Missing '{' after '\\E'.", stream) encoding_name = __snap_until(stream, "}").strip() return codec_db.get_supported_unicode_character_set( encoding_name, stream) else: stream.seek(position) return None
def do(sh, PatternDict, snap_expression=None, snap_set_expression=None): """Parse a case fold expression of the form \C(..){ R } or \C{ R }. Assume that '\C' has been snapped already from the stream. See function ucs_case_fold_parser.get_fold_set() for details about case folding. snap_expression != None, then snap_expression is the function to parse a RE and the caller expects a state machine. snap_set_expression != None, then snap_set_expression is the function to parse a character set and caller expects a NumberSet object. """ pos = sh.tell() skip_whitespace(sh) # -- parse the optional options in '(' ')' brackets if not check(sh, "("): # By default 'single' and 'multi' character case folds are active if snap_set_expression != None: flag_txt = "s" else: flag_txt = "sm" else: flag_txt = read_until_character(sh, ")") if flag_txt == "": sh.seek(pos) error_msg("Missing closing ')' in case fold expression.", sh) flag_txt = flag_txt.replace(" ", "").replace("\t", "").replace("\n", "") for letter in flag_txt: if letter not in "smt": sh.seek(pos) error_msg("Letter '%s' not permitted as case fold option.\n" % letter + \ "Options are: 's' for simple case fold.\n" + \ " 'm' for multi character sequence case fold.\n" + \ " 't' for special turkish case fold rules.", sh) if snap_set_expression != None and letter == "m": sh.seek(pos) error_msg("Option 'm' not permitted as case fold option in set expression.\n" + \ "Set expressions cannot absorb multi character sequences.", sh) skip_whitespace(sh) # -- parse the expression in '{' '}' which is subject to case folding if not check(sh, "{"): sh.seek(pos) error_msg("Missing '{' for case fold expression.", sh) skip_whitespace(sh) if snap_set_expression != None: trigger_set = snap_set_expression(sh, PatternDict) if trigger_set == None: error_msg( "Missing character set for case fold in set expression.\n" + "The content in '\\C{content}' should start with '[' or '[:'.", sh) # -- perform the case fold for Sets! for interval in trigger_set.get_intervals(PromiseToTreatWellF=True): for i in range(interval.begin, interval.end): fold = ucs_case_fold.get_fold_set(i, flag_txt) for x in fold: assert type(x) != list trigger_set.add_interval(Interval(x, x + 1)) result = trigger_set else: sm = snap_expression(sh, PatternDict) if sm == None: error_msg( "Missing expression for case fold '\C'.\n" + "The content in '\\C{content}' should start with '[' or '[:'.", sh) # -- perform the case fold for State Machines! for state_idx, state in sm.states.items(): transitions = state.transitions() for target_state_idx, trigger_set in transitions.get_map().items(): __add_case_fold(sm, flag_txt, trigger_set, state_idx, target_state_idx) result = sm if not check(sh, "}"): sh.seek(pos) error_msg("Missing '}' for case fold expression.", sh) return result
def do(fh): """Parses pattern definitions of the form: [ \t] => grid 4; [:intersection([:alpha:], [\X064-\X066]):] => space 1; In other words the right hand side *must* be a character set. """ indentation_setup = IndentationSetup(fh) # NOTE: Catching of EOF happens in caller: parse_section(...) # skip_whitespace(fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, ">"): indentation_setup.seal() indentation_setup.consistency_check(fh) return indentation_setup # A regular expression state machine pattern_str, state_machine = regular_expression.parse(fh) skip_whitespace(fh) if not check(fh, "=>"): error_msg("Missing '=>' after character set definition.", fh) skip_whitespace(fh) identifier = read_identifier(fh) if identifier == "": error_msg("Missing identifier for indentation element definition.", fh) verify_word_in_list( identifier, ["space", "grid", "bad", "newline", "suppressor"], "Unrecognized indentation specifier '%s'." % identifier, fh) trigger_set = None if identifier in ["space", "bad", "grid"]: if len(state_machine.states) != 2: error_msg("For indentation '%s' only patterns are addmissible which\n" % identifier + \ "can be matched by a single character, e.g. \" \" or [a-z].", fh) transition_map = state_machine.get_init_state().transitions( ).get_map() assert len(transition_map) == 1 trigger_set = transition_map.values()[0] skip_whitespace(fh) if identifier == "space": value = read_integer(fh) if value != None: indentation_setup.specify_space(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? variable = read_identifier(fh) if variable != "": indentation_setup.specify_space(pattern_str, trigger_set, variable, fh) else: indentation_setup.specify_space(pattern_str, trigger_set, 1, fh) elif identifier == "grid": value = read_integer(fh) if value != None: indentation_setup.specify_grid(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? skip_whitespace(fh) variable = read_identifier(fh) if variable != "": indentation_setup.specify_grid(pattern_str, trigger_set, variable, fh) else: error_msg( "Missing integer or variable name after keyword 'grid'.", fh) elif identifier == "bad": indentation_setup.specify_bad(pattern_str, trigger_set, fh) elif identifier == "newline": indentation_setup.specify_newline(pattern_str, state_machine, fh) elif identifier == "suppressor": indentation_setup.specify_suppressor(pattern_str, state_machine, fh) else: assert False, "Unreachable code reached." if not check(fh, ";"): error_msg( "Missing ';' after indentation '%s' specification." % identifier, fh)
def do(fh): """Parses pattern definitions of the form: [ \t] => grid 4; [:intersection([:alpha:], [\X064-\X066]):] => space 1; In other words the right hand side *must* be a character set. """ indentation_setup = IndentationSetup(fh) # NOTE: Catching of EOF happens in caller: parse_section(...) # skip_whitespace(fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, ">"): indentation_setup.seal() indentation_setup.consistency_check(fh) return indentation_setup # A regular expression state machine pattern_str, state_machine = regular_expression.parse(fh) skip_whitespace(fh) if not check(fh, "=>"): error_msg("Missing '=>' after character set definition.", fh) skip_whitespace(fh) identifier = read_identifier(fh) if identifier == "": error_msg("Missing identifier for indentation element definition.", fh) verify_word_in_list( identifier, ["space", "grid", "bad", "newline", "suppressor"], "Unrecognized indentation specifier '%s'." % identifier, fh, ) trigger_set = None if identifier in ["space", "bad", "grid"]: if len(state_machine.states) != 2: error_msg( "For indentation '%s' only patterns are addmissible which\n" % identifier + 'can be matched by a single character, e.g. " " or [a-z].', fh, ) transition_map = state_machine.get_init_state().transitions().get_map() assert len(transition_map) == 1 trigger_set = transition_map.values()[0] skip_whitespace(fh) if identifier == "space": value = read_integer(fh) if value != None: indentation_setup.specify_space(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? variable = read_identifier(fh) if variable != "": indentation_setup.specify_space(pattern_str, trigger_set, variable, fh) else: indentation_setup.specify_space(pattern_str, trigger_set, 1, fh) elif identifier == "grid": value = read_integer(fh) if value != None: indentation_setup.specify_grid(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? skip_whitespace(fh) variable = read_identifier(fh) if variable != "": indentation_setup.specify_grid(pattern_str, trigger_set, variable, fh) else: error_msg("Missing integer or variable name after keyword 'grid'.", fh) elif identifier == "bad": indentation_setup.specify_bad(pattern_str, trigger_set, fh) elif identifier == "newline": indentation_setup.specify_newline(pattern_str, state_machine, fh) elif identifier == "suppressor": indentation_setup.specify_suppressor(pattern_str, state_machine, fh) else: assert False, "Unreachable code reached." if not check(fh, ";"): error_msg("Missing ';' after indentation '%s' specification." % identifier, fh)
def do(sh, PatternDict, snap_expression=None, snap_set_expression=None): """Parse a case fold expression of the form \C(..){ R } or \C{ R }. Assume that '\C' has been snapped already from the stream. See function ucs_case_fold_parser.get_fold_set() for details about case folding. snap_expression != None, then snap_expression is the function to parse a RE and the caller expects a state machine. snap_set_expression != None, then snap_set_expression is the function to parse a character set and caller expects a NumberSet object. """ pos = sh.tell() skip_whitespace(sh) # -- parse the optional options in '(' ')' brackets if not check(sh, "("): # By default 'single' and 'multi' character case folds are active if snap_set_expression != None: flag_txt = "s" else: flag_txt = "sm" else: flag_txt = read_until_character(sh, ")") if flag_txt == "": sh.seek(pos) error_msg("Missing closing ')' in case fold expression.", sh) flag_txt = flag_txt.replace(" ", "").replace("\t", "").replace("\n", "") for letter in flag_txt: if letter not in "smt": sh.seek(pos) error_msg("Letter '%s' not permitted as case fold option.\n" % letter + \ "Options are: 's' for simple case fold.\n" + \ " 'm' for multi character sequence case fold.\n" + \ " 't' for special turkish case fold rules.", sh) if snap_set_expression != None and letter == "m": sh.seek(pos) error_msg("Option 'm' not permitted as case fold option in set expression.\n" + \ "Set expressions cannot absorb multi character sequences.", sh) skip_whitespace(sh) # -- parse the expression in '{' '}' which is subject to case folding if not check(sh, "{"): sh.seek(pos) error_msg("Missing '{' for case fold expression.", sh) skip_whitespace(sh) if snap_set_expression != None: trigger_set = snap_set_expression(sh, PatternDict) if trigger_set == None: error_msg("Missing character set for case fold in set expression.\n" + "The content in '\\C{content}' should start with '[' or '[:'.", sh) # -- perform the case fold for Sets! for interval in trigger_set.get_intervals(PromiseToTreatWellF=True): for i in range(interval.begin, interval.end): fold = ucs_case_fold.get_fold_set(i, flag_txt) for x in fold: assert type(x) != list trigger_set.add_interval(Interval(x, x+1)) result = trigger_set else: sm = snap_expression(sh, PatternDict) if sm == None: error_msg("Missing expression for case fold '\C'.\n" + "The content in '\\C{content}' should start with '[' or '[:'.", sh) # -- perform the case fold for State Machines! for state_idx, state in sm.states.items(): transitions = state.transitions() for target_state_idx, trigger_set in transitions.get_map().items(): __add_case_fold(sm, flag_txt, trigger_set, state_idx, target_state_idx) result = sm if not check(sh, "}"): sh.seek(pos) error_msg("Missing '}' for case fold expression.", sh) return result