def __init__(self, patterns, ignore, input,scantype="standard"): """Patterns is [(terminal,regex)...] Ignore is [terminal,...]; Input is a string""" self.tokens = [] self.restrictions = [] self.input = input self.ignore = ignore self.pos = 0 self.scantype = scantype if self.scantype == "flex" and have_star_scan: StarScan.prepare(input) self.scan = self.compiled_scan self.token = self.compiled_token self.__del__ = StarScan.cleanup elif self.scantype == "flex": print "Warning: using Python scanner" self.scantype = "standard" if self.scantype != "flex": self.scan = self.interp_scan self.token = self.interp_token # The stored patterns are a pair (compiled regex,source # regex). If the patterns variable passed in to the # constructor is None, we assume that the class already has a # proper .patterns list constructed if patterns is not None: self.patterns = [] for k,r in patterns: self.patterns.append( (k, re.compile(r)) )
def __init__(self, patterns, ignore, input, scantype="standard"): """Initialize the scanner. Parameters: patterns : [(terminal, uncompiled regex), ...] or None ignore : [terminal,...] input : string If patterns is None, we assume that the subclass has defined self.patterns : [(terminal, compiled regex), ...]. Note that the patterns parameter expects uncompiled regexes, whereas the self.patterns field expects compiled regexes. """ self.tokens = [ ] # [(begin char pos, end char pos, token name, matched text), ...] self.restrictions = [] self.input = input self.pos = 0 self.ignore = ignore self.scantype = scantype self.first_line_number = 1 if self.scantype == "flex" and have_star_scan: StarScan.prepare(input) self.scan = self.compiled_scan self.token = self.compiled_token self.__del__ = StarScan.cleanup elif self.scantype == "flex": self.scantype = "standard" if self.scantype != "flex": self.scan = self.interp_scan self.token = self.interp_token if patterns is not None: # Compile the regex strings into regex objects self.patterns = [] for terminal, regex in patterns: self.patterns.append((terminal, re.compile(regex))) if self.using_java_regex: # different access method for regexes self.patterns = map(lambda a: (a[0], a[1].matcher(self.input)), self.patterns)
def __init__(self, patterns, ignore, input, scantype="standard"): """Initialize the scanner. Parameters: patterns : [(terminal, uncompiled regex), ...] or None ignore : [terminal,...] input : string If patterns is None, we assume that the subclass has defined self.patterns : [(terminal, compiled regex), ...]. Note that the patterns parameter expects uncompiled regexes, whereas the self.patterns field expects compiled regexes. """ self.tokens = [] # [(begin char pos, end char pos, token name, matched text), ...] self.restrictions = [] self.input = input self.pos = 0 self.ignore = ignore self.scantype = scantype self.first_line_number = 1 if self.scantype == "flex" and have_star_scan: StarScan.prepare(input) self.scan = self.compiled_scan self.token = self.compiled_token self.__del__ = StarScan.cleanup elif self.scantype == "flex": self.scantype = "standard" if self.scantype != "flex": self.scan = self.interp_scan self.token = self.interp_token if patterns is not None: # Compile the regex strings into regex objects self.patterns = [] for terminal, regex in patterns: self.patterns.append( (terminal, re.compile(regex)) ) if self.using_java_regex: # different access method for regexes self.patterns = map(lambda a: (a[0],a[1].matcher(self.input)),self.patterns)