def __init__(self, patterns, ignore, input,scantype="standard"):
"""Patterns is [(terminal,regex)...]
Ignore is [terminal,...];
Input is a string"""
self.tokens = []
self.restrictions = []
self.input = input
self.ignore = ignore
self.pos = 0
self.scantype = scantype
if self.scantype == "flex" and have_star_scan:
StarScan.prepare(input)
self.scan = self.compiled_scan
self.token = self.compiled_token
self.__del__ = StarScan.cleanup
elif self.scantype == "flex":
print "Warning: using Python scanner"
self.scantype = "standard"
if self.scantype != "flex":
self.scan = self.interp_scan
self.token = self.interp_token
# The stored patterns are a pair (compiled regex,source
# regex). If the patterns variable passed in to the
# constructor is None, we assume that the class already has a
# proper .patterns list constructed
if patterns is not None:
self.patterns = []
for k,r in patterns:
self.patterns.append( (k, re.compile(r)) )
def __init__(self, patterns, ignore, input,scantype="standard"):
"""Patterns is [(terminal,regex)...]
Ignore is [terminal,...];
Input is a string"""
self.tokens = []
self.restrictions = []
self.input = input
self.ignore = ignore
self.pos = 0
self.scantype = scantype
if self.scantype == "flex" and have_star_scan:
StarScan.prepare(input)
self.scan = self.compiled_scan
self.token = self.compiled_token
self.__del__ = StarScan.cleanup
elif self.scantype == "flex":
print "Warning: using Python scanner"
self.scantype = "standard"
if self.scantype != "flex":
self.scan = self.interp_scan
self.token = self.interp_token
# The stored patterns are a pair (compiled regex,source
# regex). If the patterns variable passed in to the
# constructor is None, we assume that the class already has a
# proper .patterns list constructed
if patterns is not None:
self.patterns = []
for k,r in patterns:
self.patterns.append( (k, re.compile(r)) )
def compiled_token(self,i,restrict=0): """Get the i'th token, and if i is one past the end, then scan for another token; restrict is a list of tokens that are allowed, or 0 for any token.""" try: return StarScan.token(i) except IndexError: raise NoMoreTokens()
def compiled_token(self,i,restrict=0): """Get the i'th token, and if i is one past the end, then scan for another token; restrict is a list of tokens that are allowed, or 0 for any token.""" try: return StarScan.token(i) except IndexError: raise NoMoreTokens()
def compiled_scan(self,restrict):
token = StarScan.scan()
print "Calling compiled scan, got %s" % `token`
if token[2] not in restrict:
msg = "Bad Token"
if restrict:
msg = "Trying to find one of "+join(restrict,", ")
raise SyntaxError(self.pos,msg)
self.tokens.append(token)
self.restrictions.append(restrict)
return
def __repr__(self):
"""Print the last 10 tokens that have been scanned in"""
output = ''
if self.scantype != "flex":
for t in self.tokens[-10:]:
output = '%s\n (@%s) %s = %s' % (output,t[0],t[2],repr(t[3]))
else:
out_tokens = StarScan.last_ten()
for t in out_tokens:
output = '%s\n (~line %s) %s = %s' % (output,t[0],t[2],repr(t[3]))
return output
def compiled_scan(self,restrict):
"""Should scan another token and add it to the list, self.tokens,
and add the restriction to self.restrictions"""
token = StarScan.scan()
if token[2] not in restrict:
msg = "Bad Token"
if restrict:
msg = "Trying to find one of "+join(restrict,", ")
raise SyntaxError(self.pos, msg)
self.tokens.append(token)
self.restrictions.append(restrict)
return
def compiled_scan(self,restrict):
"""Should scan another token and add it to the list, self.tokens,
and add the restriction to self.restrictions"""
token = StarScan.scan()
if token[2] not in restrict:
msg = "Bad Token"
if restrict:
msg = "Trying to find one of "+join(restrict,", ")
raise SyntaxError(self.pos, msg)
self.tokens.append(token)
self.restrictions.append(restrict)
return
def __init__(self, patterns, ignore, input, scantype="standard"):
"""Initialize the scanner.
Parameters:
patterns : [(terminal, uncompiled regex), ...] or None
ignore : [terminal,...]
input : string
If patterns is None, we assume that the subclass has
defined self.patterns : [(terminal, compiled regex), ...].
Note that the patterns parameter expects uncompiled regexes,
whereas the self.patterns field expects compiled regexes.
"""
self.tokens = [
] # [(begin char pos, end char pos, token name, matched text), ...]
self.restrictions = []
self.input = input
self.pos = 0
self.ignore = ignore
self.scantype = scantype
self.first_line_number = 1
if self.scantype == "flex" and have_star_scan:
StarScan.prepare(input)
self.scan = self.compiled_scan
self.token = self.compiled_token
self.__del__ = StarScan.cleanup
elif self.scantype == "flex":
self.scantype = "standard"
if self.scantype != "flex":
self.scan = self.interp_scan
self.token = self.interp_token
if patterns is not None:
# Compile the regex strings into regex objects
self.patterns = []
for terminal, regex in patterns:
self.patterns.append((terminal, re.compile(regex)))
if self.using_java_regex:
# different access method for regexes
self.patterns = map(lambda a: (a[0], a[1].matcher(self.input)),
self.patterns)
def __init__(self, patterns, ignore, input, scantype="standard"):
"""Initialize the scanner.
Parameters:
patterns : [(terminal, uncompiled regex), ...] or None
ignore : [terminal,...]
input : string
If patterns is None, we assume that the subclass has
defined self.patterns : [(terminal, compiled regex), ...].
Note that the patterns parameter expects uncompiled regexes,
whereas the self.patterns field expects compiled regexes.
"""
self.tokens = [] # [(begin char pos, end char pos, token name, matched text), ...]
self.restrictions = []
self.input = input
self.pos = 0
self.ignore = ignore
self.scantype = scantype
self.first_line_number = 1
if self.scantype == "flex" and have_star_scan:
StarScan.prepare(input)
self.scan = self.compiled_scan
self.token = self.compiled_token
self.__del__ = StarScan.cleanup
elif self.scantype == "flex":
self.scantype = "standard"
if self.scantype != "flex":
self.scan = self.interp_scan
self.token = self.interp_token
if patterns is not None:
# Compile the regex strings into regex objects
self.patterns = []
for terminal, regex in patterns:
self.patterns.append( (terminal, re.compile(regex)) )
if self.using_java_regex:
# different access method for regexes
self.patterns = map(lambda a: (a[0],a[1].matcher(self.input)),self.patterns)
def compiled_token(self,i,restrict=0):
try:
return StarScan.token(i)
except IndexError:
raise NoMoreTokens()
