def tokenize(self, line):
"""
Create tokens from string.
@param line: string to scan
"""
self.rv = []
GenericScanner.tokenize(self, line)
return self.rv
def tokenize(self, input):
self.tokens = []
lines=input.split("\n")
for line in lines:
#remove inline comments
line=line.split("//")[0]
GenericScanner.tokenize(self, line)
self.lineno+=1
t=Token(type='NEWLINE',lineno=self.lineno)
self.tokens.append(t)
return self.tokens
def tokenize(self, input):
self.tokens = []
lines = input.split("\n")
for line in lines:
#remove inline comments
line = line.split("//")[0]
GenericScanner.tokenize(self, line)
self.lineno += 1
t = Token(type='NEWLINE', lineno=self.lineno)
self.tokens.append(t)
return self.tokens
def tokenize(self, input_):
self.rv = []
GenericScanner.tokenize(self, input_)
return self.rv
def __init__(self):
GenericScanner.__init__(self)
def __init__(self):
GenericScanner.__init__(self)
def __init__(self, operators=''):
self.operators = operators
GenericScanner.__init__(self)
def tokenize(self, input):
self.tokens = []
GenericScanner.tokenize(self, input)
return self.tokens
def __init__(self):
self.tokens=[]
GenericScanner.__init__(self)
self.lineno=1
def tokenize(self, input, outputFile): self.rv = [] self.outputFile = outputFile GenericScanner.tokenize(self, input) return self.rv
def __init__(self, keywordRegexString=keywordRegex): GenericScanner.__init__(self) self.keywordRegex = re.compile(keywordRegexString)
def __init__(self,lineno):
GenericScanner.__init__(self)
self.lineno = lineno
def __init__(self, operators=''):
self.operators = operators
GenericScanner.__init__(self)
def __init__(self):
GenericScanner.__init__(self)
self.charOffset = 0
def __init__(self):
GenericScanner.__init__(self)
self.lineno = 1
def __init__(self, flags=0):
GenericScanner.__init__(self, flags)
self.lineno = 1
self.filename = ''
def tokenize(self, input):
self.rv = []
GenericScanner.tokenize(self, input)
self.line_count=0
return self.rv
def __init__(self, flags=0):
GenericScanner.__init__(self,flags)
self.lineno=1
def tokenize(self, input):
self.rv = []
GenericScanner.tokenize(self, input)
return self.rv
def __init__(self): GenericScanner.__init__(self) self.labelingOrCoding = None self.blockLabelOrCoding = None
