def get_tok_punct(self):
""" get punctuation non-tokenized, non-whitespace
:returns: True if token found, result in self.tok
Checks for ;(SEMICOLON), .(PERIOD)
else labeled (PUNCT)
"""
line = self.line
if len(line) == 0:
return False
match = re.match(r'^\S', line)
if not match:
return False
tok_str = line[0]
if tok_str == ";":
tok_type = SelectStreamToken.SEMICOLON
elif tok_str == ".":
tok_type = SelectStreamToken.PERIOD
else:
tok_type = SelectStreamToken.PUNCT
self.tok = SelectStreamToken(tok_type, tok_str)
self.line = line[len(tok_str) + 1:]
return True
def get_tok_word(self):
""" get word from input ([a-z_]\w*)
:returns: True if token found, result in self.tok
"""
line = self.line
if len(line) == 0:
return False
ib = 0
iend = ib
tok_str = ""
ch = line[ib]
if not re.match(r'[a-zA-Z_]', ch):
return False
tok_str = ch
iend += 1
while iend < len(line):
ch = line[iend]
if not re.match(r'\w', ch):
break # end of word
tok_str += ch
iend += 1
self.tok = SelectStreamToken(SelectStreamToken.WORD, str=tok_str)
self.line = line[iend:]
return True
def get_tok(self):
""" Get next input token
Ignores comments(Unquoted # to end of line)
Tokens end at whitespace, end of line, ";",
character not part of the token (e.g. ",", ";")
"""
while True:
if self.line is None:
self.line = self.get_line() # Get next line
if self.line is None:
return SelectStreamToken(SelectStreamToken.EOF)
self.line = self.line.lstrip()
if self.get_tok_comment():
continue # Ignore comment
if self.get_tok_word():
return self.tok
if self.get_tok_quote():
return self.tok
if self.get_tok_number():
return self.tok
if self.get_tok_punct(): # incl SEMICOLON, PERIOD
return self.tok
if self.get_tok_eol():
return self.tok
def get_tok_str(self, delim=None, multi_line=False, esc='\\'):
""" Get string, creating a SelectStreamToken in self.tok
adjusting self.line to after string trailing delimiter
:delim: String trailing delimiter default: REQUIRED
:multi_line: string can be multi-line default: single line
:esc: escape character defaule "\"
"""
if delim is None:
raise SelectError("get_tok_str: missing required delim")
line = self.get_line()
if line is None or not line.startswith(delim):
self.line = line
return False
tok_str = ""
iend = 0
iend += len(delim) # Start after beginning delimiter
line = line[iend:]
iend = 0
while True:
if line is None:
line = self.get_line()
iend = 0
if line is None:
break # EOF
while iend < len(line):
ch = line[iend]
if esc is not None and ch == esc:
iend += 1
if iend >= len(line):
ch = "\n" # Escaped newline
else:
ch = line[iend]
iend += 1
continue # Go on to character after escaped
rem_line = line[iend:]
if rem_line.startswith(delim):
self.tok = SelectStreamToken(SelectStreamToken.QSTRING,
str=tok_str,
delim=delim)
iend += len(delim)
self.line = line[iend:]
return True
ch = line[iend]
tok_str += ch
iend += 1
if multi_line:
tok_str += "\n" # Add in newline, separating lines
line = self.get_line()
iend = 0
if line is None:
break # EOF
continue
break # Single line
raise SelectError("Missing delimiter (%s)" % delim)
def get_tok_comment(self):
""" Check if we are at a comment
Returns a token, which is can be ignored
comment format: "#" to rest of line
"""
if self.line.startswith("#"):
tok = SelectStreamToken(SelectStreamToken.COMMENT, str=self.line)
self.line = "" # To end of line
return tok
def get_tok_eol(self):
""" get End of Line
:returns: True if token found, result in self.tok
"""
line = self.line
if len(line) == 0:
tok_str = "\n"
self.tok = SelectStreamToken(SelectStreamToken.EOL, tok_str)
self.line = None
return True
return False
def get_tok_number(self):
""" get nu ber from input ([+-]?\d*(\.\d*)?
:returns: True if token found, result in self.tok
"""
line = self.line
if len(line) == 0:
return False
match = re.match(r'^[+-]?(\d+|(\d*\.\d*))', line)
if not match:
return False
tok_str = match.group()
self.tok = SelectStreamToken(SelectStreamToken.NUMBER, tok_str)
self.line = line[len(tok_str) + 1:]
return True