def print_tokens(srcfile):
from antlr4 import FileStream, CommonTokenStream
from dynamod.parser.DynamodLexer import DynamodLexer
input = FileStream(srcfile)
lexer = DynamodLexer(input)
stream = CommonTokenStream(lexer)
stream.fill()
for token in stream.getTokens(0, 9999999):
print(str(token))
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(
ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.getColumn()
msg = "Invalid tokens or characters at index " + str(
pos) + " in path '" + path + "'"
raise Exception(msg, e)
tokens = tokenStream.getTokens()
elements = list()
n = len(tokens)
i = 0
while i < n:
el = tokens[i]
next = None
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
i += 1
next = tokens[i]
invert = next.type == XPathLexer.BANG
if invert:
i += 1
next = tokens[i]
pathElement = self.getXPathElement(next, anywhere)
pathElement.invert = invert
elements.append(pathElement)
i += 1
elif el.type in [
XPathLexer.TOKEN_REF, XPathLexer.RULE_REF,
XPathLexer.WILDCARD
]:
elements.append(self.getXPathElement(el, False))
i += 1
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element " + str(el))
return elements
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.getColumn()
msg = "Invalid tokens or characters at index " + str(pos) + " in path '" + path + "'"
raise Exception(msg, e)
tokens = tokenStream.getTokens()
elements = list()
n = len(tokens)
i = 0
while i < n:
el = tokens[i]
next = None
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
i += 1
next = tokens[i]
invert = next.type == XPathLexer.BANG
if invert:
i += 1
next = tokens[i]
pathElement = self.getXPathElement(next, anywhere)
pathElement.invert = invert
elements.append(pathElement)
i += 1
elif el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD]:
elements.append(self.getXPathElement(el, False))
i += 1
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element " + str(el))
return elements
