def emit(self, token=None):
"""
The standard method called to automatically emit a token at the
outermost lexical rule. The token object should point into the
char buffer start..stop. If there is a text override in 'text',
use that to set the token's text. Override this method to emit
custom Token objects.
If you are building trees, then you should also override
Parser or TreeParser.getMissingSymbol().
"""
if token is None:
token = CommonToken(
input=self.input,
type=self._state.type,
channel=self._state.channel,
start=self._state.tokenStartCharIndex,
stop=self.getCharIndex()-1
)
token.line = self._state.tokenStartLine
token.text = self._state.text
token.charPositionInLine = self._state.tokenStartCharPositionInLine
self._state.token = token
return token
def emit(self, token=None):
"""
The standard method called to automatically emit a token at the
outermost lexical rule. The token object should point into the
char buffer start..stop. If there is a text override in 'text',
use that to set the token's text. Override this method to emit
custom Token objects.
If you are building trees, then you should also override
Parser or TreeParser.getMissingSymbol().
"""
if token is None:
token = CommonToken(
input=self.input,
type=self._state.type,
channel=self._state.channel,
start=self._state.tokenStartCharIndex,
stop=self.getCharIndex()-1
)
token.line = self._state.tokenStartLine
token.text = self._state.text
token.charPositionInLine = self._state.tokenStartCharPositionInLine
self._state.token = token
return token
def makeEOFToken(self):
eof = CommonToken(
type=EOF, channel=DEFAULT_CHANNEL,
input=self.input,
start=self.input.index(), stop=self.input.index())
eof.line = self.input.line
eof.charPositionInLine = self.input.charPositionInLine
return eof
def makeEOFToken(self):
eof = CommonToken(type=EOF,
channel=DEFAULT_CHANNEL,
input=self.input,
start=self.input.index(),
stop=self.input.index())
eof.line = self.input.line
eof.charPositionInLine = self.input.charPositionInLine
return eof
def getMissingSymbol(self, input, e, expectedTokenType, follow):
tokenText = "<missing " + self.tokenNames[expectedTokenType] + ">"
t = CommonToken(type=expectedTokenType, text=tokenText)
current = input.LT(1)
if current.type == EOF:
current = input.LT(-1)
if current is not None:
t.line = current.line
t.charPositionInLine = current.charPositionInLine
t.channel = DEFAULT_CHANNEL
return t
def parseNode(self):
# "%label:" prefix
label = None
if self.ttype == PERCENT:
self.ttype = self.tokenizer.nextToken()
if self.ttype != ID:
return None
label = self.tokenizer.sval
self.ttype = self.tokenizer.nextToken()
if self.ttype != COLON:
return None
self.ttype = self.tokenizer.nextToken(
) # move to ID following colon
# Wildcard?
if self.ttype == DOT:
self.ttype = self.tokenizer.nextToken()
wildcardPayload = CommonToken(0, ".")
node = WildcardTreePattern(wildcardPayload)
if label is not None:
node.label = label
return node
# "ID" or "ID[arg]"
if self.ttype != ID:
return None
tokenName = self.tokenizer.sval
self.ttype = self.tokenizer.nextToken()
if tokenName == "nil":
return self.adaptor.nil()
text = tokenName
# check for arg
arg = None
if self.ttype == ARG:
arg = self.tokenizer.sval
text = arg
self.ttype = self.tokenizer.nextToken()
# create node
treeNodeType = self.wizard.getTokenType(tokenName)
if treeNodeType == INVALID_TOKEN_TYPE:
return None
node = self.adaptor.createFromType(treeNodeType, text)
if label is not None and isinstance(node, TreePattern):
node.label = label
if arg is not None and isinstance(node, TreePattern):
node.hasTextArg = True
return node
def __init__(self, payload):
"""
We need to work around a bug in the antlr runtime where if you
pass an imaginary token to a custom AST node type, it passes the
integer type of the token instead of a token instance.
"""
if isinstance(payload, (int, long)):
payload = CommonToken(type=payload)
super(AstNode, self).__init__(payload)
def emit(self, token=None):
"""
The standard method called to automatically emit a token at the
outermost lexical rule. The token object should point into the
char buffer start..stop. If there is a text override in 'text',
use that to set the token's text.
"""
if token is None:
token = CommonToken(input=self.input,
type=self.type,
channel=self.channel,
start=self.tokenStartCharIndex,
stop=self.getCharIndex() - 1)
token.line = self.tokenStartLine
token.text = self.text
token.charPositionInLine = self.tokenStartCharPositionInLine
self.token = token
return token
def emit(self, token=None):
"""
The standard method called to automatically emit a token at the
outermost lexical rule. The token object should point into the
char buffer start..stop. If there is a text override in 'text',
use that to set the token's text.
"""
if token is None:
token = CommonToken(
input=self.input,
type=self.type,
channel=self.channel,
start=self.tokenStartCharIndex,
stop=self.getCharIndex()-1
)
token.line = self.tokenStartLine
token.text = self.text
token.charPositionInLine = self.tokenStartCharPositionInLine
self.token = token
return token
def extractInformationFromTreeNodeStream(self, nodes):
from antlr3.tree import Tree, CommonTree
from antlr3.tokens import CommonToken
self.node = nodes.LT(1)
adaptor = nodes.adaptor
payload = adaptor.getToken(self.node)
if payload is not None:
self.token = payload
if payload.line <= 0:
# imaginary node; no line/pos info; scan backwards
i = -1
priorNode = nodes.LT(i)
while priorNode is not None:
priorPayload = adaptor.getToken(priorNode)
if priorPayload is not None and priorPayload.line > 0:
# we found the most recent real line / pos info
self.line = priorPayload.line
self.charPositionInLine = priorPayload.charPositionInLine
self.approximateLineInfo = True
break
i -= 1
priorNode = nodes.LT(i)
else: # node created from real token
self.line = payload.line
self.charPositionInLine = payload.charPositionInLine
elif isinstance(self.node, Tree):
self.line = self.node.line
self.charPositionInLine = self.node.charPositionInLine
if isinstance(self.node, CommonTree):
self.token = self.node.token
else:
type = adaptor.getType(self.node)
text = adaptor.getText(self.node)
self.token = CommonToken(type=type, text=text)
