def p_error(p):
assert isinstance(p, tokenizer.token)
if p.ttype == tokenizer.tokentype.EOF:
raise errors.ParsingError('unexpected EOF', p.lexer.source,
len(p.lexer.source))
else:
raise errors.ParsingError('unexpected token %r' % p.value,
p.lexer.source, p.lexpos)
def makeheredoc(tokenizer, redirnode, lineno, killleading):
redirword = remove_escape(string_quote_removal(redirnode.output.word))
#redirword = redirnode.output.word
document = []
startpos = tokenizer._shell_input_line_index
#fullline = self.tok.readline(bool(redirword.output.flags & flags.word.QUOTED))
fullline = tokenizer.readline(False)
while fullline:
if killleading:
while fullline[0] == '\t':
fullline = fullline[1:]
if not fullline:
continue
if fullline[:-1] == redirword and fullline[len(redirword)] == '\n':
document.append(fullline[:-1])
# document_done
break
document.append(fullline)
#fullline = self.readline(bool(redirnode.flags & flags.word.QUOTED))
fullline = tokenizer.readline(False)
if not fullline:
raise errors.ParsingError(
"here-document at line %d delimited by end-of-file (wanted %r)" %
(lineno, redirword), tokenizer._shell_input_line,
tokenizer._shell_input_line_index)
document = ''.join(document)
endpos = tokenizer._shell_input_line_index - 1
assert hasattr(redirnode, 'heredoc')
num_of_lines = document.count('\n') + 1
redirnode.heredoc = ast.node(kind='heredoc',
value=document,
pos=(startpos, endpos),
lineno=num_of_lines)
# if the heredoc immediately follows this node, fix its end pos
if redirnode.pos[1] + 1 == startpos:
redirnode.pos = (redirnode.pos[0], endpos)
return document
def _expandwordinternal(parserobj, wordtoken, qheredocument, qdoublequotes, quoted, isexp):
istring = ''
parts = []
tindex = [0]
sindex = [0]
string = wordtoken.value
def nextchar():
sindex[0] += 1
if sindex[0] < len(string):
return string[sindex[0]]
def peekchar():
if sindex[0]+1 < len(string):
return string[sindex[0]+1]
while True:
if sindex[0] == len(string):
break
# goto finished_with_string
c = string[sindex[0]]
if c in '<>':
if (nextchar() != '(' or qheredocument or qdoublequotes or
(wordtoken.flags & set([flags.word.DQUOTE, flags.word.NOPROCSUB]))):
sindex[0] -= 1
# goto add_character
sindex[0] += 1
istring += c
else:
tindex = sindex[0] + 1
node, sindex[0] = _extractprocesssubst(parserobj, string, tindex)
parts.append(ast.node(kind='processsubstitution', command=node,
pos=(tindex - 2, sindex[0])))
istring += string[tindex - 2:sindex[0]]
# goto dollar_add_string
# TODO
# elif c == '=':
# pass
# elif c == ':':
# pass
elif c == '~':
if (wordtoken.flags & set([flags.word.NOTILDE, flags.word.DQUOTE]) or
(sindex[0] > 0 and not (wordtoken.flags & flags.word.NOTILDE)) or
qdoublequotes or qheredocument):
wordtoken.flags.clear()
wordtoken.flags.add(flags.word.ITILDE)
sindex[0] += 1
istring += c
else:
stopatcolon = wordtoken.flags & set([flags.word.ASSIGNRHS,
flags.word.ASSIGNMENT,
flags.word.TILDEEXP])
expand = True
for i in range(sindex[0], len(string)):
r = string[i]
if r == '/':
break
if r in "\\'\"":
expand = False
break
if stopatcolon and r == ':':
break
else:
# go one past the end if we didn't exit early
i += 1
if i > sindex[0] and expand:
node = ast.node(kind='tilde', value=string[sindex[0]:i],
pos=(sindex[0], i))
parts.append(node)
istring += string[sindex[0]:i]
sindex[0] = i
elif c == '$' and len(string) > 1:
tindex = sindex[0]
node, sindex[0] = _paramexpand(parserobj, string, sindex[0], wordtoken.lineno)
if node:
parts.append(node)
istring += string[tindex:sindex[0]]
elif c == '`':
tindex = sindex[0]
# bare instance of ``
if nextchar() == '`':
sindex[0] += 1
istring += '``'
else:
x = _stringextract(string, sindex[0], "`")
if x == -1:
raise errors.ParsingError('bad substitution: no closing "`" '
'in %s' % string)
else:
if wordtoken.flags & flags.word.NOCOMSUB:
pass
else:
sindex[0] = x
word = string[tindex+1:sindex[0]]
command, ttindex = _recursiveparse(parserobj, word, 0, lineno=0)
_adjustpositions(command, tindex+1, len(string), wordtoken.lineno-1)
ttindex += 1 # ttindex is on the closing char
# assert sindex[0] == ttindex
# go one past the closing `
sindex[0] += 1
node = ast.node(kind='commandsubstitution',
command=command,
pos=(tindex, sindex[0]),
lineno=wordtoken.lineno)
parts.append(node)
istring += string[tindex:sindex[0]]
elif c == '\\':
istring += string[sindex[0]+1:sindex[0]+2]
sindex[0] += 2
elif c == '"':
sindex[0] += 1
continue
# 8513
#if qdoublequotes or qheredocument:
# sindex[0] += 1
#else:
# tindex = sindex[0] + 1
# parts, sindex[0] = _stringextractdoublequoted(string, sindex[0])
# if tindex == 1 and sindex[0] == len(string):
# quotedstate = 'wholly'
# else:
# quotedstate = 'partially'
elif c == "'":
# entire string surronded by single quotes, no expansion is
# going to happen
if sindex[0] == 0 and string[-1] == "'":
return [], string[1:-1]
# check if we're inside double quotes
if not qdoublequotes:
# look for the closing ', we know we have one or otherwise
# this wouldn't tokenize due to unmatched '
tindex = sindex[0]
sindex[0] = string.find("'", sindex[0]) + 1
istring += string[tindex+1:sindex[0]-1]
else:
# this is a single quote inside double quotes, add it
istring += c
sindex[0] += 1
else:
istring += string[sindex[0]:sindex[0]+1]
sindex[0] += 1
if parts:
class v(ast.nodevisitor):
def visitnode(self, node):
assert node.pos[1] + wordtoken.lexpos <= wordtoken.endlexpos
node.pos = (node.pos[0] + wordtoken.lexpos,
node.pos[1] + wordtoken.lexpos)
visitor = v()
for node in parts:
visitor.visit(node)
return parts, istring