def p_redirection_heredoc(p):
'''redirection : LESS_LESS WORD
| NUMBER LESS_LESS WORD
| REDIR_WORD LESS_LESS WORD
| LESS_LESS_MINUS WORD
| NUMBER LESS_LESS_MINUS WORD
| REDIR_WORD LESS_LESS_MINUS WORD'''
parserobj = p.context
assert isinstance(parserobj, _parser)
output = bast.node(kind='word',
word=p[len(p) - 1],
parts=[],
pos=p.lexspan(len(p) - 1))
if len(p) == 3:
p[0] = bast.node(kind='redirect',
input=None,
type=p[1],
heredoc=None,
output=output,
pos=(p.lexpos(1), p.endlexpos(2)))
else:
p[0] = bast.node(kind='redirect',
input=p[1],
type=p[2],
heredoc=None,
output=output,
pos=(p.lexpos(1), p.endlexpos(3)))
if p.slice[len(p) - 2].ttype == tokenizer.tokentype.LESS_LESS:
parserobj.redirstack.append((p[0], False))
else:
parserobj.redirstack.append((p[0], True))
def p_redirection(p):
'''redirection : GREATER WORD
| LESS WORD
| NUMBER GREATER WORD
| NUMBER LESS WORD
| REDIR_WORD GREATER WORD
| REDIR_WORD LESS WORD
| GREATER_GREATER WORD
| NUMBER GREATER_GREATER WORD
| REDIR_WORD GREATER_GREATER WORD
| GREATER_BAR WORD
| NUMBER GREATER_BAR WORD
| REDIR_WORD GREATER_BAR WORD
| LESS_GREATER WORD
| NUMBER LESS_GREATER WORD
| REDIR_WORD LESS_GREATER WORD
| LESS_LESS_LESS WORD
| NUMBER LESS_LESS_LESS WORD
| REDIR_WORD LESS_LESS_LESS WORD
| LESS_AND NUMBER
| NUMBER LESS_AND NUMBER
| REDIR_WORD LESS_AND NUMBER
| GREATER_AND NUMBER
| NUMBER GREATER_AND NUMBER
| REDIR_WORD GREATER_AND NUMBER
| LESS_AND WORD
| NUMBER LESS_AND WORD
| REDIR_WORD LESS_AND WORD
| GREATER_AND WORD
| NUMBER GREATER_AND WORD
| REDIR_WORD GREATER_AND WORD
| GREATER_AND DASH
| NUMBER GREATER_AND DASH
| REDIR_WORD GREATER_AND DASH
| LESS_AND DASH
| NUMBER LESS_AND DASH
| REDIR_WORD LESS_AND DASH
| AND_GREATER WORD
| AND_GREATER_GREATER WORD'''
parserobj = p.context
if len(p) == 3:
output = p[2]
if p.slice[2].ttype == tokenizer.tokentype.WORD:
output = _expandword(parserobj, p.slice[2])
p[0] = bast.node(kind='redirect',
input=None,
type=p[1],
heredoc=None,
output=output,
pos=(p.lexpos(1), p.endlexpos(2)))
else:
output = p[3]
if p.slice[3].ttype == tokenizer.tokentype.WORD:
output = _expandword(parserobj, p.slice[3])
p[0] = bast.node(kind='redirect',
input=p[1],
type=p[2],
heredoc=None,
output=output,
pos=(p.lexpos(1), p.endlexpos(3)))
def p_shell_command(p):
'''shell_command : for_command
| case_command
| WHILE compound_list DO compound_list DONE
| UNTIL compound_list DO compound_list DONE
| select_command
| if_command
| subshell
| group_command
| arith_command
| cond_command
| arith_for_command'''
if len(p) == 2:
p[0] = p[1]
else:
# while or until
assert p[2].kind == 'list'
parts = _makeparts(p)
kind = parts[0].word
assert kind in ('while', 'until')
p[0] = bast.node(
kind='compound',
redirects=[],
list=[bast.node(kind=kind, parts=parts, pos=_partsspan(parts))],
pos=_partsspan(parts))
assert p[0].kind == 'compound'
def _expandword(parser, tokenword):
if parser._expansionlimit == -1:
# we enter this branch in the following conditions:
# - currently parsing a substitution as a result of an expansion
# - the previous expansion had limit == 0
#
# this means that this node is a descendant of a substitution in an
# unexpanded word and will be filtered in the limit == 0 condition below
#
# (the reason we even expand when limit == 0 is to get quote removal)
node = bast.node(kind='word',
word=tokenword,
pos=(tokenword.lexpos, tokenword.endlexpos),
parts=[])
return node
else:
quoted = bool(tokenword.flags & flags.word.QUOTED)
doublequoted = quoted and tokenword.value[0] == '"'
# TODO set qheredocument
parts, expandedword = subst._expandwordinternal(
parser, tokenword, 0, doublequoted, 0, 0)
# limit reached, don't include substitutions (still expanded to get
# quote removal though)
if parser._expansionlimit == 0:
parts = [node for node in parts if 'substitution' not in node.kind]
node = bast.node(kind='word',
word=expandedword,
pos=(tokenword.lexpos, tokenword.endlexpos),
parts=parts)
return node
def p_group_command(p):
'''group_command : LEFT_CURLY compound_list RIGHT_CURLY'''
lcurly = bast.node(kind='reservedword', word=p[1], pos=p.lexspan(1))
rcurly = bast.node(kind='reservedword', word=p[3], pos=p.lexspan(3))
parts = [lcurly, p[2], rcurly]
p[0] = bast.node(kind='compound',
list=parts,
redirects=[],
pos=_partsspan(parts))
def p_subshell(p):
'''subshell : LEFT_PAREN compound_list RIGHT_PAREN'''
lparen = bast.node(kind='reservedword', word=p[1], pos=p.lexspan(1))
rparen = bast.node(kind='reservedword', word=p[3], pos=p.lexspan(3))
parts = [lparen, p[2], rparen]
p[0] = bast.node(kind='compound',
list=parts,
redirects=[],
pos=_partsspan(parts))
def p_list0(p):
'''list0 : list1 NEWLINE newline_list
| list1 AMPERSAND newline_list
| list1 SEMICOLON newline_list'''
parts = p[1]
if len(parts) > 1 or p.slice[2].ttype != tokenizer.tokentype.NEWLINE:
parts.append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2)))
p[0] = bast.node(kind='list', parts=parts, pos=_partsspan(parts))
else:
p[0] = parts[0]
def p_if_command(p):
'''if_command : IF compound_list THEN compound_list FI
| IF compound_list THEN compound_list ELSE compound_list FI
| IF compound_list THEN compound_list elif_clause FI'''
# we currently don't distinguish the various lists that make up the
# command, because it's not needed later on. if there will be a need
# we can always add different nodes for elif/else.
parts = _makeparts(p)
p[0] = bast.node(
kind='compound',
redirects=[],
list=[bast.node(kind='if', parts=parts, pos=_partsspan(parts))],
pos=_partsspan(parts))
def _extractcommandsubst(parserobj, string, sindex, sxcommand=False):
if string[sindex] == '(':
raise NotImplementedError('arithmetic expansion')
#return _extractdelimitedstring(parserobj, string, sindex, '$(', '(', '(', sxcommand=True)
else:
node, si = _parsedolparen(parserobj, string, sindex)
si += 1
return bast.node(kind='commandsubstitution', command=node, pos=(sindex - 2, si)), si
def p_pipeline(p):
'''pipeline : pipeline BAR newline_list pipeline
| pipeline BAR_AND newline_list pipeline
| command'''
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = p[1]
p[0].append(bast.node(kind='pipe', pipe=p[2], pos=p.lexspan(2)))
p[0].extend(p[len(p) - 1])
def p_compound_list(p):
'''compound_list : list
| newline_list list1'''
if len(p) == 2:
p[0] = p[1]
else:
parts = p[2]
if len(parts) > 1:
p[0] = bast.node(kind='list', parts=parts, pos=_partsspan(parts))
else:
p[0] = parts[0]
def p_simple_list(p):
'''simple_list : simple_list1
| simple_list1 AMPERSAND
| simple_list1 SEMICOLON'''
tok = p.lexer
heredoc.gatherheredocuments(tok)
if len(p) == 3 or len(p[1]) > 1:
parts = p[1]
if len(p) == 3:
parts.append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2)))
p[0] = bast.node(kind='list', parts=parts, pos=_partsspan(parts))
else:
assert len(p[1]) == 1
p[0] = p[1][0]
if (len(p) == 2 and p.lexer._parserstate & flags.parser.CMDSUBST
and p.lexer._current_token.nopos() == p.lexer._shell_eof_token):
# accept the input
p.accept()
def p_elif_clause(p):
'''elif_clause : ELIF compound_list THEN compound_list
| ELIF compound_list THEN compound_list ELSE compound_list
| ELIF compound_list THEN compound_list elif_clause'''
parts = []
for i in range(1, len(p)):
if isinstance(p[i], bast.node):
parts.append(p[i])
else:
parts.append(
bast.node(kind='reservedword', word=p[i], pos=p.lexspan(i)))
p[0] = parts
def p_simple_list1(p):
'''simple_list1 : simple_list1 AND_AND newline_list simple_list1
| simple_list1 OR_OR newline_list simple_list1
| simple_list1 AMPERSAND simple_list1
| simple_list1 SEMICOLON simple_list1
| pipeline_command'''
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = p[1]
p[0].append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2)))
p[0].extend(p[len(p) - 1])
def p_function_def(p):
'''function_def : WORD LEFT_PAREN RIGHT_PAREN newline_list function_body
| FUNCTION WORD LEFT_PAREN RIGHT_PAREN newline_list function_body
| FUNCTION WORD newline_list function_body'''
parts = _makeparts(p)
body = parts[-1]
name = parts[bast.findfirstkind(parts, 'word')]
p[0] = bast.node(kind='function',
name=name,
body=body,
parts=parts,
pos=_partsspan(parts))
def p_for_command(p):
'''for_command : FOR WORD newline_list DO compound_list DONE
| FOR WORD newline_list LEFT_CURLY compound_list RIGHT_CURLY
| FOR WORD SEMICOLON newline_list DO compound_list DONE
| FOR WORD SEMICOLON newline_list LEFT_CURLY compound_list RIGHT_CURLY
| FOR WORD newline_list IN word_list list_terminator newline_list DO compound_list DONE
| FOR WORD newline_list IN word_list list_terminator newline_list LEFT_CURLY compound_list RIGHT_CURLY
| FOR WORD newline_list IN list_terminator newline_list DO compound_list DONE
| FOR WORD newline_list IN list_terminator newline_list LEFT_CURLY compound_list RIGHT_CURLY'''
parts = _makeparts(p)
# find the operatornode that we might have there due to
# list_terminator/newline_list and convert it to a reservedword so its
# considered as part of the for loop
for i, part in enumerate(parts):
if part.kind == 'operator' and part.op == ';':
parts[i] = bast.node(kind='reservedword', word=';', pos=part.pos)
break # there could be only one in there...
p[0] = bast.node(
kind='compound',
redirects=[],
list=[bast.node(kind='for', parts=parts, pos=_partsspan(parts))],
pos=_partsspan(parts))
def p_list1(p):
'''list1 : list1 AND_AND newline_list list1
| list1 OR_OR newline_list list1
| list1 AMPERSAND newline_list list1
| list1 SEMICOLON newline_list list1
| list1 NEWLINE newline_list list1
| pipeline_command'''
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = p[1]
# XXX newline
p[0].append(bast.node(kind='operator', op=p[2], pos=p.lexspan(2)))
p[0].extend(p[len(p) - 1])
def _paramexpand(parserobj, string, sindex):
node = None
zindex = sindex + 1
c = string[zindex] if zindex < len(string) else None
if c and c in '0123456789$#?-!*@':
# XXX 7685
node = bast.node(kind='parameter', value=c,
pos=(sindex, zindex+1))
elif c == '{':
# XXX 7863
# TODO not start enough, doesn't consider escaping
zindex = string.find('}', zindex + 1)
node = bast.node(kind='parameter', value=string[sindex + 2:zindex],
pos=(sindex, zindex+1))
# TODO
# return _parameterbraceexpand(string, zindex)
elif c == '(':
return _extractcommandsubst(parserobj, string, zindex + 1)
elif c == '[':
raise NotImplementedError('arithmetic substitution')
#return _extractarithmeticsubst(string, zindex + 1)
else:
tindex = zindex
for zindex in range(tindex, len(string) + 1):
if zindex == len(string):
break
if not string[zindex].isalnum() and not string[zindex] == '_':
break
temp1 = string[sindex:zindex]
if temp1:
return (bast.node(kind='parameter', value=temp1[1:], pos=(sindex, zindex)),
zindex)
if zindex < len(string):
zindex += 1
return node, zindex
def p_command(p):
'''command : simple_command
| shell_command
| shell_command redirection_list
| function_def
| coproc'''
if isinstance(p[1], bast.node):
p[0] = p[1]
if len(p) == 3:
assert p[0].kind == 'compound'
p[0].redirects.extend(p[2])
assert p[0].pos[0] < p[0].redirects[-1].pos[1]
p[0].pos = (p[0].pos[0], p[0].redirects[-1].pos[1])
else:
p[0] = bast.node(kind='command', parts=p[1], pos=_partsspan(p[1]))
def p_pipeline_command(p):
'''pipeline_command : pipeline
| BANG pipeline_command
| timespec pipeline_command
| timespec list_terminator
| BANG list_terminator'''
if len(p) == 2:
if len(p[1]) == 1:
p[0] = p[1][0]
else:
p[0] = bast.node(kind='pipeline',
parts=p[1],
pos=(p[1][0].pos[0], p[1][-1].pos[1]))
else:
# XXX timespec
node = bast.node(kind='reservedword', word='!', pos=p.lexspan(1))
if p[2].kind == 'pipeline':
p[0] = p[2]
p[0].parts.insert(0, node)
p[0].pos = (p[0].parts[0].pos[0], p[0].parts[-1].pos[1])
else:
p[0] = bast.node(kind='pipeline',
parts=[node, p[2]],
pos=(node.pos[0], p[2].pos[1]))
def makeheredoc(tokenizer, redirnode, lineno, killleading):
# redirword = string_quote_removal(redirectnode.word)
redirword = redirnode.output.word
document = []
startpos = tokenizer._shell_input_line_index
#fullline = self.tok.readline(bool(redirword.output.flags & flags.word.QUOTED))
fullline = tokenizer.readline(False)
while fullline:
if killleading:
while fullline[0] == '\t':
fullline = fullline[1:]
if not fullline:
continue
if fullline[:-1] == redirword and fullline[len(redirword)] == '\n':
document.append(fullline[:-1])
# document_done
break
document.append(fullline)
#fullline = self.readline(bool(redirnode.flags & flags.word.QUOTED))
fullline = tokenizer.readline(False)
if not fullline:
raise errors.ParsingError(
"here-document at line %d delimited by end-of-file (wanted %r)" %
(lineno, redirword), tokenizer._shell_input_line,
tokenizer._shell_input_line_index)
document = ''.join(document)
endpos = tokenizer._shell_input_line_index - 1
assert hasattr(redirnode, 'heredoc')
redirnode.heredoc = bast.node(kind='heredoc',
value=document,
pos=(startpos, endpos))
# if the heredoc immediately follows this node, fix its end pos
if redirnode.pos[1] + 1 == startpos:
redirnode.pos = (redirnode.pos[0], endpos)
return document
def _makeparts(p):
parts = []
for i in range(1, len(p)):
if isinstance(p[i], bast.node):
parts.append(p[i])
elif isinstance(p[i], list):
parts.extend(p[i])
elif isinstance(p.slice[i], tokenizer.token):
if p.slice[i].ttype == tokenizer.tokentype.WORD:
parserobj = p.context
parts.append(_expandword(parserobj, p.slice[i]))
else:
parts.append(
bast.node(kind='reservedword', word=p[i],
pos=p.lexspan(i)))
else:
pass
return parts
def normalize_command(node, current=None):
bash_grammar = BashGrammar()
bash_grammar.name2type = bg.name2type
if not node or not node.parts:
return
input = node.parts
num_tokens = len(node.parts)
bast_node = input[0]
if bast_node.kind == 'assignment':
normalize(bast_node, current, 'assignment')
elif bast_node.kind == 'redirect':
normalize(bast_node, current, 'redirect')
elif bast_node.kind == 'commandsubstitution':
normalize(bast_node, current, 'commandsubstitution')
elif bast_node.kind == 'word' and not bast_node.parts:
token = normalize_word(bast_node)
head = UtilityNode(token,
parent=current,
lsb=current.get_right_child())
if current:
current.add_child(head)
# If utility grammar is not known, parse into a simple two-level tree
if not bg.consume(token):
raise errors.LintParsingError(
"Warning: grammar not found - utility {}".format(token),
num_tokens, 0)
for bast_node in input[1:]:
if bast_node.kind == 'word' and (
not bast_node.parts or
(bast_node.parts[0].kind == 'parameter'
and bast_node.word.startswith('-'))):
token = normalize_word(bast_node)
if token.startswith('-'):
child = FlagNode(token,
parent=head,
lsb=head.get_right_child())
else:
child = ArgumentNode(token,
arg_type='Unknown',
parent=head,
lsb=head.get_right_child())
head.add_child(child)
else:
normalize(bast_node, head)
return
current, i = head, 1
bash_grammar.grammar = {
head.value: copy.deepcopy(bg.grammar[head.value])
}
bash_grammar.consume(head.value)
while i < len(input):
bast_node = input[i]
# '--': signal the end of options
if bast_node.kind == 'word' and bast_node.word == '--':
op = OperatorNode('--',
parent=current,
lsb=current.get_right_child())
current.add_child(op)
bash_grammar.push('--', OPERATOR_S)
i += 1
continue
# examine each possible next states in order
matched = False
for next_state in bash_grammar.next_states:
if next_state.is_compound_flag():
# Next state is a flag
if bast_node.kind != 'word' or (
bast_node.parts and
not (bast_node.word.startswith('-') and
bast_node.parts[0].kind == 'parameter')):
continue
if is_parenthesis(bast_node, current):
flag = FlagNode(bast_node.word,
parent=current,
lsb=current.get_right_child())
current.add_child(flag)
matched = True
i += 1
break
elif is_unary_logic_op(bast_node, current):
flag = UnaryLogicOpNode(
bast_node.word,
parent=current,
lsb=current.get_right_child())
current.add_child(flag)
matched = True
i += 1
break
elif is_binary_logic_op(bast_node, current):
flag = BinaryLogicOpNode(
bast_node.word,
parent=current,
lsb=current.get_right_child())
current.add_child(flag)
matched = True
i += 1
break
else:
token = normalize_word(bast_node)
try:
result = bash_grammar.push(
token, COMPOUND_FLAG_S)
except ValueError as e:
raise errors.FlagError(e.args[0], num_tokens,
i)
if result:
for flag_token, flag_arg in result:
flag = FlagNode(
flag_token,
parent=current,
lsb=current.get_right_child())
current.add_child(flag)
if flag_arg == '__OPEN__':
# Incomplete AST, expecting flag argument
current = flag
elif flag_arg is not None:
# Argument is specified with flag
argument = ArgumentNode(
flag_arg[0],
arg_type=flag_arg[1],
parent=flag,
lsb=flag.get_right_child())
flag.add_child(argument)
matched = True
i += 1
break
elif next_state.is_command():
# Next state is a nested bash command
new_command_node = bast.node(kind="command",
word="",
parts=[],
pos=(-1, -1))
if next_state.type == ARG_COMMAND_S:
if bast_node.kind == 'word' and not bast_node.parts:
token = normalize_word(bast_node)
if constants.with_quotation(token):
subcommand = token[1:-1]
start_pos = bast_node.pos[0] + 1
tree = safe_bashlex_parse(
subcommand,
start_pos=start_pos,
verbose=verbose)
if tree is None:
raise errors.SubCommandError(
'Error in subcommand string: {}'.
format(token), num_tokens, i)
normalize(tree[0], current)
bash_grammar.push(token, next_state.type)
i += 1
else:
continue
else:
normalize(bast_node, current, 'command')
i += 1
elif next_state.type == EXEC_COMMAND_S:
new_input = []
j = i
while j < len(input):
if hasattr(input[j], 'word') and \
input[j].word in next_state.stop_tokens:
break
else:
new_input.append(input[j])
j += 1
new_command_node.parts = new_input
normalize_command(new_command_node, current)
if j < len(input):
current.value += ('::' + input[j].word)
bash_grammar.push(input[j], EXEC_COMMAND_S)
else:
if verbose:
print(
"Warning: -exec missing stop token - ; added"
)
current.value += ('::' + ';')
bash_grammar.push(';', EXEC_COMMAND_S)
i = j + 1
else:
# Interpret all of the rest of the tokens as content of the nested command
new_command_node.parts = input[i:]
normalize_command(new_command_node, current)
bash_grammar.push('', next_state.type)
i = len(input)
current = current.utility
matched = True
break
elif next_state.is_argument():
# Next state is an argument
if bast_node.kind == 'word' and not bast_node.parts:
token = normalize_word(bast_node)
if next_state.is_list and next_state.list_separator != ' ':
list_separator = next_state.list_separator
argument = ArgumentNode(
token,
arg_type=next_state.arg_type,
parent=current,
lsb=current.get_right_child(),
list_members=token.split(list_separator),
list_separator=list_separator)
else:
argument = ArgumentNode(
token,
arg_type=next_state.arg_type,
parent=current,
lsb=current.get_right_child())
current.add_child(argument)
status = bash_grammar.push(token, ARG_S)
else:
normalize(bast_node, current, next_state.arg_type)
status = bash_grammar.push('', ARG_S)
if status != '__SAME_PARENT__':
current = current.utility
i += 1
matched = True
break
if not matched:
if bast_node.kind == 'redirect' or bast_node.kind == 'operator':
i += 1
matched = True
else:
raise errors.LintParsingError('Unmatched token',
num_tokens, i)
if bash_grammar.allow_eof():
post_process_command(head)
return
else:
raise errors.LintParsingError('Incomplete command', num_tokens,
i)
else:
if bast_node.parts:
normalize(bast_node, current)
else:
raise errors.LintParsingError(
'Utility needs to be a BAST node of "Word" type" {}'.
format(bast_node), num_tokens, 0)
def _expandwordinternal(parserobj, wordtoken, qheredocument, qdoublequotes, quoted, isexp):
# bash/subst.c L8132
istring = ''
parts = []
tindex = [0]
sindex = [0]
string = wordtoken.value
def nextchar():
sindex[0] += 1
if sindex[0] < len(string):
return string[sindex[0]]
def peekchar():
if sindex[0]+1 < len(string):
return string[sindex[0]+1]
while True:
if sindex[0] == len(string):
break
# goto finished_with_string
c = string[sindex[0]]
if c in '<>':
if (nextchar() != '(' or qheredocument or qdoublequotes or
(wordtoken.flags & set([flags.word.DQUOTE, flags.word.NOPROCSUB]))):
sindex[0] -= 1
# goto add_character
sindex[0] += 1
istring += c
else:
tindex = sindex[0] + 1
node, sindex[0] = _extractprocesssubst(parserobj, string, tindex)
parts.append(bast.node(kind='processsubstitution', command=node,
pos=(tindex - 2, sindex[0])))
istring += string[tindex - 2:sindex[0]]
# goto dollar_add_string
# TODO
# elif c == '=':
# pass
# elif c == ':':
# pass
elif c == '~':
if (wordtoken.flags & set([flags.word.NOTILDE, flags.word.DQUOTE]) or
(sindex[0] > 0 and not (wordtoken.flags & flags.word.NOTILDE)) or
qdoublequotes or qheredocument):
wordtoken.flags.clear()
wordtoken.flags.add(flags.word.ITILDE)
sindex[0] += 1
istring += c
else:
stopatcolon = wordtoken.flags & set([flags.word.ASSIGNRHS,
flags.word.ASSIGNMENT,
flags.word.TILDEEXP])
expand = True
for i in range(sindex[0], len(string)):
r = string[i]
if r == '/':
break
if r in "\\'\"":
expand = False
break
if stopatcolon and r == ':':
break
else:
# go one past the end if we didn't exit early
i += 1
if i > sindex[0] and expand:
node = bast.node(kind='tilde', value=string[sindex[0]:i],
pos=(sindex[0], i))
parts.append(node)
istring += string[sindex[0]:i]
sindex[0] = i
elif c == '$' and len(string) > 1:
tindex = sindex[0]
node, sindex[0] = _paramexpand(parserobj, string, sindex[0])
if node:
parts.append(node)
istring += string[tindex:sindex[0]]
elif c == '`':
tindex = sindex[0]
# bare instance of ``
if nextchar() == '`':
sindex[0] += 1
istring += '``'
else:
x = _stringextract(string, sindex[0], "`")
if x == -1:
raise errors.ParsingError('bad substitution: no closing "`" '
'in %s' % string)
else:
if wordtoken.flags & flags.word.NOCOMSUB:
pass
else:
sindex[0] = x
word = string[tindex+1:sindex[0]]
command, ttindex = _recursiveparse(parserobj, word, 0)
_adjustpositions(command, tindex+1, len(string))
ttindex += 1 # ttindex is on the closing char
# assert sindex[0] == ttindex
# go one past the closing `
sindex[0] += 1
node = bast.node(kind='commandsubstitution',
command=command,
pos=(tindex, sindex[0]))
parts.append(node)
istring += string[tindex:sindex[0]]
elif c == '\\':
istring += string[sindex[0]+1:sindex[0]+2]
sindex[0] += 2
elif c == '"':
sindex[0] += 1
continue
# 8513
#if qdoublequotes or qheredocument:
# sindex[0] += 1
#else:
# tindex = sindex[0] + 1
# parts, sindex[0] = _stringextractdoublequoted(string, sindex[0])
# if tindex == 1 and sindex[0] == len(string):
# quotedstate = 'wholly'
# else:
# quotedstate = 'partially'
elif c == "'":
# entire string surronded by single quotes, no expansion is
# going to happen
if sindex[0] == 0 and string[-1] == "'":
return [], string[1:-1]
# check if we're inside double quotes
if not qdoublequotes:
# look for the closing ', we know we have one or otherwise
# this wouldn't tokenize due to unmatched '
tindex = sindex[0]
sindex[0] = string.find("'", sindex[0]) + 1
istring += string[tindex+1:sindex[0]-1]
else:
# this is a single quote inside double quotes, add it
istring += c
sindex[0] += 1
else:
istring += string[sindex[0]:sindex[0]+1]
sindex[0] += 1
if parts:
class v(bast.nodevisitor):
def visitnode(self, node):
assert node.pos[1] + wordtoken.lexpos <= wordtoken.endlexpos
node.pos = (node.pos[0] + wordtoken.lexpos,
node.pos[1] + wordtoken.lexpos)
visitor = v()
for node in parts:
visitor.visit(node)
return parts, istring
def p_list_terminator(p):
'''list_terminator : NEWLINE
| SEMICOLON
| EOF'''
if p[1] == ';':
p[0] = bast.node(kind='operator', op=';', pos=p.lexspan(1))
