def testPipeline2(self):
Banner('ls | cut -d . -f 1 | head')
p = process.Pipeline()
p.Add(_ExtProc(['ls']))
p.Add(_ExtProc(['cut', '-d', '.', '-f', '1']))
p.Add(_ExtProc(['head']))
print(p.Run(_WAITER))
ex = InitExecutor()
# Simulating subshell for each command
w1 = ast.CompoundWord()
w1.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls')))
node1 = ast.SimpleCommand()
node1.words = [w1]
w2 = ast.CompoundWord()
w2.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, 'head')))
node2 = ast.SimpleCommand()
node2.words = [w2]
w3 = ast.CompoundWord()
w3.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, 'sort')))
w4 = ast.CompoundWord()
w4.parts.append(ast.LiteralPart(ast.token(Id.Lit_Chars, '--reverse')))
node3 = ast.SimpleCommand()
node3.words = [w3, w4]
p = process.Pipeline()
p.Add(Process(process.SubProgramThunk(ex, node1)))
p.Add(Process(process.SubProgramThunk(ex, node2)))
p.Add(Process(process.SubProgramThunk(ex, node3)))
print(p.Run(_WAITER))
def _MakeSimpleCommand(self, prefix_bindings, suffix_words, redirects):
# FOO=(1 2 3) ls is not allowed
for k, v, _ in prefix_bindings:
if word.HasArrayPart(v):
self.AddErrorContext(
'Unexpected array literal in binding: %s', v, word=v)
return None
# echo FOO=(1 2 3) is not allowed
# NOTE: Other checks can be inserted here. Can resolve builtins,
# functions, aliases, static PATH, etc.
for w in suffix_words:
kv = word.LooksLikeAssignment(w)
if kv:
k, v = kv
if word.HasArrayPart(v):
self.AddErrorContext('Unexpected array literal: %s', v, word=v)
return None
words2 = self._BraceExpand(suffix_words)
# NOTE: Must do tilde detection after brace expansion, e.g.
# {~bob,~jane}/src should work, even though ~ isn't the leading character
# of the initial word.
words3 = self._TildeDetectAll(words2)
node = ast.SimpleCommand()
node.words = words3
node.redirects = redirects
for name, val, left_spid in prefix_bindings:
pair = ast.env_pair(name, val)
pair.spids.append(left_spid)
node.more_env.append(pair)
return node
def _MakeSimpleCommand(self, prefix_bindings, suffix_words, redirects):
# FOO=(1 2 3) ls is not allowed
for k, _, v, _ in prefix_bindings:
if word.HasArrayPart(v):
self.AddErrorContext(
'Unexpected array literal in binding: %s', v, word=v)
return None
# echo FOO=(1 2 3) is not allowed
# NOTE: Other checks can be inserted here. Can resolve builtins,
# functions, aliases, static PATH, etc.
for w in suffix_words:
kov = word.LooksLikeAssignment(w)
if kov:
_, _, v = kov
if word.HasArrayPart(v):
self.AddErrorContext('Unexpected array literal: %s', v, word=w)
return None
# NOTE: # In bash, {~bob,~jane}/src works, even though ~ isn't the leading
# character of the initial word.
# However, this means we must do tilde detection AFTER brace EXPANSION, not
# just after brace DETECTION like we're doing here.
# The BracedWordTree instances have to be expanded into CompoundWord instances
# for the tilde detection to work.
words2 = braces.BraceDetectAll(suffix_words)
words3 = word.TildeDetectAll(words2)
node = ast.SimpleCommand()
node.words = words3
node.redirects = redirects
for name, op, val, left_spid in prefix_bindings:
if op != assign_op_e.Equal:
# NOTE: Using spid of RHS for now, since we don't have one for op.
self.AddErrorContext('Expected = in environment binding, got +=',
word=val)
return None
pair = ast.env_pair(name, val)
pair.spids.append(left_spid)
node.more_env.append(pair)
return node
def ParseSimpleCommand(self):
"""
Fixed transcription of the POSIX grammar (TODO: port to grammar/Shell.g)
io_file : '<' filename
| LESSAND filename
...
io_here : DLESS here_end
| DLESSDASH here_end
redirect : IO_NUMBER (io_redirect | io_here)
prefix_part : ASSIGNMENT_WORD | redirect
cmd_part : WORD | redirect
assign_kw : Declare | Export | Local | Readonly
# Without any words it is parsed as a command, not an assigment
assign_listing : assign_kw
# Now we have something to do (might be changing assignment flags too)
# NOTE: any prefixes should be a warning, but they are allowed in shell.
assignment : prefix_part* assign_kw (WORD | ASSIGNMENT_WORD)+
# an external command, a function call, or a builtin -- a "word_command"
word_command : prefix_part* cmd_part+
simple_command : assign_listing
| assignment
| proc_command
Simple imperative algorithm:
1) Read a list of words and redirects. Append them to separate lists.
2) Look for the first non-assignment word. If it's declare, etc., then
keep parsing words AND assign words. Otherwise, just parse words.
3) If there are no non-assignment words, then it's a global assignment.
{ redirects, global assignments } OR
{ redirects, prefix_bindings, words } OR
{ redirects, ERROR_prefix_bindings, keyword, assignments, words }
THEN CHECK that prefix bindings don't have any array literal parts!
global assignment and keyword assignments can have the of course.
well actually EXPORT shouldn't have them either -- WARNING
3 cases we want to warn: prefix_bindings for assignment, and array literal
in prefix bindings, or export
A command can be an assignment word, word, or redirect on its own.
ls
>out.txt
>out.txt FOO=bar # this touches the file, and hten
Or any sequence:
ls foo bar
<in.txt ls foo bar >out.txt
<in.txt ls >out.txt foo bar
Or add one or more environment bindings:
VAR=val env
>out.txt VAR=val env
here_end vs filename is a matter of whether we test that it's quoted. e.g.
<<EOF vs <<'EOF'.
"""
result = self._ScanSimpleCommand()
if not result: return None
redirects, words = result
if not words: # e.g. >out.txt # redirect without words
node = ast.SimpleCommand()
node.redirects = redirects
return node
prefix_bindings, suffix_words = self._SplitSimpleCommandPrefix(words)
if not suffix_words: # ONE=1 TWO=2 (with no other words)
if redirects:
binding1 = prefix_bindings[0]
_, _, _, spid = binding1
self.AddErrorContext('Got redirects in global assignment',
span_id=spid)
return None
pairs = []
for lhs, op, rhs, spid in prefix_bindings:
p = ast.assign_pair(ast.LhsName(lhs), op, rhs)
p.spids.append(spid)
pairs.append(p)
node = ast.Assignment(Id.Assign_None, [], pairs)
left_spid = word.LeftMostSpanForWord(words[0])
node.spids.append(left_spid) # no keyword spid to skip past
return node
kind, kw_token = word.KeywordToken(suffix_words[0])
if kind == Kind.Assign:
# Here we StaticEval suffix_words[1] to see if it's a command like
# 'typeset -p'. Then it becomes a SimpleCommand node instead of an
# Assignment. Note we're not handling duplicate flags like 'typeset
# -pf'. I see this in bashdb (bash debugger) but it can just be changed
# to 'typeset -p -f'.
is_command = False
if len(suffix_words) > 1:
ok, val, _ = word.StaticEval(suffix_words[1])
if ok and (kw_token.id, val) in self._ASSIGN_COMMANDS:
is_command = True
if is_command: # declare -f, declare -p, typeset -p, etc.
node = self._MakeSimpleCommand(prefix_bindings, suffix_words,
redirects)
return node
else: # declare str='', declare -a array=()
if redirects:
# Attach the error location to the keyword. It would be more precise
# to attach it to the
self.AddErrorContext('Got redirects in assignment', token=kw_token)
return None
if prefix_bindings: # FOO=bar local spam=eggs not allowed
# Use the location of the first value. TODO: Use the whole word before
# splitting.
_, _, v0, _ = prefix_bindings[0]
self.AddErrorContext(
'Invalid prefix bindings in assignment: %s', prefix_bindings,
word=v0)
return None
node = self._MakeAssignment(kw_token.id, suffix_words)
if not node: return None
node.spids.append(kw_token.span_id)
return node
elif kind == Kind.ControlFlow:
if redirects:
self.AddErrorContext('Got redirects in control flow: %s', redirects)
return None
if prefix_bindings: # FOO=bar local spam=eggs not allowed
# Use the location of the first value. TODO: Use the whole word before
# splitting.
_, _, v0, _ = prefix_bindings[0]
self.AddErrorContext(
'Invalid prefix bindings in control flow: %s', prefix_bindings,
word=v0)
return None
# Attach the token for errors. (Assignment may not need it.)
if len(suffix_words) == 1:
arg_word = None
elif len(suffix_words) == 2:
arg_word = suffix_words[1]
else:
self.AddErrorContext('Too many arguments')
return None
return ast.ControlFlow(kw_token, arg_word)
else:
node = self._MakeSimpleCommand(prefix_bindings, suffix_words, redirects)
return node
def ParseSimpleCommand(self):
"""
Fixed transcription of the POSIX grammar (TODO: port to grammar/Shell.g)
io_file : '<' filename
| LESSAND filename
...
io_here : DLESS here_end
| DLESSDASH here_end
redirect : IO_NUMBER (io_redirect | io_here)
prefix_part : ASSIGNMENT_WORD | redirect
cmd_part : WORD | redirect
assign_kw : Declare | Export | Local | Readonly
# Without any words it is parsed as a command, not an assigment
assign_listing : assign_kw
# Now we have something to do (might be changing assignment flags too)
# NOTE: any prefixes should be a warning, but they are allowed in shell.
assignment : prefix_part* assign_kw (WORD | ASSIGNMENT_WORD)+
# an external command, a function call, or a builtin -- a "word_command"
word_command : prefix_part* cmd_part+
simple_command : assign_listing
| assignment
| proc_command
Simple imperative algorithm:
1) Read a list of words and redirects. Append them to separate lists.
2) Look for the first non-assignment word. If it's declare, etc., then
keep parsing words AND assign words. Otherwise, just parse words.
3) If there are no non-assignment words, then it's a global assignment.
{ redirects, global assignments } OR
{ redirects, prefix_bindings, words } OR
{ redirects, ERROR_prefix_bindings, keyword, assignments, words }
THEN CHECK that prefix bindings don't have any array literal parts!
global assignment and keyword assignments can have the of course.
well actually EXPORT shouldn't have them either -- WARNING
3 cases we want to warn: prefix_bindings for assignment, and array literal
in prefix bindings, or export
A command can be an assignment word, word, or redirect on its own.
ls
>out.txt
>out.txt FOO=bar # this touches the file, and hten
Or any sequence:
ls foo bar
<in.txt ls foo bar >out.txt
<in.txt ls >out.txt foo bar
Or add one or more environment bindings:
VAR=val env
>out.txt VAR=val env
here_end vs filename is a matter of whether we test that it's quoted. e.g.
<<EOF vs <<'EOF'.
"""
result = self._ScanSimpleCommand()
if not result: return None
redirects, words = result
if not words: # e.g. >out.txt # redirect without words
node = ast.SimpleCommand()
node.redirects = redirects
return node
prefix_bindings, suffix_words = self._SplitSimpleCommandPrefix(words)
if not suffix_words: # ONE=1 TWO=2 (with no other words)
# TODO: Have a strict mode to prevent this?
if redirects: # >out.txt g=foo
print('WARNING: Got redirects in assignment: %s', redirects)
pairs = []
for lhs, rhs, spid in prefix_bindings:
p = ast.assign_pair(ast.LeftVar(lhs), rhs)
p.spids.append(spid)
pairs.append(p)
node = ast.Assignment(Id.Assign_None, pairs)
left_spid = word.LeftMostSpanForWord(words[0])
node.spids.append(left_spid) # no keyword spid to skip past
return node
assign_kw, keyword_spid = word.AssignmentBuiltinId(suffix_words[0])
if assign_kw == Id.Undefined_Tok:
node = self._MakeSimpleCommand(prefix_bindings, suffix_words, redirects)
return node
if redirects:
# TODO: Make it a warning, or do it in the second stage?
print(
'WARNING: Got redirects in assignment: %s' % redirects, file=sys.stderr)
if prefix_bindings: # FOO=bar local spam=eggs not allowed
# Use the location of the first value. TODO: Use the whole word before
# splitting.
_, v0, _ = prefix_bindings[0]
self.AddErrorContext(
'Invalid prefix bindings in assignment: %s', prefix_bindings,
word=v0)
return None
node = self._MakeAssignment(assign_kw, suffix_words)
if not node: return None
node.spids.append(keyword_spid)
return node