def substring_expansion(self, param_parts):
self.write_debug("Performing substring expansion '%s'" % param_parts,
"substring_expansion")
# get length, offset and end
if len(param_parts) == 3:
# ${parameter:offset:length}
param = param_parts[0]
offset = int(param_parts[1])
length = int(param_parts[2])
if length >= 0:
end = offset + length
else:
end = length
elif len(param_parts) == 2:
# ${parameter:offset}
param = param_parts[0]
offset = int(param_parts[1])
end = None
else:
raise ValueError("Unknown substring expansion %s" %
":".join(param_parts))
# Get output
val = self.list_var_expansion(param)
if val:
# For lists, use array indexing
if type(val) == list:
out = pybash_helper.to_str(val[offset:end], single_line=True)
# For other types, convert to string and index using characters
else:
out = pybash_helper.to_str(val, single_line=True)[offset:end]
return out
else:
return ""
def expand_substitution_operators(self, param_parts):
self.write_debug(
"Performing substitution expansion '%s'" % param_parts,
"expand_substitution_operators")
# Operation is determined by the first character of the second element
# case: ${parameter:-word}
if param_parts[1][0] == '-':
# If parameter is unset or null, the expansion of word is substituted. Otherwise, the value of parameter is substituted.
if param_parts[0] in self.locals:
return pybash_helper.to_str(self.locals[param_parts[0]],
single_line=True)
else:
return param_parts[1][1:]
# case: ${parameter:=word}
elif param_parts[1][0] == '=':
# If parameter is unset or null, the expansion of word is assigned to parameter. The value of parameter is then substituted.
if param_parts[0] in self.locals:
return pybash_helper.to_str(self.locals[param_parts[0]],
single_line=True)
else:
out = param_parts[1][1:]
self.locals[param_parts[0]] = out
return out
# case: ${parameter:?word}
elif param_parts[1][0] == '?':
# If parameter is null or unset, the expansion of word (or a message to that effect if word is not present) is written to the standard error and the shell, if it is not interactive, exits. Otherwise, the value of parameter is substituted.
# TODO: this will only write to shell stderr
if param_parts[0] in self.locals:
return pybash_helper.to_str(self.locals[param_parts[0]],
single_line=True)
else:
if len(param_parts[1]) > 1:
self.stderr_write("pybash: %s: %s" %
(param_parts[0], param_parts[1][1:]))
else:
self.stderr_write("pybash: %s: parameter null or not set" %
(param_parts[0]))
return ""
# case: ${parameter:+word}
elif param_parts[1][0] == '+':
# If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted.
if param_parts[0] in self.locals:
return param_parts[1][1:]
else:
return ""
# otherwise: This is a Substring Expansion
# ${parameter:offset} / ${parameter:offset:length}
else:
# - expands to up to "length" characters of the value of "parameter" starting at the character specified by "offset"
raise ValueError("Unknown substitution operation %s" %
":".join(param_parts))
def existing_var_expansion(self, param):
self.write_debug("Performing existing variable expansion '%s'" % param,
"existing_var_expansion")
if param.endswith('*') or param.endswith('@'):
prefix = param[1:-1]
matches = [key for key in self.locals if key.startswith(prefix)]
return pybash_helper.to_str(matches, single_line=True)
elif param.endswith('[@]') or param.endswith('[*]'):
val = self.list_var_expansion(param[1:])
if type(val) == list:
return pybash_helper.to_str([i for i in range(len(val))],
single_line=True)
else:
return "0"
else:
raise ValueError("Invalid prefix / list index expansion: '%s'" %
param)
def shell_data(var):
"""
Function to convert a python variable to a format compatible with shell commands
1. If input is None, return None
2. If input is a file, return the file
3. For other types, return pybash_helper.to_str(var)
Args:
var: Variable to be converted
Returns:
Converted variable
"""
if var is not None:
if pipe_util.isfile(var):
shell_data = var
else:
shell_data = pybash_helper.to_str(var)
else:
shell_data = None
return shell_data
def run_python_cmd(self,
cmd,
std_pipe=None,
input_var=None,
stdout=None,
stderr=None):
"""
Function to execute a python command, accepting an input variable and returning:
1. Result of python command
2. stdout of python command
3. stderr of python command
The python command is executed using exec(cmd, self.globals, self.local) in order to maintain
a separate "variable space" from the Pybash program.
Unlike run_shell_cmd(), run_python_cmd() is blocking. A future improvement could be to execute
the python command in a subprocess, and allow streaming data through std_pipe.
Args:
cmd (str): the shell command to execute
std_pipe (list): list used to pass [input_var, stdout, stderr] if all three
are available / required
input_var: The input variable to the python command, which acts similarily to stdin for
a shell command. This may be:
1. File-like object (open python file or pipe), which will be read and converted
to a string
2. Python variable that will be passed directly
3. None: no input is provided to the python command
stdout: The destination of the result(s) of the python command.
This may be:
1. File-like object (open python file or pipe), result(s) of python command will
be written here
2. subprocess.PIPE: a new collections.deque object will be created and result(s)
of python command will be written here
3. collections.deque object, result(s) of python command will be written here
4. None: stdout will be written to sys.stdout (i.e. the terminal)
stderr: The destination of the standard error generated by the python command.
This may be:
1. File-like object (open python file or pipe), stderr of python command will
be written here
2. subprocess.PIPE: a new collections.deque object will be created and stderr
of python command will be written here
3. collections.deque object, stderr of python command will be written here
4. None: stdout will be written to sys.stdout (i.e. the terminal)
Returns:
tuple: File-like objects for stdout, stderr (stdout, stderr, None)
.. note:: Since run_python_cmd() does not execute in a subprocess, no process is returned by this function.
stdout and stderr are collections.deque objects, which typically have only one element. Additional elements
are added by using redirects. For example, you can redirect the command's stderr to the stdout deque object.
The stdout deque may contain (one or both):
a) the result of evaluating a python statement (e.g. if cmd = '2+4', stdout = 6)
b) the stdout resulting from executing the python statement (e.g. if cmd = 'print("foo")', stdout = 'foo')
For example, if executing a function that contains a print() statement as well as returning a value, the
stdout deque will contain both the return value and the printed text.
The stderr deque will contain the errors generated by the python command (e.g. exception text)
"""
# Expand the std_pipe, handle redirects (create deque objects instead of os.pipe())
input_var, stdout, stderr = pipe_util.expand_std_pipe(
std_pipe, input_var, stdout, stderr)
self.write_debug(
"Expanded std_pipe: %s" % [type(input_var), stdout, stderr],
"run_python_cmd")
####################################################################
# Step 1) Get input data, initialize __inputvar__ and __outputvar__ in self.locals
# If the input_data is a file descriptor, read string
if input_var and pipe_util.isfile(input_var):
self.write_debug(
"Reading file object %s and storing as input_var" % input_var,
"run_python_cmd")
input_var = pipe_util.read_close_fd(input_var)
# Initialize the special __inputvar__ variable
# Python 3.x: Use globals so commands like '[@[f] for f in @] for'
# Python 2.x Use locals since this does not seem to be a problem
if sys.version_info >= (3, 0):
self.globals['__inputvar__'] = input_var
else:
self.locals['__inputvar__'] = input_var
# Initialize __outputvar__ to None in self.locals - may have been set by a previous cmd
self.locals['__outputvar__'] = None
####################################################################
# Step 2: Compile the command
# a) Check to see if the command references the input variable
if '@' in cmd:
cmd = cmd.replace("@", "__inputvar__")
# b) Check to see if this is command that can be assigned to a variable
# - there is probably a better way to do this without using a try-catch
# TODO: ugly nested try-catch
try:
assignment_cmd = "__outputvar__ = " + cmd
# self.write_debug("test: %s" % assignment_cmd, "run_python_cmd")
cmd_c = compile(assignment_cmd, '<pybash>', 'exec')
cmd = assignment_cmd
capture_output_var = True
except (SyntaxError, TypeError) as e:
# c) Attempt to compile original cmd
try:
cmd_c = compile(cmd, '<pybash>', 'exec')
capture_output_var = False
except (SyntaxError, TypeError) as e:
self.print_error("Could not compile: %s" % cmd)
self.stderr_write(str(e) + '\n')
self.stderr_write(traceback.format_exc())
return None, None, None
self.write_debug("Successfully compiled python: %s" % (cmd),
"run_python_cmd")
####################################################################
# Step 3: Execute the command
# a) Create StringIO for out/error
# - after doing this, don't print anything until out/err are restored!
if sys.version_info >= (3, 0):
out = StringIO()
err = StringIO()
else:
out = StringIO.StringIO()
err = StringIO.StringIO()
# Capture out/err
sys.stdout = out
sys.stderr = err
# b) run command in try-catch
# - if any errors occur, they will get added to std_pipe[2]
try:
exec(cmd_c, self.globals, self.locals)
except Exception as e:
sys.stderr.write(str(e) + '\n')
pass
# c) restore orig out/err
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
# get out/err values and close
stdout_val = out.getvalue()
stderr_val = err.getvalue()
out.close()
err.close()
self.write_debug("Restored stdout/stderr", "run_python_cmd")
####################################################################
# Step 4: Add output to pipe or print to sys.stdout/sys.stderr
# Define source lists for output + error pipes
# - __outputvar__ may not be defined if something went wrong
try:
stdout_src_list = [self.locals['__outputvar__'], stdout_val]
except UnboundLocalError:
stdout_src_list = [None, stdout_val]
stderr_src_list = [stderr_val]
# Define the mappings between the (name, output pipe, output print function, output source list)
output_mapping = [
("stdout", stdout, self.stdout_write, stdout_src_list),
("stderr", stderr, self.stderr_write, stderr_src_list)
]
# Process each output mapping
for name, pipe, print_fn, src_list in output_mapping:
# Process each source in the list
for src in src_list:
if not src:
continue
# If pipe is a deque, appendleft
if type(pipe) == deque:
self.write_debug("Adding src to %s queue" % name,
"run_python_cmd")
pipe.appendleft(src)
# If pipe is a file-like object, write to file
elif pipe_util.isfile(pipe):
self.write_debug(
"Writing %s src to file %s" % (name, pipe),
"run_python_cmd")
pipe.write(pybash_helper.to_str(src))
# If pipe is None, print using print function
elif pipe is None:
self.write_debug(
"Printing %s with function %s" % (name, print_fn),
"run_python_cmd")
print_fn(src)
# Otherwise, unrecognized pipe type
else:
raise ValueError(
"Unrecognized pipe type for %s: %s" % name, type(pipe))
# Close any open file handles and return output
# - there is no subprocess for python commands
output = [stdout, stderr, None]
for i in range(2):
if pipe_util.isfile(output[i]):
if output[i].closed is False:
output[i].close()
# Replace with None so that run_cmd() knows not to expect anything
output[i] = None
return output
def parameter_expansion(self, cmd):
"""
Function to perform bash-like parameter expansion
Args:
cmd (str): Command string which may contain bash-like parameters
Returns:
(str): Modified command string with expanded parameters
"""
# Loop until all parameter expansions have been processed
expansion_limit = 100
expansion_counter = 0
while True:
param_match = re.search(self.shell_param_regex, cmd)
param_brace_match = re.search(self.shell_param_brace_regex, cmd)
# If nothing left, break
if not param_match and not param_brace_match:
break
if expansion_counter > expansion_limit:
self.write_error("Parameter expansion limit reached")
break
try:
# First try parameter expansions in curly braces
if param_brace_match:
start, end = param_brace_match.start(
), param_brace_match.end()
# Get the full parameter to expand
param = param_brace_match.groups()[0]
self.write_debug(
"Performing brace parameter expansion for param %s" %
param, "parameter_expansion")
shvar_re = '[a-zA-Z0-9\[\]@\*]+?'
substitution_re = shvar_re + ':[-=\?\+]' + shvar_re
substring_re = shvar_re + ':' + shvar_re
substring_re2 = substring_re + ':' + shvar_re
# Check for ':' operator for substitution operators
if re.match(substitution_re, param):
out = self.expand_substitution_operators(
param.split(':'))
# Check for ':' operator for substring expansion
elif re.match(substring_re, param) or re.match(
substring_re2, param):
out = self.substring_expansion(param.split(':'))
# Check for '!' operator
elif param.startswith('!'):
out = self.existing_var_expansion(param)
# Check for '#' operator at start of param
elif param.startswith('#'):
out = self.length_expansion(param)
else:
# Default case: ${parameter} or ${paramter[index]}
var = self.list_var_expansion(param)
if var:
out = pybash_helper.to_str(var, single_line=True)
else:
out = ""
# Second try simple parameter expansion
if param_match:
start, end = param_match.start(), param_match.end()
param = param_match.groups()[0]
self.write_debug(
"Performing simple parameter expansion for param %s" %
param, "parameter_expansion")
if param in self.locals:
val = self.locals[param]
if type(val) == list:
if len(val) >= 1:
out = str(val[0])
else:
out = ""
else:
out = pybash_helper.to_str(self.locals[param],
single_line=True)
else:
out = ""
# Perform substitution
# TODO: this will overwrite any environment variables
cmd = cmd[0:start] + out + cmd[end:]
except Exception as e:
self.print_error('Invalid parameter expansion: %s' % cmd)
self.stderr_write(e)
self.stderr_write(traceback.format_exc())
break
return cmd