class TarExtractor(object):
def __init__(self, timeout=None):
self.timeout = timeout
self.tmp_dir = None
self.created_tmp_dir = False
TAR_FLAGS = {
"application/x-xz": "-J",
"application/x-gzip": "-I igzip" if which("igzip") else "-z",
"application/gzip": "-I igzip" if which("igzip") else "-z",
"application/x-bzip2": "-j",
"application/x-tar": ""
}
def _tar_flag_for_content_type(self, content_type):
flag = self.TAR_FLAGS.get(content_type)
if flag is None:
raise InvalidContentType(content_type)
return flag
def from_path(self, path, extract_dir=None, content_type=None):
if os.path.isdir(path):
self.tmp_dir = path
else:
self.content_type = content_type or content_type_from_file(path)
tar_flag = self._tar_flag_for_content_type(self.content_type)
self.tmp_dir = tempfile.mkdtemp(prefix="insights-",
dir=extract_dir)
self.created_tmp_dir = True
command = "tar --delay-directory-restore %s -x --exclude=*/dev/null -f %s -C %s" % (
tar_flag, path, self.tmp_dir)
logging.debug("Extracting files in '%s'", self.tmp_dir)
subproc.call(command, timeout=self.timeout)
return self
def validate(self):
if not blacklist.allow_command(self.cmd):
log.warning("WARNING: Skipping command %s", self.cmd)
raise dr.SkipComponent()
if not which(shlex.split(self.cmd)[0], env=self.create_env()):
raise ContentException("Couldn't execute: %s" % self.cmd)
def __init__(self, *cmds, **kwargs):
"""
cmds (list): one or more commands. Each command will be shlex.split if
it isn't already split.
bufsize (int): size of the I/O buffers to use between processes running
the commands. -1 means to use system defaults. 0 means no buffering.
Defaults to -1.
env (dict): environment in which to execute commands. Defaults to
os.environ.
timeout (int): number of seconds to wait before killing the command.
Defaults to None, which waits forever.
signum (int): signal to send the command on timeout. Defaults to
signal.SIGKILL
"""
self.bufsize = kwargs.get("bufsize", -1)
self.env = kwargs.get("env", os.environ)
timeout = kwargs.get("timeout")
signum = kwargs.get("signum", signal.SIGKILL)
cmds = [shlex.split(c) if not isinstance(c, list) else c for c in cmds]
timeout_command = which("timeout", env=self.env)
if timeout:
if timeout_command:
to = shlex.split("timeout -s {0} {1}".format(signum, timeout))
to.extend(cmds[0])
cmds[0] = to
else:
# TODO: Should this raise an exception instead?
log.warn("Timeout specified but timeout command unavailable.")
self.cmds = cmds
def write(self, dst):
fs.ensure_path(os.path.dirname(dst))
args = self.create_args()
if args:
p = Pipeline(*args, env=SAFE_ENV)
p.write(dst)
else:
call([which("cp", env=SAFE_ENV), self.path, dst], env=SAFE_ENV)
def stream(command, stdin=None, env=os.environ, timeout=None):
"""
Yields a generator of a command's output. For line oriented commands only.
Args:
command (str or list): a command without pipes. If it's not a list,
``shlex.split`` is applied.
stdin (file like object): stream to use as the command's standard input.
env (dict): The environment in which to execute the command. PATH should
be defined.
timeout (int): Amount of time in seconds to give the command to complete.
The ``timeout`` utility must be installed to use this feature.
Yields:
The output stream for the command. It should typically be wrapped in a
``reader``.
"""
if not isinstance(command, list):
command = shlex.split(command)
cmd = which(command[0])
if cmd is None:
path = env.get("PATH", "")
raise Exception("Command [%s] not in PATH [%s]" % (command[0], path))
command[0] = cmd
if timeout:
if not timeout_command[0]:
raise Exception(
"Timeout specified but timeout command not available.")
command = timeout_command + [str(timeout)] + command
output = None
try:
output = Popen(command, env=env, stdin=stdin, **stream_options)
yield output.stdout
finally:
if output:
output.wait()
from insights.util import which
stream_options = {
"bufsize": -1, # use OS defaults. Non buffered if not set.
"universal_newlines": True, # convert all to "\n"
"stdout": PIPE, # pipe to Popen.stdout instead of literally stdout
"stderr": STDOUT # redirect stderr to stdout for all processes
}
def reader(stream):
for line in stream:
yield line.rstrip("\n")
timeout_command = [which("timeout"), "-s", str(signal.SIGKILL)]
@contextmanager
def stream(command, stdin=None, env=os.environ, timeout=None):
"""
Yields a generator of a command's output. For line oriented commands only.
Args:
command (str or list): a command without pipes. If it's not a list,
``shlex.split`` is applied.
stdin (file like object): stream to use as the command's standard input.
env (dict): The environment in which to execute the command. PATH should
be defined.
timeout (int): Amount of time in seconds to give the command to complete.
The ``timeout`` utility must be installed to use this feature.
def write(self, dst):
fs.ensure_path(os.path.dirname(dst))
call([which("cp", env=SAFE_ENV), self.path, dst], env=SAFE_ENV)
def validate(self):
if not blacklist.allow_command(self.cmd):
raise dr.SkipComponent()
if not which(shlex.split(self.cmd)[0]):
raise ContentException("Couldn't execute: %s" % self.cmd)
