def __init__(self):
myRead, hisWrite = os.pipe()
hisRead, myWrite = os.pipe()
try:
# Some imports in vdsm assume /usr/share/vdsm is in your PYTHONPATH
env = os.environ.copy()
env['PYTHONPATH'] = "%s:%s" % (env.get("PYTHONPATH",
""), constants.P_VDSM)
env['PYTHONPATH'] = ":".join(
map(os.path.abspath, env['PYTHONPATH'].split(":")))
self.process = CPopen(
[constants.EXT_PYTHON, __file__,
str(hisRead),
str(hisWrite)],
close_fds=False,
env=env)
self.proxy = CrabRPCProxy(myRead, myWrite)
except:
os.close(myWrite)
os.close(myRead)
raise
finally:
os.close(hisRead)
os.close(hisWrite)
def execCmd(command, sudo=False, cwd=None, data=None, raw=False, logErr=True,
printable=None, env=None, sync=True, nice=None, ioclass=None,
ioclassdata=None, setsid=False, execCmdLogger=logging.root,
deathSignal=0, childUmask=None):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
if ioclass is not None:
cmd = command
command = [ioniceCmdPath.cmd, '-c', str(ioclass)]
if ioclassdata is not None:
command.extend(("-n", str(ioclassdata)))
command = command + cmd
if nice is not None:
command = [niceCmdPath.cmd, '-n', str(nice)] + command
if setsid:
command = [setsidCmdPath.cmd] + command
if sudo:
command = [sudoCmdPath.cmd, SUDO_NON_INTERACTIVE_FLAG] + command
if not printable:
printable = command
cmdline = repr(subprocess.list2cmdline(printable))
execCmdLogger.debug("%s (cwd %s)", cmdline, cwd)
p = CPopen(command, close_fds=True, cwd=cwd, env=env,
deathSignal=deathSignal, childUmask=childUmask)
p = AsyncProc(p)
if not sync:
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug("%s: <err> = %s; <rc> = %d",
{True: "SUCCESS", False: "FAILED"}[p.returncode == 0],
repr(err), p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return (p.returncode, out, err)
def testCwd(self):
cwd = "/proc"
p = CPopen(["python", "-c", "import os; print os.getcwd()"], cwd=cwd)
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read().strip(), cwd)
def testEcho(self):
data = "Hello"
p = CPopen([EXT_ECHO, "-n", data])
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read(), data)
def testEcho(self):
data = "Hello"
p = CPopen([EXT_ECHO, "-n", data])
out, err = p.communicate()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(out, data)
def testUserFileWithRedirect(self):
data = "it works!"
with tempfile.TemporaryFile() as f:
p = CPopen(["echo", "-n", data], stdout=f, stderr=f)
p.wait()
f.seek(0)
self.assertEqual(data, f.read())
def testNoStreams(self):
p = CPopen(['true'], stdin=None, stdout=None, stderr=None)
self.assertIsNone(p.stdin)
self.assertIsNone(p.stdout)
self.assertIsNone(p.stderr)
p.wait()
self.assertEquals(p.returncode, 0)
def execCmd(command, sudo=False, cwd=None, data=None, raw=False,
printable=None, env=None, sync=True, nice=None, ioclass=None,
ioclassdata=None, setsid=False, execCmdLogger=logging.root,
deathSignal=0, childUmask=None):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
if ioclass is not None:
command = cmdutils.ionice(command, ioclass=ioclass,
ioclassdata=ioclassdata)
if nice is not None:
command = cmdutils.nice(command, nice=nice)
if setsid:
command = cmdutils.setsid(command)
if sudo:
command = cmdutils.sudo(command)
# Unsubscriptable objects (e.g. generators) need conversion
if not callable(getattr(command, '__getitem__', None)):
command = tuple(command)
if not printable:
printable = command
execCmdLogger.debug("%s (cwd %s)", _list2cmdline(printable), cwd)
p = CPopen(command, close_fds=True, cwd=cwd, env=env,
deathSignal=deathSignal, childUmask=childUmask)
if not sync:
p = AsyncProc(p)
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug("%s: <err> = %r; <rc> = %d",
"SUCCESS" if p.returncode == 0 else "FAILED",
err, p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return p.returncode, out, err
def execCmd(command, cwd=None, data=None, raw=False,
env=None, sync=True, deathSignal=0, childUmask=None):
"""
Executes an external command,
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
cmdline = repr(subprocess.list2cmdline(command))
logger.info("%s (cwd %s)", cmdline, cwd)
p = CPopen(command, close_fds=True, cwd=cwd, env=env,
deathSignal=deathSignal, childUmask=childUmask)
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
logger.debug("%s: <err> = %s; <rc> = %d",
{True: "SUCCESS", False: "FAILED"}[p.returncode == 0],
repr(err), p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return (p.returncode, out, err)
def _subTest(self, name, params, *args, **kwargs):
p = CPopen(["python", EXT_HELPER, name] + params,
*args, **kwargs)
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read().strip(), "True")
def testEcho(self):
data = "Hello"
p = CPopen([EXT_ECHO, "-n", data])
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read(), data)
def testNoStreams(self):
p = CPopen(['true'], stdin=None, stdout=None, stderr=None)
self.assertEqual(p.stdin, None)
self.assertEqual(p.stdout, None)
self.assertEqual(p.stderr, None)
p.wait()
self.assertEquals(p.returncode, 0)
def execCmd(command, sudo=False, cwd=None, data=None, raw=False,
printable=None, env=None, sync=True, nice=None, ioclass=None,
ioclassdata=None, setsid=False, execCmdLogger=logging.root,
deathSignal=0, childUmask=None):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
if ioclass is not None:
command = cmdutils.ionice(command, ioclass=ioclass,
ioclassdata=ioclassdata)
if nice is not None:
command = cmdutils.nice(command, nice=nice)
if setsid:
command = cmdutils.setsid(command)
if sudo:
command = cmdutils.sudo(command)
# Unsubscriptable objects (e.g. generators) need conversion
if not callable(getattr(command, '__getitem__', None)):
command = tuple(command)
if not printable:
printable = command
execCmdLogger.debug("%s (cwd %s)", _list2cmdline(printable), cwd)
p = CPopen(command, close_fds=True, cwd=cwd, env=env,
deathSignal=deathSignal, childUmask=childUmask)
if not sync:
p = AsyncProc(p)
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug("%s: <err> = %r; <rc> = %d",
"SUCCESS" if p.returncode == 0 else "FAILED",
err, p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return p.returncode, out, err
def testCwd(self):
cwd = "/proc"
p = CPopen(["python", "-c", "import os; print os.getcwd()"], cwd=cwd)
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read().strip(), cwd)
def testCat(self):
path = "/etc/passwd"
p = CPopen(["cat", path])
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
with open(path, "r") as f:
self.assertEquals(p.stdout.read(), f.read())
def testCat(self):
path = "/etc/passwd"
p = CPopen(["cat", path])
out, err = p.communicate()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
with open(path, "r") as f:
self.assertEquals(out, f.read())
def testStdin(self):
data = "Hello World"
p = CPopen(["cat"])
out, err = p.communicate(data)
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(out, data)
def testCwd(self):
cwd = "/proc"
p = CPopen(["python", "-c", "import os; print os.getcwd()"],
cwd=cwd)
out, err = p.communicate()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(out.strip(), cwd)
def testCloseOnExecStderr(self):
# Unset close-on-exec on stderr fd
with tempfile.NamedTemporaryFile() as f:
set_close_on_exec(f.fileno())
p = CPopen(["sh", "-c", "echo -n data >&2"], stderr=f.fileno())
p.wait()
f.seek(0)
self.assertEqual(f.read(), "data")
def testCat(self):
path = "/etc/passwd"
p = CPopen(["cat", path])
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
with open(path, "r") as f:
self.assertEquals(p.stdout.read(), f.read())
def testCloseFDs(self):
with open("/dev/zero") as f:
p = CPopen(["sleep", "1"], close_fds=True)
try:
child_fds = set(os.listdir("/proc/%s/fd" % p.pid))
finally:
p.kill()
p.wait()
self.assertEqual(child_fds, set(["0", "1", "2"]))
def execCmd(command, sudo=False, cwd=None, data=None, raw=False, logErr=True,
printable=None, env=None, sync=True, nice=None, ioclass=None,
ioclassdata=None, setsid=False, execCmdLogger=logging.root,
deathSignal=0):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: don't define a deathSignal when sync=False
"""
if ioclass is not None:
cmd = command
command = [constants.EXT_IONICE, '-c', str(ioclass)]
if ioclassdata is not None:
command.extend(("-n", str(ioclassdata)))
command = command + cmd
if nice is not None:
command = [constants.EXT_NICE, '-n', str(nice)] + command
if setsid:
command = [constants.EXT_SETSID] + command
if sudo:
command = [constants.EXT_SUDO, SUDO_NON_INTERACTIVE_FLAG] + command
if not printable:
printable = command
cmdline = repr(subprocess.list2cmdline(printable))
execCmdLogger.debug("%s (cwd %s)", cmdline, cwd)
p = CPopen(command, close_fds=True, cwd=cwd, env=env,
deathSignal=deathSignal)
p = AsyncProc(p)
if not sync:
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug("%s: <err> = %s; <rc> = %d",
{True: "SUCCESS", False: "FAILED"}[p.returncode == 0],
repr(err), p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return (p.returncode, out, err)
def testUnicodeArg(self):
data = u'hello'
cmd = [EXT_ECHO, "-n", data]
p = CPopen(cmd)
out, err = p.communicate()
p2 = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out2, err2 = p2.communicate()
self.assertEquals(out, out2)
def testPipeline(self):
# echo -n 'it works!' | cat
data = "it works!"
p2 = CPopen(["cat"], stdin=PIPE, stdout=PIPE)
try:
p1 = CPopen(["echo", "-n", data], stdin=PIPE, stdout=p2.stdin)
p1.wait()
finally:
out, err = p2.communicate()
self.assertEqual(data, out)
def testCloseOnExecStdin(self):
# Unset close-on-exec on stdin fd
data = "data"
with tempfile.NamedTemporaryFile() as f:
f.write(data)
f.flush()
set_close_on_exec(f.fileno())
p = CPopen(["cat", f.name], stdin=f.fileno(), stdout=PIPE)
out, err = p.communicate()
self.assertEqual(out, data)
def testUnicodeArg(self):
data = u'hello'
cmd = [EXT_ECHO, "-n", data]
p = CPopen(cmd)
p.wait()
p2 = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p2.wait()
self.assertEquals(p.stdout.read(), p2.stdout.read())
class PoolHandler(object):
log = logging.getLogger("Storage.RepoFileHelper.PoolHandler")
def __init__(self):
myRead, hisWrite = os.pipe()
hisRead, myWrite = os.pipe()
try:
# Some imports in vdsm assume /usr/share/vdsm is in your PYTHONPATH
env = os.environ.copy()
env['PYTHONPATH'] = "%s:%s" % (env.get("PYTHONPATH",
""), constants.P_VDSM)
env['PYTHONPATH'] = ":".join(
map(os.path.abspath, env['PYTHONPATH'].split(":")))
self.process = CPopen(
[constants.EXT_PYTHON, __file__,
str(hisRead),
str(hisWrite)],
close_fds=False,
env=env)
self.proxy = CrabRPCProxy(myRead, myWrite)
except:
os.close(myWrite)
os.close(myRead)
raise
finally:
os.close(hisRead)
os.close(hisWrite)
def stop(self):
try:
os.kill(self.process.pid, signal.SIGKILL)
except:
pass
self.process.poll()
# Don't try to read if the process is in D state
if (self.process.returncode is not None
and self.process.returncode != 0):
err = self.process.stderr.read()
out = self.process.stdout.read()
self.log.debug("Pool handler existed, OUT: '%s' ERR: '%s'", out,
err)
try:
zombiereaper.autoReapPID(self.process.pid)
except AttributeError:
if zombiereaper is not None:
raise
def __del__(self):
self.stop()
def testNoCloseFds(self):
with open("/dev/zero") as f:
p = CPopen(["sleep", "1"], close_fds=False)
try:
child_fds = set(os.listdir("/proc/%s/fd" % p.pid))
finally:
p.kill()
p.wait()
# We cannot know which fds will be inherited in the child since the
# test framework may open some fds.
self.assertTrue(str(f.fileno()) in child_fds)
def testStdin(self):
data = "Hello World"
p = CPopen(["cat"])
p.stdin.write(data)
p.stdin.flush()
p.stdin.close()
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read(), data)
def testStdin(self):
data = "Hello World"
p = CPopen(["cat"])
p.stdin.write(data)
p.stdin.flush()
p.stdin.close()
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read(), data)
def testInheritParentFdsCloseFds(self):
# From Python docs: If close_fds is true, all file descriptors except
# 0, 1 and 2 will be closed before the child process is executed.
with open("/dev/zero") as f:
p = CPopen(["sleep", "1"], stdin=None, stdout=None, stderr=None,
close_fds=True)
try:
child_fds = set(os.listdir("/proc/%s/fd" % p.pid))
finally:
p.kill()
p.wait()
self.assertEqual(child_fds, set(["0", "1", "2"]))
def testUmaskTmpfile(self):
name = os.tempnam()
p = CPopen(['touch', name], childUmask=0o007)
p.wait()
data = os.stat(name)
os.unlink(name)
self.assertTrue(data.st_mode & stat.S_IROTH == 0,
"%s is world-readable" % name)
self.assertTrue(data.st_mode & stat.S_IWOTH == 0,
"%s is world-writeable" % name)
self.assertTrue(data.st_mode & stat.S_IXOTH == 0,
"%s is world-executable" % name)
def testUnicodeArg(self):
data = u'hello'
cmd = [EXT_ECHO, "-n", data]
p = CPopen(cmd)
p.wait()
p2 = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p2.wait()
self.assertEquals(p.stdout.read(), p2.stdout.read())
class PoolHandler(object):
log = logging.getLogger("RepoFileHelper.PoolHandler")
def __init__(self):
myRead, hisWrite = os.pipe()
hisRead, myWrite = os.pipe()
try:
# Some imports in vdsm assume /usr/share/vdsm is in your PYTHONPATH
env = os.environ.copy()
env['PYTHONPATH'] = "%s:%s" % (
env.get("PYTHONPATH", ""), constants.P_VDSM)
env['PYTHONPATH'] = ":".join(map(os.path.abspath,
env['PYTHONPATH'].split(":")))
self.process = BetterPopen([constants.EXT_PYTHON, __file__,
str(hisRead), str(hisWrite)],
close_fds=False, env=env)
self.proxy = CrabRPCProxy(myRead, myWrite)
except:
os.close(myWrite)
os.close(myRead)
raise
finally:
os.close(hisRead)
os.close(hisWrite)
def stop(self):
try:
os.kill(self.process.pid, signal.SIGKILL)
except:
pass
self.process.poll()
# Don't try to read if the process is in D state
if (self.process.returncode is not None and
self.process.returncode != 0):
err = self.process.stderr.read()
out = self.process.stdout.read()
self.log.debug("Pool handler existed, OUT: '%s' ERR: '%s'",
out, err)
try:
zombieReaper.autoReapPID(self.process.pid)
except AttributeError:
if zombieReaper is not None:
raise
def __del__(self):
self.stop()
def testInheritParentFds(self):
# From Python docs: With the default settings of None, no redirection
# will occur; the child's file handles will be inherited from the
# parent.
with open("/dev/zero") as f:
p = CPopen(["sleep", "1"], stdin=None, stdout=None, stderr=None,
close_fds=False)
try:
child_fds = set(os.listdir("/proc/%s/fd" % p.pid))
finally:
p.kill()
p.wait()
expected_fds = set(["0", "1", "2", str(f.fileno())])
self.assertEqual(child_fds, expected_fds)
def execCmd(command, sudo=False, cwd=None, data=None, raw=False,
printable=None, env=None, sync=True, nice=None, ioclass=None,
ioclassdata=None, setsid=False, execCmdLogger=logging.root,
deathSignal=0, childUmask=None, resetCpuAffinity=True):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
command = cmdutils.wrap_command(command, with_ioclass=ioclass,
ioclassdata=ioclassdata, with_nice=nice,
with_setsid=setsid, with_sudo=sudo,
reset_cpu_affinity=resetCpuAffinity)
# Unsubscriptable objects (e.g. generators) need conversion
if not callable(getattr(command, '__getitem__', None)):
command = tuple(command)
if not printable:
printable = command
execCmdLogger.debug(cmdutils.command_log_line(printable, cwd=cwd))
p = CPopen(command, close_fds=True, cwd=cwd, env=env,
deathSignal=deathSignal, childUmask=childUmask)
if not sync:
p = AsyncProc(p)
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug(cmdutils.retcode_log_line(p.returncode, err=err))
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return p.returncode, out, err
def __init__(self, cmd, cwd=None):
self._aborted = False
self._progress = 0.0
self._stdout = bytearray()
self._stderr = bytearray()
cmd = cmdutils.wrap_command(cmd,
with_nice=utils.NICENESS.HIGH,
with_ioclass=utils.IOCLASS.IDLE)
_log.debug(cmdutils.command_log_line(cmd, cwd=cwd))
self._command = CPopen(cmd, cwd=cwd, deathSignal=signal.SIGKILL)
self._stream = utils.CommandStream(self._command, self._recvstdout,
self._recvstderr)
def testUmaskTmpfile(self):
tmp_dir = tempfile.mkdtemp()
name = os.path.join(tmp_dir, "file.txt")
p = CPopen(['touch', name], childUmask=0o007)
p.wait()
data = os.stat(name)
os.unlink(name)
self.assertTrue(data.st_mode & stat.S_IROTH == 0,
"%s is world-readable" % name)
self.assertTrue(data.st_mode & stat.S_IWOTH == 0,
"%s is world-writeable" % name)
self.assertTrue(data.st_mode & stat.S_IXOTH == 0,
"%s is world-executable" % name)
posix.rmdir(tmp_dir)
def testNonASCIIUnicodeArg(self):
data = u'\u05e9\u05dc\u05d5\u05dd'
# If the default encoding is not utf-8 the test *should* fail as non
# ascii conversion shouldn't work
if sys.getfilesystemencoding() != "UTF-8":
raise SkipTest("The default encoding isn't unicode")
cmd = [EXT_ECHO, "-n", data]
p = CPopen(cmd)
p.wait()
p2 = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p2.wait()
self.assertEquals(p.stdout.read(), p2.stdout.read())
def testUmaskTmpfile(self):
tmp_dir = tempfile.mkdtemp()
try:
name = os.path.join(tmp_dir, "file.txt")
p = CPopen(['touch', name], childUmask=0o007)
p.wait()
data = os.stat(name)
self.assertTrue(data.st_mode & stat.S_IROTH == 0,
"%s is world-readable" % name)
self.assertTrue(data.st_mode & stat.S_IWOTH == 0,
"%s is world-writeable" % name)
self.assertTrue(data.st_mode & stat.S_IXOTH == 0,
"%s is world-executable" % name)
finally:
shutil.rmtree(tmp_dir)
def testUmaskTmpfile(self):
warnings.filterwarnings(
'ignore',
'^tempnam is a potential security risk.*')
name = os.tempnam()
p = CPopen(['touch', name], childUmask=0o007)
p.wait()
data = os.stat(name)
os.unlink(name)
self.assertTrue(data.st_mode & stat.S_IROTH == 0,
"%s is world-readable" % name)
self.assertTrue(data.st_mode & stat.S_IWOTH == 0,
"%s is world-writeable" % name)
self.assertTrue(data.st_mode & stat.S_IXOTH == 0,
"%s is world-executable" % name)
def testStdinEpoll(self):
import select
data = "Hello World"
p = CPopen(["cat"])
ep = select.epoll()
ep.register(p.stdin, select.EPOLLOUT)
fd, ev = ep.poll(1)[0]
ep.close()
os.write(fd, data)
p.stdin.close()
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read(), data)
def _run(self):
self._log.debug("Starting IOProcess...")
myRead, hisWrite = os.pipe()
hisRead, myWrite = os.pipe()
self._partialLogs = ""
cmd = [
self.IOPROCESS_EXE, "--read-pipe-fd",
str(hisRead), "--write-pipe-fd",
str(hisWrite), "--max-threads",
str(self._max_threads)
]
if self._DEBUG_VALGRIND:
cmd = ["valgrind", "--log-file=ioprocess.valgrind.log",
"--leak-check=full", "--tool=memcheck"] + cmd + \
["--keep-fds"]
p = CPopen(cmd)
os.close(hisRead)
os.close(hisWrite)
setNonBlocking(myRead)
setNonBlocking(myWrite)
self._startCommunication(p, myRead, myWrite)
def testNoEnt(self):
try:
CPopen(['there-is-no-executable-with-this/funny/name'])
except OSError as ose:
self.assertEquals(ose.errno, errno.ENOENT)
else:
self.fail('OSError not raised')
def testStdinEpoll(self):
import select
data = "Hello World"
p = CPopen(["cat"])
ep = select.epoll()
ep.register(p.stdin, select.EPOLLOUT)
fd, ev = ep.poll(1)[0]
ep.close()
os.write(fd, data)
p.stdin.close()
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read(), data)
def testNonASCIIUnicodeArg(self):
data = u'\u05e9\u05dc\u05d5\u05dd'
# If the default encoding is not utf-8 the test *should* fail as non
# ascii conversion shouldn't work
if sys.getfilesystemencoding() != "UTF-8":
raise SkipTest("The default encoding isn't unicode")
cmd = [EXT_ECHO, "-n", data]
p = CPopen(cmd)
p.wait()
p2 = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p2.wait()
self.assertEquals(p.stdout.read(), p2.stdout.read())
def testProcessDiesAfterBeingTracked(self):
p = CPopen(["sleep", "1"])
zombiereaper.autoReapPID(p.pid)
# wait for the grim reaper to arrive
sleep(4)
# Throws error because pid is not found or is not child
self.assertRaises(OSError, os.waitpid, p.pid, os.WNOHANG)
def testProcessDiedBeforeBeingTracked(self):
p = CPopen(["sleep", "0"])
# wait for the process to die
sleep(1)
zombiereaper.autoReapPID(p.pid)
# Throws error because pid is not found or is not child
self.assertRaises(OSError, os.waitpid, p.pid, os.WNOHANG)
def execCmd(command,
sudo=False,
cwd=None,
data=None,
raw=False,
logErr=True,
printable=None,
env=None,
sync=True,
nice=None,
ioclass=None,
ioclassdata=None,
setsid=False,
execCmdLogger=logging.root):
"""
Executes an external command, optionally via sudo.
"""
if ioclass is not None:
cmd = command
command = [constants.EXT_IONICE, '-c', str(ioclass)]
if ioclassdata is not None:
command.extend(("-n", str(ioclassdata)))
command = command + cmd
if nice is not None:
command = [constants.EXT_NICE, '-n', str(nice)] + command
if setsid:
command = [constants.EXT_SETSID] + command
if sudo:
command = [constants.EXT_SUDO, SUDO_NON_INTERACTIVE_FLAG] + command
if not printable:
printable = command
cmdline = repr(subprocess.list2cmdline(printable))
execCmdLogger.debug("%s (cwd %s)", cmdline, cwd)
p = BetterPopen(command, close_fds=True, cwd=cwd, env=env)
p = AsyncProc(p)
if not sync:
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug("%s: <err> = %s; <rc> = %d", {
True: "SUCCESS",
False: "FAILED"
}[p.returncode == 0], repr(err), p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return (p.returncode, out, err)
class QemuImgOperation(object):
REGEXPR = re.compile(r'\s*\(([\d.]+)/100%\)\s*')
def __init__(self, cmd, cwd=None):
self._aborted = False
self._progress = 0.0
self._stdout = bytearray()
self._stderr = bytearray()
cmd = cmdutils.wrap_command(cmd,
with_nice=utils.NICENESS.HIGH,
with_ioclass=utils.IOCLASS.IDLE)
_log.debug(cmdutils.command_log_line(cmd, cwd=cwd))
self._command = CPopen(cmd, cwd=cwd, deathSignal=signal.SIGKILL)
self._stream = utils.CommandStream(self._command, self._recvstdout,
self._recvstderr)
def _recvstderr(self, buffer):
self._stderr += buffer
def _recvstdout(self, buffer):
self._stdout += buffer
# Checking the presence of '\r' before splitting will prevent
# generating the array when it's not needed.
try:
idx = self._stdout.rindex('\r')
except ValueError:
return
# qemu-img updates progress by printing \r (0.00/100%) to standard out.
# The output could end with a partial progress so we must discard
# everything after the last \r and then try to parse a progress record.
valid_progress = self._stdout[:idx]
last_progress = valid_progress.rsplit('\r', 1)[-1]
# No need to keep old progress information around
del self._stdout[:idx + 1]
m = self.REGEXPR.match(last_progress)
if m is None:
raise ValueError('Unable to parse: "%r"' % last_progress)
self._progress = float(m.group(1))
@property
def progress(self):
return self._progress
@property
def error(self):
return str(self._stderr)
@property
def finished(self):
return self._command.poll() is not None
def wait(self, timeout=None):
self._stream.receive(timeout=timeout)
if not self._stream.closed:
return
self._command.wait()
if self._aborted:
raise utils.ActionStopped()
cmdutils.retcode_log_line(self._command.returncode, self.error)
if self._command.returncode != 0:
raise QImgError(self._command.returncode, "", self.error)
def abort(self):
if self._command.poll() is None:
self._aborted = True
self._command.terminate()
def execCmd(command,
sudo=False,
cwd=None,
data=None,
raw=False,
printable=None,
env=None,
sync=True,
nice=None,
ioclass=None,
ioclassdata=None,
setsid=False,
execCmdLogger=logging.root,
deathSignal=0,
childUmask=None,
resetCpuAffinity=True):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
if ioclass is not None:
command = cmdutils.ionice(command,
ioclass=ioclass,
ioclassdata=ioclassdata)
if nice is not None:
command = cmdutils.nice(command, nice=nice)
if setsid:
command = cmdutils.setsid(command)
if sudo:
command = cmdutils.sudo(command)
# warning: the order of commands matters. If we add taskset
# after sudo, we'll need to configure sudoers to allow both
# 'sudo <command>' and 'sudo taskset <command>', which is
# impractical. On the other hand, using 'taskset sudo <command>'
# is much simpler and delivers the same end result.
if resetCpuAffinity and _USING_CPU_AFFINITY:
# only VDSM itself should be bound
command = cmdutils.taskset(command, _ANY_CPU)
# Unsubscriptable objects (e.g. generators) need conversion
if not callable(getattr(command, '__getitem__', None)):
command = tuple(command)
if not printable:
printable = command
execCmdLogger.debug(cmdutils.command_log_line(printable, cwd=cwd))
p = CPopen(command,
close_fds=True,
cwd=cwd,
env=env,
deathSignal=deathSignal,
childUmask=childUmask)
if not sync:
p = AsyncProc(p)
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug(cmdutils.retcode_log_line(p.returncode, err=err))
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return p.returncode, out, err
def spawn():
procPtr[0] = CPopen(["sleep", "10"], deathSignal=signal.SIGKILL)
def testUmaskChange(self):
p = CPopen(['umask'], childUmask=0o007)
p.wait()
out = p.stdout.readlines()
self.assertEquals(out[0].strip(), '0007')
def execCmd(command,
sudo=False,
cwd=None,
data=None,
raw=False,
printable=None,
env=None,
sync=True,
nice=None,
ioclass=None,
ioclassdata=None,
setsid=False,
execCmdLogger=logging.root,
deathSignal=0,
childUmask=None,
resetCpuAffinity=True):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
command = cmdutils.wrap_command(command,
with_ioclass=ioclass,
ioclassdata=ioclassdata,
with_nice=nice,
with_setsid=setsid,
with_sudo=sudo,
reset_cpu_affinity=resetCpuAffinity)
# Unsubscriptable objects (e.g. generators) need conversion
if not callable(getattr(command, '__getitem__', None)):
command = tuple(command)
if not printable:
printable = command
execCmdLogger.debug(cmdutils.command_log_line(printable, cwd=cwd))
p = CPopen(command,
close_fds=True,
cwd=cwd,
env=env,
deathSignal=deathSignal,
childUmask=childUmask)
if not sync:
p = AsyncProc(p)
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug(cmdutils.retcode_log_line(p.returncode, err=err))
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return p.returncode, out, err
def _subTest(self, name, params, *args, **kwargs):
p = CPopen(["python", __file__, name] + params, *args, **kwargs)
p.wait()
self.assertTrue(p.returncode == 0,
"Process failed: %s" % os.strerror(p.returncode))
self.assertEquals(p.stdout.read().strip(), "True")
def execCmd(command,
sudo=False,
cwd=None,
data=None,
raw=False,
logErr=True,
printable=None,
env=None,
sync=True,
nice=None,
ioclass=None,
ioclassdata=None,
setsid=False,
execCmdLogger=logging.root,
deathSignal=0,
childUmask=None):
"""
Executes an external command, optionally via sudo.
IMPORTANT NOTE: the new process would receive `deathSignal` when the
controlling thread dies, which may not be what you intended: if you create
a temporary thread, spawn a sync=False sub-process, and have the thread
finish, the new subprocess would die immediately.
"""
if ioclass is not None:
cmd = command
command = [ioniceCmdPath.cmd, '-c', str(ioclass)]
if ioclassdata is not None:
command.extend(("-n", str(ioclassdata)))
command = command + cmd
if nice is not None:
command = [niceCmdPath.cmd, '-n', str(nice)] + command
if setsid:
command = [setsidCmdPath.cmd] + command
if sudo:
command = [sudoCmdPath.cmd, SUDO_NON_INTERACTIVE_FLAG] + command
if not printable:
printable = command
cmdline = repr(subprocess.list2cmdline(printable))
execCmdLogger.debug("%s (cwd %s)", cmdline, cwd)
p = CPopen(command,
close_fds=True,
cwd=cwd,
env=env,
deathSignal=deathSignal,
childUmask=childUmask)
p = AsyncProc(p)
if not sync:
if data is not None:
p.stdin.write(data)
p.stdin.flush()
return p
(out, err) = p.communicate(data)
if out is None:
# Prevent splitlines() from barfing later on
out = ""
execCmdLogger.debug("%s: <err> = %s; <rc> = %d", {
True: "SUCCESS",
False: "FAILED"
}[p.returncode == 0], repr(err), p.returncode)
if not raw:
out = out.splitlines(False)
err = err.splitlines(False)
return (p.returncode, out, err)