def __init__(self, logger, cmd, cwd):
"""
Start a subprocess for `cmd' in working directory `cwd'.
Output is sent to `logger', or stdout if logger is None.
"""
def print_logger(line):
print(line)
self.cmd = cmd
self.killed = False
self.completed = False
self.logger = logger.info if logger else print_logger
# report we are starting
self.logger("Subprocess starting: %s (%s)" %
(" ".join(self.cmd), self.cmd))
# start process
self.process = subprocess.Popen(
cmd,
cwd=cwd,
# Set buffering parameters
bufsize=1, # 1 = line buffering
universal_newlines=True, # translate ^M output by ssh -tt
# Don't inherit our fds after fork()
close_fds=True,
# I/O redirection: block stdin, read stdout/stderr separately
stdin=open("/dev/null"),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.pid = self.process.pid
self.exit_status = None
# output source streams
self.output_streams = {
self.STDOUT: self.process.stdout,
self.STDERR: self.process.stderr
}
# output buffer
self.output_buffers = {self.STDOUT: "", self.STDERR: ""}
# output sink
self.output_loggers = {
self.STDOUT: logger.debug if logger else print_logger,
self.STDERR: logger.warn if logger else print_logger
}
# Set fds to non-blocking to allow <4k reads. This is needed if the process
# alternates between stdout and stderr output.
for f in list(self.output_streams.values()):
flag = fcntl.fcntl(f, fcntl.F_GETFL)
fcntl.fcntl(f, fcntl.F_SETFL, flag | os.O_NONBLOCK)
def disk_usage(*paths):
"""
Return the disk usage in bytes by the file(s) in ``paths``.
"""
cmd = ['du', '-s', '-b']
proc = subprocess.Popen(cmd + list(paths), stdout=subprocess.PIPE)
sout = communicate_returning_strings(proc)[0]
if sout:
return sum([int(s.split('\t')[0]) for s in sout.strip().split('\n')])
else:
return 0
def spawn_process(cmd,
logger,
cwd=None,
env=None,
max_tries=2,
max_timeout=30):
"""
DEPRECATED -- spawn_process leads to custom, and thus bad, output handling. Use
support.subprocessgroup.SubProcessGroup instead.
Tries to spawn a process.
If it hits an OSError due to lack of memory or too many open files, it
will keep trying max_tries times, waiting timeout seconds between each.
If successful, the process object is returned. Otherwise, we eventually
propagate the exception.
"""
logger.debug(
"support.utilities.spawn_process is DEPRECATED. Please use support.subprocessgroup.SubProcessGroup"
)
# Make sure the working directory exists.
create_directory(cwd)
trycounter = 0
while True:
logger.debug("Spawning subprocess: cmd=%s, cwd=%s, env=%s" %
(cmd, cwd, env))
try:
process = subprocess.Popen(cmd,
cwd=cwd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as e:
logger.warn("Failed to spawn external process %s (%s)" %
(" ".join(cmd), str(e)))
if trycounter < max_tries:
timeout = randint(1, max_timeout)
logger.warn("Retrying in %d seconds (%d more retries)." %
(timeout, max_tries - trycounter - 1))
trycounter += 1
time.sleep(timeout)
else:
raise
else:
break
return process
def read_initscript(logger, filename, shell="/bin/sh"):
"""
Return a dict of the environment after sourcing the given script in a shell.
"""
if not os.path.exists(filename):
logger.warn("Environment initialisation script not found!")
return {}
else:
logger.debug("Reading environment from %s" % filename)
p = subprocess.Popen(['. %s ; env' % (filename)],
shell=True,
executable=shell,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
so, se = communicate_returning_strings(p)
environment = [x.split('=', 1) for x in so.strip().split('\n')]
environment = [x for x in environment if len(x) == 2]
return dict(environment)
def spawn_process(cmd,
logger,
cwd=None,
env=None,
max_tries=2,
max_timeout=30):
"""
Tries to spawn a process.
If it hits an OSError due to lack of memory or too many open files, it
will keep trying max_tries times, waiting timeout seconds between each.
If successful, the process object is returned. Otherwise, we eventually
propagate the exception.
"""
trycounter = 0
while True:
logger.debug("Spawning subprocess: cmd=%s, cwd=%s, env=%s" %
(cmd, cwd, env))
try:
process = subprocess.Popen(cmd,
cwd=cwd,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError, e:
logger.warn("Failed to spawn external process %s (%s)" %
(" ".join(cmd), str(e)))
if trycounter < max_tries:
timeout = randint(1, max_timeout)
logger.warn("Retrying in %d seconds (%d more retries)." %
(timeout, max_tries - trycounter - 1))
trycounter += 1
sleep(timeout)
else:
raise
else:
break