def retry(fn, description, pace=1.0, timeout=60.0,
retry_on_false = False, catch=[], propagate=[], verbose=False):
"""Keeping running function until success.
Arguments:
fn: the function we wish to run
retry_on_falsish: if true, retry if we get a falsish result from fn
catch: exceptions types to ignore and retry
pace: number of seconds to pace out attempts to call fn
timeout: give up after this number of seconds
fail_callback: function to call on failure
Run fn, retrying in the event of a exceptions on the catch
list for up to timeout seconds, waiting pace seconds between attempts"""
start_time = time()
count = 0
while 1:
count += 1
delta_t = time() - start_time
if verbose:
print 'RETRY:', description, 'iteration', count, 'used', delta_t,
print 'timeout', timeout
try:
elapsed = time() - start_time
with time_limit(timeout-elapsed, 'run '+description):
result = fn()
if verbose:
print 'RETRY:', description, 'iteration', count, 'finished'
if catch is not None or result:
return result
except Exception, exc:
matches = [x for x in catch if isinstance(exc, x)]
propagates = [x for x in propagate if isinstance(exc, x)]
delta_t = time() - start_time
if delta_t < timeout and matches and not propagates:
if verbose:
print 'RETRY:', description, 'iteration', count,
print 'failed with', repr(exc )
else:
raise
if verbose:
print 'RETRY: sleeping', pace, 'seconds after iteration', count,
print 'of', description, 'failed'
sleep(pace)
def retry_until_true(fn, description='seeking truth', pace=1.0, timeout=60.0,
false_callback=lambda: None, verbose=False):
"""Keep running function until it returns a truish value.
Arguments:
fn: the function we wish to return true
description: text describing what we are doing
pace: number of seconds to pace out attempts to call fn
timeout: give up aftr this number of seconds
false_callback: function to call if fn is false
Returns:
the result of fn, which will be a true value
Raises:
TimeoutStillFalseError
"""
start_time= time()
count = 0
while 1:
count += 1
if verbose:
print 'RETRY:', description, 'iteration', count,
print 'timeout', timeout
delta1_t = time() - start_time
with time_limit(timeout-delta1_t, 'run '+description):
result = fn()
delta_t = time() - start_time
if verbose:
print 'RETRY:', description, 'truish' if result else 'falsish',
print 'on iteration', count
if result:
print
return result
sleep(pace)
def run(args, timeout=60, host=None, split=False, word_split=False,
line_split=False,
ignore_failure=False, verify=True,
cwd=None, user=None, env={}, shell=False, stderr=False, echo=False,
verbose=False, announce_interval = 20, wait=True,
stdin_push='', output_callback=None,
error_callback=None):
"""Run command with args, or raise SubprocessTimeout after timeout seconds.
If host is specified, ssh to that machine. Let's hope your ssh configuration
works.
If split is true, convert output to a list of lines, where each
line is a list of words.
If word_split is true, convert output to a list of whitespace separated words.
If line_split is true, convert output to a list of lines.
If ignore_failure is true, do not raise exceptions for non-zero exit codes.
If cwd is true, run commands in that working directory using a shell.
If env is a dictionary, set those environment variables.
If shell is true, run through a shell (implied by cwd or env).
If stderr is true, return stderr as well as stdout. Otherwise or by default
return just stdout.
If echo is true, echo stdout/stderr through to sys.stdout/sys.stderr
If verbose is true, print arguments timing and exit code.
See http://stackoverflow.com/questions/1191374/subprocess-with-timeout
If verify and host are set, then make sure the connection to the host
works before trying it.
If wait is false, then simply launch the command and return straight away.
"""
description = ' '.join(args)
if host and verify:
verify_connection(host, user, timeout=timeout, verbose=verbose)
spargs = space_escape(args)
if host:
shell_prefixes = []
if cwd:
shell_prefixes.extend(['cd', cwd, '&&'])
cwd = None
for key, value in env.iteritems():
shell_prefixes.append("%s=%s" % (key, (space_escape([value])[0])))
env = None
shell = False
args = ['ssh', '-oPasswordAuthentication=no', '-l' + (user if user else 'root'), host] + shell_prefixes + spargs
description += ' on '+host
if verbose:
print 'RUN:', repr(args)
process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=shell,
env=env, cwd=cwd)
fd2output = {}
fd2file = {}
poller = poll()
def register_and_append(file_obj, eventmask):
"""Record file_obj for poll operations"""
poller.register(file_obj.fileno(), eventmask)
fd2file[file_obj.fileno()] = file_obj
out = fd2output[file_obj.fileno()] = []
return out
def close_unregister_and_remove(fdes):
"""fdes is finished"""
poller.unregister(fdes)
fd2file[fdes].close()
fd2file.pop(fdes)
def throw_timeout(delay):
"""Throw exception for after delay"""
try:
out = run(['ps', '--no-headers', '-o', 'pid', '--ppid',
str(process.pid)])
except SubprocessError:
out = ''
pids = [process.pid] + [int(p) for p in out.split()]
for pid in pids:
try:
kill(pid, SIGKILL)
except OSError:
if verbose:
print 'WARNING: unable to kill subprocess', pid
raise TimeoutError(description, timeout, delay)
if not wait:
return
start = time()
with time_limit(timeout, 'launch '+' '.join(args),
timeout_callback=throw_timeout):
if stdin_push:
process.stdin.write(stdin_push)
process.stdin.flush()
process.stdin.close()
stdout_list = register_and_append(process.stdout, POLLIN | POLLPRI)
stderr_list = register_and_append(process.stderr, POLLIN | POLLPRI)
announce = time() + announce_interval
while fd2file:
if time() > announce:
announce = time() + announce_interval
if verbose:
print 'NOTE: waiting', time() - start, 'of', timeout, \
'seconds for', ' '.join(args)
try:
ready = poller.poll(20)
except error, eparam:
if eparam.args[0] == EINTR:
continue
raise
for fdes, mode in ready:
if not mode & (POLLIN | POLLPRI):
close_unregister_and_remove(fdes)
continue
if fdes not in fd2file:
print 'operation on unexpected FD', fdes
continue
data = read(fdes, 4096)
if not data:
close_unregister_and_remove(fdes)
fd2output[fdes].append(data)
fileobj = fd2file[fdes]
if fileobj == process.stdout:
if echo:
for line in data.splitlines():
print 'STDOUT:', line
if output_callback:
output_callback(data)
if fileobj == process.stderr:
if echo:
for line in data.splitlines():
print 'STDERR:', line
if error_callback:
error_callback(data)
process.wait()
output = ''.join(stdout_list), ''.join(stderr_list)
exit_code = process.returncode
def run(args,
timeout=60,
host=None,
split=False,
word_split=False,
line_split=False,
ignore_failure=False,
verify=True,
cwd=None,
user=None,
env={},
shell=False,
stderr=False,
echo=False,
verbose=False,
announce_interval=20,
wait=True,
stdin_push='',
output_callback=None,
error_callback=None):
"""Run command with args, or raise SubprocessTimeout after timeout seconds.
If host is specified, ssh to that machine. Let's hope your ssh configuration
works.
If split is true, convert output to a list of lines, where each
line is a list of words.
If word_split is true, convert output to a list of whitespace separated words.
If line_split is true, convert output to a list of lines.
If ignore_failure is true, do not raise exceptions for non-zero exit codes.
If cwd is true, run commands in that working directory using a shell.
If env is a dictionary, set those environment variables.
If shell is true, run through a shell (implied by cwd or env).
If stderr is true, return stderr as well as stdout. Otherwise or by default
return just stdout.
If echo is true, echo stdout/stderr through to sys.stdout/sys.stderr
If verbose is true, print arguments timing and exit code.
See http://stackoverflow.com/questions/1191374/subprocess-with-timeout
If verify and host are set, then make sure the connection to the host
works before trying it.
If wait is false, then simply launch the command and return straight away.
"""
description = ' '.join(args)
if host and verify:
verify_connection(host, user, timeout=timeout, verbose=verbose)
spargs = space_escape(args)
if host:
shell_prefixes = []
if cwd:
shell_prefixes.extend(['cd', cwd, '&&'])
cwd = None
for key, value in env.iteritems():
shell_prefixes.append("%s=%s" % (key, (space_escape([value])[0])))
env = None
shell = False
args = [
'ssh', '-oPasswordAuthentication=no', '-l' +
(user if user else 'root'), host
] + shell_prefixes + spargs
description += ' on ' + host
if verbose:
print 'RUN:', repr(args)
process = Popen(args,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
shell=shell,
env=env,
cwd=cwd)
fd2output = {}
fd2file = {}
poller = poll()
def register_and_append(file_obj, eventmask):
"""Record file_obj for poll operations"""
poller.register(file_obj.fileno(), eventmask)
fd2file[file_obj.fileno()] = file_obj
out = fd2output[file_obj.fileno()] = []
return out
def close_unregister_and_remove(fdes):
"""fdes is finished"""
poller.unregister(fdes)
fd2file[fdes].close()
fd2file.pop(fdes)
def throw_timeout(delay):
"""Throw exception for after delay"""
try:
out = run([
'ps', '--no-headers', '-o', 'pid', '--ppid',
str(process.pid)
])
except SubprocessError:
out = ''
pids = [process.pid] + [int(p) for p in out.split()]
for pid in pids:
try:
kill(pid, SIGKILL)
except OSError:
if verbose:
print 'WARNING: unable to kill subprocess', pid
raise TimeoutError(description, timeout, delay)
if not wait:
return
start = time()
with time_limit(timeout,
'launch ' + ' '.join(args),
timeout_callback=throw_timeout):
if stdin_push:
process.stdin.write(stdin_push)
process.stdin.flush()
process.stdin.close()
stdout_list = register_and_append(process.stdout, POLLIN | POLLPRI)
stderr_list = register_and_append(process.stderr, POLLIN | POLLPRI)
announce = time() + announce_interval
while fd2file:
if time() > announce:
announce = time() + announce_interval
if verbose:
print 'NOTE: waiting', time() - start, 'of', timeout, \
'seconds for', ' '.join(args)
try:
ready = poller.poll(20)
except error, eparam:
if eparam.args[0] == EINTR:
continue
raise
for fdes, mode in ready:
if not mode & (POLLIN | POLLPRI):
close_unregister_and_remove(fdes)
continue
if fdes not in fd2file:
print 'operation on unexpected FD', fdes
continue
data = read(fdes, 4096)
if not data:
close_unregister_and_remove(fdes)
fd2output[fdes].append(data)
fileobj = fd2file[fdes]
if fileobj == process.stdout:
if echo:
for line in data.splitlines():
print 'STDOUT:', line
if output_callback:
output_callback(data)
if fileobj == process.stderr:
if echo:
for line in data.splitlines():
print 'STDERR:', line
if error_callback:
error_callback(data)
process.wait()
output = ''.join(stdout_list), ''.join(stderr_list)
exit_code = process.returncode
