def retry(fn, description, pace=1.0, timeout=60.0, retry_on_false = False, catch=[], propagate=[], verbose=False): """Keeping running function until success. Arguments: fn: the function we wish to run retry_on_falsish: if true, retry if we get a falsish result from fn catch: exceptions types to ignore and retry pace: number of seconds to pace out attempts to call fn timeout: give up after this number of seconds fail_callback: function to call on failure Run fn, retrying in the event of a exceptions on the catch list for up to timeout seconds, waiting pace seconds between attempts""" start_time = time() count = 0 while 1: count += 1 delta_t = time() - start_time if verbose: print 'RETRY:', description, 'iteration', count, 'used', delta_t, print 'timeout', timeout try: elapsed = time() - start_time with time_limit(timeout-elapsed, 'run '+description): result = fn() if verbose: print 'RETRY:', description, 'iteration', count, 'finished' if catch is not None or result: return result except Exception, exc: matches = [x for x in catch if isinstance(exc, x)] propagates = [x for x in propagate if isinstance(exc, x)] delta_t = time() - start_time if delta_t < timeout and matches and not propagates: if verbose: print 'RETRY:', description, 'iteration', count, print 'failed with', repr(exc ) else: raise if verbose: print 'RETRY: sleeping', pace, 'seconds after iteration', count, print 'of', description, 'failed' sleep(pace)
def retry_until_true(fn, description='seeking truth', pace=1.0, timeout=60.0, false_callback=lambda: None, verbose=False): """Keep running function until it returns a truish value. Arguments: fn: the function we wish to return true description: text describing what we are doing pace: number of seconds to pace out attempts to call fn timeout: give up aftr this number of seconds false_callback: function to call if fn is false Returns: the result of fn, which will be a true value Raises: TimeoutStillFalseError """ start_time= time() count = 0 while 1: count += 1 if verbose: print 'RETRY:', description, 'iteration', count, print 'timeout', timeout delta1_t = time() - start_time with time_limit(timeout-delta1_t, 'run '+description): result = fn() delta_t = time() - start_time if verbose: print 'RETRY:', description, 'truish' if result else 'falsish', print 'on iteration', count if result: print return result sleep(pace)
def run(args, timeout=60, host=None, split=False, word_split=False, line_split=False, ignore_failure=False, verify=True, cwd=None, user=None, env={}, shell=False, stderr=False, echo=False, verbose=False, announce_interval = 20, wait=True, stdin_push='', output_callback=None, error_callback=None): """Run command with args, or raise SubprocessTimeout after timeout seconds. If host is specified, ssh to that machine. Let's hope your ssh configuration works. If split is true, convert output to a list of lines, where each line is a list of words. If word_split is true, convert output to a list of whitespace separated words. If line_split is true, convert output to a list of lines. If ignore_failure is true, do not raise exceptions for non-zero exit codes. If cwd is true, run commands in that working directory using a shell. If env is a dictionary, set those environment variables. If shell is true, run through a shell (implied by cwd or env). If stderr is true, return stderr as well as stdout. Otherwise or by default return just stdout. If echo is true, echo stdout/stderr through to sys.stdout/sys.stderr If verbose is true, print arguments timing and exit code. See http://stackoverflow.com/questions/1191374/subprocess-with-timeout If verify and host are set, then make sure the connection to the host works before trying it. If wait is false, then simply launch the command and return straight away. """ description = ' '.join(args) if host and verify: verify_connection(host, user, timeout=timeout, verbose=verbose) spargs = space_escape(args) if host: shell_prefixes = [] if cwd: shell_prefixes.extend(['cd', cwd, '&&']) cwd = None for key, value in env.iteritems(): shell_prefixes.append("%s=%s" % (key, (space_escape([value])[0]))) env = None shell = False args = ['ssh', '-oPasswordAuthentication=no', '-l' + (user if user else 'root'), host] + shell_prefixes + spargs description += ' on '+host if verbose: print 'RUN:', repr(args) process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=shell, env=env, cwd=cwd) fd2output = {} fd2file = {} poller = poll() def register_and_append(file_obj, eventmask): """Record file_obj for poll operations""" poller.register(file_obj.fileno(), eventmask) fd2file[file_obj.fileno()] = file_obj out = fd2output[file_obj.fileno()] = [] return out def close_unregister_and_remove(fdes): """fdes is finished""" poller.unregister(fdes) fd2file[fdes].close() fd2file.pop(fdes) def throw_timeout(delay): """Throw exception for after delay""" try: out = run(['ps', '--no-headers', '-o', 'pid', '--ppid', str(process.pid)]) except SubprocessError: out = '' pids = [process.pid] + [int(p) for p in out.split()] for pid in pids: try: kill(pid, SIGKILL) except OSError: if verbose: print 'WARNING: unable to kill subprocess', pid raise TimeoutError(description, timeout, delay) if not wait: return start = time() with time_limit(timeout, 'launch '+' '.join(args), timeout_callback=throw_timeout): if stdin_push: process.stdin.write(stdin_push) process.stdin.flush() process.stdin.close() stdout_list = register_and_append(process.stdout, POLLIN | POLLPRI) stderr_list = register_and_append(process.stderr, POLLIN | POLLPRI) announce = time() + announce_interval while fd2file: if time() > announce: announce = time() + announce_interval if verbose: print 'NOTE: waiting', time() - start, 'of', timeout, \ 'seconds for', ' '.join(args) try: ready = poller.poll(20) except error, eparam: if eparam.args[0] == EINTR: continue raise for fdes, mode in ready: if not mode & (POLLIN | POLLPRI): close_unregister_and_remove(fdes) continue if fdes not in fd2file: print 'operation on unexpected FD', fdes continue data = read(fdes, 4096) if not data: close_unregister_and_remove(fdes) fd2output[fdes].append(data) fileobj = fd2file[fdes] if fileobj == process.stdout: if echo: for line in data.splitlines(): print 'STDOUT:', line if output_callback: output_callback(data) if fileobj == process.stderr: if echo: for line in data.splitlines(): print 'STDERR:', line if error_callback: error_callback(data) process.wait() output = ''.join(stdout_list), ''.join(stderr_list) exit_code = process.returncode
def run(args, timeout=60, host=None, split=False, word_split=False, line_split=False, ignore_failure=False, verify=True, cwd=None, user=None, env={}, shell=False, stderr=False, echo=False, verbose=False, announce_interval=20, wait=True, stdin_push='', output_callback=None, error_callback=None): """Run command with args, or raise SubprocessTimeout after timeout seconds. If host is specified, ssh to that machine. Let's hope your ssh configuration works. If split is true, convert output to a list of lines, where each line is a list of words. If word_split is true, convert output to a list of whitespace separated words. If line_split is true, convert output to a list of lines. If ignore_failure is true, do not raise exceptions for non-zero exit codes. If cwd is true, run commands in that working directory using a shell. If env is a dictionary, set those environment variables. If shell is true, run through a shell (implied by cwd or env). If stderr is true, return stderr as well as stdout. Otherwise or by default return just stdout. If echo is true, echo stdout/stderr through to sys.stdout/sys.stderr If verbose is true, print arguments timing and exit code. See http://stackoverflow.com/questions/1191374/subprocess-with-timeout If verify and host are set, then make sure the connection to the host works before trying it. If wait is false, then simply launch the command and return straight away. """ description = ' '.join(args) if host and verify: verify_connection(host, user, timeout=timeout, verbose=verbose) spargs = space_escape(args) if host: shell_prefixes = [] if cwd: shell_prefixes.extend(['cd', cwd, '&&']) cwd = None for key, value in env.iteritems(): shell_prefixes.append("%s=%s" % (key, (space_escape([value])[0]))) env = None shell = False args = [ 'ssh', '-oPasswordAuthentication=no', '-l' + (user if user else 'root'), host ] + shell_prefixes + spargs description += ' on ' + host if verbose: print 'RUN:', repr(args) process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=shell, env=env, cwd=cwd) fd2output = {} fd2file = {} poller = poll() def register_and_append(file_obj, eventmask): """Record file_obj for poll operations""" poller.register(file_obj.fileno(), eventmask) fd2file[file_obj.fileno()] = file_obj out = fd2output[file_obj.fileno()] = [] return out def close_unregister_and_remove(fdes): """fdes is finished""" poller.unregister(fdes) fd2file[fdes].close() fd2file.pop(fdes) def throw_timeout(delay): """Throw exception for after delay""" try: out = run([ 'ps', '--no-headers', '-o', 'pid', '--ppid', str(process.pid) ]) except SubprocessError: out = '' pids = [process.pid] + [int(p) for p in out.split()] for pid in pids: try: kill(pid, SIGKILL) except OSError: if verbose: print 'WARNING: unable to kill subprocess', pid raise TimeoutError(description, timeout, delay) if not wait: return start = time() with time_limit(timeout, 'launch ' + ' '.join(args), timeout_callback=throw_timeout): if stdin_push: process.stdin.write(stdin_push) process.stdin.flush() process.stdin.close() stdout_list = register_and_append(process.stdout, POLLIN | POLLPRI) stderr_list = register_and_append(process.stderr, POLLIN | POLLPRI) announce = time() + announce_interval while fd2file: if time() > announce: announce = time() + announce_interval if verbose: print 'NOTE: waiting', time() - start, 'of', timeout, \ 'seconds for', ' '.join(args) try: ready = poller.poll(20) except error, eparam: if eparam.args[0] == EINTR: continue raise for fdes, mode in ready: if not mode & (POLLIN | POLLPRI): close_unregister_and_remove(fdes) continue if fdes not in fd2file: print 'operation on unexpected FD', fdes continue data = read(fdes, 4096) if not data: close_unregister_and_remove(fdes) fd2output[fdes].append(data) fileobj = fd2file[fdes] if fileobj == process.stdout: if echo: for line in data.splitlines(): print 'STDOUT:', line if output_callback: output_callback(data) if fileobj == process.stderr: if echo: for line in data.splitlines(): print 'STDERR:', line if error_callback: error_callback(data) process.wait() output = ''.join(stdout_list), ''.join(stderr_list) exit_code = process.returncode