def _app(t, path):
with open(path) as fio:
datas = [json.loads(x) for x in fio.read().splitlines()]
pair = False
pretty = True
index = 0
max_lines = _max_lines = 10
hidden_keys = _hidden_keys = [
'direction', 'name', 'fntype', 'time', 'stack', 'cwd'
]
while True:
_print(
t,
_visualize(index, path, datas, hidden_keys, max_lines, pair,
pretty))
char = pager.getch()
if char == 'g':
char += pager.getch()
with t.location(0, 0):
if char == 'q':
return
elif char == '?':
_print(t, _help, True)
elif char == 'j':
index = min(index + 1, len(datas) - 1)
elif char == 'k':
index = max(index - 1, 0)
elif char == 'G':
index = len(datas) - 1
elif char == 'gg':
index = 0
elif char == 'p':
pair = not pair
elif char == 'f':
pretty = not pretty
elif char == 't':
if max_lines == _max_lines:
max_lines = 1e10
else:
max_lines = _max_lines
elif char == 'a':
if hidden_keys == _hidden_keys:
hidden_keys = []
else:
hidden_keys = _hidden_keys
elif char == 'c':
if 'cwd' in hidden_keys:
hidden_keys.remove('cwd')
else:
hidden_keys.append('cwd')
def retry(launch, cmd=None, yes=False, everything=False):
"""
retry any arg which is not running and has not logged "exited 0".
"""
launch = launch.replace('launch=', '')
text = params(launch)
data = json.loads(text)
if cmd:
new_data = json.loads(shell.run(cmd, stdin=text))
for k in data:
if data[k] != new_data[k]:
logging.info('\nold: %s', json.dumps({k: data[k]}))
logging.info('new: %s', json.dumps({k: new_data[k]}))
if not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
data = new_data
labels_to_restart = []
for val in status(launch):
state, label = val.split()
label = label.split('label=', 1)[-1]
if state == 'failed':
logging.info('going to retry failed label=%s', label)
labels_to_restart.append(label)
elif state == 'missing':
logging.info('going to retry missing label=%s', label)
labels_to_restart.append(label)
elif everything:
logging.info('going to retry label=%s', label)
labels_to_restart.append(label)
if labels_to_restart:
if not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
logging.info('restarting:')
for label in labels_to_restart:
logging.info(' %s', label)
args_to_restart = [
arg for arg, label in zip(data['args'], data['labels'])
if label in labels_to_restart
]
return new(name=data['name'],
arg=args_to_restart,
label=labels_to_restart,
tag=data['tags'],
**util.dicts.drop(data, ['name', 'args', 'labels', 'tags']))
else:
logging.info('nothing to retry')
def _app(t, path):
with open(path) as fio:
datas = [json.loads(x) for x in fio.read().splitlines()]
pair = False
pretty = True
index = 0
max_lines = _max_lines = 10
hidden_keys = _hidden_keys = ['direction', 'name', 'fntype', 'time', 'stack', 'cwd']
while True:
_print(t, _visualize(index, path, datas, hidden_keys, max_lines, pair, pretty))
char = pager.getch()
if char == 'g':
char += pager.getch()
with t.location(0, 0):
if char == 'q':
return
elif char == '?':
_print(t, _help, True)
elif char == 'j':
index = min(index + 1, len(datas) - 1)
elif char == 'k':
index = max(index - 1, 0)
elif char == 'G':
index = len(datas) - 1
elif char == 'gg':
index = 0
elif char == 'p':
pair = not pair
elif char == 'f':
pretty = not pretty
elif char == 't':
if max_lines == _max_lines:
max_lines = 1e10
else:
max_lines = _max_lines
elif char == 'a':
if hidden_keys == _hidden_keys:
hidden_keys = []
else:
hidden_keys = _hidden_keys
elif char == 'c':
if 'cwd' in hidden_keys:
hidden_keys.remove('cwd')
else:
hidden_keys.append('cwd')
def retry(launch, cmd=None, yes=False, everything=False):
"""
retry any arg which is not running and has not logged "exited 0".
"""
launch = launch.replace('launch=', '')
text = params(launch)
data = json.loads(text)
if cmd:
new_data = json.loads(shell.run(cmd, stdin=text))
for k in data:
if data[k] != new_data[k]:
logging.info('\nold: %s', json.dumps({k: data[k]}))
logging.info('new: %s', json.dumps({k: new_data[k]}))
if not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
data = new_data
labels_to_restart = []
for val in status(launch):
state, label = val.split()
label = label.split('label=', 1)[-1]
if state == 'failed':
logging.info('going to retry failed label=%s', label)
labels_to_restart.append(label)
elif state == 'missing':
logging.info('going to retry missing label=%s', label)
labels_to_restart.append(label)
elif everything:
logging.info('going to retry label=%s', label)
labels_to_restart.append(label)
if labels_to_restart:
if not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
logging.info('restarting:')
for label in labels_to_restart:
logging.info(' %s', label)
args_to_restart = [arg
for arg, label in zip(data['args'], data['labels'])
if label in labels_to_restart]
return new(name=data['name'],
arg=args_to_restart,
label=labels_to_restart,
tag=data['tags'],
**util.dicts.drop(data, ['name', 'args', 'labels', 'tags']))
else:
logging.info('nothing to retry')
def scp(src,
dst,
name=None,
group=None,
yes=False,
max_threads=0,
user='******'):
assert ':' in src + dst, 'you didnt specify a remote path, which starts with ":"'
if name:
_ips = list(ip(name))
else:
_ips = list(ips(group))
assert _ips, 'didnt find instances'
logging.info('targeting:')
for _ip in _ips:
logging.info(_ip)
logging.info('going to scp: %s to %s', src, dst)
if is_cli and not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
justify = max(len(_ip) for _ip in _ips)
def run(_ip, color):
if color:
color = getattr(util.colors, color)
else:
color = lambda x: x
name = (_ip + ': ').ljust(justify + 2)
def fn():
host = user + '@' + _ip
_src = host + src if src.startswith(':') else src
_dst = host + dst if dst.startswith(':') else dst
try:
shell.run(
'scp',
ssh_args,
_src,
_dst,
callback=lambda x: print(color(name + x), flush=True))
except:
failures.append(util.colors.red('failure: ') + _ip)
else:
successes.append(util.colors.green('success: ') + _ip)
return fn
failures = []
successes = []
pool.thread.wait(*map(
run, _ips,
itertools.cycle(util.colors._colors) if len(_ips) > 1 else [False]),
max_threads=max_threads)
logging.info('\nresults:')
for msg in successes + failures:
logging.info(' ' + msg)
if failures:
sys.exit(1)
def __call__(self, pagenum):
prompt = self.get_prompt(pagenum)
pager.echo(prompt)
options = self.get_options()
ch = pager.getch()
if ch in options:
return getattr(self, options[ch])(ch)
pager.echo('\r' + ' ' * (len(prompt) - 1) + '\r')
def __call__(self, pagenum):
prompt = self.get_prompt(pagenum)
pager.echo(prompt)
options = self.get_options()
ch = pager.getch()
if ch in options:
return getattr(self, options[ch])(ch)
pager.echo('\r' + ' '*(len(prompt)-1) + '\r')
def delete_table(name, yes=False):
"""
delete tables
"""
logging.info('going to delete table: %s', name)
if is_cli and not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
_client().delete_table(TableName=name)
def _prompt(pagenum, linesleft):
"""
Show default prompt to continue and process keypress.
It assumes terminal/console understands carriage return \r character.
"""
if linesleft > 1:
return False
prompt = "[Press esc or Q to stop listing, or any other key for more results... ] "
pager.echo(prompt)
try:
if pager.getch() in [pager.ESC_, pager.CTRL_C_, 'q', 'Q']:
pager.echo('\r' + ' '*(len(prompt)-1) + '\r')
return False
except KeyboardInterrupt:
# pager is supposed to catch ctrl+c but it doesn't appear to
sys.exit(0)
pager.echo('\r' + ' '*(len(prompt)-1) + '\r')
def pull(src, dst, name, filter=None, yes=False):
_ip = list(ip(name))[0]
logging.info('targeting:\n %s', _ip)
script = _tar_script(src, filter, echo_only=True)
cmd = ('cat %(script)s |ssh' + ssh_args +
'ubuntu@%(_ip)s bash -s') % locals()
logging.info('going to pull:')
logging.info(util.strings.indent(shell.check_output(cmd), 1))
shell.check_call('rm -rf', os.path.dirname(script))
if is_cli and not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
script = _tar_script(src, filter)
cmd = ('cd %(dst)s && cat %(script)s | ssh' + ssh_args +
'ubuntu@%(_ip)s bash -s | tar xf -') % locals()
try:
shell.check_call(cmd)
except:
logging.info('failure for: %s', _ip)
sys.exit(1)
finally:
shell.check_call('rm -rf', os.path.dirname(script))
def _print(t, text, wait=False):
print(t.clear)
print(text)
if wait:
pager.getch()
def _print(t, text, wait=False):
print(t.clear)
print(text)
if wait:
pager.getch()
def push(src,
dst,
name=None,
group=None,
filter=None,
yes=False,
max_threads=0,
user='******'):
if name:
_ips = list(ip(name))
else:
_ips = list(ips(group))
logging.info('targeting:')
for _ip in _ips:
logging.info(' %s', _ip)
logging.info(
'going to push:\n%s',
util.strings.indent(
shell.run('bash', _tar_script(src, filter, echo_only=True)), 1))
if is_cli and not yes:
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
script = _tar_script(src, filter)
failures = []
successes = []
justify = max(len(_ip) for _ip in _ips)
def run(_ip, color):
if color:
color = getattr(util.colors, color)
else:
color = lambda x: x
_name = (_ip + ': ').ljust(justify + 2)
def fn():
try:
shell.run(
'bash',
script,
'|ssh',
ssh_args,
user + '@' + _ip,
'"mkdir -p',
dst,
'&& cd',
dst,
'&& tar xf -"',
callback=lambda x: print(color(_name + x), flush=True))
except:
failures.append(util.colors.red('failure: ') + _ip)
else:
successes.append(util.colors.green('success: ') + _ip)
return fn
pool.thread.wait(*map(
run, _ips,
itertools.cycle(util.colors._colors) if len(_ips) > 1 else [False]),
max_threads=max_threads)
shell.check_call('rm -rf', os.path.dirname(script))
logging.info('\nresults:')
for msg in successes + failures:
logging.info(' ' + msg)
if failures:
sys.exit(1)
def ssh(
name=None,
group=None,
quiet: 'less output' = False,
cmd:
'cmd to run on remote host, can also be a file which will be read' = '',
yes: 'no prompt to proceed' = False,
max_threads: 'max ssh connections' = 20,
timeout: 'seconds before ssh cmd considered failed' = None,
no_tty: 'when backgrounding a process, you dont want a tty' = False,
user: '******' = 'ubuntu',
key: 'speficy ssh key' = None,
batch_mode:
'operate like there are many instances, even if only one' = False,
prefixed:
'when running against a single host, should streaming output be prefixed with name and ip' = False,
error_message:
'error message to print for a failed host, something like: {id} {name} {ip} {ipv4_private} failed' = '',
first_n=None,
last_n=None):
# tty means that when you ^C to exit, the remote processes are killed. this is usually what you want, ie no lingering `tail -f` instances.
# no_tty is the opposite, which is good for backgrounding processes, for example: `ec2 ssh $host -nyc 'bash cmd.sh </dev/null &>cmd.log &'
# TODO backgrounding appears to succeed, but ec2 ssh never exits, when targeting more than 1 host?
assert name or group
@_retry
def f():
if name:
x = list(ip(name))
else:
x = list(ips(group))
assert x, 'didnt find any ips'
return x
_ips = f()
if first_n:
_ips = _ips[:int(first_n)]
elif last_n:
_ips = _ips[-int(last_n):]
if os.path.exists(cmd):
with open(cmd) as f:
cmd = f.read()
if cmd == '-':
cmd = sys.stdin.read()
if cmd and 'set -e' not in cmd:
if cmd.startswith('#!'):
lines = cmd.splitlines()
lines.insert(1, 'set -e')
cmd = '\n'.join(lines)
else:
cmd = 'set -e\n' + cmd
if not (quiet and yes):
for _ip in _ips:
logging.info(_ip)
ssh_cmd = ('ssh -A' + (' -_ip {} '.format(key) if key else '') +
(' -tt ' if not no_tty or not cmd else ' -T ') +
ssh_args).split()
if timeout:
ssh_cmd = ['timeout', '{}s'.format(timeout)] + ssh_cmd
make_ssh_cmd = lambda _ip: ssh_cmd + [
user + '@' + _ip, _remote_cmd(cmd, _ip)
]
if is_cli and not yes and not (len(_ips) == 1 and not cmd):
logging.info('\nwould you like to proceed? y/n\n')
assert pager.getch() == 'y', 'abort'
try:
if len(_ips) > 1 or batch_mode:
failures = []
successes = []
results = []
def run(_ip):
def fn():
try:
shell.run(*make_ssh_cmd(_ip),
callback=_make_callback(_ip, quiet, results),
echo=False,
raw_cmd=True,
stream=False,
hide_stderr=quiet)
except:
if error_message:
print(error_message.format(_ip=_ip), flush=True)
msg = util.colors.red('failure: ') + _ip
failures.append(msg)
else:
msg = util.colors.green('success: ') + _ip
successes.append(msg)
if not quiet:
logging.info(msg)
return fn
pool.thread.wait(*map(run, _ips), max_threads=max_threads)
# TODO would be really nice to see these results, plus unknowns:, when ^C to exit early
if not quiet:
logging.info('\nresults:')
for msg in successes + failures:
logging.info(' ' + msg)
logging.info('\ntotals:')
logging.info(
util.colors.green(' successes: ') + str(len(successes)))
logging.info(
util.colors.red(' failures: ') + str(len(failures)))
if failures:
sys.exit(1)
else:
return results
elif cmd:
return shell.run(
*make_ssh_cmd(_ips[0]),
echo=False,
stream=not prefixed,
hide_stderr=quiet,
raw_cmd=True,
callback=_make_callback(_ips[0], quiet) if prefixed else None)
else:
subprocess.check_call(ssh_cmd + [user + '@' + _ips[0]])
except:
sys.exit(1)