def add_jumpbox(host):
if jb.cluster.locate(host):
return
print('Connecting to jumpbox:', host)
try:
t = ssh.connection_worker(host, None, init_data['auth'])
retries = 3
while not t.is_authenticated() and retries > 0:
# Try interactive password authentication
print('Failed to authenticate to Jumpbox (%s)' % host)
jb_user = raw_input('Enter username for [%s]: ' % host)
jb_passwd = getpass.getpass('Enter password for %s@%s: ' %
(jb_user, host))
reauth = AuthManager(jb_user,
auth_file=None,
include_agent=False,
include_userkeys=False,
default_password=jb_passwd)
t = ssh.connection_worker(host, None, reauth)
retries -= 1
except Exception as e:
print(host, repr(e))
finally:
jb.cluster.connections[host] = t
if len(sys.argv) < 2:
print('Usage: %s [--password=<password> ] host1 [...]' % sys.argv[0])
sys.exit(1)
# Setup stuff that the shell module would
password = '******'
socket.setdefaulttimeout(10)
hosts = sys.argv[1:]
if hosts[0].startswith('--password='******'root', default_password=password)
# Create a Console for streaming the results of running command output
# This can be suppressed via the console.quiet() call
console = RadSSHConsole()
# Create a list of tuples for the connections
# Leave the 2nd element as None to let RadSSH do the socket connections
connections = [(x, None) for x in hosts]
print('\nConnecting...')
C = Cluster(connections, login, console=console)
for host, status in C.status():
print('%14s : %s' % (str(host), status))
# Cluster connections and authentications established
def run(self):
jb = None
defaults = config.load_default_settings()
original_paramiko_agent = paramiko.Agent
with hp.a_temp_file() as fle:
if self.proxy and self.proxy_ssh_key:
fle.write("keyfile|{0}|{1}\n".format(self.proxy, self.proxy_ssh_key).encode('utf-8'))
if self.ssh_key:
fle.write("keyfile|*|{0}\n".format(self.ssh_key).encode('utf-8'))
fle.close()
auth_file = fle.name if (self.ssh_key or self.proxy_ssh_key) else None
if self.proxy:
jb = plugins.load_plugin(jumpbox.__file__)
jb.init(auth=AuthManager(self.proxy_ssh_user, auth_file=auth_file), defaults=defaults)
login = AuthManager(self.ssh_user, auth_file=auth_file, include_agent=True)
keys = {}
for key in login.agent_connection.get_keys():
ident = str(uuid.uuid1())
identity = type("Identity", (object, ), {
"__str__": lambda s: ident
, "get_name": lambda s: key.get_name()
, "asbytes": lambda s: key.asbytes()
, "sign_ssh_data": lambda s, *args, **kwargs: key.sign_ssh_data(*args, **kwargs)
})()
keys[identity] = key
login.deferred_keys[identity] = key
try:
outputs = defaultdict(lambda: {"stdout": [], "stderr": []})
class TwoQueue(object):
def __init__(self):
self.q = queue.Queue(300)
def put(self, thing):
(host, is_stderr), line = thing
outputs[host][["stdout", "stderr"][is_stderr]].append(line)
self.q.put(thing)
def __getattr__(self, key):
if key in ("q", "put"):
return object.__getattribute__(self, key)
else:
return getattr(self.q, key)
console = RadSSHConsole(q=TwoQueue())
connections = [(ip, None) for ip in self.ips]
if jb:
jb.add_jumpbox(self.proxy)
connections = list((ip, socket) for _, ip, socket in jb.do_jumpbox_connections(self.proxy, self.ips))
cluster = None
try:
log.info("Connecting")
authenticated = False
for _ in hp.until(timeout=120):
if authenticated:
break
cluster = Cluster(connections, login, console=console, defaults=defaults)
for _ in hp.until(timeout=10, step=0.5):
if not any(cluster.pending):
break
if cluster.pending:
raise BespinError("Timedout waiting to connect to some hosts", waiting_for=cluster.pending.keys())
for _ in hp.until(timeout=10, step=0.5):
connections = list(cluster.connections.values())
if any(isinstance(connection, socket.gaierror) for connection in connections):
raise BespinError("Some connections failed!", failures=[conn for conn in connections if isinstance(conn, socket.gaierror)])
if all(conn.authenticated for conn in connections):
break
authenticated = all(conn.authenticated for conn in cluster.connections.values())
if not authenticated:
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
log.info("Failed to authenticate will try to reconnect in 5 seconds\tunauthenticate=%s", unauthenticated)
time.sleep(5)
# Try to reauth if not authenticated yet
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
if unauthenticated:
for host in unauthenticated:
print('{0:14s} : {1}'.format(str(host), cluster.connections[host]))
raise BespinError("Timedout waiting to authenticate all the hosts, do you have an ssh-agent running?", unauthenticated=unauthenticated)
failed = []
for host, status in cluster.connections.items():
print('{0:14s} : {1}'.format(str(host), status))
if type(status) is socket.gaierror:
failed.append(host)
if failed:
raise BespinError("Failed to connect to some hosts", failed=failed)
cluster.run_command(self.command)
error = False
for host, job in cluster.last_result.items():
if not job.completed or job.result.return_code not in self.acceptable_return_codes:
log.error('%s -%s', host, cluster.connections[host])
log.error('%s, %s', job, job.result.status)
error = True
if error:
raise BespinError("Failed to run the commands")
return outputs
finally:
if cluster:
cluster.close_connections()
finally:
paramiko.Agent = original_paramiko_agent
def run(self):
jb = None
defaults = config.load_default_settings()
defaults['hostkey.verify'] = 'ignore'
original_paramiko_agent = paramiko.Agent
with hp.a_temp_file() as fle:
if self.proxy and self.proxy_ssh_key:
fle.write("keyfile|{0}|{1}\n".format(self.proxy, self.proxy_ssh_key).encode('utf-8'))
if self.ssh_key:
fle.write("keyfile|*|{0}\n".format(self.ssh_key).encode('utf-8'))
fle.close()
auth_file = fle.name if (self.ssh_key or self.proxy_ssh_key) else None
if self.proxy:
jb = plugins.load_plugin(jumpbox.__file__)
jb.init(auth=AuthManager(self.proxy_ssh_user, auth_file=auth_file), defaults=defaults)
login = AuthManager(self.ssh_user, auth_file=auth_file, include_agent=True)
keys = {}
for key in login.agent_connection.get_keys():
ident = str(uuid.uuid1())
identity = type("Identity", (object, ), {
"__str__": lambda s: ident
, "get_name": lambda s: key.get_name()
, "asbytes": lambda s: key.asbytes()
, "sign_ssh_data": lambda s, *args, **kwargs: key.sign_ssh_data(*args, **kwargs)
})()
keys[identity] = key
login.deferred_keys[identity] = key
# Diry dirty hack
# Waiting for https://github.com/radssh/radssh/pull/10
paramiko.Agent = type("AgentConnection", (object, ), {"get_keys": lambda *args: keys.keys()})
try:
console = RadSSHConsole()
connections = [(ip, None) for ip in self.ips]
if jb:
jb.add_jumpbox(self.proxy)
connections = list((ip, socket) for _, ip, socket in jb.do_jumpbox_connections(self.proxy, self.ips))
cluster = None
try:
log.info("Connecting")
authenticated = False
for _ in hp.until(timeout=120):
if authenticated:
break
cluster = Cluster(connections, login, console=console, defaults=defaults)
for _ in hp.until(timeout=10, step=0.5):
if not any(cluster.pending):
break
if cluster.pending:
raise BespinError("Timedout waiting to connect to some hosts", waiting_for=cluster.pending.keys())
for _ in hp.until(timeout=10, step=0.5):
if all(conn.authenticated for conn in cluster.connections.values()):
break
authenticated = all(conn.authenticated for conn in cluster.connections.values())
if not authenticated:
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
log.info("Failed to authenticate will try to reconnect in 5 seconds\tunauthenticate=%s", unauthenticated)
time.sleep(5)
# Try to reauth if not authenticated yet
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
if unauthenticated:
for host in unauthenticated:
print('{0:14s} : {1}'.format(str(host), cluster.connections[host]))
raise BespinError("Timedout waiting to authenticate all the hosts, do you have an ssh-agent running?", unauthenticated=unauthenticated)
failed = []
for host, status in cluster.connections.items():
print('{0:14s} : {1}'.format(str(host), status))
if type(status) is socket.gaierror:
failed.append(host)
if failed:
raise BespinError("Failed to connect to some hosts", failed=failed)
cluster.run_command(self.command)
error = False
for host, job in cluster.last_result.items():
if not job.completed or job.result.return_code not in self.acceptable_return_codes:
log.error('%s -%s', host, cluster.connections[host])
log.error('%s, %s', job, job.result.status)
error = True
if error:
raise BespinError("Failed to run the commands")
finally:
if cluster:
cluster.close_connections()
finally:
paramiko.Agent = original_paramiko_agent
def run(self):
jb = None
defaults = config.load_default_settings()
defaults['hostkey.verify'] = 'ignore'
original_paramiko_agent = paramiko.Agent
with hp.a_temp_file() as fle:
if self.proxy and self.proxy_ssh_key:
fle.write("keyfile|{0}|{1}\n".format(self.proxy, self.proxy_ssh_key).encode('utf-8'))
if self.ssh_key:
fle.write("keyfile|*|{0}\n".format(self.ssh_key).encode('utf-8'))
fle.close()
auth_file = fle.name if (self.ssh_key or self.proxy_ssh_key) else None
if self.proxy:
jb = plugins.load_plugin(jumpbox.__file__)
jb.init(auth=AuthManager(self.proxy_ssh_user, auth_file=auth_file), defaults=defaults)
login = AuthManager(self.ssh_user, auth_file=auth_file, include_agent=True)
keys = {}
for key in login.agent_connection.get_keys():
ident = str(uuid.uuid1())
identity = type("Identity", (object, ), {
"__str__": lambda s: ident
, "get_name": lambda s: key.get_name()
, "asbytes": lambda s: key.asbytes()
, "sign_ssh_data": lambda s, *args, **kwargs: key.sign_ssh_data(*args, **kwargs)
})()
keys[identity] = key
login.deferred_keys[identity] = key
try:
outputs = defaultdict(lambda: {"stdout": [], "stderr": []})
class TwoQueue(object):
def __init__(self):
self.q = queue.Queue(300)
def put(self, thing):
(host, is_stderr), line = thing
outputs[host][["stdout", "stderr"][is_stderr]].append(line)
self.q.put(thing)
def __getattr__(self, key):
if key in ("q", "put"):
return object.__getattribute__(self, key)
else:
return getattr(self.q, key)
console = RadSSHConsole(q=TwoQueue())
connections = [(ip, None) for ip in self.ips]
if jb:
jb.add_jumpbox(self.proxy)
connections = list((ip, socket) for _, ip, socket in jb.do_jumpbox_connections(self.proxy, self.ips))
cluster = None
try:
log.info("Connecting")
authenticated = False
for _ in hp.until(timeout=120):
if authenticated:
break
cluster = Cluster(connections, login, console=console, defaults=defaults)
for _ in hp.until(timeout=10, step=0.5):
if not any(cluster.pending):
break
if cluster.pending:
raise BespinError("Timedout waiting to connect to some hosts", waiting_for=cluster.pending.keys())
for _ in hp.until(timeout=10, step=0.5):
connections = list(cluster.connections.values())
if any(isinstance(connection, socket.gaierror) for connection in connections):
raise BespinError("Some connections failed!", failures=[conn for conn in connections if isinstance(conn, socket.gaierror)])
if all(conn.authenticated for conn in connections):
break
authenticated = all(conn.authenticated for conn in cluster.connections.values())
if not authenticated:
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
log.info("Failed to authenticate will try to reconnect in 5 seconds\tunauthenticate=%s", unauthenticated)
time.sleep(5)
# Try to reauth if not authenticated yet
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
if unauthenticated:
for host in unauthenticated:
print('{0:14s} : {1}'.format(str(host), cluster.connections[host]))
raise BespinError("Timedout waiting to authenticate all the hosts, do you have an ssh-agent running?", unauthenticated=unauthenticated)
failed = []
for host, status in cluster.connections.items():
print('{0:14s} : {1}'.format(str(host), status))
if type(status) is socket.gaierror:
failed.append(host)
if failed:
raise BespinError("Failed to connect to some hosts", failed=failed)
cluster.run_command(self.command)
error = False
for host, job in cluster.last_result.items():
if not job.completed or job.result.return_code not in self.acceptable_return_codes:
log.error('%s -%s', host, cluster.connections[host])
log.error('%s, %s', job, job.result.status)
error = True
if error:
raise BespinError("Failed to run the commands")
return outputs
finally:
if cluster:
cluster.close_connections()
finally:
paramiko.Agent = original_paramiko_agent