def run(self):
jb = None
defaults = config.load_default_settings()
defaults['hostkey.verify'] = 'ignore'
original_paramiko_agent = paramiko.Agent
with hp.a_temp_file() as fle:
if self.proxy and self.proxy_ssh_key:
fle.write("keyfile|{0}|{1}\n".format(self.proxy, self.proxy_ssh_key).encode('utf-8'))
if self.ssh_key:
fle.write("keyfile|*|{0}\n".format(self.ssh_key).encode('utf-8'))
fle.close()
auth_file = fle.name if (self.ssh_key or self.proxy_ssh_key) else None
if self.proxy:
jb = plugins.load_plugin(jumpbox.__file__)
jb.init(auth=AuthManager(self.proxy_ssh_user, auth_file=auth_file), defaults=defaults)
login = AuthManager(self.ssh_user, auth_file=auth_file, include_agent=True)
keys = {}
for key in login.agent_connection.get_keys():
ident = str(uuid.uuid1())
identity = type("Identity", (object, ), {
"__str__": lambda s: ident
, "get_name": lambda s: key.get_name()
, "asbytes": lambda s: key.asbytes()
, "sign_ssh_data": lambda s, *args, **kwargs: key.sign_ssh_data(*args, **kwargs)
})()
keys[identity] = key
login.deferred_keys[identity] = key
# Diry dirty hack
# Waiting for https://github.com/radssh/radssh/pull/10
paramiko.Agent = type("AgentConnection", (object, ), {"get_keys": lambda *args: keys.keys()})
try:
console = RadSSHConsole()
connections = [(ip, None) for ip in self.ips]
if jb:
jb.add_jumpbox(self.proxy)
connections = list((ip, socket) for _, ip, socket in jb.do_jumpbox_connections(self.proxy, self.ips))
cluster = None
try:
log.info("Connecting")
authenticated = False
for _ in hp.until(timeout=120):
if authenticated:
break
cluster = Cluster(connections, login, console=console, defaults=defaults)
for _ in hp.until(timeout=10, step=0.5):
if not any(cluster.pending):
break
if cluster.pending:
raise BespinError("Timedout waiting to connect to some hosts", waiting_for=cluster.pending.keys())
for _ in hp.until(timeout=10, step=0.5):
if all(conn.authenticated for conn in cluster.connections.values()):
break
authenticated = all(conn.authenticated for conn in cluster.connections.values())
if not authenticated:
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
log.info("Failed to authenticate will try to reconnect in 5 seconds\tunauthenticate=%s", unauthenticated)
time.sleep(5)
# Try to reauth if not authenticated yet
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
if unauthenticated:
for host in unauthenticated:
print('{0:14s} : {1}'.format(str(host), cluster.connections[host]))
raise BespinError("Timedout waiting to authenticate all the hosts, do you have an ssh-agent running?", unauthenticated=unauthenticated)
failed = []
for host, status in cluster.connections.items():
print('{0:14s} : {1}'.format(str(host), status))
if type(status) is socket.gaierror:
failed.append(host)
if failed:
raise BespinError("Failed to connect to some hosts", failed=failed)
cluster.run_command(self.command)
error = False
for host, job in cluster.last_result.items():
if not job.completed or job.result.return_code not in self.acceptable_return_codes:
log.error('%s -%s', host, cluster.connections[host])
log.error('%s, %s', job, job.result.status)
error = True
if error:
raise BespinError("Failed to run the commands")
finally:
if cluster:
cluster.close_connections()
finally:
paramiko.Agent = original_paramiko_agent
def run(self):
jb = None
defaults = config.load_default_settings()
original_paramiko_agent = paramiko.Agent
with hp.a_temp_file() as fle:
if self.proxy and self.proxy_ssh_key:
fle.write("keyfile|{0}|{1}\n".format(self.proxy, self.proxy_ssh_key).encode('utf-8'))
if self.ssh_key:
fle.write("keyfile|*|{0}\n".format(self.ssh_key).encode('utf-8'))
fle.close()
auth_file = fle.name if (self.ssh_key or self.proxy_ssh_key) else None
if self.proxy:
jb = plugins.load_plugin(jumpbox.__file__)
jb.init(auth=AuthManager(self.proxy_ssh_user, auth_file=auth_file), defaults=defaults)
login = AuthManager(self.ssh_user, auth_file=auth_file, include_agent=True)
keys = {}
for key in login.agent_connection.get_keys():
ident = str(uuid.uuid1())
identity = type("Identity", (object, ), {
"__str__": lambda s: ident
, "get_name": lambda s: key.get_name()
, "asbytes": lambda s: key.asbytes()
, "sign_ssh_data": lambda s, *args, **kwargs: key.sign_ssh_data(*args, **kwargs)
})()
keys[identity] = key
login.deferred_keys[identity] = key
try:
outputs = defaultdict(lambda: {"stdout": [], "stderr": []})
class TwoQueue(object):
def __init__(self):
self.q = queue.Queue(300)
def put(self, thing):
(host, is_stderr), line = thing
outputs[host][["stdout", "stderr"][is_stderr]].append(line)
self.q.put(thing)
def __getattr__(self, key):
if key in ("q", "put"):
return object.__getattribute__(self, key)
else:
return getattr(self.q, key)
console = RadSSHConsole(q=TwoQueue())
connections = [(ip, None) for ip in self.ips]
if jb:
jb.add_jumpbox(self.proxy)
connections = list((ip, socket) for _, ip, socket in jb.do_jumpbox_connections(self.proxy, self.ips))
cluster = None
try:
log.info("Connecting")
authenticated = False
for _ in hp.until(timeout=120):
if authenticated:
break
cluster = Cluster(connections, login, console=console, defaults=defaults)
for _ in hp.until(timeout=10, step=0.5):
if not any(cluster.pending):
break
if cluster.pending:
raise BespinError("Timedout waiting to connect to some hosts", waiting_for=cluster.pending.keys())
for _ in hp.until(timeout=10, step=0.5):
connections = list(cluster.connections.values())
if any(isinstance(connection, socket.gaierror) for connection in connections):
raise BespinError("Some connections failed!", failures=[conn for conn in connections if isinstance(conn, socket.gaierror)])
if all(conn.authenticated for conn in connections):
break
authenticated = all(conn.authenticated for conn in cluster.connections.values())
if not authenticated:
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
log.info("Failed to authenticate will try to reconnect in 5 seconds\tunauthenticate=%s", unauthenticated)
time.sleep(5)
# Try to reauth if not authenticated yet
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
if unauthenticated:
for host in unauthenticated:
print('{0:14s} : {1}'.format(str(host), cluster.connections[host]))
raise BespinError("Timedout waiting to authenticate all the hosts, do you have an ssh-agent running?", unauthenticated=unauthenticated)
failed = []
for host, status in cluster.connections.items():
print('{0:14s} : {1}'.format(str(host), status))
if type(status) is socket.gaierror:
failed.append(host)
if failed:
raise BespinError("Failed to connect to some hosts", failed=failed)
cluster.run_command(self.command)
error = False
for host, job in cluster.last_result.items():
if not job.completed or job.result.return_code not in self.acceptable_return_codes:
log.error('%s -%s', host, cluster.connections[host])
log.error('%s, %s', job, job.result.status)
error = True
if error:
raise BespinError("Failed to run the commands")
return outputs
finally:
if cluster:
cluster.close_connections()
finally:
paramiko.Agent = original_paramiko_agent
def run(self):
jb = None
defaults = config.load_default_settings()
defaults['hostkey.verify'] = 'ignore'
original_paramiko_agent = paramiko.Agent
with hp.a_temp_file() as fle:
if self.proxy and self.proxy_ssh_key:
fle.write("keyfile|{0}|{1}\n".format(self.proxy, self.proxy_ssh_key).encode('utf-8'))
if self.ssh_key:
fle.write("keyfile|*|{0}\n".format(self.ssh_key).encode('utf-8'))
fle.close()
auth_file = fle.name if (self.ssh_key or self.proxy_ssh_key) else None
if self.proxy:
jb = plugins.load_plugin(jumpbox.__file__)
jb.init(auth=AuthManager(self.proxy_ssh_user, auth_file=auth_file), defaults=defaults)
login = AuthManager(self.ssh_user, auth_file=auth_file, include_agent=True)
keys = {}
for key in login.agent_connection.get_keys():
ident = str(uuid.uuid1())
identity = type("Identity", (object, ), {
"__str__": lambda s: ident
, "get_name": lambda s: key.get_name()
, "asbytes": lambda s: key.asbytes()
, "sign_ssh_data": lambda s, *args, **kwargs: key.sign_ssh_data(*args, **kwargs)
})()
keys[identity] = key
login.deferred_keys[identity] = key
try:
outputs = defaultdict(lambda: {"stdout": [], "stderr": []})
class TwoQueue(object):
def __init__(self):
self.q = queue.Queue(300)
def put(self, thing):
(host, is_stderr), line = thing
outputs[host][["stdout", "stderr"][is_stderr]].append(line)
self.q.put(thing)
def __getattr__(self, key):
if key in ("q", "put"):
return object.__getattribute__(self, key)
else:
return getattr(self.q, key)
console = RadSSHConsole(q=TwoQueue())
connections = [(ip, None) for ip in self.ips]
if jb:
jb.add_jumpbox(self.proxy)
connections = list((ip, socket) for _, ip, socket in jb.do_jumpbox_connections(self.proxy, self.ips))
cluster = None
try:
log.info("Connecting")
authenticated = False
for _ in hp.until(timeout=120):
if authenticated:
break
cluster = Cluster(connections, login, console=console, defaults=defaults)
for _ in hp.until(timeout=10, step=0.5):
if not any(cluster.pending):
break
if cluster.pending:
raise BespinError("Timedout waiting to connect to some hosts", waiting_for=cluster.pending.keys())
for _ in hp.until(timeout=10, step=0.5):
connections = list(cluster.connections.values())
if any(isinstance(connection, socket.gaierror) for connection in connections):
raise BespinError("Some connections failed!", failures=[conn for conn in connections if isinstance(conn, socket.gaierror)])
if all(conn.authenticated for conn in connections):
break
authenticated = all(conn.authenticated for conn in cluster.connections.values())
if not authenticated:
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
log.info("Failed to authenticate will try to reconnect in 5 seconds\tunauthenticate=%s", unauthenticated)
time.sleep(5)
# Try to reauth if not authenticated yet
unauthenticated = [host for host, conn in cluster.connections.items() if not conn.authenticated]
if unauthenticated:
for host in unauthenticated:
print('{0:14s} : {1}'.format(str(host), cluster.connections[host]))
raise BespinError("Timedout waiting to authenticate all the hosts, do you have an ssh-agent running?", unauthenticated=unauthenticated)
failed = []
for host, status in cluster.connections.items():
print('{0:14s} : {1}'.format(str(host), status))
if type(status) is socket.gaierror:
failed.append(host)
if failed:
raise BespinError("Failed to connect to some hosts", failed=failed)
cluster.run_command(self.command)
error = False
for host, job in cluster.last_result.items():
if not job.completed or job.result.return_code not in self.acceptable_return_codes:
log.error('%s -%s', host, cluster.connections[host])
log.error('%s, %s', job, job.result.status)
error = True
if error:
raise BespinError("Failed to run the commands")
return outputs
finally:
if cluster:
cluster.close_connections()
finally:
paramiko.Agent = original_paramiko_agent