def make_client(auth, hosts):
if auth['method'] == 'password':
return ParallelSSHClient(hosts.values(),
user=auth['user'],
password=auth['details']['password'])
raise RuntimeError('unsupported auth method: %s' % auth['method'])
def setup_server(server, key):
client = ParallelSSHClient([server],
user='******',
pkey=key,
allow_agent=False)
output = client.run_command('rm -rf ~/server_config')
utils.print_pssh_output(output)
output = client.run_command('mkdir ~/server_config')
utils.print_pssh_output(output)
for f in os.listdir(SERVER_CONFIG_DIR):
dest_path = get_server_config_path(f.split('/')[-1])
local_path = os.path.join(SERVER_CONFIG_DIR, f)
fileUpload.upload_file_no_pssh([server], key, local_path, dest_path)
print(f'Wrote file {local_path} to {server}')
output = client.run_command('bash ~/server_config/setup_new_server.sh')
utils.print_pssh_output(output)
with open(utils.get_project_path('all_servers.txt')) as f:
hostnames = {host.strip() for host in f.readlines()}
hostnames.add(server)
hostnames = sorted(hostnames)
with open(utils.get_project_path('all_servers.txt'), 'w') as f:
f.write('\n'.join(hostnames) + '\n')
print(
f"Added {server} to the fleet. Don't forget to commit autogenerated changes to all_servers.txt."
)
def command(self, command, timeout=60, sudo=False):
"""Execute a command on the instances. This will be done using an ssh command and potentially with sudo"""
logger.debug(f'Executing {command} with sudo {sudo}.')
client = ParallelSSHClient([i.ip for i in self.hosts], pkey=self.keysfile)
output = client.run_command(command, read_timeout=timeout, sudo=sudo)
client.join()
return output
async def ssh(ctx, host, user, password, command):
client = ParallelSSHClient(host, user=user, password=password)
output = client.run_command(command)
for host_output in output:
output = "\n".join(host_output.stdout)
await ctx.send(output)
exit_code = host_output.exit_code
def shell_cmd(cloud_name, machine_id, cmd):
from pssh.clients import ParallelSSHClient
##util.message("# " + str(cmd), "info")
aa, bb, cc, dd, describe, ff, gg = read(cloud_name, machine_id)
if describe == None:
util.message("Node " + str(machine_id) + " not found", "error")
return
key_name = describe['key_name']
host = describe['public_ip']
hosts = host.split()
username, pkey = key.read(key_name)
if username == None:
util.message("key file not found", "error")
return
##util.message("host=" + str(hosts) + ", user="******", pkey=" + str(pkey), "info")
client = ParallelSSHClient(hosts, user=username, pkey=pkey)
output = client.run_command(cmd, use_pty=True, read_timeout=3600)
for host_out in output:
try:
for line in host_out.stdout:
print(line)
except:
time.sleep(3)
continue
return
def connect_ssh(self, retries=6, user="******"):
kwargs = {"num_retries": retries, "timeout": 10000}
if hasattr(self, "host_config"):
kwargs["host_config"] = self.host_config
else:
kwargs["user"] = user
self.client = ParallelSSHClient(self.hosts, **kwargs)
def _create_cluster_ssh_conns(self):
hostips = [n["publicIpAddress"] for n in self.cluster_nodes]
hostconfigs = [HostConfig(port=n["port"]) for n in self.cluster_nodes]
self._all_ssh = ParallelSSHClient(hostips,
host_config=hostconfigs,
user=self.admin_username)
self._master_ssh = ParallelSSHClient(hostips[:1],
host_config=hostconfigs[:1],
user=self.admin_username)
self._master_scp = SSHClient(hostips[0],
port=hostconfigs[0].port,
user=self.admin_username)
def sar(command, directory, hosts, filenames):
# "keyname": "/Users/mb/Downloads/bilal-us-east.pem.txt"
config = json.load(open('config.json'))
client = ParallelSSHClient(hosts,
user=config["username"],
password="",
pkey=config["keyname"],
timeout=600,
pool_size=64)
if command == "start":
start_parallel(client)
elif command == "terminate":
stop_parallel(client)
elif command == "collectcsv":
count = 0
for host in hosts:
sar_helper(host, directory, filenames[count], command, config)
count += 1
# getfiles_parallel(client, framework, len(hosts), confFile, hosts)
else:
export_parallel(client)
def __init__(self, external_init_dict=None):
self.logger = logging.getLogger(constants.logging_name)
init_dict = {}
clsname = self.__class__.__name__
if external_init_dict is not None:
self.logger.debug(
"{}: initializing from external dict".format(clsname))
init_dict = external_init_dict
else:
self.logger.debug(
"{}: initializing with default values".format(clsname))
self.hostname = constants.hpc_hostname
self.user = constants.user
self.home_dir = os.path.join(constants.cc_working_dir, self.user)
self.src_data_path = init_dict.get("src_data_path", "./data")
self.template_path = constants.template_path
self.logger.debug("Host being used is {}, under username {}".format(
self.hostname, self.user))
self.keypath = init_dict.get("ssh_key_filename",
constants.ssh_key_filename)
self.client = ParallelSSHClient([self.hostname],
pkey=self.keypath,
user=self.user,
keepalive_seconds=300)
self.remote_abs_working_folder = None
self.remote_working_folder = None
self.active_dataset_name = None
self.live_job_id = None
def test_agent_forwarding(self):
client = ParallelSSHClient(['localhost'], forward_ssh_agent=True,
port=self.port)
output = client.run_command(self.cmd)
stdout = [list(output['localhost'].stdout) for k in output]
expected_stdout = [[self.resp]]
self.assertListEqual(stdout, expected_stdout)
def _exec_scp(self):
client = ParallelSSHClient(self.hosts, port=self.port)
output = client.copy_file(self.source, self.destination, True)
joinall(output, raise_error=True)
nice_output = dict()
for host in output:
nice_output[host] = {'stdout': [], 'stderr': []}
return nice_output
def useSCP(args):
enable_host_logger()
hosts = utils.collect_hostnames(args.servers)
client = ParallelSSHClient(hosts,
user='******',
pkey=args.key,
allow_agent=False)
upload_file(client, args.file, args.destination)
def tryssh(url):
try:
client = ParallelSSHClient([url], user="******")
client.run_command('ls')
except pssh.exceptions.AuthenticationException:
return True
except pssh.exceptions.ConnectionErrorException:
return False
def get_client():
return ParallelSSHClient(
list(host_selected.keys()),
host_config=host_selected,
num_retries=num_retries,
retry_delay=retry_delay,
pool_size=10 if len(host_selected) < 10 else len(host_selected),
)
def ssh_all_hosts(self, command):
""" Sends given command to remote hosts via ssh
Parameters:
command: command to be sent over ssh """
hosts = self.addresses
client = ParallelSSHClient(hosts, user='******')
client.run_command(command)
client.join()
def ssh(hosts, command):
client = ParallelSSHClient(hosts, user='******', password='******')
output = client.run_command(command)
for host, host_output in output.items():
for line in host_output.stdout:
print line
f = open("Computer_Parseble_OUTPUT", "w")
f.write(str(output))
f.close()
def generate_client(self, hosts):
host_config = {}
for host in hosts:
host_config[host] = {}
host_config[host]['user'] = sv.HOSTS_CONFIG[host]['user']
host_config[host]['password'] = sv.HOSTS_CONFIG[host]['password']
if sv.HOSTS_CONFIG[host].get('port'):
host_config[host]['port'] = sv.HOSTS_CONFIG[host]['port']
return ParallelSSHClient(hosts, host_config=host_config)
def fix_hostnames():
hosts = get_ips()
client = ParallelSSHClient(hosts, user=USER)
host_args = [{
"cmd":
"sudo hostnamectl set-hostname synthetic-bot-up-%s" % (i, )
} for i in range(len(hosts))]
output = client.run_command("%(cmd)s", host_args=host_args)
client.join()
def get_pssh_client(host,
private_key_path,
single_copy=False,
port=8022,
user='******'):
if single_copy and isinstance(host, str):
logger.debug('Trying single copy')
return SSHClient(host, port=port, pkey=private_key_path, user=user)
return ParallelSSHClient(host, port=port, pkey=private_key_path, user=user)
def defineConnection(user, password, hosts):
print(f'Defining connection with hosts ({hosts})...', flush=True)
clients = ParallelSSHClient(hosts, user=user, password=password)
output = clients.connect_auth()
joinall(output, raise_error=True)
print('OK!', flush=True)
return clients
def go():
command = 'uname -a'
nodes = ['localhost']
pw = getpass.getpass()
client = ParallelSSHClient(nodes, password=pw, timeout=3)
client.run_command(command)
output = client.get_last_output()
for node in output:
for line in output[node]['stdout']:
print('{0} {1}'.format(node, line))
def login_with_pool(self, ipaddress, port, user_passwd_pair_list, pool_size=10):
for user_passwd_pair in user_passwd_pair_list:
try:
client = ParallelSSHClient(hosts=[ipaddress], port=port, user=user_passwd_pair[0],
password=user_passwd_pair[1], num_retries=0, timeout=self.timeout,
pool_size=pool_size)
output = client.run_command('whoami', timeout=self.timeout)
log_success("SSH", ipaddress, port, user_passwd_pair)
except Exception as E:
logger.debug('AuthenticationException: ssh')
continue
finally:
pass
def run_command(command,
hosts,
user,
verbose=False,
proxy_host=None,
timeout=10,
**kwargs):
"""Run ssh command using Parallel SSH."""
result = {"0": [], "1": []}
if proxy_host:
client = ParallelSSHClient(hosts,
user='******',
pkey=SSH_KEY,
proxy_host=proxy_host,
proxy_user=user,
proxy_pkey=SSH_KEY,
timeout=timeout)
else:
client = ParallelSSHClient(hosts,
user=user,
pkey=SSH_KEY,
timeout=timeout)
output = client.run_command(command, stop_on_errors=False, **kwargs)
client.join(output)
# output = pssh.output.HostOutput objects list
for host in output:
if host.exit_code == 0:
if verbose and host.stdout:
for line in host.stdout:
print(line)
result['0'].append(host.host)
elif host.host is not None:
result['1'].append(host.host)
# find hosts that have raised Exception (Authentication, Connection)
# host.exception = pssh.exceptions.* & host.host = None
failed_hosts = list(set(hosts) - set(sum(result.values(), [])))
if failed_hosts:
result['1'].extend(failed_hosts)
return result
def ssh(host_list, cmd, **kargs):
try:
client = ParallelSSHClient(host_list,
user='******',
pkey=f'{sq(args.ssh_key)}')
output = client.run_command(cmd, **kargs)
for host in output:
logger.info(host)
for line in output[host]['stdout']:
logger.info(line)
except:
logger.info('cannot connect to all the hosts')
return
def __init__(self, timeout, log: Log, hostNames=None, hostFile=None, quiet=False, only=[]):
if hostFile and hostNames:
raise ValueError('Cannot specify both "hostNames" and "hostFile"')
elif hostNames:
self.client = ParallelSSHClient(list(hostNames), timeout=timeout)
self.hostConfig = None
elif hostFile:
self.hostConfig = getHostConfig(hostFile)
self.client = ParallelSSHClient(
list(self.hostConfig.keys()), host_config=self.hostConfig, timeout=timeout)
else:
raise ValueError('One of "hostNames" or "hostFile" is required')
if only:
for host in list(self.client.hosts):
if str(host) in only or self._hostName(host) in only:
continue
self.client.hosts.remove(host)
self.quiet = quiet
self.log = log
self.pool = ThreadPoolExecutor()
self.justification = max(
len(self._hostName(host)) for host in (hostNames or self.hostConfig.keys()))
def _exec_ssh(self, cmd):
client = ParallelSSHClient(self.hosts,
user=self.username,
pkey=self.pkey,
password=self.password,
port=self.port)
output = client.run_command(cmd, sudo=self.sudo)
nice_output = dict()
for host_output in output:
host = host_output.host
nice_output[host] = {'stdout': [], 'stderr': []}
nice_output[host]['stdout'] = list(host_output.stdout)
nice_output[host]['stderr'] = list(host_output.stderr)
return nice_output
def parallel_ssh(host, user, password, command):
#enable_host_logger()
client = ParallelSSHClient([host], user=user, password=password)
output = client.run_command(command)
client.join()
stdout = ""
for host_output in output:
if host_output.exit_code != 0:
raise Exception("host returned exit code " +
str(host_output.exit_code))
stdout_li = list(host_output.stdout)
for line in stdout_li:
stdout += line + "\n"
return stdout
def login(self, ipaddress, port, user_passwd_pair_list):
parallel_logger = logging.getLogger("parallel")
parallel_logger.setLevel('CRITICAL')
for user_passwd_pair in user_passwd_pair_list:
try:
client = ParallelSSHClient(hosts=[ipaddress], port=port, user=user_passwd_pair[0],
password=user_passwd_pair[1], num_retries=0, timeout=self.timeout)
output = client.run_command('whoami', timeout=self.timeout)
log_success("SSH", ipaddress, port, user_passwd_pair)
except Exception as E:
logger.debug('AuthenticationException: ssh')
continue
finally:
pass
def run_command(command,
hosts,
user,
verbose=False,
proxy_host=None,
timeout=10,
**kwargs):
"""Run ssh command using Parallel SSH."""
result = {"0": [], "1": []}
if proxy_host:
client = ParallelSSHClient(hosts,
user='******',
proxy_host=proxy_host,
proxy_user=user,
timeout=timeout)
else:
client = ParallelSSHClient(hosts, user=user, timeout=timeout)
output = client.run_command(command, stop_on_errors=False, **kwargs)
client.join(output)
for host in hosts:
if host not in output:
# Pssh AuthenticationException duplicate output dict key
# {'saclay.iot-lab.info': {'exception': ...},
# {'saclay.iot-lab.info_qzhtyxlt': {'exception': ...}}
site = next(iter(sorted(output)))
raise OpenA8SshAuthenticationException(site)
result['0' if output[host]['exit_code'] == 0 else '1'].append(host)
if verbose:
for host in hosts:
# Pssh >= 1.0.0: stdout is None instead of generator object
# when you have ConnectionErrorException
stdout = output[host].get('stdout')
if stdout:
for _ in stdout:
pass
return result
def condor_retire(self):
"""Retire the instances from condor. This means that jobs keep running on them, but they will
not accept new jobs."""
status = self.condor_status()
commands = {}
for host in self.hosts:
for s in status:
if s['host']==host.name:
commands[host.name] = s['fullhost']
commandlist = [f"condor_off -startd -peaceful {c}" for c in commands.values()]
logger.debug(commandlist)
server = ['server']*len(commandlist)
client = ParallelSSHClient(server, pkey=self.keysfile)
output = client.run_command('%s', host_args=commandlist, sudo=True)
